?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
kernel/extp{read,write}v: Change ioccnt from u_int to int.
[dragonfly.git] / sys / netinet / in_pcb.c
blob50c42560dd933af5719305f0a883ef7ffdc71205
1 /*
2 * Copyright (c) 2004 Jeffrey M. Hsu. All rights reserved.
3 * Copyright (c) 2004 The DragonFly Project. All rights reserved.
4 *
5 * This code is derived from software contributed to The DragonFly Project
6 * by Jeffrey M. Hsu.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 * 3. Neither the name of The DragonFly Project nor the names of its
17 * contributors may be used to endorse or promote products derived
18 * from this software without specific, prior written permission.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
23 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
24 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
25 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
26 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
27 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
28 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
29 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
30 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
31 * SUCH DAMAGE.
32 */
33
34 /*
35 * Copyright (c) 1982, 1986, 1991, 1993, 1995
36 * The Regents of the University of California. All rights reserved.
37 *
38 * Redistribution and use in source and binary forms, with or without
39 * modification, are permitted provided that the following conditions
40 * are met:
41 * 1. Redistributions of source code must retain the above copyright
42 * notice, this list of conditions and the following disclaimer.
43 * 2. Redistributions in binary form must reproduce the above copyright
44 * notice, this list of conditions and the following disclaimer in the
45 * documentation and/or other materials provided with the distribution.
46 * 3. Neither the name of the University nor the names of its contributors
47 * may be used to endorse or promote products derived from this software
48 * without specific prior written permission.
49 *
50 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
51 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
52 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
53 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
54 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
55 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
56 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
57 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
58 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
59 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
60 * SUCH DAMAGE.
61 *
62 * @(#)in_pcb.c 8.4 (Berkeley) 5/24/95
63 * $FreeBSD: src/sys/netinet/in_pcb.c,v 1.59.2.27 2004/01/02 04:06:42 ambrisko Exp $
64 */
65
66 #include "opt_ipsec.h"
67 #include "opt_inet6.h"
68
69 #include <sys/param.h>
70 #include <sys/systm.h>
71 #include <sys/malloc.h>
72 #include <sys/mbuf.h>
73 #include <sys/domain.h>
74 #include <sys/protosw.h>
75 #include <sys/socket.h>
76 #include <sys/socketvar.h>
77 #include <sys/proc.h>
78 #include <sys/priv.h>
79 #include <sys/jail.h>
80 #include <sys/kernel.h>
81 #include <sys/sysctl.h>
82
83 #include <sys/thread2.h>
84 #include <sys/socketvar2.h>
85 #include <sys/msgport2.h>
86
87 #include <machine/limits.h>
88
89 #include <net/if.h>
90 #include <net/if_types.h>
91 #include <net/route.h>
92 #include <net/netisr2.h>
93 #include <net/toeplitz2.h>
94
95 #include <netinet/in.h>
96 #include <netinet/in_pcb.h>
97 #include <netinet/in_var.h>
98 #include <netinet/ip_var.h>
99 #ifdef INET6
100 #include <netinet/ip6.h>
101 #include <netinet6/ip6_var.h>
102 #endif /* INET6 */
103
104 #ifdef IPSEC
105 #include <netinet6/ipsec.h>
106 #include <netproto/key/key.h>
107 #include <netproto/ipsec/esp_var.h>
108 #endif
109
110 #ifdef FAST_IPSEC
111 #if defined(IPSEC) || defined(IPSEC_ESP)
112 #error "Bad idea: don't compile with both IPSEC and FAST_IPSEC!"
113 #endif
114
115 #include <netproto/ipsec/ipsec.h>
116 #include <netproto/ipsec/key.h>
117 #define IPSEC
118 #endif /* FAST_IPSEC */
119
120 #define INP_LOCALGROUP_SIZMIN 8
121 #define INP_LOCALGROUP_SIZMAX 256
122
123 static struct inpcb *in_pcblookup_local(struct inpcbporthead *porthash,
124 struct in_addr laddr, u_int lport_arg, int wild_okay,
125 struct ucred *cred);
126
127 struct in_addr zeroin_addr;
128
129 /*
130 * These configure the range of local port addresses assigned to
131 * "unspecified" outgoing connections/packets/whatever.
132 */
133 int ipport_lowfirstauto = IPPORT_RESERVED - 1; /* 1023 */
134 int ipport_lowlastauto = IPPORT_RESERVEDSTART; /* 600 */
135
136 int ipport_firstauto = IPPORT_RESERVED; /* 1024 */
137 int ipport_lastauto = IPPORT_USERRESERVED; /* 5000 */
138
139 int ipport_hifirstauto = IPPORT_HIFIRSTAUTO; /* 49152 */
140 int ipport_hilastauto = IPPORT_HILASTAUTO; /* 65535 */
141
142 #define RANGECHK(var, min, max) \
143 if ((var) < (min)) { (var) = (min); } \
144 else if ((var) > (max)) { (var) = (max); }
145
146 int udpencap_enable = 1; /* enabled by default */
147 int udpencap_port = 4500; /* triggers decapsulation */
148
149 /*
150 * Per-netisr inpcb markers.
151 * NOTE: they should only be used in netisrs.
152 */
153 static struct inpcb *in_pcbmarkers;
154 static struct inpcontainer *in_pcbcontainer_markers;
155
156 static int
157 sysctl_net_ipport_check(SYSCTL_HANDLER_ARGS)
158 {
159 int error;
160
161 error = sysctl_handle_int(oidp, oidp->oid_arg1, oidp->oid_arg2, req);
162 if (!error) {
163 RANGECHK(ipport_lowfirstauto, 1, IPPORT_RESERVED - 1);
164 RANGECHK(ipport_lowlastauto, 1, IPPORT_RESERVED - 1);
165
166 RANGECHK(ipport_firstauto, IPPORT_RESERVED, USHRT_MAX);
167 RANGECHK(ipport_lastauto, IPPORT_RESERVED, USHRT_MAX);
168
169 RANGECHK(ipport_hifirstauto, IPPORT_RESERVED, USHRT_MAX);
170 RANGECHK(ipport_hilastauto, IPPORT_RESERVED, USHRT_MAX);
171 }
172 return (error);
173 }
174
175 #undef RANGECHK
176
177 SYSCTL_NODE(_net_inet_ip, IPPROTO_IP, portrange, CTLFLAG_RW, 0, "IP Ports");
178
179 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowfirst, CTLTYPE_INT|CTLFLAG_RW,
180 &ipport_lowfirstauto, 0, &sysctl_net_ipport_check, "I", "");
181 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowlast, CTLTYPE_INT|CTLFLAG_RW,
182 &ipport_lowlastauto, 0, &sysctl_net_ipport_check, "I", "");
183 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, first, CTLTYPE_INT|CTLFLAG_RW,
184 &ipport_firstauto, 0, &sysctl_net_ipport_check, "I", "");
185 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, last, CTLTYPE_INT|CTLFLAG_RW,
186 &ipport_lastauto, 0, &sysctl_net_ipport_check, "I", "");
187 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hifirst, CTLTYPE_INT|CTLFLAG_RW,
188 &ipport_hifirstauto, 0, &sysctl_net_ipport_check, "I", "");
189 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hilast, CTLTYPE_INT|CTLFLAG_RW,
190 &ipport_hilastauto, 0, &sysctl_net_ipport_check, "I", "");
191
192 /* Initialized by ip_init() */
193 int ip_porthash_trycount;
194 SYSCTL_INT(_net_inet_ip, OID_AUTO, porthash_trycount, CTLFLAG_RW,
195 &ip_porthash_trycount, 0,
196 "Number of tries to find local port matching hash of 4-tuple");
197
198 /*
199 * in_pcb.c: manage the Protocol Control Blocks.
200 *
201 * NOTE: It is assumed that most of these functions will be called from
202 * a critical section. XXX - There are, unfortunately, a few exceptions
203 * to this rule that should be fixed.
204 *
205 * NOTE: The caller should initialize the cpu field to the cpu running the
206 * protocol stack associated with this inpcbinfo.
207 */
208
209 void
210 in_pcbinfo_init(struct inpcbinfo *pcbinfo, int cpu, boolean_t shared)
211 {
212 KASSERT(cpu >= 0 && cpu < netisr_ncpus, ("invalid cpu%d", cpu));
213 pcbinfo->cpu = cpu;
214
215 LIST_INIT(&pcbinfo->pcblisthead);
216 pcbinfo->portsave = kmalloc(sizeof(*pcbinfo->portsave), M_PCB,
217 M_WAITOK | M_ZERO);
218
219 if (shared) {
220 pcbinfo->infotoken = kmalloc(sizeof(struct lwkt_token),
221 M_PCB, M_WAITOK);
222 lwkt_token_init(pcbinfo->infotoken, "infotoken");
223 } else {
224 pcbinfo->infotoken = NULL;
225 }
226 }
227
228 void
229 in_pcbportinfo_set(struct inpcbinfo *pcbinfo, struct inpcbportinfo *portinfo,
230 int portinfo_cnt)
231 {
232
233 KASSERT(portinfo_cnt > 0, ("invalid portinfo_cnt %d", portinfo_cnt));
234 pcbinfo->portinfo = portinfo;
235 pcbinfo->portinfo_cnt = portinfo_cnt;
236 }
237
238 struct baddynamicports baddynamicports;
239
240 /*
241 * Check if the specified port is invalid for dynamic allocation.
242 */
243 int
244 in_baddynamic(u_int16_t port, u_int16_t proto)
245 {
246 switch (proto) {
247 case IPPROTO_TCP:
248 return (DP_ISSET(baddynamicports.tcp, port));
249 case IPPROTO_UDP:
250 #ifdef IPSEC
251 /* Cannot preset this as it is a sysctl */
252 if (port == udpencap_port)
253 return (1);
254 #endif
255 return (DP_ISSET(baddynamicports.udp, port));
256 default:
257 return (0);
258 }
259 }
260
261 void
262 in_pcbonlist(struct inpcb *inp)
263 {
264 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
265
266 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
267 ("not in the correct netisr"));
268 KASSERT((inp->inp_flags & INP_ONLIST) == 0, ("already on pcblist"));
269 inp->inp_flags |= INP_ONLIST;
270
271 GET_PCBINFO_TOKEN(pcbinfo);
272 LIST_INSERT_HEAD(&pcbinfo->pcblisthead, inp, inp_list);
273 pcbinfo->ipi_count++;
274 REL_PCBINFO_TOKEN(pcbinfo);
275 }
276
277 void
278 in_pcbofflist(struct inpcb *inp)
279 {
280 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
281
282 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
283 ("not in the correct netisr"));
284 KASSERT(inp->inp_flags & INP_ONLIST, ("not on pcblist"));
285 inp->inp_flags &= ~INP_ONLIST;
286
287 GET_PCBINFO_TOKEN(pcbinfo);
288 LIST_REMOVE(inp, inp_list);
289 KASSERT(pcbinfo->ipi_count > 0,
290 ("invalid inpcb count %d", pcbinfo->ipi_count));
291 pcbinfo->ipi_count--;
292 REL_PCBINFO_TOKEN(pcbinfo);
293 }
294
295 /*
296 * Allocate a PCB and associate it with the socket.
297 */
298 int
299 in_pcballoc(struct socket *so, struct inpcbinfo *pcbinfo)
300 {
301 struct inpcb *inp;
302 #ifdef IPSEC
303 int error;
304 #endif
305
306 inp = kmalloc(pcbinfo->ipi_size, M_PCB, M_WAITOK|M_ZERO|M_NULLOK);
307 if (inp == NULL)
308 return (ENOMEM);
309 inp->inp_lgrpindex = -1;
310 inp->inp_gencnt = ++pcbinfo->ipi_gencnt;
311 inp->inp_pcbinfo = pcbinfo;
312 inp->inp_socket = so;
313 #ifdef IPSEC
314 error = ipsec_init_policy(so, &inp->inp_sp);
315 if (error != 0) {
316 kfree(inp, M_PCB);
317 return (error);
318 }
319 #endif
320 #ifdef INET6
321 if (INP_CHECK_SOCKAF(so, AF_INET6)) {
322 if (ip6_auto_flowlabel)
323 inp->inp_flags |= IN6P_AUTOFLOWLABEL;
324 inp->inp_af = AF_INET6;
325 } else
326 #endif
327 inp->inp_af = AF_INET;
328 soreference(so);
329 so->so_pcb = inp;
330
331 in_pcbonlist(inp);
332 return (0);
333 }
334
335 /*
336 * Unlink a pcb with the intention of moving it to another cpu with a
337 * different pcbinfo. While unlinked nothing should attempt to dereference
338 * inp_pcbinfo, NULL it out so we assert if it does.
339 */
340 void
341 in_pcbunlink_flags(struct inpcb *inp, struct inpcbinfo *pcbinfo, int flags)
342 {
343 KASSERT(inp->inp_pcbinfo == pcbinfo, ("pcbinfo mismatch"));
344 KASSERT((inp->inp_flags & (flags | INP_CONNECTED)) == 0,
345 ("already linked"));
346
347 in_pcbofflist(inp);
348 inp->inp_pcbinfo = NULL;
349 }
350
351 void
352 in_pcbunlink(struct inpcb *inp, struct inpcbinfo *pcbinfo)
353 {
354 in_pcbunlink_flags(inp, pcbinfo, INP_WILDCARD);
355 }
356
357 /*
358 * Relink a pcb into a new pcbinfo.
359 */
360 void
361 in_pcblink_flags(struct inpcb *inp, struct inpcbinfo *pcbinfo, int flags)
362 {
363 KASSERT(inp->inp_pcbinfo == NULL, ("has pcbinfo"));
364 KASSERT((inp->inp_flags & (flags | INP_CONNECTED)) == 0,
365 ("already linked"));
366
367 inp->inp_pcbinfo = pcbinfo;
368 in_pcbonlist(inp);
369 }
370
371 void
372 in_pcblink(struct inpcb *inp, struct inpcbinfo *pcbinfo)
373 {
374 return in_pcblink_flags(inp, pcbinfo, INP_WILDCARD);
375 }
376
377 static boolean_t
378 in_pcbporthash_update(struct inpcbportinfo *portinfo,
379 struct inpcb *inp, u_short lport, struct ucred *cred, int wild)
380 {
381 struct inpcbporthead *porthash;
382
383 /*
384 * This has to be atomic. If the porthash is shared across multiple
385 * protocol threads, e.g. tcp and udp, then the token must be held.
386 */
387 porthash = in_pcbporthash_head(portinfo, lport);
388 GET_PORTHASH_TOKEN(porthash);
389
390 if (in_pcblookup_local(porthash, inp->inp_laddr, lport,
391 wild, cred) != NULL) {
392 REL_PORTHASH_TOKEN(porthash);
393 return FALSE;
394 }
395 inp->inp_lport = lport;
396 in_pcbinsporthash(porthash, inp);
397
398 REL_PORTHASH_TOKEN(porthash);
399 return TRUE;
400 }
401
402 static int
403 in_pcbsetlport(struct inpcb *inp, int wild, struct ucred *cred)
404 {
405 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
406 struct inpcbportinfo *portinfo;
407 u_short first, last, lport, step, first0, last0;
408 int count, error;
409 int portinfo_first, portinfo_idx;
410 uint32_t cut;
411
412 inp->inp_flags |= INP_ANONPORT;
413
414 step = pcbinfo->portinfo_cnt;
415 portinfo_first = mycpuid % pcbinfo->portinfo_cnt;
416 portinfo_idx = portinfo_first;
417
418 if (inp->inp_flags & INP_HIGHPORT) {
419 first0 = ipport_hifirstauto; /* sysctl */
420 last0 = ipport_hilastauto;
421 } else if (inp->inp_flags & INP_LOWPORT) {
422 if (cred &&
423 (error =
424 priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT, 0))) {
425 inp->inp_laddr.s_addr = INADDR_ANY;
426 return error;
427 }
428 first0 = ipport_lowfirstauto; /* 1023 */
429 last0 = ipport_lowlastauto; /* 600 */
430 } else {
431 first0 = ipport_firstauto; /* sysctl */
432 last0 = ipport_lastauto;
433 }
434 if (first0 > last0) {
435 lport = last0;
436 last0 = first0;
437 first0 = lport;
438 }
439 KKASSERT(last0 >= first0);
440
441 cut = karc4random();
442 loop:
443 portinfo = &pcbinfo->portinfo[portinfo_idx];
444 first = first0;
445 last = last0;
446
447 /*
448 * Simple check to ensure all ports are not used up causing
449 * a deadlock here.
450 */
451 in_pcbportrange(&last, &first, portinfo->offset, step);
452 lport = last - first;
453 count = lport / step;
454
455 lport = rounddown(cut % lport, step) + first;
456 KKASSERT(lport % step == portinfo->offset);
457
458 for (;;) {
459 if (count-- < 0) { /* completely used? */
460 error = EADDRNOTAVAIL;
461 break;
462 }
463
464 if (__predict_false(lport < first || lport > last)) {
465 lport = first;
466 KKASSERT(lport % step == portinfo->offset);
467 }
468
469 if (in_pcbporthash_update(portinfo, inp, htons(lport),
470 cred, wild)) {
471 error = 0;
472 break;
473 }
474
475 lport += step;
476 KKASSERT(lport % step == portinfo->offset);
477 }
478
479 if (error) {
480 /* Try next portinfo */
481 portinfo_idx++;
482 portinfo_idx %= pcbinfo->portinfo_cnt;
483 if (portinfo_idx != portinfo_first)
484 goto loop;
485 inp->inp_laddr.s_addr = INADDR_ANY;
486 }
487 return error;
488 }
489
490 int
491 in_pcbbind(struct inpcb *inp, struct sockaddr *nam, struct thread *td)
492 {
493 struct socket *so = inp->inp_socket;
494 struct sockaddr_in jsin;
495 struct ucred *cred = NULL;
496 int wild = 0;
497
498 if (TAILQ_EMPTY(&in_ifaddrheads[mycpuid])) /* XXX broken! */
499 return (EADDRNOTAVAIL);
500 if (inp->inp_lport != 0 || inp->inp_laddr.s_addr != INADDR_ANY)
501 return (EINVAL); /* already bound */
502
503 if (!(so->so_options & (SO_REUSEADDR|SO_REUSEPORT)))
504 wild = 1; /* neither SO_REUSEADDR nor SO_REUSEPORT is set */
505 if (td->td_proc)
506 cred = td->td_proc->p_ucred;
507
508 if (nam != NULL) {
509 struct sockaddr_in *sin = (struct sockaddr_in *)nam;
510 struct inpcbinfo *pcbinfo;
511 struct inpcbportinfo *portinfo;
512 struct inpcbporthead *porthash;
513 struct inpcb *t;
514 u_short lport, lport_ho;
515 int reuseport = (so->so_options & SO_REUSEPORT);
516 int error;
517
518 if (nam->sa_len != sizeof *sin)
519 return (EINVAL);
520 #ifdef notdef
521 /*
522 * We should check the family, but old programs
523 * incorrectly fail to initialize it.
524 */
525 if (sin->sin_family != AF_INET)
526 return (EAFNOSUPPORT);
527 #endif
528 if (!prison_replace_wildcards(td, nam))
529 return (EINVAL);
530
531 lport = sin->sin_port;
532 if (IN_MULTICAST(ntohl(sin->sin_addr.s_addr))) {
533 /*
534 * Treat SO_REUSEADDR as SO_REUSEPORT for multicast;
535 * allow complete duplication of binding if
536 * SO_REUSEPORT is set, or if SO_REUSEADDR is set
537 * and a multicast address is bound on both
538 * new and duplicated sockets.
539 */
540 if (so->so_options & SO_REUSEADDR)
541 reuseport = SO_REUSEADDR | SO_REUSEPORT;
542 } else if (sin->sin_addr.s_addr != INADDR_ANY) {
543 sin->sin_port = 0; /* yech... */
544 bzero(&sin->sin_zero, sizeof sin->sin_zero);
545 if (ifa_ifwithaddr((struct sockaddr *)sin) == NULL)
546 return (EADDRNOTAVAIL);
547 }
548
549 inp->inp_laddr = sin->sin_addr;
550
551 jsin.sin_family = AF_INET;
552 jsin.sin_addr.s_addr = inp->inp_laddr.s_addr;
553 if (!prison_replace_wildcards(td, (struct sockaddr *)&jsin)) {
554 inp->inp_laddr.s_addr = INADDR_ANY;
555 return (EINVAL);
556 }
557 inp->inp_laddr.s_addr = jsin.sin_addr.s_addr;
558
559 if (lport == 0) {
560 /* Auto-select local port */
561 return in_pcbsetlport(inp, wild, cred);
562 }
563 lport_ho = ntohs(lport);
564
565 /* GROSS */
566 if (lport_ho < IPPORT_RESERVED && cred &&
567 (error =
568 priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT, 0))) {
569 inp->inp_laddr.s_addr = INADDR_ANY;
570 return (error);
571 }
572
573 /*
574 * Locate the proper portinfo based on lport
575 */
576 pcbinfo = inp->inp_pcbinfo;
577 portinfo =
578 &pcbinfo->portinfo[lport_ho % pcbinfo->portinfo_cnt];
579 KKASSERT((lport_ho % pcbinfo->portinfo_cnt) ==
580 portinfo->offset);
581
582 /*
583 * This has to be atomic. If the porthash is shared across
584 * multiple protocol threads, e.g. tcp and udp then the token
585 * must be held.
586 */
587 porthash = in_pcbporthash_head(portinfo, lport);
588 GET_PORTHASH_TOKEN(porthash);
589
590 if (so->so_cred->cr_uid != 0 &&
591 !IN_MULTICAST(ntohl(sin->sin_addr.s_addr))) {
592 t = in_pcblookup_local(porthash, sin->sin_addr, lport,
593 INPLOOKUP_WILDCARD, cred);
594 if (t &&
595 (so->so_cred->cr_uid !=
596 t->inp_socket->so_cred->cr_uid)) {
597 inp->inp_laddr.s_addr = INADDR_ANY;
598 error = EADDRINUSE;
599 goto done;
600 }
601 }
602 if (cred && !prison_replace_wildcards(td, nam)) {
603 inp->inp_laddr.s_addr = INADDR_ANY;
604 error = EADDRNOTAVAIL;
605 goto done;
606 }
607 t = in_pcblookup_local(porthash, sin->sin_addr, lport,
608 wild, cred);
609 if (t && !(reuseport & t->inp_socket->so_options)) {
610 inp->inp_laddr.s_addr = INADDR_ANY;
611 error = EADDRINUSE;
612 goto done;
613 }
614 inp->inp_lport = lport;
615 in_pcbinsporthash(porthash, inp);
616 error = 0;
617 done:
618 REL_PORTHASH_TOKEN(porthash);
619 return (error);
620 } else {
621 jsin.sin_family = AF_INET;
622 jsin.sin_addr.s_addr = inp->inp_laddr.s_addr;
623 if (!prison_replace_wildcards(td, (struct sockaddr *)&jsin)) {
624 inp->inp_laddr.s_addr = INADDR_ANY;
625 return (EINVAL);
626 }
627 inp->inp_laddr.s_addr = jsin.sin_addr.s_addr;
628
629 return in_pcbsetlport(inp, wild, cred);
630 }
631 }
632
633 static struct inpcb *
634 in_pcblookup_localremote(struct inpcbporthead *porthash, struct in_addr laddr,
635 u_short lport, struct in_addr faddr, u_short fport, struct ucred *cred)
636 {
637 struct inpcb *inp;
638 struct inpcbport *phd;
639 struct inpcb *match = NULL;
640
641 /*
642 * If the porthashbase is shared across several cpus, it must
643 * have been locked.
644 */
645 ASSERT_PORTHASH_TOKEN_HELD(porthash);
646
647 /*
648 * Best fit PCB lookup.
649 *
650 * First see if this local port is in use by looking on the
651 * port hash list.
652 */
653 LIST_FOREACH(phd, porthash, phd_hash) {
654 if (phd->phd_port == lport)
655 break;
656 }
657 if (phd != NULL) {
658 LIST_FOREACH(inp, &phd->phd_pcblist, inp_portlist) {
659 #ifdef INET6
660 if (!INP_ISIPV4(inp))
661 continue;
662 #endif
663 if (inp->inp_laddr.s_addr != INADDR_ANY &&
664 inp->inp_laddr.s_addr != laddr.s_addr)
665 continue;
666
667 if (inp->inp_faddr.s_addr != INADDR_ANY &&
668 inp->inp_faddr.s_addr != faddr.s_addr)
669 continue;
670
671 if (inp->inp_fport != 0 && inp->inp_fport != fport)
672 continue;
673
674 if (cred == NULL ||
675 cred->cr_prison ==
676 inp->inp_socket->so_cred->cr_prison) {
677 match = inp;
678 break;
679 }
680 }
681 }
682 return (match);
683 }
684
685 static boolean_t
686 in_pcbporthash_update4(struct inpcbportinfo *portinfo,
687 struct inpcb *inp, u_short lport, const struct sockaddr_in *sin,
688 struct ucred *cred)
689 {
690 struct inpcbporthead *porthash;
691
692 /*
693 * This has to be atomic. If the porthash is shared across multiple
694 * protocol threads, e.g. tcp and udp, then the token must be held.
695 */
696 porthash = in_pcbporthash_head(portinfo, lport);
697 GET_PORTHASH_TOKEN(porthash);
698
699 if (in_pcblookup_localremote(porthash, inp->inp_laddr,
700 lport, sin->sin_addr, sin->sin_port, cred) != NULL) {
701 REL_PORTHASH_TOKEN(porthash);
702 return FALSE;
703 }
704 inp->inp_lport = lport;
705 in_pcbinsporthash(porthash, inp);
706
707 REL_PORTHASH_TOKEN(porthash);
708 return TRUE;
709 }
710
711 int
712 in_pcbbind_remote(struct inpcb *inp, const struct sockaddr *remote,
713 struct thread *td)
714 {
715 struct proc *p = td->td_proc;
716 const struct sockaddr_in *sin = (const struct sockaddr_in *)remote;
717 struct sockaddr_in jsin;
718 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
719 struct ucred *cred = NULL;
720 u_short first, last, lport;
721 int count, hash_count;
722 int error, selfconn = 0;
723 int cpuid = mycpuid;
724 uint32_t hash_base = 0, hash;
725
726 ASSERT_NETISR_NCPUS(cpuid);
727
728 if (TAILQ_EMPTY(&in_ifaddrheads[cpuid])) /* XXX broken! */
729 return (EADDRNOTAVAIL);
730
731 KKASSERT(inp->inp_laddr.s_addr != INADDR_ANY);
732 if (inp->inp_lport != 0)
733 return (EINVAL); /* already bound */
734
735 KKASSERT(p);
736 cred = p->p_ucred;
737
738 jsin.sin_family = AF_INET;
739 jsin.sin_addr.s_addr = inp->inp_laddr.s_addr;
740 if (!prison_replace_wildcards(td, (struct sockaddr *)&jsin)) {
741 inp->inp_laddr.s_addr = INADDR_ANY;
742 return (EINVAL);
743 }
744 inp->inp_laddr.s_addr = jsin.sin_addr.s_addr;
745
746 hash_count = ip_porthash_trycount;
747 if (hash_count > 0) {
748 hash_base = toeplitz_piecemeal_addr(sin->sin_addr.s_addr) ^
749 toeplitz_piecemeal_addr(inp->inp_laddr.s_addr) ^
750 toeplitz_piecemeal_port(sin->sin_port);
751 } else {
752 hash_count = 0;
753 }
754
755 inp->inp_flags |= INP_ANONPORT;
756
757 if (inp->inp_flags & INP_HIGHPORT) {
758 first = ipport_hifirstauto; /* sysctl */
759 last = ipport_hilastauto;
760 } else if (inp->inp_flags & INP_LOWPORT) {
761 if (cred &&
762 (error =
763 priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT, 0))) {
764 inp->inp_laddr.s_addr = INADDR_ANY;
765 return (error);
766 }
767 first = ipport_lowfirstauto; /* 1023 */
768 last = ipport_lowlastauto; /* 600 */
769 } else {
770 first = ipport_firstauto; /* sysctl */
771 last = ipport_lastauto;
772 }
773 if (first > last) {
774 lport = last;
775 last = first;
776 first = lport;
777 }
778 KKASSERT(last >= first);
779
780 count = last - first;
781 lport = (karc4random() % count) + first;
782 count += hash_count;
783
784 /*
785 * Simple check to ensure all ports are not used up causing
786 * a deadlock here.
787 */
788 for (;;) {
789 u_short lport_no;
790
791 if (count-- < 0) { /* completely used? */
792 error = EADDRNOTAVAIL;
793 break;
794 }
795
796 if (__predict_false(lport < first || lport > last))
797 lport = first;
798 lport_no = htons(lport);
799
800 /* This could happen on loopback interface */
801 if (__predict_false(sin->sin_port == lport_no &&
802 sin->sin_addr.s_addr == inp->inp_laddr.s_addr)) {
803 if (!selfconn) {
804 ++count; /* don't count this try */
805 selfconn = 1;
806 }
807 goto next;
808 }
809
810 if (hash_count) {
811 --hash_count;
812 hash = hash_base ^
813 toeplitz_piecemeal_port(lport_no);
814 if (netisr_hashcpu(hash) != cpuid && hash_count)
815 goto next;
816 }
817
818 if (in_pcbporthash_update4(
819 &pcbinfo->portinfo[lport % pcbinfo->portinfo_cnt],
820 inp, lport_no, sin, cred)) {
821 error = 0;
822 break;
823 }
824 next:
825 ++lport;
826 }
827
828 if (error)
829 inp->inp_laddr.s_addr = INADDR_ANY;
830 return (error);
831 }
832
833 /*
834 * Transform old in_pcbconnect() into an inner subroutine for new
835 * in_pcbconnect(): Do some validity-checking on the remote
836 * address (in mbuf 'nam') and then determine local host address
837 * (i.e., which interface) to use to access that remote host.
838 *
839 * This preserves definition of in_pcbconnect(), while supporting a
840 * slightly different version for T/TCP. (This is more than
841 * a bit of a kludge, but cleaning up the internal interfaces would
842 * have forced minor changes in every protocol).
843 */
844 int
845 in_pcbladdr_find(struct inpcb *inp, struct sockaddr *nam,
846 struct sockaddr_in **plocal_sin, struct thread *td, int find)
847 {
848 struct in_ifaddr *ia;
849 struct ucred *cred = NULL;
850 struct sockaddr_in *sin = (struct sockaddr_in *)nam;
851 struct sockaddr *jsin;
852 int jailed = 0, alloc_route = 0;
853
854 if (nam->sa_len != sizeof *sin)
855 return (EINVAL);
856 if (sin->sin_family != AF_INET)
857 return (EAFNOSUPPORT);
858 if (sin->sin_port == 0)
859 return (EADDRNOTAVAIL);
860 if (td && td->td_proc && td->td_proc->p_ucred)
861 cred = td->td_proc->p_ucred;
862 if (cred && cred->cr_prison)
863 jailed = 1;
864 if (!TAILQ_EMPTY(&in_ifaddrheads[mycpuid])) {
865 ia = TAILQ_FIRST(&in_ifaddrheads[mycpuid])->ia;
866 /*
867 * If the destination address is INADDR_ANY,
868 * use the primary local address.
869 * If the supplied address is INADDR_BROADCAST,
870 * and the primary interface supports broadcast,
871 * choose the broadcast address for that interface.
872 */
873 if (sin->sin_addr.s_addr == INADDR_ANY)
874 sin->sin_addr = IA_SIN(ia)->sin_addr;
875 else if (sin->sin_addr.s_addr == (u_long)INADDR_BROADCAST &&
876 (ia->ia_ifp->if_flags & IFF_BROADCAST))
877 sin->sin_addr = satosin(&ia->ia_broadaddr)->sin_addr;
878 }
879 if (find) {
880 struct route *ro;
881
882 ia = NULL;
883 /*
884 * If route is known or can be allocated now,
885 * our src addr is taken from the i/f, else punt.
886 * Note that we should check the address family of the cached
887 * destination, in case of sharing the cache with IPv6.
888 */
889 ro = &inp->inp_route;
890 if (ro->ro_rt &&
891 (!(ro->ro_rt->rt_flags & RTF_UP) ||
892 ro->ro_dst.sa_family != AF_INET ||
893 satosin(&ro->ro_dst)->sin_addr.s_addr !=
894 sin->sin_addr.s_addr ||
895 inp->inp_socket->so_options & SO_DONTROUTE)) {
896 RTFREE(ro->ro_rt);
897 ro->ro_rt = NULL;
898 }
899 if (!(inp->inp_socket->so_options & SO_DONTROUTE) && /*XXX*/
900 (ro->ro_rt == NULL ||
901 ro->ro_rt->rt_ifp == NULL)) {
902 /* No route yet, so try to acquire one */
903 bzero(&ro->ro_dst, sizeof(struct sockaddr_in));
904 ro->ro_dst.sa_family = AF_INET;
905 ro->ro_dst.sa_len = sizeof(struct sockaddr_in);
906 ((struct sockaddr_in *) &ro->ro_dst)->sin_addr =
907 sin->sin_addr;
908 rtalloc(ro);
909 alloc_route = 1;
910 }
911 /*
912 * If we found a route, use the address
913 * corresponding to the outgoing interface
914 * unless it is the loopback (in case a route
915 * to our address on another net goes to loopback).
916 */
917 if (ro->ro_rt &&
918 !(ro->ro_rt->rt_ifp->if_flags & IFF_LOOPBACK)) {
919 if (jailed) {
920 if (jailed_ip(cred->cr_prison,
921 ro->ro_rt->rt_ifa->ifa_addr)) {
922 ia = ifatoia(ro->ro_rt->rt_ifa);
923 }
924 } else {
925 ia = ifatoia(ro->ro_rt->rt_ifa);
926 }
927 }
928 if (ia == NULL) {
929 u_short fport = sin->sin_port;
930
931 sin->sin_port = 0;
932 ia = ifatoia(ifa_ifwithdstaddr(sintosa(sin)));
933 if (ia && jailed && !jailed_ip(cred->cr_prison,
934 sintosa(&ia->ia_addr)))
935 ia = NULL;
936 if (ia == NULL)
937 ia = ifatoia(ifa_ifwithnet(sintosa(sin)));
938 if (ia && jailed && !jailed_ip(cred->cr_prison,
939 sintosa(&ia->ia_addr)))
940 ia = NULL;
941 sin->sin_port = fport;
942 if (ia == NULL &&
943 !TAILQ_EMPTY(&in_ifaddrheads[mycpuid]))
944 ia = TAILQ_FIRST(&in_ifaddrheads[mycpuid])->ia;
945 if (ia && jailed && !jailed_ip(cred->cr_prison,
946 sintosa(&ia->ia_addr)))
947 ia = NULL;
948
949 if (!jailed && ia == NULL)
950 goto fail;
951 }
952 /*
953 * If the destination address is multicast and an outgoing
954 * interface has been set as a multicast option, use the
955 * address of that interface as our source address.
956 */
957 if (!jailed && IN_MULTICAST(ntohl(sin->sin_addr.s_addr)) &&
958 inp->inp_moptions != NULL) {
959 struct ip_moptions *imo;
960 struct ifnet *ifp;
961
962 imo = inp->inp_moptions;
963 if ((ifp = imo->imo_multicast_ifp) != NULL) {
964 struct in_ifaddr_container *iac;
965
966 ia = NULL;
967 TAILQ_FOREACH(iac,
968 &in_ifaddrheads[mycpuid], ia_link) {
969 if (iac->ia->ia_ifp == ifp) {
970 ia = iac->ia;
971 break;
972 }
973 }
974 if (ia == NULL)
975 goto fail;
976 }
977 }
978 /*
979 * Don't do pcblookup call here; return interface in plocal_sin
980 * and exit to caller, that will do the lookup.
981 */
982 if (ia == NULL && jailed) {
983 if ((jsin = prison_get_nonlocal(
984 cred->cr_prison, AF_INET, NULL)) != NULL ||
985 (jsin = prison_get_local(
986 cred->cr_prison, AF_INET, NULL)) != NULL) {
987 *plocal_sin = satosin(jsin);
988 } else {
989 /* IPv6 only Jail */
990 goto fail;
991 }
992 } else {
993 *plocal_sin = &ia->ia_addr;
994 }
995 }
996 return (0);
997 fail:
998 if (alloc_route)
999 in_pcbresetroute(inp);
1000 return (EADDRNOTAVAIL);
1001 }
1002
1003 int
1004 in_pcbladdr(struct inpcb *inp, struct sockaddr *nam,
1005 struct sockaddr_in **plocal_sin, struct thread *td)
1006 {
1007 return in_pcbladdr_find(inp, nam, plocal_sin, td,
1008 (inp->inp_laddr.s_addr == INADDR_ANY));
1009 }
1010
1011 /*
1012 * Outer subroutine:
1013 * Connect from a socket to a specified address.
1014 * Both address and port must be specified in argument sin.
1015 * If don't have a local address for this socket yet,
1016 * then pick one.
1017 */
1018 int
1019 in_pcbconnect(struct inpcb *inp, struct sockaddr *nam, struct thread *td)
1020 {
1021 struct sockaddr_in *if_sin;
1022 struct sockaddr_in *sin = (struct sockaddr_in *)nam;
1023 int error;
1024
1025 if_sin = NULL; /* avoid gcc warnings */
1026
1027 /* Call inner routine to assign local interface address. */
1028 if ((error = in_pcbladdr(inp, nam, &if_sin, td)) != 0)
1029 return (error);
1030
1031 if (in_pcblookup_hash(inp->inp_pcbinfo, sin->sin_addr, sin->sin_port,
1032 inp->inp_laddr.s_addr ?
1033 inp->inp_laddr : if_sin->sin_addr,
1034 inp->inp_lport, FALSE, NULL) != NULL) {
1035 return (EADDRINUSE);
1036 }
1037 if (inp->inp_laddr.s_addr == INADDR_ANY) {
1038 if (inp->inp_lport == 0) {
1039 error = in_pcbbind(inp, NULL, td);
1040 if (error)
1041 return (error);
1042 }
1043 inp->inp_laddr = if_sin->sin_addr;
1044 }
1045 inp->inp_faddr = sin->sin_addr;
1046 inp->inp_fport = sin->sin_port;
1047 in_pcbinsconnhash(inp);
1048 return (0);
1049 }
1050
1051 void
1052 in_pcbdisconnect(struct inpcb *inp)
1053 {
1054
1055 in_pcbremconnhash(inp);
1056 inp->inp_faddr.s_addr = INADDR_ANY;
1057 inp->inp_fport = 0;
1058 }
1059
1060 void
1061 in_pcbdetach(struct inpcb *inp)
1062 {
1063 struct socket *so = inp->inp_socket;
1064 struct inpcbinfo *ipi = inp->inp_pcbinfo;
1065
1066 #ifdef IPSEC
1067 ipsec4_delete_pcbpolicy(inp);
1068 #endif /*IPSEC*/
1069 inp->inp_gencnt = ++ipi->ipi_gencnt;
1070 KKASSERT((so->so_state & SS_ASSERTINPROG) == 0);
1071 in_pcbremlists(inp);
1072 so->so_pcb = NULL;
1073 sofree(so); /* remove pcb ref */
1074 if (inp->inp_options)
1075 m_free(inp->inp_options);
1076 if (inp->inp_route.ro_rt)
1077 rtfree(inp->inp_route.ro_rt);
1078 ip_freemoptions(inp->inp_moptions);
1079 kfree(inp, M_PCB);
1080 }
1081
1082 /*
1083 * The socket may have an invalid PCB, i.e. NULL. For example, a TCP
1084 * socket received RST.
1085 */
1086 static int
1087 in_setsockaddr(struct socket *so, struct sockaddr **nam)
1088 {
1089 struct inpcb *inp;
1090 struct sockaddr_in *sin;
1091
1092 KASSERT(curthread->td_type == TD_TYPE_NETISR, ("not in netisr"));
1093 inp = so->so_pcb;
1094 if (!inp)
1095 return (ECONNRESET);
1096
1097 sin = kmalloc(sizeof *sin, M_SONAME, M_WAITOK | M_ZERO);
1098 sin->sin_family = AF_INET;
1099 sin->sin_len = sizeof *sin;
1100 sin->sin_port = inp->inp_lport;
1101 sin->sin_addr = inp->inp_laddr;
1102
1103 *nam = (struct sockaddr *)sin;
1104 return (0);
1105 }
1106
1107 void
1108 in_setsockaddr_dispatch(netmsg_t msg)
1109 {
1110 int error;
1111
1112 error = in_setsockaddr(msg->base.nm_so, msg->peeraddr.nm_nam);
1113 lwkt_replymsg(&msg->lmsg, error);
1114 }
1115
1116 /*
1117 * The socket may have an invalid PCB, i.e. NULL. For example, a TCP
1118 * socket received RST.
1119 */
1120 int
1121 in_setpeeraddr(struct socket *so, struct sockaddr **nam)
1122 {
1123 struct inpcb *inp;
1124 struct sockaddr_in *sin;
1125
1126 KASSERT(curthread->td_type == TD_TYPE_NETISR, ("not in netisr"));
1127 inp = so->so_pcb;
1128 if (!inp)
1129 return (ECONNRESET);
1130
1131 sin = kmalloc(sizeof *sin, M_SONAME, M_WAITOK | M_ZERO);
1132 sin->sin_family = AF_INET;
1133 sin->sin_len = sizeof *sin;
1134 sin->sin_port = inp->inp_fport;
1135 sin->sin_addr = inp->inp_faddr;
1136
1137 *nam = (struct sockaddr *)sin;
1138 return (0);
1139 }
1140
1141 void
1142 in_setpeeraddr_dispatch(netmsg_t msg)
1143 {
1144 int error;
1145
1146 error = in_setpeeraddr(msg->base.nm_so, msg->peeraddr.nm_nam);
1147 lwkt_replymsg(&msg->lmsg, error);
1148 }
1149
1150 void
1151 in_pcbnotifyall(struct inpcbinfo *pcbinfo, struct in_addr faddr, int err,
1152 inp_notify_t notify)
1153 {
1154 struct inpcb *inp, *marker;
1155
1156 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
1157 ("not in the correct netisr"));
1158 marker = in_pcbmarker();
1159
1160 /*
1161 * NOTE:
1162 * - If INP_PLACEMARKER is set we must ignore the rest of the
1163 * structure and skip it.
1164 * - It is safe to nuke inpcbs here, since we are in their own
1165 * netisr.
1166 */
1167 GET_PCBINFO_TOKEN(pcbinfo);
1168
1169 LIST_INSERT_HEAD(&pcbinfo->pcblisthead, marker, inp_list);
1170 while ((inp = LIST_NEXT(marker, inp_list)) != NULL) {
1171 LIST_REMOVE(marker, inp_list);
1172 LIST_INSERT_AFTER(inp, marker, inp_list);
1173
1174 if (inp->inp_flags & INP_PLACEMARKER)
1175 continue;
1176 #ifdef INET6
1177 if (!INP_ISIPV4(inp))
1178 continue;
1179 #endif
1180 if (inp->inp_faddr.s_addr != faddr.s_addr ||
1181 inp->inp_socket == NULL)
1182 continue;
1183 (*notify)(inp, err); /* can remove inp from list! */
1184 }
1185 LIST_REMOVE(marker, inp_list);
1186
1187 REL_PCBINFO_TOKEN(pcbinfo);
1188 }
1189
1190 void
1191 in_pcbpurgeif0(struct inpcbinfo *pcbinfo, struct ifnet *ifp)
1192 {
1193 struct inpcb *inp, *marker;
1194
1195 /*
1196 * We only need to make sure that we are in netisr0, where all
1197 * multicast operation happen. We could check inpcbinfo which
1198 * does not belong to netisr0 by holding the inpcbinfo's token.
1199 * In this case, the pcbinfo must be able to be shared, i.e.
1200 * pcbinfo->infotoken is not NULL.
1201 */
1202 ASSERT_NETISR0;
1203 KASSERT(pcbinfo->cpu == 0 || pcbinfo->infotoken != NULL,
1204 ("pcbinfo could not be shared"));
1205
1206 /*
1207 * Get a marker for the current netisr (netisr0).
1208 *
1209 * It is possible that the multicast address deletion blocks,
1210 * which could cause temporary token releasing. So we use
1211 * inpcb marker here to get a coherent view of the inpcb list.
1212 *
1213 * While, on the other hand, moptions are only added and deleted
1214 * in netisr0, so we would not see staled moption or miss moption
1215 * even if the token was released due to the blocking multicast
1216 * address deletion.
1217 */
1218 marker = in_pcbmarker();
1219
1220 GET_PCBINFO_TOKEN(pcbinfo);
1221
1222 LIST_INSERT_HEAD(&pcbinfo->pcblisthead, marker, inp_list);
1223 while ((inp = LIST_NEXT(marker, inp_list)) != NULL) {
1224 struct ip_moptions *imo;
1225
1226 LIST_REMOVE(marker, inp_list);
1227 LIST_INSERT_AFTER(inp, marker, inp_list);
1228
1229 if (inp->inp_flags & INP_PLACEMARKER)
1230 continue;
1231 imo = inp->inp_moptions;
1232 if (INP_ISIPV4(inp) && imo != NULL) {
1233 int i, gap;
1234
1235 /*
1236 * Unselect the outgoing interface if it is being
1237 * detached.
1238 */
1239 if (imo->imo_multicast_ifp == ifp)
1240 imo->imo_multicast_ifp = NULL;
1241
1242 /*
1243 * Drop multicast group membership if we joined
1244 * through the interface being detached.
1245 */
1246 for (i = 0, gap = 0; i < imo->imo_num_memberships;
1247 i++) {
1248 if (imo->imo_membership[i]->inm_ifp == ifp) {
1249 /*
1250 * NOTE:
1251 * This could block and the pcbinfo
1252 * token could be passively released.
1253 */
1254 in_delmulti(imo->imo_membership[i]);
1255 gap++;
1256 } else if (gap != 0)
1257 imo->imo_membership[i - gap] =
1258 imo->imo_membership[i];
1259 }
1260 imo->imo_num_memberships -= gap;
1261 }
1262 }
1263 LIST_REMOVE(marker, inp_list);
1264
1265 REL_PCBINFO_TOKEN(pcbinfo);
1266 }
1267
1268 /*
1269 * Check for alternatives when higher level complains
1270 * about service problems. For now, invalidate cached
1271 * routing information. If the route was created dynamically
1272 * (by a redirect), time to try a default gateway again.
1273 */
1274 void
1275 in_losing(struct inpcb *inp)
1276 {
1277 struct rtentry *rt;
1278 struct rt_addrinfo rtinfo;
1279
1280 if ((rt = inp->inp_route.ro_rt)) {
1281 bzero(&rtinfo, sizeof(struct rt_addrinfo));
1282 rtinfo.rti_info[RTAX_DST] = rt_key(rt);
1283 rtinfo.rti_info[RTAX_GATEWAY] = rt->rt_gateway;
1284 rtinfo.rti_info[RTAX_NETMASK] = rt_mask(rt);
1285 rtinfo.rti_flags = rt->rt_flags;
1286 rt_missmsg(RTM_LOSING, &rtinfo, rt->rt_flags, 0);
1287 if (rt->rt_flags & RTF_DYNAMIC) {
1288 rtrequest(RTM_DELETE, rt_key(rt), rt->rt_gateway,
1289 rt_mask(rt), rt->rt_flags, NULL);
1290 }
1291 inp->inp_route.ro_rt = NULL;
1292 rtfree(rt);
1293 /*
1294 * A new route can be allocated
1295 * the next time output is attempted.
1296 */
1297 }
1298 }
1299
1300 /*
1301 * After a routing change, flush old routing
1302 * and allocate a (hopefully) better one.
1303 */
1304 void
1305 in_rtchange(struct inpcb *inp, int err)
1306 {
1307 if (inp->inp_route.ro_rt) {
1308 rtfree(inp->inp_route.ro_rt);
1309 inp->inp_route.ro_rt = NULL;
1310 /*
1311 * A new route can be allocated the next time
1312 * output is attempted.
1313 */
1314 }
1315 }
1316
1317 /*
1318 * Lookup a PCB based on the local address and port.
1319 */
1320 static struct inpcb *
1321 in_pcblookup_local(struct inpcbporthead *porthash, struct in_addr laddr,
1322 u_int lport_arg, int wild_okay, struct ucred *cred)
1323 {
1324 struct inpcb *inp;
1325 int matchwild = 3, wildcard;
1326 u_short lport = lport_arg;
1327 struct inpcbport *phd;
1328 struct inpcb *match = NULL;
1329
1330 /*
1331 * If the porthashbase is shared across several cpus, it must
1332 * have been locked.
1333 */
1334 ASSERT_PORTHASH_TOKEN_HELD(porthash);
1335
1336 /*
1337 * Best fit PCB lookup.
1338 *
1339 * First see if this local port is in use by looking on the
1340 * port hash list.
1341 */
1342 LIST_FOREACH(phd, porthash, phd_hash) {
1343 if (phd->phd_port == lport)
1344 break;
1345 }
1346 if (phd != NULL) {
1347 /*
1348 * Port is in use by one or more PCBs. Look for best
1349 * fit.
1350 */
1351 LIST_FOREACH(inp, &phd->phd_pcblist, inp_portlist) {
1352 wildcard = 0;
1353 #ifdef INET6
1354 if (!INP_ISIPV4(inp))
1355 continue;
1356 #endif
1357 if (inp->inp_faddr.s_addr != INADDR_ANY)
1358 wildcard++;
1359 if (inp->inp_laddr.s_addr != INADDR_ANY) {
1360 if (laddr.s_addr == INADDR_ANY)
1361 wildcard++;
1362 else if (inp->inp_laddr.s_addr != laddr.s_addr)
1363 continue;
1364 } else {
1365 if (laddr.s_addr != INADDR_ANY)
1366 wildcard++;
1367 }
1368 if (wildcard && !wild_okay)
1369 continue;
1370 if (wildcard < matchwild &&
1371 (cred == NULL ||
1372 cred->cr_prison ==
1373 inp->inp_socket->so_cred->cr_prison)) {
1374 match = inp;
1375 matchwild = wildcard;
1376 if (matchwild == 0) {
1377 break;
1378 }
1379 }
1380 }
1381 }
1382 return (match);
1383 }
1384
1385 struct inpcb *
1386 in_pcblocalgroup_last(const struct inpcbinfo *pcbinfo,
1387 const struct inpcb *inp)
1388 {
1389 const struct inp_localgrphead *hdr;
1390 const struct inp_localgroup *grp;
1391 int i;
1392
1393 if (pcbinfo->localgrphashbase == NULL)
1394 return NULL;
1395
1396 GET_PCBINFO_TOKEN(pcbinfo);
1397
1398 hdr = &pcbinfo->localgrphashbase[
1399 INP_PCBLOCALGRPHASH(inp->inp_lport, pcbinfo->localgrphashmask)];
1400
1401 LIST_FOREACH(grp, hdr, il_list) {
1402 if (grp->il_af == inp->inp_af &&
1403 grp->il_lport == inp->inp_lport &&
1404 memcmp(&grp->il_dependladdr,
1405 &inp->inp_inc.inc_ie.ie_dependladdr,
1406 sizeof(grp->il_dependladdr)) == 0) {
1407 break;
1408 }
1409 }
1410 if (grp == NULL || grp->il_inpcnt == 1) {
1411 REL_PCBINFO_TOKEN(pcbinfo);
1412 return NULL;
1413 }
1414
1415 KASSERT(grp->il_inpcnt >= 2,
1416 ("invalid localgroup inp count %d", grp->il_inpcnt));
1417 for (i = 0; i < grp->il_inpcnt; ++i) {
1418 if (grp->il_inp[i] == inp) {
1419 int last = grp->il_inpcnt - 1;
1420
1421 if (i == last)
1422 last = grp->il_inpcnt - 2;
1423 REL_PCBINFO_TOKEN(pcbinfo);
1424 return grp->il_inp[last];
1425 }
1426 }
1427 REL_PCBINFO_TOKEN(pcbinfo);
1428 return NULL;
1429 }
1430
1431 static struct inpcb *
1432 inp_localgroup_lookup(const struct inpcbinfo *pcbinfo,
1433 struct in_addr laddr, uint16_t lport, uint32_t pkt_hash)
1434 {
1435 struct inpcb *local_wild = NULL;
1436 const struct inp_localgrphead *hdr;
1437 const struct inp_localgroup *grp;
1438
1439 ASSERT_PCBINFO_TOKEN_HELD(pcbinfo);
1440
1441 hdr = &pcbinfo->localgrphashbase[
1442 INP_PCBLOCALGRPHASH(lport, pcbinfo->localgrphashmask)];
1443
1444 /*
1445 * Order of socket selection:
1446 * 1. non-wild.
1447 * 2. wild.
1448 *
1449 * NOTE: Local group does not contain jailed sockets
1450 */
1451 LIST_FOREACH(grp, hdr, il_list) {
1452 #ifdef INET6
1453 if (grp->il_af != AF_INET)
1454 continue;
1455 #endif
1456 if (grp->il_lport == lport) {
1457 int idx;
1458
1459 /*
1460 * Modulo-N is used here, which greatly reduces
1461 * completion queue token contention, thus more
1462 * cpu time is saved.
1463 */
1464 idx = netisr_hashlsb(pkt_hash) % grp->il_inpcnt;
1465 if (grp->il_laddr.s_addr == laddr.s_addr)
1466 return grp->il_inp[idx];
1467 else if (grp->il_laddr.s_addr == INADDR_ANY)
1468 local_wild = grp->il_inp[idx];
1469 }
1470 }
1471 if (local_wild != NULL)
1472 return local_wild;
1473 return NULL;
1474 }
1475
1476 /*
1477 * Lookup PCB in hash list.
1478 */
1479 struct inpcb *
1480 in_pcblookup_pkthash(struct inpcbinfo *pcbinfo, struct in_addr faddr,
1481 u_int fport_arg, struct in_addr laddr, u_int lport_arg,
1482 boolean_t wildcard, struct ifnet *ifp, const struct mbuf *m)
1483 {
1484 struct inpcbhead *head;
1485 struct inpcb *inp, *jinp=NULL;
1486 u_short fport = fport_arg, lport = lport_arg;
1487
1488 /*
1489 * First look for an exact match.
1490 */
1491 head = &pcbinfo->hashbase[INP_PCBCONNHASH(faddr.s_addr, fport,
1492 laddr.s_addr, lport, pcbinfo->hashmask)];
1493 LIST_FOREACH(inp, head, inp_hash) {
1494 #ifdef INET6
1495 if (!INP_ISIPV4(inp))
1496 continue;
1497 #endif
1498 if (in_hosteq(inp->inp_faddr, faddr) &&
1499 in_hosteq(inp->inp_laddr, laddr) &&
1500 inp->inp_fport == fport && inp->inp_lport == lport) {
1501 /* found */
1502 if (inp->inp_socket == NULL ||
1503 inp->inp_socket->so_cred->cr_prison == NULL) {
1504 return (inp);
1505 } else {
1506 if (jinp == NULL)
1507 jinp = inp;
1508 }
1509 }
1510 }
1511 if (jinp != NULL)
1512 return (jinp);
1513
1514 if (wildcard) {
1515 struct inpcb *local_wild = NULL;
1516 struct inpcb *jinp_wild = NULL;
1517 struct inpcontainer *ic;
1518 struct inpcontainerhead *chead;
1519 struct sockaddr_in jsin;
1520 struct ucred *cred;
1521
1522 GET_PCBINFO_TOKEN(pcbinfo);
1523
1524 /*
1525 * Check local group first
1526 */
1527 if (pcbinfo->localgrphashbase != NULL &&
1528 m != NULL && (m->m_flags & M_HASH)) {
1529 inp = inp_localgroup_lookup(pcbinfo,
1530 laddr, lport, m->m_pkthdr.hash);
1531 if (inp != NULL) {
1532 REL_PCBINFO_TOKEN(pcbinfo);
1533 return inp;
1534 }
1535 }
1536
1537 /*
1538 * Order of socket selection:
1539 * 1. non-jailed, non-wild.
1540 * 2. non-jailed, wild.
1541 * 3. jailed, non-wild.
1542 * 4. jailed, wild.
1543 */
1544 jsin.sin_family = AF_INET;
1545 chead = &pcbinfo->wildcardhashbase[
1546 INP_PCBWILDCARDHASH(lport, pcbinfo->wildcardhashmask)];
1547 LIST_FOREACH(ic, chead, ic_list) {
1548 inp = ic->ic_inp;
1549 if (inp->inp_flags & INP_PLACEMARKER)
1550 continue;
1551
1552 jsin.sin_addr.s_addr = laddr.s_addr;
1553 #ifdef INET6
1554 if (!INP_ISIPV4(inp))
1555 continue;
1556 #endif
1557 if (inp->inp_socket != NULL)
1558 cred = inp->inp_socket->so_cred;
1559 else
1560 cred = NULL;
1561 if (cred != NULL && jailed(cred)) {
1562 if (jinp != NULL)
1563 continue;
1564 else
1565 if (!jailed_ip(cred->cr_prison,
1566 (struct sockaddr *)&jsin))
1567 continue;
1568 }
1569 if (inp->inp_lport == lport) {
1570 if (inp->inp_laddr.s_addr == laddr.s_addr) {
1571 if (cred != NULL && jailed(cred)) {
1572 jinp = inp;
1573 } else {
1574 REL_PCBINFO_TOKEN(pcbinfo);
1575 return (inp);
1576 }
1577 }
1578 if (inp->inp_laddr.s_addr == INADDR_ANY) {
1579 if (cred != NULL && jailed(cred))
1580 jinp_wild = inp;
1581 else
1582 local_wild = inp;
1583 }
1584 }
1585 }
1586
1587 REL_PCBINFO_TOKEN(pcbinfo);
1588
1589 if (local_wild != NULL)
1590 return (local_wild);
1591 if (jinp != NULL)
1592 return (jinp);
1593 return (jinp_wild);
1594 }
1595
1596 /*
1597 * Not found.
1598 */
1599 return (NULL);
1600 }
1601
1602 struct inpcb *
1603 in_pcblookup_hash(struct inpcbinfo *pcbinfo, struct in_addr faddr,
1604 u_int fport_arg, struct in_addr laddr, u_int lport_arg,
1605 boolean_t wildcard, struct ifnet *ifp)
1606 {
1607 return in_pcblookup_pkthash(pcbinfo, faddr, fport_arg,
1608 laddr, lport_arg, wildcard, ifp, NULL);
1609 }
1610
1611 /*
1612 * Insert PCB into connection hash table.
1613 */
1614 void
1615 in_pcbinsconnhash(struct inpcb *inp)
1616 {
1617 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
1618 struct inpcbhead *bucket;
1619 u_int32_t hashkey_faddr, hashkey_laddr;
1620
1621 #ifdef INET6
1622 if (INP_ISIPV6(inp)) {
1623 hashkey_faddr = inp->in6p_faddr.s6_addr32[3] /* XXX JH */;
1624 hashkey_laddr = inp->in6p_laddr.s6_addr32[3] /* XXX JH */;
1625 } else {
1626 #endif
1627 hashkey_faddr = inp->inp_faddr.s_addr;
1628 hashkey_laddr = inp->inp_laddr.s_addr;
1629 #ifdef INET6
1630 }
1631 #endif
1632
1633 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
1634 ("not in the correct netisr"));
1635 ASSERT_INP_NOTINHASH(inp);
1636 inp->inp_flags |= INP_CONNECTED;
1637
1638 /*
1639 * Insert into the connection hash table.
1640 */
1641 bucket = &pcbinfo->hashbase[INP_PCBCONNHASH(hashkey_faddr,
1642 inp->inp_fport, hashkey_laddr, inp->inp_lport, pcbinfo->hashmask)];
1643 LIST_INSERT_HEAD(bucket, inp, inp_hash);
1644 }
1645
1646 /*
1647 * Remove PCB from connection hash table.
1648 */
1649 void
1650 in_pcbremconnhash(struct inpcb *inp)
1651 {
1652 struct inpcbinfo *pcbinfo __debugvar = inp->inp_pcbinfo;
1653
1654 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
1655 ("not in the correct netisr"));
1656 KASSERT(inp->inp_flags & INP_CONNECTED, ("inp not connected"));
1657
1658 LIST_REMOVE(inp, inp_hash);
1659 inp->inp_flags &= ~INP_CONNECTED;
1660 }
1661
1662 /*
1663 * Insert PCB into port hash table.
1664 */
1665 void
1666 in_pcbinsporthash(struct inpcbporthead *pcbporthash, struct inpcb *inp)
1667 {
1668 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
1669 struct inpcbport *phd;
1670
1671 /*
1672 * If the porthashbase is shared across several cpus, it must
1673 * have been locked.
1674 */
1675 ASSERT_PORTHASH_TOKEN_HELD(pcbporthash);
1676
1677 /*
1678 * Insert into the port hash table.
1679 */
1680
1681 /* Go through port list and look for a head for this lport. */
1682 LIST_FOREACH(phd, pcbporthash, phd_hash) {
1683 if (phd->phd_port == inp->inp_lport)
1684 break;
1685 }
1686
1687 /* If none exists, use saved one and tack it on. */
1688 if (phd == NULL) {
1689 KKASSERT(pcbinfo->portsave != NULL);
1690 phd = pcbinfo->portsave;
1691 pcbinfo->portsave = NULL;
1692 phd->phd_port = inp->inp_lport;
1693 LIST_INIT(&phd->phd_pcblist);
1694 LIST_INSERT_HEAD(pcbporthash, phd, phd_hash);
1695 }
1696
1697 inp->inp_porthash = pcbporthash;
1698 inp->inp_phd = phd;
1699 LIST_INSERT_HEAD(&phd->phd_pcblist, inp, inp_portlist);
1700
1701 /*
1702 * Malloc one inpcbport for later use. It is safe to use
1703 * "wait" malloc here (port token would be released, if
1704 * malloc ever blocked), since all changes to the porthash
1705 * are done.
1706 */
1707 if (pcbinfo->portsave == NULL) {
1708 pcbinfo->portsave = kmalloc(sizeof(*pcbinfo->portsave),
1709 M_PCB, M_INTWAIT | M_ZERO);
1710 }
1711 }
1712
1713 void
1714 in_pcbinsporthash_lport(struct inpcb *inp)
1715 {
1716 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
1717 struct inpcbportinfo *portinfo;
1718 struct inpcbporthead *porthash;
1719 u_short lport_ho;
1720
1721 /* Locate the proper portinfo based on lport */
1722 lport_ho = ntohs(inp->inp_lport);
1723 portinfo = &pcbinfo->portinfo[lport_ho % pcbinfo->portinfo_cnt];
1724 KKASSERT((lport_ho % pcbinfo->portinfo_cnt) == portinfo->offset);
1725
1726 porthash = in_pcbporthash_head(portinfo, inp->inp_lport);
1727 GET_PORTHASH_TOKEN(porthash);
1728 in_pcbinsporthash(porthash, inp);
1729 REL_PORTHASH_TOKEN(porthash);
1730 }
1731
1732 void
1733 in_pcbremporthash(struct inpcb *inp)
1734 {
1735 struct inpcbporthead *porthash;
1736 struct inpcbport *phd;
1737
1738 if (inp->inp_phd == NULL)
1739 return;
1740 KASSERT(inp->inp_lport != 0, ("inpcb has no lport"));
1741
1742 porthash = inp->inp_porthash;
1743 KASSERT(porthash != NULL, ("no porthash"));
1744
1745 GET_PORTHASH_TOKEN(porthash);
1746
1747 phd = inp->inp_phd;
1748 LIST_REMOVE(inp, inp_portlist);
1749 if (LIST_FIRST(&phd->phd_pcblist) == NULL) {
1750 LIST_REMOVE(phd, phd_hash);
1751 kfree(phd, M_PCB);
1752 }
1753
1754 REL_PORTHASH_TOKEN(porthash);
1755
1756 inp->inp_phd = NULL;
1757 /* NOTE: Don't whack inp_lport, which may be used later */
1758 }
1759
1760 static struct inp_localgroup *
1761 inp_localgroup_alloc(u_char af, uint16_t port,
1762 const union in_dependaddr *addr, int size)
1763 {
1764 struct inp_localgroup *grp;
1765
1766 grp = kmalloc(__offsetof(struct inp_localgroup, il_inp[size]),
1767 M_TEMP, M_INTWAIT | M_ZERO);
1768 grp->il_af = af;
1769 grp->il_lport = port;
1770 grp->il_dependladdr = *addr;
1771 grp->il_inpsiz = size;
1772
1773 return grp;
1774 }
1775
1776 static void
1777 inp_localgroup_free(struct inp_localgroup *grp)
1778 {
1779 kfree(grp, M_TEMP);
1780 }
1781
1782 static void
1783 inp_localgroup_destroy(struct inp_localgroup *grp)
1784 {
1785 LIST_REMOVE(grp, il_list);
1786 inp_localgroup_free(grp);
1787 }
1788
1789 static void
1790 inp_localgroup_copy(struct inp_localgroup *grp,
1791 const struct inp_localgroup *old_grp)
1792 {
1793 int i;
1794
1795 KASSERT(old_grp->il_inpcnt < grp->il_inpsiz,
1796 ("invalid new local group size %d and old local group count %d",
1797 grp->il_inpsiz, old_grp->il_inpcnt));
1798 for (i = 0; i < old_grp->il_inpcnt; ++i)
1799 grp->il_inp[i] = old_grp->il_inp[i];
1800 grp->il_inpcnt = old_grp->il_inpcnt;
1801 }
1802
1803 static void
1804 in_pcbinslocalgrphash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo)
1805 {
1806 struct inp_localgrphead *hdr;
1807 struct inp_localgroup *grp, *grp_alloc = NULL;
1808 struct ucred *cred;
1809 int i, idx;
1810
1811 ASSERT_PCBINFO_TOKEN_HELD(pcbinfo);
1812
1813 if (pcbinfo->localgrphashbase == NULL)
1814 return;
1815
1816 /*
1817 * XXX don't allow jailed socket to join local group
1818 */
1819 if (inp->inp_socket != NULL)
1820 cred = inp->inp_socket->so_cred;
1821 else
1822 cred = NULL;
1823 if (cred != NULL && jailed(cred))
1824 return;
1825
1826 hdr = &pcbinfo->localgrphashbase[
1827 INP_PCBLOCALGRPHASH(inp->inp_lport, pcbinfo->localgrphashmask)];
1828
1829 again:
1830 LIST_FOREACH(grp, hdr, il_list) {
1831 if (grp->il_af == inp->inp_af &&
1832 grp->il_lport == inp->inp_lport &&
1833 memcmp(&grp->il_dependladdr,
1834 &inp->inp_inc.inc_ie.ie_dependladdr,
1835 sizeof(grp->il_dependladdr)) == 0) {
1836 break;
1837 }
1838 }
1839 if (grp == NULL) {
1840 /*
1841 * Create a new local group
1842 */
1843 if (grp_alloc == NULL) {
1844 grp_alloc = inp_localgroup_alloc(inp->inp_af,
1845 inp->inp_lport, &inp->inp_inc.inc_ie.ie_dependladdr,
1846 INP_LOCALGROUP_SIZMIN);
1847 /*
1848 * Local group allocation could block and the
1849 * local group w/ the same property might have
1850 * been added by others when we were blocked;
1851 * check again.
1852 */
1853 goto again;
1854 } else {
1855 /* Local group has been allocated; link it */
1856 grp = grp_alloc;
1857 grp_alloc = NULL;
1858 LIST_INSERT_HEAD(hdr, grp, il_list);
1859 }
1860 } else if (grp->il_inpcnt == grp->il_inpsiz) {
1861 if (grp->il_inpsiz >= INP_LOCALGROUP_SIZMAX) {
1862 static int limit_logged = 0;
1863
1864 if (!limit_logged) {
1865 limit_logged = 1;
1866 kprintf("local group port %d, "
1867 "limit reached\n", ntohs(grp->il_lport));
1868 }
1869 if (grp_alloc != NULL) {
1870 /*
1871 * This would happen if the local group
1872 * w/ the same property was expanded when
1873 * our local group allocation blocked.
1874 */
1875 inp_localgroup_free(grp_alloc);
1876 }
1877 return;
1878 }
1879
1880 /*
1881 * Expand this local group
1882 */
1883 if (grp_alloc == NULL ||
1884 grp->il_inpcnt >= grp_alloc->il_inpsiz) {
1885 if (grp_alloc != NULL)
1886 inp_localgroup_free(grp_alloc);
1887 grp_alloc = inp_localgroup_alloc(grp->il_af,
1888 grp->il_lport, &grp->il_dependladdr,
1889 grp->il_inpsiz * 2);
1890 /*
1891 * Local group allocation could block and the
1892 * local group w/ the same property might have
1893 * been expanded by others when we were blocked;
1894 * check again.
1895 */
1896 goto again;
1897 }
1898
1899 /*
1900 * Save the old local group, link the new one, and then
1901 * destroy the old local group
1902 */
1903 inp_localgroup_copy(grp_alloc, grp);
1904 LIST_INSERT_HEAD(hdr, grp_alloc, il_list);
1905 inp_localgroup_destroy(grp);
1906
1907 grp = grp_alloc;
1908 grp_alloc = NULL;
1909 } else {
1910 /*
1911 * Found the local group
1912 */
1913 if (grp_alloc != NULL) {
1914 /*
1915 * This would happen if the local group w/ the
1916 * same property was added or expanded when our
1917 * local group allocation blocked.
1918 */
1919 inp_localgroup_free(grp_alloc);
1920 grp_alloc = NULL;
1921 }
1922 }
1923
1924 KASSERT(grp->il_inpcnt < grp->il_inpsiz,
1925 ("invalid local group size %d and count %d",
1926 grp->il_inpsiz, grp->il_inpcnt));
1927
1928 /*
1929 * Keep the local group sorted by the inpcb local group index
1930 * in ascending order.
1931 *
1932 * This eases the multi-process userland application which uses
1933 * SO_REUSEPORT sockets and binds process to the owner cpu of
1934 * the SO_REUSEPORT socket:
1935 * If we didn't sort the local group by the inpcb local group
1936 * index and one of the process owning an inpcb in this local
1937 * group restarted, e.g. crashed and restarted by watchdog,
1938 * other processes owning a inpcb in this local group would have
1939 * to detect that event, refetch its socket's owner cpu, and
1940 * re-bind.
1941 */
1942 idx = grp->il_inpcnt;
1943 for (i = 0; i < idx; ++i) {
1944 struct inpcb *oinp = grp->il_inp[i];
1945
1946 if (oinp->inp_lgrpindex > i) {
1947 if (inp->inp_lgrpindex < 0) {
1948 inp->inp_lgrpindex = i;
1949 } else if (inp->inp_lgrpindex != i) {
1950 if (bootverbose) {
1951 kprintf("inp %p: grpidx %d, "
1952 "assigned to %d, cpu%d\n",
1953 inp, inp->inp_lgrpindex, i,
1954 mycpuid);
1955 }
1956 }
1957 grp->il_inp[i] = inp;
1958
1959 /* Pull down inpcbs */
1960 for (; i < grp->il_inpcnt; ++i) {
1961 struct inpcb *oinp1 = grp->il_inp[i + 1];
1962
1963 grp->il_inp[i + 1] = oinp;
1964 oinp = oinp1;
1965 }
1966 grp->il_inpcnt++;
1967 return;
1968 }
1969 }
1970
1971 if (inp->inp_lgrpindex < 0) {
1972 inp->inp_lgrpindex = idx;
1973 } else if (inp->inp_lgrpindex != idx) {
1974 if (bootverbose) {
1975 kprintf("inp %p: grpidx %d, assigned to %d, cpu%d\n",
1976 inp, inp->inp_lgrpindex, idx, mycpuid);
1977 }
1978 }
1979 grp->il_inp[idx] = inp;
1980 grp->il_inpcnt++;
1981 }
1982
1983 void
1984 in_pcbinswildcardhash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo)
1985 {
1986 struct inpcontainer *ic;
1987 struct inpcontainerhead *bucket;
1988
1989 GET_PCBINFO_TOKEN(pcbinfo);
1990
1991 in_pcbinslocalgrphash_oncpu(inp, pcbinfo);
1992
1993 bucket = &pcbinfo->wildcardhashbase[
1994 INP_PCBWILDCARDHASH(inp->inp_lport, pcbinfo->wildcardhashmask)];
1995
1996 ic = kmalloc(sizeof(struct inpcontainer), M_TEMP, M_INTWAIT);
1997 ic->ic_inp = inp;
1998 LIST_INSERT_HEAD(bucket, ic, ic_list);
1999
2000 REL_PCBINFO_TOKEN(pcbinfo);
2001 }
2002
2003 /*
2004 * Insert PCB into wildcard hash table.
2005 */
2006 void
2007 in_pcbinswildcardhash(struct inpcb *inp)
2008 {
2009 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
2010
2011 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
2012 ("not in correct netisr"));
2013 ASSERT_INP_NOTINHASH(inp);
2014 inp->inp_flags |= INP_WILDCARD;
2015
2016 in_pcbinswildcardhash_oncpu(inp, pcbinfo);
2017 }
2018
2019 static void
2020 in_pcbremlocalgrphash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo)
2021 {
2022 struct inp_localgrphead *hdr;
2023 struct inp_localgroup *grp;
2024
2025 ASSERT_PCBINFO_TOKEN_HELD(pcbinfo);
2026
2027 if (pcbinfo->localgrphashbase == NULL)
2028 return;
2029
2030 hdr = &pcbinfo->localgrphashbase[
2031 INP_PCBLOCALGRPHASH(inp->inp_lport, pcbinfo->localgrphashmask)];
2032
2033 LIST_FOREACH(grp, hdr, il_list) {
2034 int i;
2035
2036 for (i = 0; i < grp->il_inpcnt; ++i) {
2037 if (grp->il_inp[i] != inp)
2038 continue;
2039
2040 if (grp->il_inpcnt == 1) {
2041 /* Destroy this local group */
2042 inp_localgroup_destroy(grp);
2043 } else {
2044 /* Pull up inpcbs */
2045 for (; i + 1 < grp->il_inpcnt; ++i)
2046 grp->il_inp[i] = grp->il_inp[i + 1];
2047 grp->il_inpcnt--;
2048 }
2049 return;
2050 }
2051 }
2052 }
2053
2054 void
2055 in_pcbremwildcardhash_oncpu(struct inpcb *inp, struct inpcbinfo *pcbinfo)
2056 {
2057 struct inpcontainer *ic;
2058 struct inpcontainerhead *head;
2059
2060 GET_PCBINFO_TOKEN(pcbinfo);
2061
2062 in_pcbremlocalgrphash_oncpu(inp, pcbinfo);
2063
2064 /* find bucket */
2065 head = &pcbinfo->wildcardhashbase[
2066 INP_PCBWILDCARDHASH(inp->inp_lport, pcbinfo->wildcardhashmask)];
2067
2068 LIST_FOREACH(ic, head, ic_list) {
2069 if (ic->ic_inp == inp)
2070 goto found;
2071 }
2072 REL_PCBINFO_TOKEN(pcbinfo);
2073 return; /* not found! */
2074
2075 found:
2076 LIST_REMOVE(ic, ic_list); /* remove container from bucket chain */
2077 REL_PCBINFO_TOKEN(pcbinfo);
2078 kfree(ic, M_TEMP); /* deallocate container */
2079 }
2080
2081 /*
2082 * Remove PCB from wildcard hash table.
2083 */
2084 void
2085 in_pcbremwildcardhash(struct inpcb *inp)
2086 {
2087 struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
2088
2089 KASSERT(&curthread->td_msgport == netisr_cpuport(pcbinfo->cpu),
2090 ("not in correct netisr"));
2091 KASSERT(inp->inp_flags & INP_WILDCARD, ("inp not wildcard"));
2092
2093 in_pcbremwildcardhash_oncpu(inp, pcbinfo);
2094 inp->inp_lgrpindex = -1;
2095 inp->inp_flags &= ~INP_WILDCARD;
2096 }
2097
2098 /*
2099 * Remove PCB from various lists.
2100 */
2101 void
2102 in_pcbremlists(struct inpcb *inp)
2103 {
2104 in_pcbremporthash(inp);
2105 if (inp->inp_flags & INP_WILDCARD) {
2106 in_pcbremwildcardhash(inp);
2107 } else if (inp->inp_flags & INP_CONNECTED) {
2108 in_pcbremconnhash(inp);
2109 }
2110
2111 if (inp->inp_flags & INP_ONLIST)
2112 in_pcbofflist(inp);
2113 }
2114
2115 int
2116 prison_xinpcb(struct thread *td, struct inpcb *inp)
2117 {
2118 struct ucred *cr;
2119
2120 if (td->td_proc == NULL)
2121 return (0);
2122 cr = td->td_proc->p_ucred;
2123 if (cr->cr_prison == NULL)
2124 return (0);
2125 if (inp->inp_socket && inp->inp_socket->so_cred &&
2126 inp->inp_socket->so_cred->cr_prison &&
2127 cr->cr_prison == inp->inp_socket->so_cred->cr_prison)
2128 return (0);
2129 return (1);
2130 }
2131
2132 int
2133 in_pcblist_range(SYSCTL_HANDLER_ARGS)
2134 {
2135 struct inpcbinfo *pcbinfo_arr = arg1;
2136 int pcbinfo_arrlen = arg2;
2137 struct inpcb *marker;
2138 int cpu, origcpu;
2139 int error, n;
2140
2141 KASSERT(pcbinfo_arrlen <= netisr_ncpus && pcbinfo_arrlen >= 1,
2142 ("invalid pcbinfo count %d", pcbinfo_arrlen));
2143
2144 /*
2145 * The process of preparing the TCB list is too time-consuming and
2146 * resource-intensive to repeat twice on every request.
2147 */
2148 n = 0;
2149 if (req->oldptr == NULL) {
2150 for (cpu = 0; cpu < pcbinfo_arrlen; ++cpu)
2151 n += pcbinfo_arr[cpu].ipi_count;
2152 req->oldidx = (n + n/8 + 10) * sizeof(struct xinpcb);
2153 return 0;
2154 }
2155
2156 if (req->newptr != NULL)
2157 return EPERM;
2158
2159 marker = kmalloc(sizeof(struct inpcb), M_TEMP, M_WAITOK|M_ZERO);
2160 marker->inp_flags |= INP_PLACEMARKER;
2161
2162 /*
2163 * OK, now we're committed to doing something. Re-fetch ipi_count
2164 * after obtaining the generation count.
2165 */
2166 error = 0;
2167 origcpu = mycpuid;
2168 for (cpu = 0; cpu < pcbinfo_arrlen && error == 0; ++cpu) {
2169 struct inpcbinfo *pcbinfo = &pcbinfo_arr[cpu];
2170 struct inpcb *inp;
2171 struct xinpcb xi;
2172 int i;
2173
2174 lwkt_migratecpu(cpu);
2175
2176 GET_PCBINFO_TOKEN(pcbinfo);
2177
2178 n = pcbinfo->ipi_count;
2179
2180 LIST_INSERT_HEAD(&pcbinfo->pcblisthead, marker, inp_list);
2181 i = 0;
2182 while ((inp = LIST_NEXT(marker, inp_list)) != NULL && i < n) {
2183 LIST_REMOVE(marker, inp_list);
2184 LIST_INSERT_AFTER(inp, marker, inp_list);
2185
2186 if (inp->inp_flags & INP_PLACEMARKER)
2187 continue;
2188 if (prison_xinpcb(req->td, inp))
2189 continue;
2190
2191 bzero(&xi, sizeof xi);
2192 xi.xi_len = sizeof xi;
2193 bcopy(inp, &xi.xi_inp, sizeof *inp);
2194 if (inp->inp_socket)
2195 sotoxsocket(inp->inp_socket, &xi.xi_socket);
2196 if ((error = SYSCTL_OUT(req, &xi, sizeof xi)) != 0)
2197 break;
2198 ++i;
2199 }
2200 LIST_REMOVE(marker, inp_list);
2201
2202 REL_PCBINFO_TOKEN(pcbinfo);
2203
2204 if (error == 0 && i < n) {
2205 bzero(&xi, sizeof xi);
2206 xi.xi_len = sizeof xi;
2207 while (i < n) {
2208 error = SYSCTL_OUT(req, &xi, sizeof xi);
2209 if (error)
2210 break;
2211 ++i;
2212 }
2213 }
2214 }
2215
2216 lwkt_migratecpu(origcpu);
2217 kfree(marker, M_TEMP);
2218 return error;
2219 }
2220
2221 int
2222 in_pcblist_ncpus(SYSCTL_HANDLER_ARGS)
2223 {
2224
2225 return (in_pcblist_range(oidp, arg1, netisr_ncpus, req));
2226 }
2227
2228 void
2229 in_savefaddr(struct socket *so, const struct sockaddr *faddr)
2230 {
2231 struct sockaddr_in *sin;
2232
2233 KASSERT(faddr->sa_family == AF_INET,
2234 ("not AF_INET faddr %d", faddr->sa_family));
2235
2236 sin = kmalloc(sizeof(*sin), M_SONAME, M_WAITOK | M_ZERO);
2237 sin->sin_family = AF_INET;
2238 sin->sin_len = sizeof(*sin);
2239 sin->sin_port = ((const struct sockaddr_in *)faddr)->sin_port;
2240 sin->sin_addr = ((const struct sockaddr_in *)faddr)->sin_addr;
2241
2242 so->so_faddr = (struct sockaddr *)sin;
2243 }
2244
2245 void
2246 in_pcbportinfo_init(struct inpcbportinfo *portinfo, int hashsize,
2247 u_short offset)
2248 {
2249 memset(portinfo, 0, sizeof(*portinfo));
2250
2251 portinfo->offset = offset;
2252 portinfo->porthashbase = phashinit(hashsize, M_PCB,
2253 &portinfo->porthashcnt);
2254 }
2255
2256 void
2257 in_pcbportrange(u_short *hi0, u_short *lo0, u_short ofs, u_short step)
2258 {
2259 int hi, lo;
2260
2261 if (step == 1)
2262 return;
2263
2264 hi = *hi0;
2265 lo = *lo0;
2266
2267 hi = rounddown(hi, step);
2268 hi += ofs;
2269 if (hi > (int)*hi0)
2270 hi -= step;
2271
2272 lo = roundup(lo, step);
2273 lo -= (step - ofs);
2274 if (lo < (int)*lo0)
2275 lo += step;
2276
2277 *hi0 = hi;
2278 *lo0 = lo;
2279 }
2280
2281 void
2282 in_pcbglobalinit(void)
2283 {
2284 int cpu;
2285
2286 in_pcbmarkers = kmalloc(netisr_ncpus * sizeof(struct inpcb), M_PCB,
2287 M_WAITOK | M_ZERO);
2288 in_pcbcontainer_markers =
2289 kmalloc(netisr_ncpus * sizeof(struct inpcontainer), M_PCB,
2290 M_WAITOK | M_ZERO);
2291
2292 for (cpu = 0; cpu < netisr_ncpus; ++cpu) {
2293 struct inpcontainer *ic = &in_pcbcontainer_markers[cpu];
2294 struct inpcb *marker = &in_pcbmarkers[cpu];
2295
2296 marker->inp_flags |= INP_PLACEMARKER;
2297 ic->ic_inp = marker;
2298 }
2299 }
2300
2301 struct inpcb *
2302 in_pcbmarker(void)
2303 {
2304
2305 ASSERT_NETISR_NCPUS(mycpuid);
2306 return &in_pcbmarkers[mycpuid];
2307 }
2308
2309 struct inpcontainer *
2310 in_pcbcontainer_marker(void)
2311 {
2312
2313 ASSERT_NETISR_NCPUS(mycpuid);
2314 return &in_pcbcontainer_markers[mycpuid];
2315 }
2316
2317 void
2318 in_pcbresetroute(struct inpcb *inp)
2319 {
2320 struct route *ro = &inp->inp_route;
2321
2322 if (ro->ro_rt != NULL)
2323 RTFREE(ro->ro_rt);
2324 bzero(ro, sizeof(*ro));
2325 }
DragonFly BSD