2 * Copyright (c) 2004 Jeffrey M. Hsu. All rights reserved.
3 * Copyright (c) 2004 The DragonFly Project. All rights reserved.
5 * This code is derived from software contributed to The DragonFly Project
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 * 3. Neither the name of The DragonFly Project nor the names of its
17 * contributors may be used to endorse or promote products derived
18 * from this software without specific, prior written permission.
20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
23 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
24 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
25 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
26 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
27 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
28 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
29 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
30 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
35 * Copyright (c) 1982, 1986, 1988, 1990, 1993, 1995
36 * The Regents of the University of California. All rights reserved.
38 * Redistribution and use in source and binary forms, with or without
39 * modification, are permitted provided that the following conditions
41 * 1. Redistributions of source code must retain the above copyright
42 * notice, this list of conditions and the following disclaimer.
43 * 2. Redistributions in binary form must reproduce the above copyright
44 * notice, this list of conditions and the following disclaimer in the
45 * documentation and/or other materials provided with the distribution.
46 * 3. Neither the name of the University nor the names of its contributors
47 * may be used to endorse or promote products derived from this software
48 * without specific prior written permission.
50 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
51 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
52 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
53 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
54 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
55 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
56 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
57 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
58 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
59 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
62 * @(#)udp_usrreq.c 8.6 (Berkeley) 5/23/95
63 * $FreeBSD: src/sys/netinet/udp_usrreq.c,v 1.64.2.18 2003/01/24 05:11:34 sam Exp $
66 #include "opt_ipsec.h"
67 #include "opt_inet6.h"
69 #include <sys/param.h>
70 #include <sys/systm.h>
71 #include <sys/kernel.h>
72 #include <sys/malloc.h>
74 #include <sys/domain.h>
77 #include <sys/protosw.h>
78 #include <sys/socket.h>
79 #include <sys/socketvar.h>
80 #include <sys/sysctl.h>
81 #include <sys/syslog.h>
82 #include <sys/in_cksum.h>
85 #include <sys/thread2.h>
86 #include <sys/socketvar2.h>
87 #include <sys/serialize.h>
89 #include <machine/stdarg.h>
92 #include <net/route.h>
93 #include <net/netmsg2.h>
94 #include <net/netisr2.h>
96 #include <netinet/in.h>
97 #include <netinet/in_systm.h>
98 #include <netinet/ip.h>
100 #include <netinet/ip6.h>
102 #include <netinet/in_pcb.h>
103 #include <netinet/in_var.h>
104 #include <netinet/ip_var.h>
106 #include <netinet6/ip6_var.h>
108 #include <netinet/ip_icmp.h>
109 #include <netinet/icmp_var.h>
110 #include <netinet/udp.h>
111 #include <netinet/udp_var.h>
114 #include <netproto/ipsec/ipsec.h>
118 #include <netinet6/ipsec.h>
121 #define MSGF_UDP_SEND MSGF_PROTO1
123 #define INP_DIRECT_DETACH INP_FLAG_PROTO2
125 #define UDP_KTR_STRING "inp=%p"
126 #define UDP_KTR_ARGS struct inpcb *inp
129 #define KTR_UDP KTR_ALL
132 KTR_INFO_MASTER(udp
);
133 KTR_INFO(KTR_UDP
, udp
, send_beg
, 0, UDP_KTR_STRING
, UDP_KTR_ARGS
);
134 KTR_INFO(KTR_UDP
, udp
, send_end
, 1, UDP_KTR_STRING
, UDP_KTR_ARGS
);
135 KTR_INFO(KTR_UDP
, udp
, send_ipout
, 2, UDP_KTR_STRING
, UDP_KTR_ARGS
);
136 KTR_INFO(KTR_UDP
, udp
, redisp_ipout_beg
, 3, UDP_KTR_STRING
, UDP_KTR_ARGS
);
137 KTR_INFO(KTR_UDP
, udp
, redisp_ipout_end
, 4, UDP_KTR_STRING
, UDP_KTR_ARGS
);
138 KTR_INFO(KTR_UDP
, udp
, send_redisp
, 5, UDP_KTR_STRING
, UDP_KTR_ARGS
);
139 KTR_INFO(KTR_UDP
, udp
, send_inswildcard
, 6, UDP_KTR_STRING
, UDP_KTR_ARGS
);
141 #define logudp(name, inp) KTR_LOG(udp_##name, inp)
144 * UDP protocol implementation.
145 * Per RFC 768, August, 1980.
148 static int udpcksum
= 1;
150 static int udpcksum
= 0; /* XXX */
152 SYSCTL_INT(_net_inet_udp
, UDPCTL_CHECKSUM
, checksum
, CTLFLAG_RW
,
153 &udpcksum
, 0, "Enable checksumming of UDP packets");
156 SYSCTL_INT(_net_inet_udp
, OID_AUTO
, log_in_vain
, CTLFLAG_RW
,
157 &log_in_vain
, 0, "Log all incoming UDP packets");
159 static int blackhole
= 0;
160 SYSCTL_INT(_net_inet_udp
, OID_AUTO
, blackhole
, CTLFLAG_RW
,
161 &blackhole
, 0, "Do not send port unreachables for refused connects");
163 static int strict_mcast_mship
= 1;
164 SYSCTL_INT(_net_inet_udp
, OID_AUTO
, strict_mcast_mship
, CTLFLAG_RW
,
165 &strict_mcast_mship
, 0, "Only send multicast to member sockets");
167 int udp_sosend_async
= 1;
168 SYSCTL_INT(_net_inet_udp
, OID_AUTO
, sosend_async
, CTLFLAG_RW
,
169 &udp_sosend_async
, 0, "UDP asynchronized pru_send");
171 int udp_sosend_prepend
= 1;
172 SYSCTL_INT(_net_inet_udp
, OID_AUTO
, sosend_prepend
, CTLFLAG_RW
,
173 &udp_sosend_prepend
, 0,
174 "Prepend enough space for proto and link header in pru_send");
176 static int udp_reuseport_ext
= 1;
177 SYSCTL_INT(_net_inet_udp
, OID_AUTO
, reuseport_ext
, CTLFLAG_RW
,
178 &udp_reuseport_ext
, 0, "SO_REUSEPORT extension");
180 struct inpcbinfo udbinfo
[MAXCPU
];
183 #define UDBHASHSIZE 16
185 CTASSERT((UDBHASHSIZE
& (UDBHASHSIZE
- 1)) == 0);
187 struct udpstat udpstat_percpu
[MAXCPU
] __cachealign
;
189 static void udp_append(struct inpcb
*last
, struct ip
*ip
,
190 struct mbuf
*n
, int off
, struct sockaddr_in
*udp_in
);
192 static int udp_connect_oncpu(struct inpcb
*inp
, struct sockaddr_in
*sin
,
193 struct sockaddr_in
*if_sin
, uint16_t hash
);
195 static boolean_t
udp_inswildcardhash(struct inpcb
*inp
,
196 struct netmsg_base
*msg
, int error
);
197 static void udp_remwildcardhash(struct inpcb
*inp
);
200 udp_lportcpu(short lport
)
202 return (ntohs(lport
) % netisr_ncpus
);
208 struct inpcbportinfo
*portinfo
;
211 portinfo
= kmalloc_cachealign(sizeof(*portinfo
) * netisr_ncpus
, M_PCB
,
214 for (cpu
= 0; cpu
< netisr_ncpus
; cpu
++) {
215 struct inpcbinfo
*uicb
= &udbinfo
[cpu
];
219 * UDP pcb list, wildcard hash table and localgroup hash
222 in_pcbinfo_init(uicb
, cpu
, TRUE
);
223 uicb
->hashbase
= hashinit(UDBHASHSIZE
, M_PCB
, &uicb
->hashmask
);
225 in_pcbportinfo_init(&portinfo
[cpu
], UDBHASHSIZE
, cpu
);
226 in_pcbportinfo_set(uicb
, portinfo
, netisr_ncpus
);
228 uicb
->wildcardhashbase
= hashinit(UDBHASHSIZE
, M_PCB
,
229 &uicb
->wildcardhashmask
);
230 uicb
->localgrphashbase
= hashinit(UDBHASHSIZE
, M_PCB
,
231 &uicb
->localgrphashmask
);
233 uicb
->ipi_size
= sizeof(struct inpcb
);
237 * Initialize UDP statistics counters for each CPU.
239 for (cpu
= 0; cpu
< netisr_ncpus
; ++cpu
)
240 bzero(&udpstat_percpu
[cpu
], sizeof(struct udpstat
));
244 sysctl_udpstat(SYSCTL_HANDLER_ARGS
)
248 for (cpu
= 0; cpu
< netisr_ncpus
; ++cpu
) {
249 if ((error
= SYSCTL_OUT(req
, &udpstat_percpu
[cpu
],
250 sizeof(struct udpstat
))))
252 if ((error
= SYSCTL_IN(req
, &udpstat_percpu
[cpu
],
253 sizeof(struct udpstat
))))
259 SYSCTL_PROC(_net_inet_udp
, UDPCTL_STATS
, stats
, (CTLTYPE_OPAQUE
| CTLFLAG_RW
),
260 0, 0, sysctl_udpstat
, "S,udpstat", "UDP statistics");
263 udp_ctloutput(netmsg_t msg
)
265 struct socket
*so
= msg
->base
.nm_so
;
266 struct sockopt
*sopt
= msg
->ctloutput
.nm_sopt
;
267 struct inpcb
*inp
= so
->so_pcb
;
270 lwkt_replymsg(&msg
->lmsg
, EINVAL
);
274 if (sopt
->sopt_level
== IPPROTO_IP
&& sopt
->sopt_dir
== SOPT_SET
) {
275 switch (sopt
->sopt_name
) {
276 case IP_MULTICAST_IF
:
277 case IP_MULTICAST_VIF
:
278 case IP_MULTICAST_TTL
:
279 case IP_MULTICAST_LOOP
:
280 case IP_ADD_MEMBERSHIP
:
281 case IP_DROP_MEMBERSHIP
:
283 * This pr_ctloutput msg will be forwarded
284 * to netisr0 to run; we can't do direct
288 * Don't optimize for the sockets whose
289 * current so_port is netisr0's msgport.
290 * These sockets could be connect(2)'ed
291 * later and the so_port will be changed.
293 inp
->inp_flags
&= ~INP_DIRECT_DETACH
;
297 return ip_ctloutput(msg
);
301 * Check multicast packets to make sure they are only sent to sockets with
302 * multicast memberships for the packet's destination address and arrival
303 * interface. Multicast packets to multicast-unaware sockets are also
306 * Returns 0 if the packet is acceptable, -1 if it is not.
309 check_multicast_membership(const struct ip
*ip
, const struct inpcb
*inp
,
310 const struct mbuf
*m
)
312 const struct ip_moptions
*mopt
;
315 if (strict_mcast_mship
== 0 ||
316 !IN_MULTICAST(ntohl(ip
->ip_dst
.s_addr
))) {
322 mopt
= inp
->inp_moptions
;
325 for (mshipno
= 0; mshipno
< mopt
->imo_num_memberships
; ++mshipno
) {
326 const struct in_multi
*maddr
= mopt
->imo_membership
[mshipno
];
328 if (ip
->ip_dst
.s_addr
== maddr
->inm_addr
.s_addr
&&
329 m
->m_pkthdr
.rcvif
== maddr
->inm_ifp
) {
336 struct udp_mcast_arg
{
342 struct sockaddr_in
*udp_in
;
346 udp_mcast_input(struct udp_mcast_arg
*arg
)
348 struct inpcb
*inp
= arg
->inp
;
349 struct inpcb
*last
= arg
->last
;
350 struct ip
*ip
= arg
->ip
;
351 struct mbuf
*m
= arg
->m
;
353 if (check_multicast_membership(ip
, inp
, m
) < 0)
354 return ERESTART
; /* caller continue */
360 /* check AH/ESP integrity. */
361 if (ipsec4_in_reject_so(m
, last
->inp_socket
))
362 ipsecstat
.in_polvio
++;
363 /* do not inject data to pcb */
367 /* check AH/ESP integrity. */
368 if (ipsec4_in_reject(m
, last
))
371 #endif /*FAST_IPSEC*/
372 if ((n
= m_copypacket(m
, M_NOWAIT
)) != NULL
)
373 udp_append(last
, ip
, n
,
374 arg
->iphlen
+ sizeof(struct udphdr
),
377 arg
->last
= last
= inp
;
380 * Don't look for additional matches if this one does
381 * not have either the SO_REUSEPORT or SO_REUSEADDR
382 * socket options set. This heuristic avoids searching
383 * through all pcbs in the common case of a non-shared
384 * port. It * assumes that an application will never
385 * clear these options after setting them.
387 if (!(last
->inp_socket
->so_options
&
388 (SO_REUSEPORT
| SO_REUSEADDR
)))
389 return EJUSTRETURN
; /* caller stop */
394 udp_input(struct mbuf
**mp
, int *offp
, int proto
)
396 struct sockaddr_in udp_in
= { sizeof udp_in
, AF_INET
};
402 struct mbuf
*opts
= NULL
;
405 struct inpcbinfo
*pcbinfo
= &udbinfo
[mycpuid
];
412 udp_stat
.udps_ipackets
++;
415 * Strip IP options, if any; should skip this,
416 * make available to user, and use on returned packets,
417 * but we don't yet have a way to check the checksum
418 * with options still present.
420 if (iphlen
> sizeof(struct ip
)) {
422 iphlen
= sizeof(struct ip
);
426 * IP and UDP headers are together in first mbuf.
427 * Already checked and pulled up in ip_demux().
429 KASSERT(m
->m_len
>= iphlen
+ sizeof(struct udphdr
),
430 ("UDP header not in one mbuf"));
432 ip
= mtod(m
, struct ip
*);
433 uh
= (struct udphdr
*)((caddr_t
)ip
+ iphlen
);
435 /* destination port of 0 is illegal, based on RFC768. */
436 if (uh
->uh_dport
== 0)
440 * Make mbuf data length reflect UDP length.
441 * If not enough data to reflect UDP length, drop.
443 len
= ntohs((u_short
)uh
->uh_ulen
);
444 if (ip
->ip_len
!= len
) {
445 if (len
> ip
->ip_len
|| len
< sizeof(struct udphdr
)) {
446 udp_stat
.udps_badlen
++;
449 m_adj(m
, len
- ip
->ip_len
);
450 /* ip->ip_len = len; */
453 * Save a copy of the IP header in case we want restore it
454 * for sending an ICMP error message in response.
459 * Checksum extended UDP header and data.
462 if (m
->m_pkthdr
.csum_flags
& CSUM_DATA_VALID
) {
463 if (m
->m_pkthdr
.csum_flags
& CSUM_PSEUDO_HDR
)
464 uh
->uh_sum
= m
->m_pkthdr
.csum_data
;
466 uh
->uh_sum
= in_pseudo(ip
->ip_src
.s_addr
,
467 ip
->ip_dst
.s_addr
, htonl((u_short
)len
+
468 m
->m_pkthdr
.csum_data
+ IPPROTO_UDP
));
469 uh
->uh_sum
^= 0xffff;
473 bcopy(((struct ipovly
*)ip
)->ih_x1
, b
, 9);
474 bzero(((struct ipovly
*)ip
)->ih_x1
, 9);
475 ((struct ipovly
*)ip
)->ih_len
= uh
->uh_ulen
;
476 uh
->uh_sum
= in_cksum(m
, len
+ sizeof(struct ip
));
477 bcopy(b
, ((struct ipovly
*)ip
)->ih_x1
, 9);
480 udp_stat
.udps_badsum
++;
482 return(IPPROTO_DONE
);
485 udp_stat
.udps_nosum
++;
487 if (IN_MULTICAST(ntohl(ip
->ip_dst
.s_addr
)) ||
488 in_broadcast(ip
->ip_dst
, m
->m_pkthdr
.rcvif
)) {
489 struct inpcbhead
*connhead
;
490 struct inpcontainer
*ic
, *ic_marker
;
491 struct inpcontainerhead
*ichead
;
492 struct udp_mcast_arg arg
;
497 * Deliver a multicast or broadcast datagram to *all* sockets
498 * for which the local and remote addresses and ports match
499 * those of the incoming datagram. This allows more than
500 * one process to receive multi/broadcasts on the same port.
501 * (This really ought to be done for unicast datagrams as
502 * well, but that would cause problems with existing
503 * applications that open both address-specific sockets and
504 * a wildcard socket listening to the same port -- they would
505 * end up receiving duplicates of every unicast datagram.
506 * Those applications open the multiple sockets to overcome an
507 * inadequacy of the UDP socket interface, but for backwards
508 * compatibility we avoid the problem here rather than
509 * fixing the interface. Maybe 4.5BSD will remedy this?)
513 * Construct sockaddr format source address.
515 udp_in
.sin_port
= uh
->uh_sport
;
516 udp_in
.sin_addr
= ip
->ip_src
;
517 arg
.udp_in
= &udp_in
;
519 * Locate pcb(s) for datagram.
520 * (Algorithm copied from raw_intr().)
525 connhead
= &pcbinfo
->hashbase
[
526 INP_PCBCONNHASH(ip
->ip_src
.s_addr
, uh
->uh_sport
,
527 ip
->ip_dst
.s_addr
, uh
->uh_dport
, pcbinfo
->hashmask
)];
528 LIST_FOREACH(inp
, connhead
, inp_hash
) {
530 if (!INP_ISIPV4(inp
))
533 if (!in_hosteq(inp
->inp_faddr
, ip
->ip_src
) ||
534 !in_hosteq(inp
->inp_laddr
, ip
->ip_dst
) ||
535 inp
->inp_fport
!= uh
->uh_sport
||
536 inp
->inp_lport
!= uh
->uh_dport
)
544 error
= udp_mcast_input(&arg
);
545 if (error
== ERESTART
)
549 if (error
== EJUSTRETURN
)
553 ichead
= &pcbinfo
->wildcardhashbase
[
554 INP_PCBWILDCARDHASH(uh
->uh_dport
,
555 pcbinfo
->wildcardhashmask
)];
556 ic_marker
= in_pcbcontainer_marker();
558 GET_PCBINFO_TOKEN(pcbinfo
);
559 LIST_INSERT_HEAD(ichead
, ic_marker
, ic_list
);
560 while ((ic
= LIST_NEXT(ic_marker
, ic_list
)) != NULL
) {
561 LIST_REMOVE(ic_marker
, ic_list
);
562 LIST_INSERT_AFTER(ic
, ic_marker
, ic_list
);
565 if (inp
->inp_flags
& INP_PLACEMARKER
)
568 if (!INP_ISIPV4(inp
))
571 if (inp
->inp_lport
!= uh
->uh_dport
)
573 if (inp
->inp_laddr
.s_addr
!= INADDR_ANY
&&
574 inp
->inp_laddr
.s_addr
!= ip
->ip_dst
.s_addr
)
582 error
= udp_mcast_input(&arg
);
583 if (error
== ERESTART
)
587 if (error
== EJUSTRETURN
)
590 LIST_REMOVE(ic_marker
, ic_list
);
591 REL_PCBINFO_TOKEN(pcbinfo
);
595 * No matching pcb found; discard datagram.
596 * (No need to send an ICMP Port Unreachable
597 * for a broadcast or multicast datgram.)
599 udp_stat
.udps_noportbcast
++;
603 /* check AH/ESP integrity. */
604 if (ipsec4_in_reject_so(m
, last
->inp_socket
)) {
605 ipsecstat
.in_polvio
++;
610 /* check AH/ESP integrity. */
611 if (ipsec4_in_reject(m
, last
))
613 #endif /*FAST_IPSEC*/
614 udp_append(last
, ip
, m
, iphlen
+ sizeof(struct udphdr
),
616 return(IPPROTO_DONE
);
619 * Locate pcb for datagram.
621 inp
= in_pcblookup_pkthash(pcbinfo
, ip
->ip_src
, uh
->uh_sport
,
622 ip
->ip_dst
, uh
->uh_dport
, TRUE
, m
->m_pkthdr
.rcvif
,
623 udp_reuseport_ext
? m
: NULL
);
626 char src
[INET_ADDRSTRLEN
], dst
[INET_ADDRSTRLEN
];
629 "Connection attempt to UDP %s:%d from %s:%d\n",
630 kinet_ntoa(ip
->ip_dst
, dst
), ntohs(uh
->uh_dport
),
631 kinet_ntoa(ip
->ip_src
, src
), ntohs(uh
->uh_sport
));
633 udp_stat
.udps_noport
++;
634 if (m
->m_flags
& (M_BCAST
| M_MCAST
)) {
635 udp_stat
.udps_noportbcast
++;
641 if (badport_bandlim(BANDLIM_ICMP_UNREACH
) < 0)
645 ip
->ip_len
+= iphlen
;
646 icmp_error(m
, ICMP_UNREACH
, ICMP_UNREACH_PORT
, 0, 0);
647 return(IPPROTO_DONE
);
649 KASSERT(INP_ISIPV4(inp
), ("not inet inpcb"));
651 if (ipsec4_in_reject_so(m
, inp
->inp_socket
)) {
652 ipsecstat
.in_polvio
++;
657 if (ipsec4_in_reject(m
, inp
))
659 #endif /*FAST_IPSEC*/
661 * Check the minimum TTL for socket.
663 if (ip
->ip_ttl
< inp
->inp_ip_minttl
)
667 * Construct sockaddr format source address.
668 * Stuff source address and datagram in user buffer.
670 udp_in
.sin_port
= uh
->uh_sport
;
671 udp_in
.sin_addr
= ip
->ip_src
;
672 if ((inp
->inp_flags
& INP_CONTROLOPTS
) ||
673 (inp
->inp_socket
->so_options
& SO_TIMESTAMP
))
674 ip_savecontrol(inp
, &opts
, ip
, m
);
675 m_adj(m
, iphlen
+ sizeof(struct udphdr
));
677 lwkt_gettoken(&inp
->inp_socket
->so_rcv
.ssb_token
);
678 if (ssb_appendaddr(&inp
->inp_socket
->so_rcv
,
679 (struct sockaddr
*)&udp_in
, m
, opts
) == 0) {
680 lwkt_reltoken(&inp
->inp_socket
->so_rcv
.ssb_token
);
681 udp_stat
.udps_fullsock
++;
684 lwkt_reltoken(&inp
->inp_socket
->so_rcv
.ssb_token
);
685 sorwakeup(inp
->inp_socket
);
686 return(IPPROTO_DONE
);
691 return(IPPROTO_DONE
);
695 * subroutine of udp_input(), mainly for source code readability.
696 * caller must properly init udp_ip6 and udp_in6 beforehand.
699 udp_append(struct inpcb
*last
, struct ip
*ip
, struct mbuf
*n
, int off
,
700 struct sockaddr_in
*udp_in
)
702 struct mbuf
*opts
= NULL
;
705 KASSERT(INP_ISIPV4(last
), ("not inet inpcb"));
707 if (last
->inp_flags
& INP_CONTROLOPTS
||
708 last
->inp_socket
->so_options
& SO_TIMESTAMP
)
709 ip_savecontrol(last
, &opts
, ip
, n
);
712 lwkt_gettoken(&last
->inp_socket
->so_rcv
.ssb_token
);
713 ret
= ssb_appendaddr(&last
->inp_socket
->so_rcv
,
714 (struct sockaddr
*)udp_in
, n
, opts
);
715 lwkt_reltoken(&last
->inp_socket
->so_rcv
.ssb_token
);
720 udp_stat
.udps_fullsock
++;
722 sorwakeup(last
->inp_socket
);
727 * Notify a udp user of an asynchronous error;
728 * just wake up so that he can collect error status.
731 udp_notify(struct inpcb
*inp
, int error
)
733 inp
->inp_socket
->so_error
= error
;
734 sorwakeup(inp
->inp_socket
);
735 sowwakeup(inp
->inp_socket
);
738 struct netmsg_udp_notify
{
739 struct netmsg_base base
;
740 inp_notify_t nm_notify
;
741 struct in_addr nm_faddr
;
746 udp_notifyall_oncpu(netmsg_t msg
)
748 struct netmsg_udp_notify
*nm
= (struct netmsg_udp_notify
*)msg
;
749 int nextcpu
, cpu
= mycpuid
;
751 ASSERT_NETISR_NCPUS(cpu
);
753 in_pcbnotifyall(&udbinfo
[cpu
], nm
->nm_faddr
, nm
->nm_arg
, nm
->nm_notify
);
756 if (nextcpu
< netisr_ncpus
)
757 lwkt_forwardmsg(netisr_cpuport(nextcpu
), &nm
->base
.lmsg
);
759 lwkt_replymsg(&nm
->base
.lmsg
, 0);
763 udp_get_inpnotify(int cmd
, const struct sockaddr
*sa
,
764 struct ip
**ip0
, int *cpuid
)
766 struct in_addr faddr
;
767 struct ip
*ip
= *ip0
;
768 inp_notify_t notify
= udp_notify
;
770 faddr
= ((const struct sockaddr_in
*)sa
)->sin_addr
;
771 if (sa
->sa_family
!= AF_INET
|| faddr
.s_addr
== INADDR_ANY
)
774 if (PRC_IS_REDIRECT(cmd
)) {
776 notify
= in_rtchange
;
777 } else if (cmd
== PRC_HOSTDEAD
) {
779 } else if ((unsigned)cmd
>= PRC_NCMDS
|| inetctlerrmap
[cmd
] == 0) {
785 /* Go through all effective netisr CPUs. */
786 *cpuid
= netisr_ncpus
;
788 const struct udphdr
*uh
;
790 uh
= (const struct udphdr
*)
791 ((caddr_t
)ip
+ (ip
->ip_hl
<< 2));
792 *cpuid
= udp_addrcpu(faddr
.s_addr
, uh
->uh_dport
,
793 ip
->ip_src
.s_addr
, uh
->uh_sport
);
802 udp_ctlinput(netmsg_t msg
)
804 struct sockaddr
*sa
= msg
->ctlinput
.nm_arg
;
805 struct ip
*ip
= msg
->ctlinput
.nm_extra
;
806 int cmd
= msg
->ctlinput
.nm_cmd
, cpuid
;
808 struct in_addr faddr
;
810 ASSERT_NETISR_NCPUS(mycpuid
);
812 notify
= udp_get_inpnotify(cmd
, sa
, &ip
, &cpuid
);
816 faddr
= ((struct sockaddr_in
*)sa
)->sin_addr
;
818 const struct udphdr
*uh
;
821 if (cpuid
!= mycpuid
)
824 uh
= (const struct udphdr
*)((caddr_t
)ip
+ (ip
->ip_hl
<< 2));
825 inp
= in_pcblookup_hash(&udbinfo
[mycpuid
], faddr
, uh
->uh_dport
,
826 ip
->ip_src
, uh
->uh_sport
, 0, NULL
);
827 if (inp
!= NULL
&& inp
->inp_socket
!= NULL
)
828 notify(inp
, inetctlerrmap
[cmd
]);
829 } else if (msg
->ctlinput
.nm_direct
) {
830 if (cpuid
!= netisr_ncpus
&& cpuid
!= mycpuid
)
833 in_pcbnotifyall(&udbinfo
[mycpuid
], faddr
, inetctlerrmap
[cmd
],
836 struct netmsg_udp_notify
*nm
;
839 nm
= kmalloc(sizeof(*nm
), M_LWKTMSG
, M_INTWAIT
);
840 netmsg_init(&nm
->base
, NULL
, &netisr_afree_rport
,
841 0, udp_notifyall_oncpu
);
842 nm
->nm_faddr
= faddr
;
843 nm
->nm_arg
= inetctlerrmap
[cmd
];
844 nm
->nm_notify
= notify
;
845 lwkt_sendmsg(netisr_cpuport(0), &nm
->base
.lmsg
);
848 lwkt_replymsg(&msg
->lmsg
, 0);
851 SYSCTL_PROC(_net_inet_udp
, UDPCTL_PCBLIST
, pcblist
, CTLFLAG_RD
, udbinfo
, 0,
852 in_pcblist_ncpus
, "S,xinpcb", "List of active UDP sockets");
855 udp_getcred(SYSCTL_HANDLER_ARGS
)
857 struct sockaddr_in addrs
[2];
858 struct ucred cred0
, *cred
= NULL
;
860 int error
, cpu
, origcpu
;
862 error
= priv_check(req
->td
, PRIV_ROOT
);
865 error
= SYSCTL_IN(req
, addrs
, sizeof addrs
);
870 cpu
= udp_addrcpu(addrs
[1].sin_addr
.s_addr
, addrs
[1].sin_port
,
871 addrs
[0].sin_addr
.s_addr
, addrs
[0].sin_port
);
873 lwkt_migratecpu(cpu
);
875 inp
= in_pcblookup_hash(&udbinfo
[cpu
],
876 addrs
[1].sin_addr
, addrs
[1].sin_port
,
877 addrs
[0].sin_addr
, addrs
[0].sin_port
, TRUE
, NULL
);
878 if (inp
== NULL
|| inp
->inp_socket
== NULL
) {
880 } else if (inp
->inp_socket
->so_cred
!= NULL
) {
881 cred0
= *(inp
->inp_socket
->so_cred
);
885 lwkt_migratecpu(origcpu
);
890 return SYSCTL_OUT(req
, cred
, sizeof(struct ucred
));
892 SYSCTL_PROC(_net_inet_udp
, OID_AUTO
, getcred
, CTLTYPE_OPAQUE
|CTLFLAG_RW
,
893 0, 0, udp_getcred
, "S,ucred", "Get the ucred of a UDP connection");
896 udp_send_redispatch(netmsg_t msg
)
898 struct mbuf
*m
= msg
->send
.nm_m
;
899 int pru_flags
= msg
->send
.nm_flags
;
900 struct inpcb
*inp
= msg
->send
.base
.nm_so
->so_pcb
;
901 struct mbuf
*m_opt
= msg
->send
.nm_control
; /* XXX save ipopt */
902 int flags
= msg
->send
.nm_priv
; /* ip_output flags */
905 logudp(redisp_ipout_beg
, inp
);
908 * - Don't use inp route cache. It should only be used in the
910 * - Access to inp_moptions should be safe, since multicast UDP
911 * datagrams are redispatched to netisr0 and inp_moptions is
912 * changed only in netisr0.
914 error
= ip_output(m
, m_opt
, NULL
, flags
, inp
->inp_moptions
, inp
);
915 if ((pru_flags
& PRUS_NOREPLY
) == 0)
916 lwkt_replymsg(&msg
->send
.base
.lmsg
, error
);
919 /* Free saved ip options, if any */
923 logudp(redisp_ipout_end
, inp
);
927 udp_send(netmsg_t msg
)
929 struct socket
*so
= msg
->send
.base
.nm_so
;
930 struct mbuf
*m
= msg
->send
.nm_m
;
931 struct sockaddr
*dstaddr
= msg
->send
.nm_addr
;
932 int pru_flags
= msg
->send
.nm_flags
;
933 struct inpcb
*inp
= so
->so_pcb
;
934 struct thread
*td
= msg
->send
.nm_td
;
939 int len
= m
->m_pkthdr
.len
;
940 struct sockaddr_in
*sin
; /* really is initialized before use */
943 KKASSERT(msg
->send
.nm_control
== NULL
);
945 logudp(send_beg
, inp
);
952 if (len
+ sizeof(struct udpiphdr
) > IP_MAXPACKET
) {
957 if (inp
->inp_lport
== 0) { /* unbound socket */
960 error
= in_pcbbind(inp
, NULL
, td
);
965 * Need to call udp_send again, after this inpcb is
966 * inserted into wildcard hash table.
968 msg
->send
.base
.lmsg
.ms_flags
|= MSGF_UDP_SEND
;
969 forwarded
= udp_inswildcardhash(inp
, &msg
->send
.base
, 0);
972 * The message is further forwarded, so we are
975 logudp(send_inswildcard
, inp
);
980 if (dstaddr
!= NULL
) { /* destination address specified */
981 if (inp
->inp_faddr
.s_addr
!= INADDR_ANY
) {
982 /* already connected */
986 sin
= (struct sockaddr_in
*)dstaddr
;
987 if (!prison_remote_ip(td
, (struct sockaddr
*)&sin
)) {
988 error
= EAFNOSUPPORT
; /* IPv6 only jail */
992 if (inp
->inp_faddr
.s_addr
== INADDR_ANY
) {
993 /* no destination specified and not already connected */
1001 * Calculate data length and get a mbuf
1002 * for UDP and IP headers.
1004 M_PREPEND(m
, sizeof(struct udpiphdr
), M_NOWAIT
);
1011 * Fill in mbuf with extended UDP header
1012 * and addresses and length put into network format.
1014 ui
= mtod(m
, struct udpiphdr
*);
1015 bzero(ui
->ui_x1
, sizeof ui
->ui_x1
); /* XXX still needed? */
1016 ui
->ui_pr
= IPPROTO_UDP
;
1019 * Set destination address.
1021 if (dstaddr
!= NULL
) { /* use specified destination */
1022 ui
->ui_dst
= sin
->sin_addr
;
1023 ui
->ui_dport
= sin
->sin_port
;
1024 } else { /* use connected destination */
1025 ui
->ui_dst
= inp
->inp_faddr
;
1026 ui
->ui_dport
= inp
->inp_fport
;
1030 * Set source address.
1032 if (inp
->inp_laddr
.s_addr
== INADDR_ANY
||
1033 IN_MULTICAST(ntohl(inp
->inp_laddr
.s_addr
))) {
1034 struct sockaddr_in
*if_sin
;
1036 if (dstaddr
== NULL
) {
1038 * connect() had (or should have) failed because
1039 * the interface had no IP address, but the
1040 * application proceeded to call send() anyways.
1046 /* Look up outgoing interface. */
1047 error
= in_pcbladdr_find(inp
, dstaddr
, &if_sin
, td
, 1);
1050 ui
->ui_src
= if_sin
->sin_addr
; /* use address of interface */
1052 ui
->ui_src
= inp
->inp_laddr
; /* use non-null bound address */
1054 ui
->ui_sport
= inp
->inp_lport
;
1055 KASSERT(inp
->inp_lport
!= 0, ("inp lport should have been bound"));
1058 * Release the original thread, since it is no longer used
1060 if (pru_flags
& PRUS_HELDTD
) {
1062 pru_flags
&= ~PRUS_HELDTD
;
1065 * Free the dest address, since it is no longer needed
1067 if (pru_flags
& PRUS_FREEADDR
) {
1068 kfree(dstaddr
, M_SONAME
);
1069 pru_flags
&= ~PRUS_FREEADDR
;
1072 ui
->ui_ulen
= htons((u_short
)len
+ sizeof(struct udphdr
));
1075 * Set up checksum and output datagram.
1078 ui
->ui_sum
= in_pseudo(ui
->ui_src
.s_addr
, ui
->ui_dst
.s_addr
,
1079 htons((u_short
)len
+ sizeof(struct udphdr
) + IPPROTO_UDP
));
1080 m
->m_pkthdr
.csum_flags
= CSUM_UDP
;
1081 m
->m_pkthdr
.csum_data
= offsetof(struct udphdr
, uh_sum
);
1082 m
->m_pkthdr
.csum_thlen
= sizeof(struct udphdr
);
1086 ((struct ip
*)ui
)->ip_len
= sizeof(struct udpiphdr
) + len
;
1087 ((struct ip
*)ui
)->ip_ttl
= inp
->inp_ip_ttl
; /* XXX */
1088 ((struct ip
*)ui
)->ip_tos
= inp
->inp_ip_tos
; /* XXX */
1089 udp_stat
.udps_opackets
++;
1091 flags
= IP_DEBUGROUTE
|
1092 (inp
->inp_socket
->so_options
& (SO_DONTROUTE
| SO_BROADCAST
));
1093 if (pru_flags
& PRUS_DONTROUTE
)
1094 flags
|= SO_DONTROUTE
;
1096 if (inp
->inp_flags
& INP_CONNECTED
) {
1098 * For connected socket, this datagram has already
1099 * been in the correct netisr; no need to rehash.
1101 KASSERT(inp
->inp_flags
& INP_HASH
, ("inpcb has no hash"));
1102 m_sethash(m
, inp
->inp_hashval
);
1106 hash
= udp_addrhash(ui
->ui_dst
.s_addr
, ui
->ui_dport
,
1107 ui
->ui_src
.s_addr
, ui
->ui_sport
);
1110 cpu
= netisr_hashcpu(hash
);
1111 if (cpu
!= mycpuid
) {
1112 struct mbuf
*m_opt
= NULL
;
1113 struct netmsg_pru_send
*smsg
;
1114 struct lwkt_port
*port
= netisr_cpuport(cpu
);
1117 * Not on the CPU that matches this UDP datagram hash;
1118 * redispatch to the correct CPU to do the ip_output().
1120 if (inp
->inp_options
!= NULL
) {
1122 * If there are ip options, then save a copy,
1123 * since accessing inp_options on other CPUs'
1126 * XXX optimize this?
1128 m_opt
= m_copym(inp
->inp_options
, 0, M_COPYALL
,
1131 if ((pru_flags
& PRUS_NOREPLY
) == 0) {
1133 * Change some parts of the original netmsg and
1134 * forward it to the target netisr.
1136 * NOTE: so_port MUST NOT be checked in the target
1140 smsg
->nm_priv
= flags
; /* ip_output flags */
1142 smsg
->nm_control
= m_opt
; /* XXX save ipopt */
1143 smsg
->base
.lmsg
.ms_flags
|= MSGF_IGNSOPORT
;
1144 smsg
->base
.nm_dispatch
= udp_send_redispatch
;
1145 lwkt_forwardmsg(port
, &smsg
->base
.lmsg
);
1148 * Recreate the netmsg, since the original mbuf
1149 * could have been changed. And send it to the
1152 * NOTE: so_port MUST NOT be checked in the target
1155 smsg
= &m
->m_hdr
.mh_sndmsg
;
1156 netmsg_init(&smsg
->base
, so
, &netisr_apanic_rport
,
1157 MSGF_IGNSOPORT
, udp_send_redispatch
);
1158 smsg
->nm_priv
= flags
; /* ip_output flags */
1159 smsg
->nm_flags
= pru_flags
;
1161 smsg
->nm_control
= m_opt
; /* XXX save ipopt */
1162 lwkt_sendmsg(port
, &smsg
->base
.lmsg
);
1165 /* This UDP datagram is redispatched; done */
1166 logudp(send_redisp
, inp
);
1171 logudp(send_ipout
, inp
);
1172 error
= ip_output(m
, inp
->inp_options
, &inp
->inp_route
, flags
,
1173 inp
->inp_moptions
, inp
);
1180 if (pru_flags
& PRUS_HELDTD
)
1182 if (pru_flags
& PRUS_FREEADDR
)
1183 kfree(dstaddr
, M_SONAME
);
1184 if ((pru_flags
& PRUS_NOREPLY
) == 0)
1185 lwkt_replymsg(&msg
->send
.base
.lmsg
, error
);
1187 logudp(send_end
, inp
);
1190 u_long udp_sendspace
= 9216; /* really max datagram size */
1191 /* 40 1K datagrams */
1192 SYSCTL_INT(_net_inet_udp
, UDPCTL_MAXDGRAM
, maxdgram
, CTLFLAG_RW
,
1193 &udp_sendspace
, 0, "Maximum outgoing UDP datagram size");
1195 u_long udp_recvspace
= 40 * (1024 +
1197 sizeof(struct sockaddr_in6
)
1199 sizeof(struct sockaddr_in
)
1202 SYSCTL_INT(_net_inet_udp
, UDPCTL_RECVSPACE
, recvspace
, CTLFLAG_RW
,
1203 &udp_recvspace
, 0, "Maximum incoming UDP datagram size");
1206 * This should never happen, since UDP socket does not support
1207 * connection acception (SO_ACCEPTCONN, i.e. listen(2)).
1210 udp_abort(netmsg_t msg __unused
)
1212 panic("udp_abort is called");
1216 udp_preattach(struct socket
*so
, int proto __unused
, struct pru_attach_info
*ai
)
1218 return soreserve(so
, udp_sendspace
, udp_recvspace
, ai
->sb_rlimit
);
1222 udp_attach(netmsg_t msg
)
1224 struct socket
*so
= msg
->attach
.base
.nm_so
;
1225 struct pru_attach_info
*ai
= msg
->attach
.nm_ai
;
1229 KASSERT(so
->so_pcb
== NULL
, ("udp socket attached"));
1232 error
= udp_preattach(so
, 0 /* don't care */, ai
);
1236 /* Post attach; do nothing */
1239 error
= in_pcballoc(so
, &udbinfo
[mycpuid
]);
1244 inp
->inp_flags
|= INP_DIRECT_DETACH
;
1245 inp
->inp_ip_ttl
= ip_defttl
;
1248 lwkt_replymsg(&msg
->attach
.base
.lmsg
, error
);
1252 udp_inswildcard_replymsg(netmsg_t msg
)
1254 lwkt_msg_t lmsg
= &msg
->lmsg
;
1256 if (lmsg
->ms_flags
& MSGF_UDP_SEND
) {
1258 /* msg is replied by udp_send() */
1260 lwkt_replymsg(lmsg
, lmsg
->ms_error
);
1265 udp_soreuseport_dispatch(netmsg_t msg
)
1267 /* This inpcb has already been in the wildcard hash. */
1268 in_pcblink_flags(msg
->base
.nm_so
->so_pcb
, &udbinfo
[mycpuid
], 0);
1269 udp_inswildcard_replymsg(msg
);
1273 udp_sosetport(struct lwkt_msg
*msg
, lwkt_port_t port
)
1275 sosetport(((struct netmsg_base
*)msg
)->nm_so
, port
);
1279 udp_inswildcardhash_oncpu(struct inpcb
*inp
, struct netmsg_base
*msg
)
1283 KASSERT(inp
->inp_pcbinfo
== &udbinfo
[mycpuid
],
1284 ("not on owner cpu"));
1286 in_pcbinswildcardhash(inp
);
1287 for (cpu
= 0; cpu
< netisr_ncpus
; ++cpu
) {
1288 if (cpu
== mycpuid
) {
1290 * This inpcb has been inserted by the above
1291 * in_pcbinswildcardhash().
1295 in_pcbinswildcardhash_oncpu(inp
, &udbinfo
[cpu
]);
1298 /* NOTE: inp_lgrpindex is _not_ assigned in jail. */
1299 if ((inp
->inp_socket
->so_options
& SO_REUSEPORT
) &&
1300 inp
->inp_lgrpindex
>= 0) {
1302 * For SO_REUSEPORT socket, redistribute it based on its
1303 * local group index.
1305 cpu
= inp
->inp_lgrpindex
% netisr_ncpus
;
1306 if (cpu
!= mycpuid
) {
1307 struct lwkt_port
*port
= netisr_cpuport(cpu
);
1308 lwkt_msg_t lmsg
= &msg
->lmsg
;
1311 * We are moving the protocol processing port the
1312 * socket is on, we have to unlink here and re-link
1313 * on the target cpu (this inpcb is still left in
1314 * the wildcard hash).
1316 in_pcbunlink_flags(inp
, &udbinfo
[mycpuid
], 0);
1317 msg
->nm_dispatch
= udp_soreuseport_dispatch
;
1320 * See the related comment in tcp_usrreq.c
1323 lwkt_setmsg_receipt(lmsg
, udp_sosetport
);
1324 lwkt_forwardmsg(port
, lmsg
);
1325 return TRUE
; /* forwarded */
1332 udp_inswildcardhash_dispatch(netmsg_t msg
)
1334 struct inpcb
*inp
= msg
->base
.nm_so
->so_pcb
;
1335 boolean_t forwarded
;
1337 KASSERT(inp
->inp_lport
!= 0, ("local port not set yet"));
1338 KASSERT(udp_lportcpu(inp
->inp_lport
) == mycpuid
, ("not target cpu"));
1340 in_pcblink(inp
, &udbinfo
[mycpuid
]);
1342 forwarded
= udp_inswildcardhash_oncpu(inp
, &msg
->base
);
1344 /* The message is further forwarded, so we are done here. */
1347 udp_inswildcard_replymsg(msg
);
1351 udp_inswildcardhash(struct inpcb
*inp
, struct netmsg_base
*msg
, int error
)
1353 lwkt_msg_t lmsg
= &msg
->lmsg
;
1356 ASSERT_INP_NOTINHASH(inp
);
1358 /* This inpcb could no longer be directly detached */
1359 inp
->inp_flags
&= ~INP_DIRECT_DETACH
;
1362 * Always clear the route cache, so we don't need to
1363 * worry about any owner CPU changes later.
1365 in_pcbresetroute(inp
);
1367 KASSERT(inp
->inp_lport
!= 0, ("local port not set yet"));
1368 cpu
= udp_lportcpu(inp
->inp_lport
);
1370 lmsg
->ms_error
= error
;
1371 if (cpu
!= mycpuid
) {
1372 struct lwkt_port
*port
= netisr_cpuport(cpu
);
1375 * We are moving the protocol processing port the socket
1376 * is on, we have to unlink here and re-link on the
1379 in_pcbunlink(inp
, &udbinfo
[mycpuid
]);
1380 msg
->nm_dispatch
= udp_inswildcardhash_dispatch
;
1382 /* See the related comment in tcp_usrreq.c tcp_connect() */
1383 lwkt_setmsg_receipt(lmsg
, udp_sosetport
);
1384 lwkt_forwardmsg(port
, lmsg
);
1385 return TRUE
; /* forwarded */
1388 return udp_inswildcardhash_oncpu(inp
, msg
);
1392 udp_bind(netmsg_t msg
)
1394 struct socket
*so
= msg
->bind
.base
.nm_so
;
1400 struct sockaddr
*nam
= msg
->bind
.nm_nam
;
1401 struct thread
*td
= msg
->bind
.nm_td
;
1402 struct sockaddr_in
*sin
;
1407 * Check "already bound" here (in_pcbbind() does the same
1408 * check though), so we don't forward a connected/bound
1409 * socket randomly which would panic in the following
1412 if (inp
->inp_lport
!= 0 ||
1413 inp
->inp_laddr
.s_addr
!= INADDR_ANY
) {
1414 error
= EINVAL
; /* already bound */
1418 if (nam
->sa_len
!= sizeof(*sin
)) {
1422 sin
= (struct sockaddr_in
*)nam
;
1424 cpu
= udp_lportcpu(sin
->sin_port
);
1425 port
= netisr_cpuport(cpu
);
1428 * See the related comment in tcp_usrreq.c tcp_usr_bind().
1429 * The exception is that we use local port based netisr
1430 * to serialize in_pcbbind().
1432 if (&curthread
->td_msgport
!= port
) {
1433 lwkt_msg_t lmsg
= &msg
->bind
.base
.lmsg
;
1435 KASSERT((msg
->bind
.nm_flags
& PRUB_RELINK
) == 0,
1436 ("already asked to relink"));
1438 in_pcbunlink(so
->so_pcb
, &udbinfo
[mycpuid
]);
1439 msg
->bind
.nm_flags
|= PRUB_RELINK
;
1442 * See the related comment in tcp_usrreq.c
1445 lwkt_setmsg_receipt(lmsg
, udp_sosetport
);
1446 lwkt_forwardmsg(port
, lmsg
);
1447 /* msg invalid now */
1450 KASSERT(so
->so_port
== port
, ("so_port is not netisr%d", cpu
));
1452 if (msg
->bind
.nm_flags
& PRUB_RELINK
) {
1453 msg
->bind
.nm_flags
&= ~PRUB_RELINK
;
1454 in_pcblink(so
->so_pcb
, &udbinfo
[mycpuid
]);
1456 KASSERT(inp
->inp_pcbinfo
== &udbinfo
[cpu
],
1457 ("pcbinfo is not udbinfo%d", cpu
));
1459 error
= in_pcbbind(inp
, nam
, td
);
1461 boolean_t forwarded
;
1463 if (sin
->sin_addr
.s_addr
!= INADDR_ANY
)
1464 inp
->inp_flags
|= INP_WASBOUND_NOTANY
;
1466 forwarded
= udp_inswildcardhash(inp
,
1467 &msg
->bind
.base
, 0);
1470 * The message is further forwarded, so
1480 lwkt_replymsg(&msg
->bind
.base
.lmsg
, error
);
1484 udp_preconnect(struct socket
*so
, const struct sockaddr
*nam __unused
,
1485 struct thread
*td __unused
)
1487 sosetstate(so
, SS_ISCONNECTED
); /* XXX */
1492 udp_connect(netmsg_t msg
)
1494 struct socket
*so
= msg
->connect
.base
.nm_so
;
1495 struct sockaddr
*nam
= msg
->connect
.nm_nam
;
1496 struct thread
*td
= msg
->connect
.nm_td
;
1498 struct sockaddr_in
*sin
= (struct sockaddr_in
*)nam
;
1499 struct sockaddr_in
*if_sin
;
1500 struct lwkt_port
*port
;
1504 KKASSERT(msg
->connect
.nm_m
== NULL
);
1512 if (msg
->connect
.nm_flags
& PRUC_RECONNECT
) {
1513 msg
->connect
.nm_flags
&= ~PRUC_RECONNECT
;
1514 in_pcblink(inp
, &udbinfo
[mycpuid
]);
1517 if (inp
->inp_faddr
.s_addr
!= INADDR_ANY
) {
1524 * Bind if we have to
1526 if (inp
->inp_lport
== 0) {
1527 error
= in_pcbbind(inp
, NULL
, td
);
1533 * Calculate the correct protocol processing thread. The connect
1534 * operation must run there.
1536 error
= in_pcbladdr(inp
, nam
, &if_sin
, td
);
1539 if (!prison_remote_ip(td
, nam
)) {
1540 error
= EAFNOSUPPORT
; /* IPv6 only jail */
1544 hash
= udp_addrhash(sin
->sin_addr
.s_addr
, sin
->sin_port
,
1545 inp
->inp_laddr
.s_addr
!= INADDR_ANY
?
1546 inp
->inp_laddr
.s_addr
: if_sin
->sin_addr
.s_addr
, inp
->inp_lport
);
1547 port
= netisr_hashport(hash
);
1548 if (port
!= &curthread
->td_msgport
) {
1549 lwkt_msg_t lmsg
= &msg
->connect
.base
.lmsg
;
1550 int nm_flags
= PRUC_RECONNECT
;
1553 * in_pcbladdr() may have allocated a route entry for us
1554 * on the current CPU, but we need a route entry on the
1555 * inpcb's owner CPU, so free it here.
1557 in_pcbresetroute(inp
);
1559 if (inp
->inp_flags
& INP_WILDCARD
) {
1561 * Remove this inpcb from the wildcard hash before
1562 * the socket's msgport changes.
1564 udp_remwildcardhash(inp
);
1567 if (so
->so_orig_port
== NULL
) {
1569 * First time change protocol processing port.
1570 * Save the current port for synchronization upon
1573 so
->so_orig_port
= &curthread
->td_msgport
;
1576 * We have changed protocol processing port more
1577 * than once. We could not do direct detach
1578 * anymore, because we lose the track of the
1579 * original protocol processing ports to perform
1580 * synchronization upon udp_detach. This should
1583 inp
->inp_flags
&= ~INP_DIRECT_DETACH
;
1587 * We are moving the protocol processing port the socket
1588 * is on, we have to unlink here and re-link on the
1591 in_pcbunlink(inp
, &udbinfo
[mycpuid
]);
1592 msg
->connect
.nm_flags
|= nm_flags
;
1594 /* See the related comment in tcp_usrreq.c tcp_connect() */
1595 lwkt_setmsg_receipt(lmsg
, udp_sosetport
);
1596 lwkt_forwardmsg(port
, lmsg
);
1597 /* msg invalid now */
1600 error
= udp_connect_oncpu(inp
, sin
, if_sin
, hash
);
1602 if (msg
->connect
.nm_flags
& PRUC_HELDTD
)
1604 if (error
&& (msg
->connect
.nm_flags
& PRUC_ASYNC
)) {
1605 if (inp
->inp_lport
== 0) {
1607 * As long as we have the local port, it is fine
1608 * for connect to fail, e.g. disconnect.
1610 so
->so_error
= error
;
1612 soclrstate(so
, SS_ISCONNECTED
);
1614 * Wake up callers blocked on this socket to make sure
1615 * that they can see this error.
1618 * sodisconnected() can't be used here, which bricks
1619 * sending and receiving.
1621 wakeup(&so
->so_timeo
);
1625 if (error
&& inp
!= NULL
&& inp
->inp_lport
!= 0 &&
1626 (inp
->inp_flags
& INP_WILDCARD
) == 0) {
1627 boolean_t forwarded
;
1629 /* Connect failed; put it to wildcard hash. */
1630 forwarded
= udp_inswildcardhash(inp
, &msg
->connect
.base
,
1634 * The message is further forwarded, so we are done
1640 lwkt_replymsg(&msg
->connect
.base
.lmsg
, error
);
1644 udp_remwildcardhash(struct inpcb
*inp
)
1648 KASSERT(inp
->inp_pcbinfo
== &udbinfo
[mycpuid
],
1649 ("not on owner cpu"));
1651 for (cpu
= 0; cpu
< netisr_ncpus
; ++cpu
) {
1652 if (cpu
== mycpuid
) {
1654 * This inpcb will be removed by the later
1655 * in_pcbremwildcardhash().
1659 in_pcbremwildcardhash_oncpu(inp
, &udbinfo
[cpu
]);
1661 in_pcbremwildcardhash(inp
);
1665 udp_connect_oncpu(struct inpcb
*inp
, struct sockaddr_in
*sin
,
1666 struct sockaddr_in
*if_sin
, uint16_t hash
)
1668 struct socket
*so
= inp
->inp_socket
;
1671 oinp
= in_pcblookup_hash(inp
->inp_pcbinfo
,
1672 sin
->sin_addr
, sin
->sin_port
,
1673 inp
->inp_laddr
.s_addr
!= INADDR_ANY
?
1674 inp
->inp_laddr
: if_sin
->sin_addr
, inp
->inp_lport
, FALSE
, NULL
);
1679 * No more errors can occur, finish adjusting the socket
1680 * and change the processing port to reflect the connected
1681 * socket. Once set we can no longer safely mess with the
1685 if (inp
->inp_flags
& INP_WILDCARD
)
1686 udp_remwildcardhash(inp
);
1688 if (inp
->inp_laddr
.s_addr
== INADDR_ANY
)
1689 inp
->inp_laddr
= if_sin
->sin_addr
;
1690 inp
->inp_faddr
= sin
->sin_addr
;
1691 inp
->inp_fport
= sin
->sin_port
;
1692 in_pcbinsconnhash(inp
);
1694 inp
->inp_flags
|= INP_HASH
;
1695 inp
->inp_hashval
= hash
;
1703 udp_detach2(struct socket
*so
)
1705 in_pcbdetach(so
->so_pcb
);
1711 udp_detach_final_dispatch(netmsg_t msg
)
1713 udp_detach2(msg
->base
.nm_so
);
1717 udp_detach_oncpu_dispatch(netmsg_t msg
)
1719 struct netmsg_base
*clomsg
= &msg
->base
;
1720 struct socket
*so
= clomsg
->nm_so
;
1721 struct inpcb
*inp
= so
->so_pcb
;
1722 struct thread
*td
= curthread
;
1723 int nextcpu
, cpuid
= mycpuid
;
1725 KASSERT(td
->td_type
== TD_TYPE_NETISR
, ("not in netisr"));
1727 if (inp
->inp_flags
& INP_WILDCARD
) {
1729 * This inp will be removed on the inp's
1730 * owner CPU later, so don't do it now.
1732 if (&td
->td_msgport
!= so
->so_port
)
1733 in_pcbremwildcardhash_oncpu(inp
, &udbinfo
[cpuid
]);
1738 * Free and clear multicast socket option,
1739 * which is only accessed in netisr0.
1741 ip_freemoptions(inp
->inp_moptions
);
1742 inp
->inp_moptions
= NULL
;
1745 nextcpu
= cpuid
+ 1;
1746 if (nextcpu
< netisr_ncpus
) {
1747 lwkt_forwardmsg(netisr_cpuport(nextcpu
), &clomsg
->lmsg
);
1750 * No one could see this inpcb now; destroy this
1751 * inpcb in its owner netisr.
1753 netmsg_init(clomsg
, so
, &netisr_apanic_rport
, 0,
1754 udp_detach_final_dispatch
);
1755 lwkt_sendmsg(so
->so_port
, &clomsg
->lmsg
);
1760 udp_detach_syncorig_dispatch(netmsg_t msg
)
1762 struct netmsg_base
*clomsg
= &msg
->base
;
1763 struct socket
*so
= clomsg
->nm_so
;
1766 * Original protocol processing port is synchronized;
1767 * destroy this inpcb in its owner netisr.
1769 netmsg_init(clomsg
, so
, &netisr_apanic_rport
, 0,
1770 udp_detach_final_dispatch
);
1771 lwkt_sendmsg(so
->so_port
, &clomsg
->lmsg
);
1775 udp_detach(netmsg_t msg
)
1777 struct socket
*so
= msg
->detach
.base
.nm_so
;
1778 struct netmsg_base
*clomsg
;
1783 lwkt_replymsg(&msg
->detach
.base
.lmsg
, EINVAL
);
1788 * Reply EJUSTRETURN ASAP, we will call sodiscard() and
1791 lwkt_replymsg(&msg
->detach
.base
.lmsg
, EJUSTRETURN
);
1793 if (netisr_ncpus
== 1) {
1794 /* Only one CPU, detach the inpcb directly. */
1800 * Remove this inpcb from the inpcb list first, so that
1801 * no one could find this inpcb from the inpcb list.
1806 * Remove this inpcb from the local port hash directly
1807 * here, so that its bound local port could be recycled
1810 in_pcbremporthash(inp
);
1812 if (inp
->inp_flags
& INP_DIRECT_DETACH
) {
1814 * Direct detaching is allowed
1816 KASSERT((inp
->inp_flags
& INP_WILDCARD
) == 0,
1817 ("in the wildcardhash"));
1818 KASSERT(inp
->inp_moptions
== NULL
, ("has mcast options"));
1819 if (so
->so_orig_port
== NULL
) {
1823 * Protocol processing port changed once, so
1824 * we need to make sure that there are nothing
1825 * left on the original protocol processing
1826 * port before we destroy this socket and inpcb.
1827 * This is more lightweight than going through
1828 * all UDP processing netisrs.
1830 clomsg
= &so
->so_clomsg
;
1831 netmsg_init(clomsg
, so
, &netisr_apanic_rport
,
1832 MSGF_IGNSOPORT
, udp_detach_syncorig_dispatch
);
1833 lwkt_sendmsg(so
->so_orig_port
, &clomsg
->lmsg
);
1839 * Go through netisrs which process UDP to make sure
1840 * no one could find this inpcb anymore.
1842 clomsg
= &so
->so_clomsg
;
1843 netmsg_init(clomsg
, so
, &netisr_apanic_rport
, MSGF_IGNSOPORT
,
1844 udp_detach_oncpu_dispatch
);
1845 lwkt_sendmsg(netisr_cpuport(0), &clomsg
->lmsg
);
1849 udp_disconnect(netmsg_t msg
)
1851 struct socket
*so
= msg
->disconnect
.base
.nm_so
;
1853 boolean_t forwarded
;
1861 if (inp
->inp_faddr
.s_addr
== INADDR_ANY
) {
1866 soclrstate(so
, SS_ISCONNECTED
); /* XXX */
1868 in_pcbdisconnect(inp
);
1869 inp
->inp_flags
&= ~INP_HASH
;
1872 * Follow traditional BSD behavior and retain the local port
1873 * binding. But, fix the old misbehavior of overwriting any
1874 * previously bound local address.
1876 if (!(inp
->inp_flags
& INP_WASBOUND_NOTANY
))
1877 inp
->inp_laddr
.s_addr
= INADDR_ANY
;
1879 if (so
->so_state
& SS_ISCLOSING
) {
1881 * If this socket is being closed, there is no need
1882 * to put this socket back into wildcard hash table.
1888 forwarded
= udp_inswildcardhash(inp
, &msg
->disconnect
.base
, 0);
1891 * The message is further forwarded, so we are done
1897 lwkt_replymsg(&msg
->disconnect
.base
.lmsg
, error
);
1901 udp_shutdown(netmsg_t msg
)
1903 struct socket
*so
= msg
->shutdown
.base
.nm_so
;
1914 lwkt_replymsg(&msg
->shutdown
.base
.lmsg
, error
);
1917 struct pr_usrreqs udp_usrreqs
= {
1918 .pru_abort
= udp_abort
,
1919 .pru_accept
= pr_generic_notsupp
,
1920 .pru_attach
= udp_attach
,
1921 .pru_bind
= udp_bind
,
1922 .pru_connect
= udp_connect
,
1923 .pru_connect2
= pr_generic_notsupp
,
1924 .pru_control
= in_control_dispatch
,
1925 .pru_detach
= udp_detach
,
1926 .pru_disconnect
= udp_disconnect
,
1927 .pru_listen
= pr_generic_notsupp
,
1928 .pru_peeraddr
= in_setpeeraddr_dispatch
,
1929 .pru_rcvd
= pr_generic_notsupp
,
1930 .pru_rcvoob
= pr_generic_notsupp
,
1931 .pru_send
= udp_send
,
1932 .pru_sense
= pru_sense_null
,
1933 .pru_shutdown
= udp_shutdown
,
1934 .pru_sockaddr
= in_setsockaddr_dispatch
,
1935 .pru_sosend
= sosendudp
,
1936 .pru_soreceive
= soreceive
,
1937 .pru_preconnect
= udp_preconnect
,
1938 .pru_preattach
= udp_preattach