2 * Copyright (c) 2004 Jeffrey M. Hsu. All rights reserved.
3 * Copyright (c) 2004 The DragonFly Project. All rights reserved.
5 * This code is derived from software contributed to The DragonFly Project
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 * 3. Neither the name of The DragonFly Project nor the names of its
17 * contributors may be used to endorse or promote products derived
18 * from this software without specific, prior written permission.
20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
23 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
24 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
25 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
26 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
27 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
28 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
29 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
30 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
35 * Copyright (c) 1982, 1986, 1988, 1990, 1993, 1995
36 * The Regents of the University of California. All rights reserved.
38 * Redistribution and use in source and binary forms, with or without
39 * modification, are permitted provided that the following conditions
41 * 1. Redistributions of source code must retain the above copyright
42 * notice, this list of conditions and the following disclaimer.
43 * 2. Redistributions in binary form must reproduce the above copyright
44 * notice, this list of conditions and the following disclaimer in the
45 * documentation and/or other materials provided with the distribution.
46 * 3. Neither the name of the University nor the names of its contributors
47 * may be used to endorse or promote products derived from this software
48 * without specific prior written permission.
50 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
51 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
52 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
53 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
54 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
55 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
56 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
57 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
58 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
59 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
62 * @(#)udp_usrreq.c 8.6 (Berkeley) 5/23/95
63 * $FreeBSD: src/sys/netinet/udp_usrreq.c,v 1.64.2.18 2003/01/24 05:11:34 sam Exp $
66 #include "opt_ipsec.h"
67 #include "opt_inet6.h"
69 #include <sys/param.h>
70 #include <sys/systm.h>
71 #include <sys/kernel.h>
72 #include <sys/malloc.h>
74 #include <sys/domain.h>
77 #include <sys/protosw.h>
78 #include <sys/socket.h>
79 #include <sys/socketvar.h>
80 #include <sys/sysctl.h>
81 #include <sys/syslog.h>
82 #include <sys/in_cksum.h>
85 #include <sys/thread2.h>
86 #include <sys/socketvar2.h>
87 #include <sys/serialize.h>
89 #include <machine/stdarg.h>
92 #include <net/route.h>
93 #include <net/netmsg2.h>
94 #include <net/netisr2.h>
96 #include <netinet/in.h>
97 #include <netinet/in_systm.h>
98 #include <netinet/ip.h>
100 #include <netinet/ip6.h>
102 #include <netinet/in_pcb.h>
103 #include <netinet/in_var.h>
104 #include <netinet/ip_var.h>
106 #include <netinet6/ip6_var.h>
108 #include <netinet/ip_icmp.h>
109 #include <netinet/icmp_var.h>
110 #include <netinet/udp.h>
111 #include <netinet/udp_var.h>
114 #include <netproto/ipsec/ipsec.h>
118 #include <netinet6/ipsec.h>
121 #define MSGF_UDP_SEND MSGF_PROTO1
123 #define INP_DIRECT_DETACH INP_FLAG_PROTO2
125 #define UDP_KTR_STRING "inp=%p"
126 #define UDP_KTR_ARGS struct inpcb *inp
129 #define KTR_UDP KTR_ALL
132 KTR_INFO_MASTER(udp
);
133 KTR_INFO(KTR_UDP
, udp
, send_beg
, 0, UDP_KTR_STRING
, UDP_KTR_ARGS
);
134 KTR_INFO(KTR_UDP
, udp
, send_end
, 1, UDP_KTR_STRING
, UDP_KTR_ARGS
);
135 KTR_INFO(KTR_UDP
, udp
, send_ipout
, 2, UDP_KTR_STRING
, UDP_KTR_ARGS
);
136 KTR_INFO(KTR_UDP
, udp
, redisp_ipout_beg
, 3, UDP_KTR_STRING
, UDP_KTR_ARGS
);
137 KTR_INFO(KTR_UDP
, udp
, redisp_ipout_end
, 4, UDP_KTR_STRING
, UDP_KTR_ARGS
);
138 KTR_INFO(KTR_UDP
, udp
, send_redisp
, 5, UDP_KTR_STRING
, UDP_KTR_ARGS
);
139 KTR_INFO(KTR_UDP
, udp
, send_inswildcard
, 6, UDP_KTR_STRING
, UDP_KTR_ARGS
);
141 #define logudp(name, inp) KTR_LOG(udp_##name, inp)
144 * UDP protocol implementation.
145 * Per RFC 768, August, 1980.
148 static int udpcksum
= 1;
150 static int udpcksum
= 0; /* XXX */
152 SYSCTL_INT(_net_inet_udp
, UDPCTL_CHECKSUM
, checksum
, CTLFLAG_RW
,
153 &udpcksum
, 0, "Enable checksumming of UDP packets");
156 SYSCTL_INT(_net_inet_udp
, OID_AUTO
, log_in_vain
, CTLFLAG_RW
,
157 &log_in_vain
, 0, "Log all incoming UDP packets");
159 static int blackhole
= 0;
160 SYSCTL_INT(_net_inet_udp
, OID_AUTO
, blackhole
, CTLFLAG_RW
,
161 &blackhole
, 0, "Do not send port unreachables for refused connects");
163 static int strict_mcast_mship
= 1;
164 SYSCTL_INT(_net_inet_udp
, OID_AUTO
, strict_mcast_mship
, CTLFLAG_RW
,
165 &strict_mcast_mship
, 0, "Only send multicast to member sockets");
167 int udp_sosend_async
= 1;
168 SYSCTL_INT(_net_inet_udp
, OID_AUTO
, sosend_async
, CTLFLAG_RW
,
169 &udp_sosend_async
, 0, "UDP asynchronized pru_send");
171 int udp_sosend_prepend
= 1;
172 SYSCTL_INT(_net_inet_udp
, OID_AUTO
, sosend_prepend
, CTLFLAG_RW
,
173 &udp_sosend_prepend
, 0,
174 "Prepend enough space for proto and link header in pru_send");
176 static int udp_reuseport_ext
= 1;
177 SYSCTL_INT(_net_inet_udp
, OID_AUTO
, reuseport_ext
, CTLFLAG_RW
,
178 &udp_reuseport_ext
, 0, "SO_REUSEPORT extension");
180 struct inpcbinfo udbinfo
[MAXCPU
];
183 #define UDBHASHSIZE 16
186 struct udpstat udpstat_percpu
[MAXCPU
] __cachealign
;
188 static void udp_append(struct inpcb
*last
, struct ip
*ip
,
189 struct mbuf
*n
, int off
, struct sockaddr_in
*udp_in
);
191 static int udp_connect_oncpu(struct inpcb
*inp
, struct sockaddr_in
*sin
,
192 struct sockaddr_in
*if_sin
, uint16_t hash
);
194 static boolean_t
udp_inswildcardhash(struct inpcb
*inp
,
195 struct netmsg_base
*msg
, int error
);
196 static void udp_remwildcardhash(struct inpcb
*inp
);
199 udp_lportcpu(short lport
)
201 return (ntohs(lport
) % netisr_ncpus
);
207 struct inpcbportinfo
*portinfo
;
210 portinfo
= kmalloc_cachealign(sizeof(*portinfo
) * netisr_ncpus
, M_PCB
,
213 for (cpu
= 0; cpu
< netisr_ncpus
; cpu
++) {
214 struct inpcbinfo
*uicb
= &udbinfo
[cpu
];
218 * UDP pcb list, wildcard hash table and localgroup hash
221 in_pcbinfo_init(uicb
, cpu
, TRUE
);
222 uicb
->hashbase
= hashinit(UDBHASHSIZE
, M_PCB
, &uicb
->hashmask
);
224 in_pcbportinfo_init(&portinfo
[cpu
], UDBHASHSIZE
, cpu
);
225 in_pcbportinfo_set(uicb
, portinfo
, netisr_ncpus
);
227 uicb
->wildcardhashbase
= hashinit(UDBHASHSIZE
, M_PCB
,
228 &uicb
->wildcardhashmask
);
229 uicb
->localgrphashbase
= hashinit(UDBHASHSIZE
, M_PCB
,
230 &uicb
->localgrphashmask
);
232 uicb
->ipi_size
= sizeof(struct inpcb
);
236 * Initialize UDP statistics counters for each CPU.
238 for (cpu
= 0; cpu
< ncpus
; ++cpu
)
239 bzero(&udpstat_percpu
[cpu
], sizeof(struct udpstat
));
243 sysctl_udpstat(SYSCTL_HANDLER_ARGS
)
247 for (cpu
= 0; cpu
< ncpus
; ++cpu
) {
248 if ((error
= SYSCTL_OUT(req
, &udpstat_percpu
[cpu
],
249 sizeof(struct udpstat
))))
251 if ((error
= SYSCTL_IN(req
, &udpstat_percpu
[cpu
],
252 sizeof(struct udpstat
))))
258 SYSCTL_PROC(_net_inet_udp
, UDPCTL_STATS
, stats
, (CTLTYPE_OPAQUE
| CTLFLAG_RW
),
259 0, 0, sysctl_udpstat
, "S,udpstat", "UDP statistics");
262 udp_ctloutput(netmsg_t msg
)
264 struct socket
*so
= msg
->base
.nm_so
;
265 struct sockopt
*sopt
= msg
->ctloutput
.nm_sopt
;
266 struct inpcb
*inp
= so
->so_pcb
;
269 lwkt_replymsg(&msg
->lmsg
, EINVAL
);
273 if (sopt
->sopt_level
== IPPROTO_IP
&& sopt
->sopt_dir
== SOPT_SET
) {
274 switch (sopt
->sopt_name
) {
275 case IP_MULTICAST_IF
:
276 case IP_MULTICAST_VIF
:
277 case IP_MULTICAST_TTL
:
278 case IP_MULTICAST_LOOP
:
279 case IP_ADD_MEMBERSHIP
:
280 case IP_DROP_MEMBERSHIP
:
282 * This pr_ctloutput msg will be forwarded
283 * to netisr0 to run; we can't do direct
287 * Don't optimize for the sockets whose
288 * current so_port is netisr0's msgport.
289 * These sockets could be connect(2)'ed
290 * later and the so_port will be changed.
292 inp
->inp_flags
&= ~INP_DIRECT_DETACH
;
296 return ip_ctloutput(msg
);
300 * Check multicast packets to make sure they are only sent to sockets with
301 * multicast memberships for the packet's destination address and arrival
302 * interface. Multicast packets to multicast-unaware sockets are also
305 * Returns 0 if the packet is acceptable, -1 if it is not.
308 check_multicast_membership(const struct ip
*ip
, const struct inpcb
*inp
,
309 const struct mbuf
*m
)
311 const struct ip_moptions
*mopt
;
314 if (strict_mcast_mship
== 0 ||
315 !IN_MULTICAST(ntohl(ip
->ip_dst
.s_addr
))) {
321 mopt
= inp
->inp_moptions
;
324 for (mshipno
= 0; mshipno
< mopt
->imo_num_memberships
; ++mshipno
) {
325 const struct in_multi
*maddr
= mopt
->imo_membership
[mshipno
];
327 if (ip
->ip_dst
.s_addr
== maddr
->inm_addr
.s_addr
&&
328 m
->m_pkthdr
.rcvif
== maddr
->inm_ifp
) {
335 struct udp_mcast_arg
{
341 struct sockaddr_in
*udp_in
;
345 udp_mcast_input(struct udp_mcast_arg
*arg
)
347 struct inpcb
*inp
= arg
->inp
;
348 struct inpcb
*last
= arg
->last
;
349 struct ip
*ip
= arg
->ip
;
350 struct mbuf
*m
= arg
->m
;
352 if (check_multicast_membership(ip
, inp
, m
) < 0)
353 return ERESTART
; /* caller continue */
359 /* check AH/ESP integrity. */
360 if (ipsec4_in_reject_so(m
, last
->inp_socket
))
361 ipsecstat
.in_polvio
++;
362 /* do not inject data to pcb */
366 /* check AH/ESP integrity. */
367 if (ipsec4_in_reject(m
, last
))
370 #endif /*FAST_IPSEC*/
371 if ((n
= m_copypacket(m
, M_NOWAIT
)) != NULL
)
372 udp_append(last
, ip
, n
,
373 arg
->iphlen
+ sizeof(struct udphdr
),
376 arg
->last
= last
= inp
;
379 * Don't look for additional matches if this one does
380 * not have either the SO_REUSEPORT or SO_REUSEADDR
381 * socket options set. This heuristic avoids searching
382 * through all pcbs in the common case of a non-shared
383 * port. It * assumes that an application will never
384 * clear these options after setting them.
386 if (!(last
->inp_socket
->so_options
&
387 (SO_REUSEPORT
| SO_REUSEADDR
)))
388 return EJUSTRETURN
; /* caller stop */
393 udp_input(struct mbuf
**mp
, int *offp
, int proto
)
395 struct sockaddr_in udp_in
= { sizeof udp_in
, AF_INET
};
401 struct mbuf
*opts
= NULL
;
404 struct inpcbinfo
*pcbinfo
= &udbinfo
[mycpuid
];
411 udp_stat
.udps_ipackets
++;
414 * Strip IP options, if any; should skip this,
415 * make available to user, and use on returned packets,
416 * but we don't yet have a way to check the checksum
417 * with options still present.
419 if (iphlen
> sizeof(struct ip
)) {
421 iphlen
= sizeof(struct ip
);
425 * IP and UDP headers are together in first mbuf.
426 * Already checked and pulled up in ip_demux().
428 KASSERT(m
->m_len
>= iphlen
+ sizeof(struct udphdr
),
429 ("UDP header not in one mbuf"));
431 ip
= mtod(m
, struct ip
*);
432 uh
= (struct udphdr
*)((caddr_t
)ip
+ iphlen
);
434 /* destination port of 0 is illegal, based on RFC768. */
435 if (uh
->uh_dport
== 0)
439 * Make mbuf data length reflect UDP length.
440 * If not enough data to reflect UDP length, drop.
442 len
= ntohs((u_short
)uh
->uh_ulen
);
443 if (ip
->ip_len
!= len
) {
444 if (len
> ip
->ip_len
|| len
< sizeof(struct udphdr
)) {
445 udp_stat
.udps_badlen
++;
448 m_adj(m
, len
- ip
->ip_len
);
449 /* ip->ip_len = len; */
452 * Save a copy of the IP header in case we want restore it
453 * for sending an ICMP error message in response.
458 * Checksum extended UDP header and data.
461 if (m
->m_pkthdr
.csum_flags
& CSUM_DATA_VALID
) {
462 if (m
->m_pkthdr
.csum_flags
& CSUM_PSEUDO_HDR
)
463 uh
->uh_sum
= m
->m_pkthdr
.csum_data
;
465 uh
->uh_sum
= in_pseudo(ip
->ip_src
.s_addr
,
466 ip
->ip_dst
.s_addr
, htonl((u_short
)len
+
467 m
->m_pkthdr
.csum_data
+ IPPROTO_UDP
));
468 uh
->uh_sum
^= 0xffff;
472 bcopy(((struct ipovly
*)ip
)->ih_x1
, b
, 9);
473 bzero(((struct ipovly
*)ip
)->ih_x1
, 9);
474 ((struct ipovly
*)ip
)->ih_len
= uh
->uh_ulen
;
475 uh
->uh_sum
= in_cksum(m
, len
+ sizeof(struct ip
));
476 bcopy(b
, ((struct ipovly
*)ip
)->ih_x1
, 9);
479 udp_stat
.udps_badsum
++;
481 return(IPPROTO_DONE
);
484 udp_stat
.udps_nosum
++;
486 if (IN_MULTICAST(ntohl(ip
->ip_dst
.s_addr
)) ||
487 in_broadcast(ip
->ip_dst
, m
->m_pkthdr
.rcvif
)) {
488 struct inpcbhead
*connhead
;
489 struct inpcontainer
*ic
, *ic_marker
;
490 struct inpcontainerhead
*ichead
;
491 struct udp_mcast_arg arg
;
496 * Deliver a multicast or broadcast datagram to *all* sockets
497 * for which the local and remote addresses and ports match
498 * those of the incoming datagram. This allows more than
499 * one process to receive multi/broadcasts on the same port.
500 * (This really ought to be done for unicast datagrams as
501 * well, but that would cause problems with existing
502 * applications that open both address-specific sockets and
503 * a wildcard socket listening to the same port -- they would
504 * end up receiving duplicates of every unicast datagram.
505 * Those applications open the multiple sockets to overcome an
506 * inadequacy of the UDP socket interface, but for backwards
507 * compatibility we avoid the problem here rather than
508 * fixing the interface. Maybe 4.5BSD will remedy this?)
512 * Construct sockaddr format source address.
514 udp_in
.sin_port
= uh
->uh_sport
;
515 udp_in
.sin_addr
= ip
->ip_src
;
516 arg
.udp_in
= &udp_in
;
518 * Locate pcb(s) for datagram.
519 * (Algorithm copied from raw_intr().)
524 connhead
= &pcbinfo
->hashbase
[
525 INP_PCBCONNHASH(ip
->ip_src
.s_addr
, uh
->uh_sport
,
526 ip
->ip_dst
.s_addr
, uh
->uh_dport
, pcbinfo
->hashmask
)];
527 LIST_FOREACH(inp
, connhead
, inp_hash
) {
529 if (!INP_ISIPV4(inp
))
532 if (!in_hosteq(inp
->inp_faddr
, ip
->ip_src
) ||
533 !in_hosteq(inp
->inp_laddr
, ip
->ip_dst
) ||
534 inp
->inp_fport
!= uh
->uh_sport
||
535 inp
->inp_lport
!= uh
->uh_dport
)
543 error
= udp_mcast_input(&arg
);
544 if (error
== ERESTART
)
548 if (error
== EJUSTRETURN
)
552 ichead
= &pcbinfo
->wildcardhashbase
[
553 INP_PCBWILDCARDHASH(uh
->uh_dport
,
554 pcbinfo
->wildcardhashmask
)];
555 ic_marker
= in_pcbcontainer_marker(mycpuid
);
557 GET_PCBINFO_TOKEN(pcbinfo
);
558 LIST_INSERT_HEAD(ichead
, ic_marker
, ic_list
);
559 while ((ic
= LIST_NEXT(ic_marker
, ic_list
)) != NULL
) {
560 LIST_REMOVE(ic_marker
, ic_list
);
561 LIST_INSERT_AFTER(ic
, ic_marker
, ic_list
);
564 if (inp
->inp_flags
& INP_PLACEMARKER
)
567 if (!INP_ISIPV4(inp
))
570 if (inp
->inp_lport
!= uh
->uh_dport
)
572 if (inp
->inp_laddr
.s_addr
!= INADDR_ANY
&&
573 inp
->inp_laddr
.s_addr
!= ip
->ip_dst
.s_addr
)
581 error
= udp_mcast_input(&arg
);
582 if (error
== ERESTART
)
586 if (error
== EJUSTRETURN
)
589 LIST_REMOVE(ic_marker
, ic_list
);
590 REL_PCBINFO_TOKEN(pcbinfo
);
594 * No matching pcb found; discard datagram.
595 * (No need to send an ICMP Port Unreachable
596 * for a broadcast or multicast datgram.)
598 udp_stat
.udps_noportbcast
++;
602 /* check AH/ESP integrity. */
603 if (ipsec4_in_reject_so(m
, last
->inp_socket
)) {
604 ipsecstat
.in_polvio
++;
609 /* check AH/ESP integrity. */
610 if (ipsec4_in_reject(m
, last
))
612 #endif /*FAST_IPSEC*/
613 udp_append(last
, ip
, m
, iphlen
+ sizeof(struct udphdr
),
615 return(IPPROTO_DONE
);
618 * Locate pcb for datagram.
620 inp
= in_pcblookup_pkthash(pcbinfo
, ip
->ip_src
, uh
->uh_sport
,
621 ip
->ip_dst
, uh
->uh_dport
, TRUE
, m
->m_pkthdr
.rcvif
,
622 udp_reuseport_ext
? m
: NULL
);
625 char src
[INET_ADDRSTRLEN
], dst
[INET_ADDRSTRLEN
];
628 "Connection attempt to UDP %s:%d from %s:%d\n",
629 kinet_ntoa(ip
->ip_dst
, dst
), ntohs(uh
->uh_dport
),
630 kinet_ntoa(ip
->ip_src
, src
), ntohs(uh
->uh_sport
));
632 udp_stat
.udps_noport
++;
633 if (m
->m_flags
& (M_BCAST
| M_MCAST
)) {
634 udp_stat
.udps_noportbcast
++;
640 if (badport_bandlim(BANDLIM_ICMP_UNREACH
) < 0)
644 ip
->ip_len
+= iphlen
;
645 icmp_error(m
, ICMP_UNREACH
, ICMP_UNREACH_PORT
, 0, 0);
646 return(IPPROTO_DONE
);
648 KASSERT(INP_ISIPV4(inp
), ("not inet inpcb"));
650 if (ipsec4_in_reject_so(m
, inp
->inp_socket
)) {
651 ipsecstat
.in_polvio
++;
656 if (ipsec4_in_reject(m
, inp
))
658 #endif /*FAST_IPSEC*/
660 * Check the minimum TTL for socket.
662 if (ip
->ip_ttl
< inp
->inp_ip_minttl
)
666 * Construct sockaddr format source address.
667 * Stuff source address and datagram in user buffer.
669 udp_in
.sin_port
= uh
->uh_sport
;
670 udp_in
.sin_addr
= ip
->ip_src
;
671 if ((inp
->inp_flags
& INP_CONTROLOPTS
) ||
672 (inp
->inp_socket
->so_options
& SO_TIMESTAMP
))
673 ip_savecontrol(inp
, &opts
, ip
, m
);
674 m_adj(m
, iphlen
+ sizeof(struct udphdr
));
676 lwkt_gettoken(&inp
->inp_socket
->so_rcv
.ssb_token
);
677 if (ssb_appendaddr(&inp
->inp_socket
->so_rcv
,
678 (struct sockaddr
*)&udp_in
, m
, opts
) == 0) {
679 lwkt_reltoken(&inp
->inp_socket
->so_rcv
.ssb_token
);
680 udp_stat
.udps_fullsock
++;
683 lwkt_reltoken(&inp
->inp_socket
->so_rcv
.ssb_token
);
684 sorwakeup(inp
->inp_socket
);
685 return(IPPROTO_DONE
);
690 return(IPPROTO_DONE
);
694 * subroutine of udp_input(), mainly for source code readability.
695 * caller must properly init udp_ip6 and udp_in6 beforehand.
698 udp_append(struct inpcb
*last
, struct ip
*ip
, struct mbuf
*n
, int off
,
699 struct sockaddr_in
*udp_in
)
701 struct mbuf
*opts
= NULL
;
704 KASSERT(INP_ISIPV4(last
), ("not inet inpcb"));
706 if (last
->inp_flags
& INP_CONTROLOPTS
||
707 last
->inp_socket
->so_options
& SO_TIMESTAMP
)
708 ip_savecontrol(last
, &opts
, ip
, n
);
711 lwkt_gettoken(&last
->inp_socket
->so_rcv
.ssb_token
);
712 ret
= ssb_appendaddr(&last
->inp_socket
->so_rcv
,
713 (struct sockaddr
*)udp_in
, n
, opts
);
714 lwkt_reltoken(&last
->inp_socket
->so_rcv
.ssb_token
);
719 udp_stat
.udps_fullsock
++;
721 sorwakeup(last
->inp_socket
);
726 * Notify a udp user of an asynchronous error;
727 * just wake up so that he can collect error status.
730 udp_notify(struct inpcb
*inp
, int error
)
732 inp
->inp_socket
->so_error
= error
;
733 sorwakeup(inp
->inp_socket
);
734 sowwakeup(inp
->inp_socket
);
737 struct netmsg_udp_notify
{
738 struct netmsg_base base
;
739 inp_notify_t nm_notify
;
740 struct in_addr nm_faddr
;
745 udp_notifyall_oncpu(netmsg_t msg
)
747 struct netmsg_udp_notify
*nm
= (struct netmsg_udp_notify
*)msg
;
748 int nextcpu
, cpu
= mycpuid
;
750 in_pcbnotifyall(&udbinfo
[cpu
], nm
->nm_faddr
, nm
->nm_arg
, nm
->nm_notify
);
753 if (nextcpu
< netisr_ncpus
)
754 lwkt_forwardmsg(netisr_cpuport(nextcpu
), &nm
->base
.lmsg
);
756 lwkt_replymsg(&nm
->base
.lmsg
, 0);
760 udp_get_inpnotify(int cmd
, const struct sockaddr
*sa
,
761 struct ip
**ip0
, int *cpuid
)
763 struct in_addr faddr
;
764 struct ip
*ip
= *ip0
;
765 inp_notify_t notify
= udp_notify
;
767 faddr
= ((const struct sockaddr_in
*)sa
)->sin_addr
;
768 if (sa
->sa_family
!= AF_INET
|| faddr
.s_addr
== INADDR_ANY
)
771 if (PRC_IS_REDIRECT(cmd
)) {
773 notify
= in_rtchange
;
774 } else if (cmd
== PRC_HOSTDEAD
) {
776 } else if ((unsigned)cmd
>= PRC_NCMDS
|| inetctlerrmap
[cmd
] == 0) {
782 /* Go through all CPUs */
785 const struct udphdr
*uh
;
787 uh
= (const struct udphdr
*)
788 ((caddr_t
)ip
+ (ip
->ip_hl
<< 2));
789 *cpuid
= udp_addrcpu(faddr
.s_addr
, uh
->uh_dport
,
790 ip
->ip_src
.s_addr
, uh
->uh_sport
);
799 udp_ctlinput(netmsg_t msg
)
801 struct sockaddr
*sa
= msg
->ctlinput
.nm_arg
;
802 struct ip
*ip
= msg
->ctlinput
.nm_extra
;
803 int cmd
= msg
->ctlinput
.nm_cmd
, cpuid
;
805 struct in_addr faddr
;
807 notify
= udp_get_inpnotify(cmd
, sa
, &ip
, &cpuid
);
811 faddr
= ((struct sockaddr_in
*)sa
)->sin_addr
;
813 const struct udphdr
*uh
;
816 if (cpuid
!= mycpuid
)
819 uh
= (const struct udphdr
*)((caddr_t
)ip
+ (ip
->ip_hl
<< 2));
820 inp
= in_pcblookup_hash(&udbinfo
[mycpuid
], faddr
, uh
->uh_dport
,
821 ip
->ip_src
, uh
->uh_sport
, 0, NULL
);
822 if (inp
!= NULL
&& inp
->inp_socket
!= NULL
)
823 notify(inp
, inetctlerrmap
[cmd
]);
824 } else if (msg
->ctlinput
.nm_direct
) {
825 if (cpuid
!= ncpus
&& cpuid
!= mycpuid
)
827 if (mycpuid
>= netisr_ncpus
)
830 in_pcbnotifyall(&udbinfo
[mycpuid
], faddr
, inetctlerrmap
[cmd
],
833 struct netmsg_udp_notify
*nm
;
836 nm
= kmalloc(sizeof(*nm
), M_LWKTMSG
, M_INTWAIT
);
837 netmsg_init(&nm
->base
, NULL
, &netisr_afree_rport
,
838 0, udp_notifyall_oncpu
);
839 nm
->nm_faddr
= faddr
;
840 nm
->nm_arg
= inetctlerrmap
[cmd
];
841 nm
->nm_notify
= notify
;
842 lwkt_sendmsg(netisr_cpuport(0), &nm
->base
.lmsg
);
845 lwkt_replymsg(&msg
->lmsg
, 0);
848 SYSCTL_PROC(_net_inet_udp
, UDPCTL_PCBLIST
, pcblist
, CTLFLAG_RD
, udbinfo
, 0,
849 in_pcblist_ncpus
, "S,xinpcb", "List of active UDP sockets");
852 udp_getcred(SYSCTL_HANDLER_ARGS
)
854 struct sockaddr_in addrs
[2];
855 struct ucred cred0
, *cred
= NULL
;
857 int error
, cpu
, origcpu
;
859 error
= priv_check(req
->td
, PRIV_ROOT
);
862 error
= SYSCTL_IN(req
, addrs
, sizeof addrs
);
867 cpu
= udp_addrcpu(addrs
[1].sin_addr
.s_addr
, addrs
[1].sin_port
,
868 addrs
[0].sin_addr
.s_addr
, addrs
[0].sin_port
);
870 lwkt_migratecpu(cpu
);
872 inp
= in_pcblookup_hash(&udbinfo
[cpu
],
873 addrs
[1].sin_addr
, addrs
[1].sin_port
,
874 addrs
[0].sin_addr
, addrs
[0].sin_port
, TRUE
, NULL
);
875 if (inp
== NULL
|| inp
->inp_socket
== NULL
) {
877 } else if (inp
->inp_socket
->so_cred
!= NULL
) {
878 cred0
= *(inp
->inp_socket
->so_cred
);
882 lwkt_migratecpu(origcpu
);
887 return SYSCTL_OUT(req
, cred
, sizeof(struct ucred
));
889 SYSCTL_PROC(_net_inet_udp
, OID_AUTO
, getcred
, CTLTYPE_OPAQUE
|CTLFLAG_RW
,
890 0, 0, udp_getcred
, "S,ucred", "Get the ucred of a UDP connection");
893 udp_send_redispatch(netmsg_t msg
)
895 struct mbuf
*m
= msg
->send
.nm_m
;
896 int pru_flags
= msg
->send
.nm_flags
;
897 struct inpcb
*inp
= msg
->send
.base
.nm_so
->so_pcb
;
898 struct mbuf
*m_opt
= msg
->send
.nm_control
; /* XXX save ipopt */
899 int flags
= msg
->send
.nm_priv
; /* ip_output flags */
902 logudp(redisp_ipout_beg
, inp
);
905 * - Don't use inp route cache. It should only be used in the
907 * - Access to inp_moptions should be safe, since multicast UDP
908 * datagrams are redispatched to netisr0 and inp_moptions is
909 * changed only in netisr0.
911 error
= ip_output(m
, m_opt
, NULL
, flags
, inp
->inp_moptions
, inp
);
912 if ((pru_flags
& PRUS_NOREPLY
) == 0)
913 lwkt_replymsg(&msg
->send
.base
.lmsg
, error
);
916 /* Free saved ip options, if any */
920 logudp(redisp_ipout_end
, inp
);
924 udp_send(netmsg_t msg
)
926 struct socket
*so
= msg
->send
.base
.nm_so
;
927 struct mbuf
*m
= msg
->send
.nm_m
;
928 struct sockaddr
*dstaddr
= msg
->send
.nm_addr
;
929 int pru_flags
= msg
->send
.nm_flags
;
930 struct inpcb
*inp
= so
->so_pcb
;
931 struct thread
*td
= msg
->send
.nm_td
;
936 int len
= m
->m_pkthdr
.len
;
937 struct sockaddr_in
*sin
; /* really is initialized before use */
940 KKASSERT(msg
->send
.nm_control
== NULL
);
942 logudp(send_beg
, inp
);
949 if (len
+ sizeof(struct udpiphdr
) > IP_MAXPACKET
) {
954 if (inp
->inp_lport
== 0) { /* unbound socket */
957 error
= in_pcbbind(inp
, NULL
, td
);
962 * Need to call udp_send again, after this inpcb is
963 * inserted into wildcard hash table.
965 msg
->send
.base
.lmsg
.ms_flags
|= MSGF_UDP_SEND
;
966 forwarded
= udp_inswildcardhash(inp
, &msg
->send
.base
, 0);
969 * The message is further forwarded, so we are
972 logudp(send_inswildcard
, inp
);
977 if (dstaddr
!= NULL
) { /* destination address specified */
978 if (inp
->inp_faddr
.s_addr
!= INADDR_ANY
) {
979 /* already connected */
983 sin
= (struct sockaddr_in
*)dstaddr
;
984 if (!prison_remote_ip(td
, (struct sockaddr
*)&sin
)) {
985 error
= EAFNOSUPPORT
; /* IPv6 only jail */
989 if (inp
->inp_faddr
.s_addr
== INADDR_ANY
) {
990 /* no destination specified and not already connected */
998 * Calculate data length and get a mbuf
999 * for UDP and IP headers.
1001 M_PREPEND(m
, sizeof(struct udpiphdr
), M_NOWAIT
);
1008 * Fill in mbuf with extended UDP header
1009 * and addresses and length put into network format.
1011 ui
= mtod(m
, struct udpiphdr
*);
1012 bzero(ui
->ui_x1
, sizeof ui
->ui_x1
); /* XXX still needed? */
1013 ui
->ui_pr
= IPPROTO_UDP
;
1016 * Set destination address.
1018 if (dstaddr
!= NULL
) { /* use specified destination */
1019 ui
->ui_dst
= sin
->sin_addr
;
1020 ui
->ui_dport
= sin
->sin_port
;
1021 } else { /* use connected destination */
1022 ui
->ui_dst
= inp
->inp_faddr
;
1023 ui
->ui_dport
= inp
->inp_fport
;
1027 * Set source address.
1029 if (inp
->inp_laddr
.s_addr
== INADDR_ANY
||
1030 IN_MULTICAST(ntohl(inp
->inp_laddr
.s_addr
))) {
1031 struct sockaddr_in
*if_sin
;
1033 if (dstaddr
== NULL
) {
1035 * connect() had (or should have) failed because
1036 * the interface had no IP address, but the
1037 * application proceeded to call send() anyways.
1043 /* Look up outgoing interface. */
1044 error
= in_pcbladdr_find(inp
, dstaddr
, &if_sin
, td
, 1);
1047 ui
->ui_src
= if_sin
->sin_addr
; /* use address of interface */
1049 ui
->ui_src
= inp
->inp_laddr
; /* use non-null bound address */
1051 ui
->ui_sport
= inp
->inp_lport
;
1052 KASSERT(inp
->inp_lport
!= 0, ("inp lport should have been bound"));
1055 * Release the original thread, since it is no longer used
1057 if (pru_flags
& PRUS_HELDTD
) {
1059 pru_flags
&= ~PRUS_HELDTD
;
1062 * Free the dest address, since it is no longer needed
1064 if (pru_flags
& PRUS_FREEADDR
) {
1065 kfree(dstaddr
, M_SONAME
);
1066 pru_flags
&= ~PRUS_FREEADDR
;
1069 ui
->ui_ulen
= htons((u_short
)len
+ sizeof(struct udphdr
));
1072 * Set up checksum and output datagram.
1075 ui
->ui_sum
= in_pseudo(ui
->ui_src
.s_addr
, ui
->ui_dst
.s_addr
,
1076 htons((u_short
)len
+ sizeof(struct udphdr
) + IPPROTO_UDP
));
1077 m
->m_pkthdr
.csum_flags
= CSUM_UDP
;
1078 m
->m_pkthdr
.csum_data
= offsetof(struct udphdr
, uh_sum
);
1079 m
->m_pkthdr
.csum_thlen
= sizeof(struct udphdr
);
1083 ((struct ip
*)ui
)->ip_len
= sizeof(struct udpiphdr
) + len
;
1084 ((struct ip
*)ui
)->ip_ttl
= inp
->inp_ip_ttl
; /* XXX */
1085 ((struct ip
*)ui
)->ip_tos
= inp
->inp_ip_tos
; /* XXX */
1086 udp_stat
.udps_opackets
++;
1088 flags
= IP_DEBUGROUTE
|
1089 (inp
->inp_socket
->so_options
& (SO_DONTROUTE
| SO_BROADCAST
));
1090 if (pru_flags
& PRUS_DONTROUTE
)
1091 flags
|= SO_DONTROUTE
;
1093 if (inp
->inp_flags
& INP_CONNECTED
) {
1095 * For connected socket, this datagram has already
1096 * been in the correct netisr; no need to rehash.
1098 KASSERT(inp
->inp_flags
& INP_HASH
, ("inpcb has no hash"));
1099 m_sethash(m
, inp
->inp_hashval
);
1103 hash
= udp_addrhash(ui
->ui_dst
.s_addr
, ui
->ui_dport
,
1104 ui
->ui_src
.s_addr
, ui
->ui_sport
);
1107 cpu
= netisr_hashcpu(hash
);
1108 if (cpu
!= mycpuid
) {
1109 struct mbuf
*m_opt
= NULL
;
1110 struct netmsg_pru_send
*smsg
;
1111 struct lwkt_port
*port
= netisr_cpuport(cpu
);
1114 * Not on the CPU that matches this UDP datagram hash;
1115 * redispatch to the correct CPU to do the ip_output().
1117 if (inp
->inp_options
!= NULL
) {
1119 * If there are ip options, then save a copy,
1120 * since accessing inp_options on other CPUs'
1123 * XXX optimize this?
1125 m_opt
= m_copym(inp
->inp_options
, 0, M_COPYALL
,
1128 if ((pru_flags
& PRUS_NOREPLY
) == 0) {
1130 * Change some parts of the original netmsg and
1131 * forward it to the target netisr.
1133 * NOTE: so_port MUST NOT be checked in the target
1137 smsg
->nm_priv
= flags
; /* ip_output flags */
1139 smsg
->nm_control
= m_opt
; /* XXX save ipopt */
1140 smsg
->base
.lmsg
.ms_flags
|= MSGF_IGNSOPORT
;
1141 smsg
->base
.nm_dispatch
= udp_send_redispatch
;
1142 lwkt_forwardmsg(port
, &smsg
->base
.lmsg
);
1145 * Recreate the netmsg, since the original mbuf
1146 * could have been changed. And send it to the
1149 * NOTE: so_port MUST NOT be checked in the target
1152 smsg
= &m
->m_hdr
.mh_sndmsg
;
1153 netmsg_init(&smsg
->base
, so
, &netisr_apanic_rport
,
1154 MSGF_IGNSOPORT
, udp_send_redispatch
);
1155 smsg
->nm_priv
= flags
; /* ip_output flags */
1156 smsg
->nm_flags
= pru_flags
;
1158 smsg
->nm_control
= m_opt
; /* XXX save ipopt */
1159 lwkt_sendmsg(port
, &smsg
->base
.lmsg
);
1162 /* This UDP datagram is redispatched; done */
1163 logudp(send_redisp
, inp
);
1168 logudp(send_ipout
, inp
);
1169 error
= ip_output(m
, inp
->inp_options
, &inp
->inp_route
, flags
,
1170 inp
->inp_moptions
, inp
);
1177 if (pru_flags
& PRUS_HELDTD
)
1179 if (pru_flags
& PRUS_FREEADDR
)
1180 kfree(dstaddr
, M_SONAME
);
1181 if ((pru_flags
& PRUS_NOREPLY
) == 0)
1182 lwkt_replymsg(&msg
->send
.base
.lmsg
, error
);
1184 logudp(send_end
, inp
);
1187 u_long udp_sendspace
= 9216; /* really max datagram size */
1188 /* 40 1K datagrams */
1189 SYSCTL_INT(_net_inet_udp
, UDPCTL_MAXDGRAM
, maxdgram
, CTLFLAG_RW
,
1190 &udp_sendspace
, 0, "Maximum outgoing UDP datagram size");
1192 u_long udp_recvspace
= 40 * (1024 +
1194 sizeof(struct sockaddr_in6
)
1196 sizeof(struct sockaddr_in
)
1199 SYSCTL_INT(_net_inet_udp
, UDPCTL_RECVSPACE
, recvspace
, CTLFLAG_RW
,
1200 &udp_recvspace
, 0, "Maximum incoming UDP datagram size");
1203 * This should never happen, since UDP socket does not support
1204 * connection acception (SO_ACCEPTCONN, i.e. listen(2)).
1207 udp_abort(netmsg_t msg __unused
)
1209 panic("udp_abort is called");
1213 udp_preattach(struct socket
*so
, int proto __unused
, struct pru_attach_info
*ai
)
1215 return soreserve(so
, udp_sendspace
, udp_recvspace
, ai
->sb_rlimit
);
1219 udp_attach(netmsg_t msg
)
1221 struct socket
*so
= msg
->attach
.base
.nm_so
;
1222 struct pru_attach_info
*ai
= msg
->attach
.nm_ai
;
1226 KASSERT(so
->so_pcb
== NULL
, ("udp socket attached"));
1229 error
= udp_preattach(so
, 0 /* don't care */, ai
);
1233 /* Post attach; do nothing */
1236 error
= in_pcballoc(so
, &udbinfo
[mycpuid
]);
1241 inp
->inp_flags
|= INP_DIRECT_DETACH
;
1242 inp
->inp_ip_ttl
= ip_defttl
;
1245 lwkt_replymsg(&msg
->attach
.base
.lmsg
, error
);
1249 udp_inswildcard_replymsg(netmsg_t msg
)
1251 lwkt_msg_t lmsg
= &msg
->lmsg
;
1253 if (lmsg
->ms_flags
& MSGF_UDP_SEND
) {
1255 /* msg is replied by udp_send() */
1257 lwkt_replymsg(lmsg
, lmsg
->ms_error
);
1262 udp_soreuseport_dispatch(netmsg_t msg
)
1264 /* This inpcb has already been in the wildcard hash. */
1265 in_pcblink_flags(msg
->base
.nm_so
->so_pcb
, &udbinfo
[mycpuid
], 0);
1266 udp_inswildcard_replymsg(msg
);
1270 udp_sosetport(struct lwkt_msg
*msg
, lwkt_port_t port
)
1272 sosetport(((struct netmsg_base
*)msg
)->nm_so
, port
);
1276 udp_inswildcardhash_oncpu(struct inpcb
*inp
, struct netmsg_base
*msg
)
1280 KASSERT(inp
->inp_pcbinfo
== &udbinfo
[mycpuid
],
1281 ("not on owner cpu"));
1283 in_pcbinswildcardhash(inp
);
1284 for (cpu
= 0; cpu
< netisr_ncpus
; ++cpu
) {
1285 if (cpu
== mycpuid
) {
1287 * This inpcb has been inserted by the above
1288 * in_pcbinswildcardhash().
1292 in_pcbinswildcardhash_oncpu(inp
, &udbinfo
[cpu
]);
1295 if (inp
->inp_socket
->so_options
& SO_REUSEPORT
) {
1297 * For SO_REUSEPORT socket, redistribute it based on its
1298 * local group index.
1300 cpu
= inp
->inp_lgrpindex
% netisr_ncpus
;
1301 if (cpu
!= mycpuid
) {
1302 struct lwkt_port
*port
= netisr_cpuport(cpu
);
1303 lwkt_msg_t lmsg
= &msg
->lmsg
;
1306 * We are moving the protocol processing port the
1307 * socket is on, we have to unlink here and re-link
1308 * on the target cpu (this inpcb is still left in
1309 * the wildcard hash).
1311 in_pcbunlink_flags(inp
, &udbinfo
[mycpuid
], 0);
1312 msg
->nm_dispatch
= udp_soreuseport_dispatch
;
1315 * See the related comment in tcp_usrreq.c
1318 lwkt_setmsg_receipt(lmsg
, udp_sosetport
);
1319 lwkt_forwardmsg(port
, lmsg
);
1320 return TRUE
; /* forwarded */
1327 udp_inswildcardhash_dispatch(netmsg_t msg
)
1329 struct inpcb
*inp
= msg
->base
.nm_so
->so_pcb
;
1330 boolean_t forwarded
;
1332 KASSERT(inp
->inp_lport
!= 0, ("local port not set yet"));
1333 KASSERT(udp_lportcpu(inp
->inp_lport
) == mycpuid
, ("not target cpu"));
1335 in_pcblink(inp
, &udbinfo
[mycpuid
]);
1337 forwarded
= udp_inswildcardhash_oncpu(inp
, &msg
->base
);
1339 /* The message is further forwarded, so we are done here. */
1342 udp_inswildcard_replymsg(msg
);
1346 udp_inswildcardhash(struct inpcb
*inp
, struct netmsg_base
*msg
, int error
)
1348 lwkt_msg_t lmsg
= &msg
->lmsg
;
1351 ASSERT_INP_NOTINHASH(inp
);
1353 /* This inpcb could no longer be directly detached */
1354 inp
->inp_flags
&= ~INP_DIRECT_DETACH
;
1357 * Always clear the route cache, so we don't need to
1358 * worry about any owner CPU changes later.
1360 in_pcbresetroute(inp
);
1362 KASSERT(inp
->inp_lport
!= 0, ("local port not set yet"));
1363 cpu
= udp_lportcpu(inp
->inp_lport
);
1365 lmsg
->ms_error
= error
;
1366 if (cpu
!= mycpuid
) {
1367 struct lwkt_port
*port
= netisr_cpuport(cpu
);
1370 * We are moving the protocol processing port the socket
1371 * is on, we have to unlink here and re-link on the
1374 in_pcbunlink(inp
, &udbinfo
[mycpuid
]);
1375 msg
->nm_dispatch
= udp_inswildcardhash_dispatch
;
1377 /* See the related comment in tcp_usrreq.c tcp_connect() */
1378 lwkt_setmsg_receipt(lmsg
, udp_sosetport
);
1379 lwkt_forwardmsg(port
, lmsg
);
1380 return TRUE
; /* forwarded */
1383 return udp_inswildcardhash_oncpu(inp
, msg
);
1387 udp_bind(netmsg_t msg
)
1389 struct socket
*so
= msg
->bind
.base
.nm_so
;
1395 struct sockaddr
*nam
= msg
->bind
.nm_nam
;
1396 struct thread
*td
= msg
->bind
.nm_td
;
1397 struct sockaddr_in
*sin
;
1402 * Check "already bound" here (in_pcbbind() does the same
1403 * check though), so we don't forward a connected/bound
1404 * socket randomly which would panic in the following
1407 if (inp
->inp_lport
!= 0 ||
1408 inp
->inp_laddr
.s_addr
!= INADDR_ANY
) {
1409 error
= EINVAL
; /* already bound */
1413 if (nam
->sa_len
!= sizeof(*sin
)) {
1417 sin
= (struct sockaddr_in
*)nam
;
1419 cpu
= udp_lportcpu(sin
->sin_port
);
1420 port
= netisr_cpuport(cpu
);
1423 * See the related comment in tcp_usrreq.c tcp_usr_bind().
1424 * The exception is that we use local port based netisr
1425 * to serialize in_pcbbind().
1427 if (&curthread
->td_msgport
!= port
) {
1428 lwkt_msg_t lmsg
= &msg
->bind
.base
.lmsg
;
1430 KASSERT((msg
->bind
.nm_flags
& PRUB_RELINK
) == 0,
1431 ("already asked to relink"));
1433 in_pcbunlink(so
->so_pcb
, &udbinfo
[mycpuid
]);
1434 msg
->bind
.nm_flags
|= PRUB_RELINK
;
1437 * See the related comment in tcp_usrreq.c
1440 lwkt_setmsg_receipt(lmsg
, udp_sosetport
);
1441 lwkt_forwardmsg(port
, lmsg
);
1442 /* msg invalid now */
1445 KASSERT(so
->so_port
== port
, ("so_port is not netisr%d", cpu
));
1447 if (msg
->bind
.nm_flags
& PRUB_RELINK
) {
1448 msg
->bind
.nm_flags
&= ~PRUB_RELINK
;
1449 in_pcblink(so
->so_pcb
, &udbinfo
[mycpuid
]);
1451 KASSERT(inp
->inp_pcbinfo
== &udbinfo
[cpu
],
1452 ("pcbinfo is not udbinfo%d", cpu
));
1454 error
= in_pcbbind(inp
, nam
, td
);
1456 boolean_t forwarded
;
1458 if (sin
->sin_addr
.s_addr
!= INADDR_ANY
)
1459 inp
->inp_flags
|= INP_WASBOUND_NOTANY
;
1461 forwarded
= udp_inswildcardhash(inp
,
1462 &msg
->bind
.base
, 0);
1465 * The message is further forwarded, so
1475 lwkt_replymsg(&msg
->bind
.base
.lmsg
, error
);
1479 udp_preconnect(struct socket
*so
, const struct sockaddr
*nam __unused
,
1480 struct thread
*td __unused
)
1482 sosetstate(so
, SS_ISCONNECTED
); /* XXX */
1487 udp_connect(netmsg_t msg
)
1489 struct socket
*so
= msg
->connect
.base
.nm_so
;
1490 struct sockaddr
*nam
= msg
->connect
.nm_nam
;
1491 struct thread
*td
= msg
->connect
.nm_td
;
1493 struct sockaddr_in
*sin
= (struct sockaddr_in
*)nam
;
1494 struct sockaddr_in
*if_sin
;
1495 struct lwkt_port
*port
;
1499 KKASSERT(msg
->connect
.nm_m
== NULL
);
1507 if (msg
->connect
.nm_flags
& PRUC_RECONNECT
) {
1508 msg
->connect
.nm_flags
&= ~PRUC_RECONNECT
;
1509 in_pcblink(inp
, &udbinfo
[mycpuid
]);
1512 if (inp
->inp_faddr
.s_addr
!= INADDR_ANY
) {
1519 * Bind if we have to
1521 if (inp
->inp_lport
== 0) {
1522 error
= in_pcbbind(inp
, NULL
, td
);
1528 * Calculate the correct protocol processing thread. The connect
1529 * operation must run there.
1531 error
= in_pcbladdr(inp
, nam
, &if_sin
, td
);
1534 if (!prison_remote_ip(td
, nam
)) {
1535 error
= EAFNOSUPPORT
; /* IPv6 only jail */
1539 hash
= udp_addrhash(sin
->sin_addr
.s_addr
, sin
->sin_port
,
1540 inp
->inp_laddr
.s_addr
!= INADDR_ANY
?
1541 inp
->inp_laddr
.s_addr
: if_sin
->sin_addr
.s_addr
, inp
->inp_lport
);
1542 port
= netisr_hashport(hash
);
1543 if (port
!= &curthread
->td_msgport
) {
1544 lwkt_msg_t lmsg
= &msg
->connect
.base
.lmsg
;
1545 int nm_flags
= PRUC_RECONNECT
;
1548 * in_pcbladdr() may have allocated a route entry for us
1549 * on the current CPU, but we need a route entry on the
1550 * inpcb's owner CPU, so free it here.
1552 in_pcbresetroute(inp
);
1554 if (inp
->inp_flags
& INP_WILDCARD
) {
1556 * Remove this inpcb from the wildcard hash before
1557 * the socket's msgport changes.
1559 udp_remwildcardhash(inp
);
1562 if (so
->so_orig_port
== NULL
) {
1564 * First time change protocol processing port.
1565 * Save the current port for synchronization upon
1568 so
->so_orig_port
= &curthread
->td_msgport
;
1571 * We have changed protocol processing port more
1572 * than once. We could not do direct detach
1573 * anymore, because we lose the track of the
1574 * original protocol processing ports to perform
1575 * synchronization upon udp_detach. This should
1578 inp
->inp_flags
&= ~INP_DIRECT_DETACH
;
1582 * We are moving the protocol processing port the socket
1583 * is on, we have to unlink here and re-link on the
1586 in_pcbunlink(inp
, &udbinfo
[mycpuid
]);
1587 msg
->connect
.nm_flags
|= nm_flags
;
1589 /* See the related comment in tcp_usrreq.c tcp_connect() */
1590 lwkt_setmsg_receipt(lmsg
, udp_sosetport
);
1591 lwkt_forwardmsg(port
, lmsg
);
1592 /* msg invalid now */
1595 error
= udp_connect_oncpu(inp
, sin
, if_sin
, hash
);
1597 if (msg
->connect
.nm_flags
& PRUC_HELDTD
)
1599 if (error
&& (msg
->connect
.nm_flags
& PRUC_ASYNC
)) {
1600 if (inp
->inp_lport
== 0) {
1602 * As long as we have the local port, it is fine
1603 * for connect to fail, e.g. disconnect.
1605 so
->so_error
= error
;
1607 soclrstate(so
, SS_ISCONNECTED
);
1609 * Wake up callers blocked on this socket to make sure
1610 * that they can see this error.
1613 * sodisconnected() can't be used here, which bricks
1614 * sending and receiving.
1616 wakeup(&so
->so_timeo
);
1620 if (error
&& inp
!= NULL
&& inp
->inp_lport
!= 0 &&
1621 (inp
->inp_flags
& INP_WILDCARD
) == 0) {
1622 boolean_t forwarded
;
1624 /* Connect failed; put it to wildcard hash. */
1625 forwarded
= udp_inswildcardhash(inp
, &msg
->connect
.base
,
1629 * The message is further forwarded, so we are done
1635 lwkt_replymsg(&msg
->connect
.base
.lmsg
, error
);
1639 udp_remwildcardhash(struct inpcb
*inp
)
1643 KASSERT(inp
->inp_pcbinfo
== &udbinfo
[mycpuid
],
1644 ("not on owner cpu"));
1646 for (cpu
= 0; cpu
< netisr_ncpus
; ++cpu
) {
1647 if (cpu
== mycpuid
) {
1649 * This inpcb will be removed by the later
1650 * in_pcbremwildcardhash().
1654 in_pcbremwildcardhash_oncpu(inp
, &udbinfo
[cpu
]);
1656 in_pcbremwildcardhash(inp
);
1660 udp_connect_oncpu(struct inpcb
*inp
, struct sockaddr_in
*sin
,
1661 struct sockaddr_in
*if_sin
, uint16_t hash
)
1663 struct socket
*so
= inp
->inp_socket
;
1666 oinp
= in_pcblookup_hash(inp
->inp_pcbinfo
,
1667 sin
->sin_addr
, sin
->sin_port
,
1668 inp
->inp_laddr
.s_addr
!= INADDR_ANY
?
1669 inp
->inp_laddr
: if_sin
->sin_addr
, inp
->inp_lport
, FALSE
, NULL
);
1674 * No more errors can occur, finish adjusting the socket
1675 * and change the processing port to reflect the connected
1676 * socket. Once set we can no longer safely mess with the
1680 if (inp
->inp_flags
& INP_WILDCARD
)
1681 udp_remwildcardhash(inp
);
1683 if (inp
->inp_laddr
.s_addr
== INADDR_ANY
)
1684 inp
->inp_laddr
= if_sin
->sin_addr
;
1685 inp
->inp_faddr
= sin
->sin_addr
;
1686 inp
->inp_fport
= sin
->sin_port
;
1687 in_pcbinsconnhash(inp
);
1689 inp
->inp_flags
|= INP_HASH
;
1690 inp
->inp_hashval
= hash
;
1698 udp_detach2(struct socket
*so
)
1700 in_pcbdetach(so
->so_pcb
);
1706 udp_detach_final_dispatch(netmsg_t msg
)
1708 udp_detach2(msg
->base
.nm_so
);
1712 udp_detach_oncpu_dispatch(netmsg_t msg
)
1714 struct netmsg_base
*clomsg
= &msg
->base
;
1715 struct socket
*so
= clomsg
->nm_so
;
1716 struct inpcb
*inp
= so
->so_pcb
;
1717 struct thread
*td
= curthread
;
1718 int nextcpu
, cpuid
= mycpuid
;
1720 KASSERT(td
->td_type
== TD_TYPE_NETISR
, ("not in netisr"));
1722 if (inp
->inp_flags
& INP_WILDCARD
) {
1724 * This inp will be removed on the inp's
1725 * owner CPU later, so don't do it now.
1727 if (&td
->td_msgport
!= so
->so_port
)
1728 in_pcbremwildcardhash_oncpu(inp
, &udbinfo
[cpuid
]);
1733 * Free and clear multicast socket option,
1734 * which is only accessed in netisr0.
1736 ip_freemoptions(inp
->inp_moptions
);
1737 inp
->inp_moptions
= NULL
;
1740 nextcpu
= cpuid
+ 1;
1741 if (nextcpu
< netisr_ncpus
) {
1742 lwkt_forwardmsg(netisr_cpuport(nextcpu
), &clomsg
->lmsg
);
1745 * No one could see this inpcb now; destroy this
1746 * inpcb in its owner netisr.
1748 netmsg_init(clomsg
, so
, &netisr_apanic_rport
, 0,
1749 udp_detach_final_dispatch
);
1750 lwkt_sendmsg(so
->so_port
, &clomsg
->lmsg
);
1755 udp_detach_syncorig_dispatch(netmsg_t msg
)
1757 struct netmsg_base
*clomsg
= &msg
->base
;
1758 struct socket
*so
= clomsg
->nm_so
;
1761 * Original protocol processing port is synchronized;
1762 * destroy this inpcb in its owner netisr.
1764 netmsg_init(clomsg
, so
, &netisr_apanic_rport
, 0,
1765 udp_detach_final_dispatch
);
1766 lwkt_sendmsg(so
->so_port
, &clomsg
->lmsg
);
1770 udp_detach(netmsg_t msg
)
1772 struct socket
*so
= msg
->detach
.base
.nm_so
;
1773 struct netmsg_base
*clomsg
;
1778 lwkt_replymsg(&msg
->detach
.base
.lmsg
, EINVAL
);
1783 * Reply EJUSTRETURN ASAP, we will call sodiscard() and
1786 lwkt_replymsg(&msg
->detach
.base
.lmsg
, EJUSTRETURN
);
1788 if (netisr_ncpus
== 1) {
1789 /* Only one CPU, detach the inpcb directly. */
1795 * Remove this inpcb from the inpcb list first, so that
1796 * no one could find this inpcb from the inpcb list.
1801 * Remove this inpcb from the local port hash directly
1802 * here, so that its bound local port could be recycled
1805 in_pcbremporthash(inp
);
1807 if (inp
->inp_flags
& INP_DIRECT_DETACH
) {
1809 * Direct detaching is allowed
1811 KASSERT((inp
->inp_flags
& INP_WILDCARD
) == 0,
1812 ("in the wildcardhash"));
1813 KASSERT(inp
->inp_moptions
== NULL
, ("has mcast options"));
1814 if (so
->so_orig_port
== NULL
) {
1818 * Protocol processing port changed once, so
1819 * we need to make sure that there are nothing
1820 * left on the original protocol processing
1821 * port before we destroy this socket and inpcb.
1822 * This is more lightweight than going through
1823 * all UDP processing netisrs.
1825 clomsg
= &so
->so_clomsg
;
1826 netmsg_init(clomsg
, so
, &netisr_apanic_rport
,
1827 MSGF_IGNSOPORT
, udp_detach_syncorig_dispatch
);
1828 lwkt_sendmsg(so
->so_orig_port
, &clomsg
->lmsg
);
1834 * Go through netisrs which process UDP to make sure
1835 * no one could find this inpcb anymore.
1837 clomsg
= &so
->so_clomsg
;
1838 netmsg_init(clomsg
, so
, &netisr_apanic_rport
, MSGF_IGNSOPORT
,
1839 udp_detach_oncpu_dispatch
);
1840 lwkt_sendmsg(netisr_cpuport(0), &clomsg
->lmsg
);
1844 udp_disconnect(netmsg_t msg
)
1846 struct socket
*so
= msg
->disconnect
.base
.nm_so
;
1848 boolean_t forwarded
;
1856 if (inp
->inp_faddr
.s_addr
== INADDR_ANY
) {
1861 soclrstate(so
, SS_ISCONNECTED
); /* XXX */
1863 in_pcbdisconnect(inp
);
1864 inp
->inp_flags
&= ~INP_HASH
;
1867 * Follow traditional BSD behavior and retain the local port
1868 * binding. But, fix the old misbehavior of overwriting any
1869 * previously bound local address.
1871 if (!(inp
->inp_flags
& INP_WASBOUND_NOTANY
))
1872 inp
->inp_laddr
.s_addr
= INADDR_ANY
;
1874 if (so
->so_state
& SS_ISCLOSING
) {
1876 * If this socket is being closed, there is no need
1877 * to put this socket back into wildcard hash table.
1883 forwarded
= udp_inswildcardhash(inp
, &msg
->disconnect
.base
, 0);
1886 * The message is further forwarded, so we are done
1892 lwkt_replymsg(&msg
->disconnect
.base
.lmsg
, error
);
1896 udp_shutdown(netmsg_t msg
)
1898 struct socket
*so
= msg
->shutdown
.base
.nm_so
;
1909 lwkt_replymsg(&msg
->shutdown
.base
.lmsg
, error
);
1912 struct pr_usrreqs udp_usrreqs
= {
1913 .pru_abort
= udp_abort
,
1914 .pru_accept
= pr_generic_notsupp
,
1915 .pru_attach
= udp_attach
,
1916 .pru_bind
= udp_bind
,
1917 .pru_connect
= udp_connect
,
1918 .pru_connect2
= pr_generic_notsupp
,
1919 .pru_control
= in_control_dispatch
,
1920 .pru_detach
= udp_detach
,
1921 .pru_disconnect
= udp_disconnect
,
1922 .pru_listen
= pr_generic_notsupp
,
1923 .pru_peeraddr
= in_setpeeraddr_dispatch
,
1924 .pru_rcvd
= pr_generic_notsupp
,
1925 .pru_rcvoob
= pr_generic_notsupp
,
1926 .pru_send
= udp_send
,
1927 .pru_sense
= pru_sense_null
,
1928 .pru_shutdown
= udp_shutdown
,
1929 .pru_sockaddr
= in_setsockaddr_dispatch
,
1930 .pru_sosend
= sosendudp
,
1931 .pru_soreceive
= soreceive
,
1932 .pru_preconnect
= udp_preconnect
,
1933 .pru_preattach
= udp_preattach