2 * Copyright (c) 2004 Jeffrey M. Hsu. All rights reserved.
3 * Copyright (c) 2004 The DragonFly Project. All rights reserved.
5 * This code is derived from software contributed to The DragonFly Project
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 * 3. Neither the name of The DragonFly Project nor the names of its
17 * contributors may be used to endorse or promote products derived
18 * from this software without specific, prior written permission.
20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
23 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
24 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
25 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
26 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
27 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
28 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
29 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
30 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
35 * Copyright (c) 1982, 1986, 1988, 1990, 1993, 1995
36 * The Regents of the University of California. All rights reserved.
38 * Redistribution and use in source and binary forms, with or without
39 * modification, are permitted provided that the following conditions
41 * 1. Redistributions of source code must retain the above copyright
42 * notice, this list of conditions and the following disclaimer.
43 * 2. Redistributions in binary form must reproduce the above copyright
44 * notice, this list of conditions and the following disclaimer in the
45 * documentation and/or other materials provided with the distribution.
46 * 3. Neither the name of the University nor the names of its contributors
47 * may be used to endorse or promote products derived from this software
48 * without specific prior written permission.
50 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
51 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
52 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
53 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
54 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
55 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
56 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
57 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
58 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
59 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
62 * @(#)udp_usrreq.c 8.6 (Berkeley) 5/23/95
63 * $FreeBSD: src/sys/netinet/udp_usrreq.c,v 1.64.2.18 2003/01/24 05:11:34 sam Exp $
66 #include "opt_ipsec.h"
67 #include "opt_inet6.h"
69 #include <sys/param.h>
70 #include <sys/systm.h>
71 #include <sys/kernel.h>
72 #include <sys/malloc.h>
74 #include <sys/domain.h>
77 #include <sys/protosw.h>
78 #include <sys/socket.h>
79 #include <sys/socketvar.h>
80 #include <sys/sysctl.h>
81 #include <sys/syslog.h>
82 #include <sys/in_cksum.h>
85 #include <sys/thread2.h>
86 #include <sys/socketvar2.h>
87 #include <sys/serialize.h>
89 #include <machine/stdarg.h>
92 #include <net/route.h>
93 #include <net/netmsg2.h>
94 #include <net/netisr2.h>
96 #include <netinet/in.h>
97 #include <netinet/in_systm.h>
98 #include <netinet/ip.h>
100 #include <netinet/ip6.h>
102 #include <netinet/in_pcb.h>
103 #include <netinet/in_var.h>
104 #include <netinet/ip_var.h>
106 #include <netinet6/ip6_var.h>
108 #include <netinet/ip_icmp.h>
109 #include <netinet/icmp_var.h>
110 #include <netinet/udp.h>
111 #include <netinet/udp_var.h>
114 #include <netproto/ipsec/ipsec.h>
118 #include <netinet6/ipsec.h>
121 #define MSGF_UDP_SEND MSGF_PROTO1
123 #define INP_DIRECT_DETACH INP_FLAG_PROTO2
125 #define UDP_KTR_STRING "inp=%p"
126 #define UDP_KTR_ARGS struct inpcb *inp
129 #define KTR_UDP KTR_ALL
132 KTR_INFO_MASTER(udp
);
133 KTR_INFO(KTR_UDP
, udp
, send_beg
, 0, UDP_KTR_STRING
, UDP_KTR_ARGS
);
134 KTR_INFO(KTR_UDP
, udp
, send_end
, 1, UDP_KTR_STRING
, UDP_KTR_ARGS
);
135 KTR_INFO(KTR_UDP
, udp
, send_ipout
, 2, UDP_KTR_STRING
, UDP_KTR_ARGS
);
136 KTR_INFO(KTR_UDP
, udp
, redisp_ipout_beg
, 3, UDP_KTR_STRING
, UDP_KTR_ARGS
);
137 KTR_INFO(KTR_UDP
, udp
, redisp_ipout_end
, 4, UDP_KTR_STRING
, UDP_KTR_ARGS
);
138 KTR_INFO(KTR_UDP
, udp
, send_redisp
, 5, UDP_KTR_STRING
, UDP_KTR_ARGS
);
139 KTR_INFO(KTR_UDP
, udp
, send_inswildcard
, 6, UDP_KTR_STRING
, UDP_KTR_ARGS
);
141 #define logudp(name, inp) KTR_LOG(udp_##name, inp)
144 * UDP protocol implementation.
145 * Per RFC 768, August, 1980.
148 static int udpcksum
= 1;
150 static int udpcksum
= 0; /* XXX */
152 SYSCTL_INT(_net_inet_udp
, UDPCTL_CHECKSUM
, checksum
, CTLFLAG_RW
,
153 &udpcksum
, 0, "Enable checksumming of UDP packets");
156 SYSCTL_INT(_net_inet_udp
, OID_AUTO
, log_in_vain
, CTLFLAG_RW
,
157 &log_in_vain
, 0, "Log all incoming UDP packets");
159 static int blackhole
= 0;
160 SYSCTL_INT(_net_inet_udp
, OID_AUTO
, blackhole
, CTLFLAG_RW
,
161 &blackhole
, 0, "Do not send port unreachables for refused connects");
163 static int strict_mcast_mship
= 1;
164 SYSCTL_INT(_net_inet_udp
, OID_AUTO
, strict_mcast_mship
, CTLFLAG_RW
,
165 &strict_mcast_mship
, 0, "Only send multicast to member sockets");
167 int udp_sosend_async
= 1;
168 SYSCTL_INT(_net_inet_udp
, OID_AUTO
, sosend_async
, CTLFLAG_RW
,
169 &udp_sosend_async
, 0, "UDP asynchronized pru_send");
171 int udp_sosend_prepend
= 1;
172 SYSCTL_INT(_net_inet_udp
, OID_AUTO
, sosend_prepend
, CTLFLAG_RW
,
173 &udp_sosend_prepend
, 0,
174 "Prepend enough space for proto and link header in pru_send");
176 static int udp_reuseport_ext
= 1;
177 SYSCTL_INT(_net_inet_udp
, OID_AUTO
, reuseport_ext
, CTLFLAG_RW
,
178 &udp_reuseport_ext
, 0, "SO_REUSEPORT extension");
180 struct inpcbinfo udbinfo
[MAXCPU
];
183 #define UDBHASHSIZE 16
186 struct udpstat udpstat_percpu
[MAXCPU
] __cachealign
;
188 static void udp_append(struct inpcb
*last
, struct ip
*ip
,
189 struct mbuf
*n
, int off
, struct sockaddr_in
*udp_in
);
191 static int udp_connect_oncpu(struct inpcb
*inp
, struct sockaddr_in
*sin
,
192 struct sockaddr_in
*if_sin
, uint16_t hash
);
194 static boolean_t
udp_inswildcardhash(struct inpcb
*inp
,
195 struct netmsg_base
*msg
, int error
);
196 static void udp_remwildcardhash(struct inpcb
*inp
);
199 udp_lportcpu(short lport
)
201 return (ntohs(lport
) % netisr_ncpus
);
207 struct inpcbportinfo
*portinfo
;
210 portinfo
= kmalloc_cachealign(sizeof(*portinfo
) * netisr_ncpus
, M_PCB
,
213 for (cpu
= 0; cpu
< netisr_ncpus
; cpu
++) {
214 struct inpcbinfo
*uicb
= &udbinfo
[cpu
];
218 * UDP pcb list, wildcard hash table and localgroup hash
221 in_pcbinfo_init(uicb
, cpu
, TRUE
);
222 uicb
->hashbase
= hashinit(UDBHASHSIZE
, M_PCB
, &uicb
->hashmask
);
224 in_pcbportinfo_init(&portinfo
[cpu
], UDBHASHSIZE
, cpu
);
225 in_pcbportinfo_set(uicb
, portinfo
, netisr_ncpus
);
227 uicb
->wildcardhashbase
= hashinit(UDBHASHSIZE
, M_PCB
,
228 &uicb
->wildcardhashmask
);
229 uicb
->localgrphashbase
= hashinit(UDBHASHSIZE
, M_PCB
,
230 &uicb
->localgrphashmask
);
232 uicb
->ipi_size
= sizeof(struct inpcb
);
236 * Initialize UDP statistics counters for each CPU.
238 for (cpu
= 0; cpu
< netisr_ncpus
; ++cpu
)
239 bzero(&udpstat_percpu
[cpu
], sizeof(struct udpstat
));
243 sysctl_udpstat(SYSCTL_HANDLER_ARGS
)
247 for (cpu
= 0; cpu
< netisr_ncpus
; ++cpu
) {
248 if ((error
= SYSCTL_OUT(req
, &udpstat_percpu
[cpu
],
249 sizeof(struct udpstat
))))
251 if ((error
= SYSCTL_IN(req
, &udpstat_percpu
[cpu
],
252 sizeof(struct udpstat
))))
258 SYSCTL_PROC(_net_inet_udp
, UDPCTL_STATS
, stats
, (CTLTYPE_OPAQUE
| CTLFLAG_RW
),
259 0, 0, sysctl_udpstat
, "S,udpstat", "UDP statistics");
262 udp_ctloutput(netmsg_t msg
)
264 struct socket
*so
= msg
->base
.nm_so
;
265 struct sockopt
*sopt
= msg
->ctloutput
.nm_sopt
;
266 struct inpcb
*inp
= so
->so_pcb
;
269 lwkt_replymsg(&msg
->lmsg
, EINVAL
);
273 if (sopt
->sopt_level
== IPPROTO_IP
&& sopt
->sopt_dir
== SOPT_SET
) {
274 switch (sopt
->sopt_name
) {
275 case IP_MULTICAST_IF
:
276 case IP_MULTICAST_VIF
:
277 case IP_MULTICAST_TTL
:
278 case IP_MULTICAST_LOOP
:
279 case IP_ADD_MEMBERSHIP
:
280 case IP_DROP_MEMBERSHIP
:
282 * This pr_ctloutput msg will be forwarded
283 * to netisr0 to run; we can't do direct
287 * Don't optimize for the sockets whose
288 * current so_port is netisr0's msgport.
289 * These sockets could be connect(2)'ed
290 * later and the so_port will be changed.
292 inp
->inp_flags
&= ~INP_DIRECT_DETACH
;
296 return ip_ctloutput(msg
);
300 * Check multicast packets to make sure they are only sent to sockets with
301 * multicast memberships for the packet's destination address and arrival
302 * interface. Multicast packets to multicast-unaware sockets are also
305 * Returns 0 if the packet is acceptable, -1 if it is not.
308 check_multicast_membership(const struct ip
*ip
, const struct inpcb
*inp
,
309 const struct mbuf
*m
)
311 const struct ip_moptions
*mopt
;
314 if (strict_mcast_mship
== 0 ||
315 !IN_MULTICAST(ntohl(ip
->ip_dst
.s_addr
))) {
321 mopt
= inp
->inp_moptions
;
324 for (mshipno
= 0; mshipno
< mopt
->imo_num_memberships
; ++mshipno
) {
325 const struct in_multi
*maddr
= mopt
->imo_membership
[mshipno
];
327 if (ip
->ip_dst
.s_addr
== maddr
->inm_addr
.s_addr
&&
328 m
->m_pkthdr
.rcvif
== maddr
->inm_ifp
) {
335 struct udp_mcast_arg
{
341 struct sockaddr_in
*udp_in
;
345 udp_mcast_input(struct udp_mcast_arg
*arg
)
347 struct inpcb
*inp
= arg
->inp
;
348 struct inpcb
*last
= arg
->last
;
349 struct ip
*ip
= arg
->ip
;
350 struct mbuf
*m
= arg
->m
;
352 if (check_multicast_membership(ip
, inp
, m
) < 0)
353 return ERESTART
; /* caller continue */
359 /* check AH/ESP integrity. */
360 if (ipsec4_in_reject_so(m
, last
->inp_socket
))
361 ipsecstat
.in_polvio
++;
362 /* do not inject data to pcb */
366 /* check AH/ESP integrity. */
367 if (ipsec4_in_reject(m
, last
))
370 #endif /*FAST_IPSEC*/
371 if ((n
= m_copypacket(m
, M_NOWAIT
)) != NULL
)
372 udp_append(last
, ip
, n
,
373 arg
->iphlen
+ sizeof(struct udphdr
),
376 arg
->last
= last
= inp
;
379 * Don't look for additional matches if this one does
380 * not have either the SO_REUSEPORT or SO_REUSEADDR
381 * socket options set. This heuristic avoids searching
382 * through all pcbs in the common case of a non-shared
383 * port. It * assumes that an application will never
384 * clear these options after setting them.
386 if (!(last
->inp_socket
->so_options
&
387 (SO_REUSEPORT
| SO_REUSEADDR
)))
388 return EJUSTRETURN
; /* caller stop */
393 udp_input(struct mbuf
**mp
, int *offp
, int proto
)
395 struct sockaddr_in udp_in
= { sizeof udp_in
, AF_INET
};
401 struct mbuf
*opts
= NULL
;
404 struct inpcbinfo
*pcbinfo
= &udbinfo
[mycpuid
];
411 udp_stat
.udps_ipackets
++;
414 * Strip IP options, if any; should skip this,
415 * make available to user, and use on returned packets,
416 * but we don't yet have a way to check the checksum
417 * with options still present.
419 if (iphlen
> sizeof(struct ip
)) {
421 iphlen
= sizeof(struct ip
);
425 * IP and UDP headers are together in first mbuf.
426 * Already checked and pulled up in ip_demux().
428 KASSERT(m
->m_len
>= iphlen
+ sizeof(struct udphdr
),
429 ("UDP header not in one mbuf"));
431 ip
= mtod(m
, struct ip
*);
432 uh
= (struct udphdr
*)((caddr_t
)ip
+ iphlen
);
434 /* destination port of 0 is illegal, based on RFC768. */
435 if (uh
->uh_dport
== 0)
439 * Make mbuf data length reflect UDP length.
440 * If not enough data to reflect UDP length, drop.
442 len
= ntohs((u_short
)uh
->uh_ulen
);
443 if (ip
->ip_len
!= len
) {
444 if (len
> ip
->ip_len
|| len
< sizeof(struct udphdr
)) {
445 udp_stat
.udps_badlen
++;
448 m_adj(m
, len
- ip
->ip_len
);
449 /* ip->ip_len = len; */
452 * Save a copy of the IP header in case we want restore it
453 * for sending an ICMP error message in response.
458 * Checksum extended UDP header and data.
461 if (m
->m_pkthdr
.csum_flags
& CSUM_DATA_VALID
) {
462 if (m
->m_pkthdr
.csum_flags
& CSUM_PSEUDO_HDR
)
463 uh
->uh_sum
= m
->m_pkthdr
.csum_data
;
465 uh
->uh_sum
= in_pseudo(ip
->ip_src
.s_addr
,
466 ip
->ip_dst
.s_addr
, htonl((u_short
)len
+
467 m
->m_pkthdr
.csum_data
+ IPPROTO_UDP
));
468 uh
->uh_sum
^= 0xffff;
472 bcopy(((struct ipovly
*)ip
)->ih_x1
, b
, 9);
473 bzero(((struct ipovly
*)ip
)->ih_x1
, 9);
474 ((struct ipovly
*)ip
)->ih_len
= uh
->uh_ulen
;
475 uh
->uh_sum
= in_cksum(m
, len
+ sizeof(struct ip
));
476 bcopy(b
, ((struct ipovly
*)ip
)->ih_x1
, 9);
479 udp_stat
.udps_badsum
++;
481 return(IPPROTO_DONE
);
484 udp_stat
.udps_nosum
++;
486 if (IN_MULTICAST(ntohl(ip
->ip_dst
.s_addr
)) ||
487 in_broadcast(ip
->ip_dst
, m
->m_pkthdr
.rcvif
)) {
488 struct inpcbhead
*connhead
;
489 struct inpcontainer
*ic
, *ic_marker
;
490 struct inpcontainerhead
*ichead
;
491 struct udp_mcast_arg arg
;
496 * Deliver a multicast or broadcast datagram to *all* sockets
497 * for which the local and remote addresses and ports match
498 * those of the incoming datagram. This allows more than
499 * one process to receive multi/broadcasts on the same port.
500 * (This really ought to be done for unicast datagrams as
501 * well, but that would cause problems with existing
502 * applications that open both address-specific sockets and
503 * a wildcard socket listening to the same port -- they would
504 * end up receiving duplicates of every unicast datagram.
505 * Those applications open the multiple sockets to overcome an
506 * inadequacy of the UDP socket interface, but for backwards
507 * compatibility we avoid the problem here rather than
508 * fixing the interface. Maybe 4.5BSD will remedy this?)
512 * Construct sockaddr format source address.
514 udp_in
.sin_port
= uh
->uh_sport
;
515 udp_in
.sin_addr
= ip
->ip_src
;
516 arg
.udp_in
= &udp_in
;
518 * Locate pcb(s) for datagram.
519 * (Algorithm copied from raw_intr().)
524 connhead
= &pcbinfo
->hashbase
[
525 INP_PCBCONNHASH(ip
->ip_src
.s_addr
, uh
->uh_sport
,
526 ip
->ip_dst
.s_addr
, uh
->uh_dport
, pcbinfo
->hashmask
)];
527 LIST_FOREACH(inp
, connhead
, inp_hash
) {
529 if (!INP_ISIPV4(inp
))
532 if (!in_hosteq(inp
->inp_faddr
, ip
->ip_src
) ||
533 !in_hosteq(inp
->inp_laddr
, ip
->ip_dst
) ||
534 inp
->inp_fport
!= uh
->uh_sport
||
535 inp
->inp_lport
!= uh
->uh_dport
)
543 error
= udp_mcast_input(&arg
);
544 if (error
== ERESTART
)
548 if (error
== EJUSTRETURN
)
552 ichead
= &pcbinfo
->wildcardhashbase
[
553 INP_PCBWILDCARDHASH(uh
->uh_dport
,
554 pcbinfo
->wildcardhashmask
)];
555 ic_marker
= in_pcbcontainer_marker();
557 GET_PCBINFO_TOKEN(pcbinfo
);
558 LIST_INSERT_HEAD(ichead
, ic_marker
, ic_list
);
559 while ((ic
= LIST_NEXT(ic_marker
, ic_list
)) != NULL
) {
560 LIST_REMOVE(ic_marker
, ic_list
);
561 LIST_INSERT_AFTER(ic
, ic_marker
, ic_list
);
564 if (inp
->inp_flags
& INP_PLACEMARKER
)
567 if (!INP_ISIPV4(inp
))
570 if (inp
->inp_lport
!= uh
->uh_dport
)
572 if (inp
->inp_laddr
.s_addr
!= INADDR_ANY
&&
573 inp
->inp_laddr
.s_addr
!= ip
->ip_dst
.s_addr
)
581 error
= udp_mcast_input(&arg
);
582 if (error
== ERESTART
)
586 if (error
== EJUSTRETURN
)
589 LIST_REMOVE(ic_marker
, ic_list
);
590 REL_PCBINFO_TOKEN(pcbinfo
);
594 * No matching pcb found; discard datagram.
595 * (No need to send an ICMP Port Unreachable
596 * for a broadcast or multicast datgram.)
598 udp_stat
.udps_noportbcast
++;
602 /* check AH/ESP integrity. */
603 if (ipsec4_in_reject_so(m
, last
->inp_socket
)) {
604 ipsecstat
.in_polvio
++;
609 /* check AH/ESP integrity. */
610 if (ipsec4_in_reject(m
, last
))
612 #endif /*FAST_IPSEC*/
613 udp_append(last
, ip
, m
, iphlen
+ sizeof(struct udphdr
),
615 return(IPPROTO_DONE
);
618 * Locate pcb for datagram.
620 inp
= in_pcblookup_pkthash(pcbinfo
, ip
->ip_src
, uh
->uh_sport
,
621 ip
->ip_dst
, uh
->uh_dport
, TRUE
, m
->m_pkthdr
.rcvif
,
622 udp_reuseport_ext
? m
: NULL
);
625 char src
[INET_ADDRSTRLEN
], dst
[INET_ADDRSTRLEN
];
628 "Connection attempt to UDP %s:%d from %s:%d\n",
629 kinet_ntoa(ip
->ip_dst
, dst
), ntohs(uh
->uh_dport
),
630 kinet_ntoa(ip
->ip_src
, src
), ntohs(uh
->uh_sport
));
632 udp_stat
.udps_noport
++;
633 if (m
->m_flags
& (M_BCAST
| M_MCAST
)) {
634 udp_stat
.udps_noportbcast
++;
640 if (badport_bandlim(BANDLIM_ICMP_UNREACH
) < 0)
644 ip
->ip_len
+= iphlen
;
645 icmp_error(m
, ICMP_UNREACH
, ICMP_UNREACH_PORT
, 0, 0);
646 return(IPPROTO_DONE
);
648 KASSERT(INP_ISIPV4(inp
), ("not inet inpcb"));
650 if (ipsec4_in_reject_so(m
, inp
->inp_socket
)) {
651 ipsecstat
.in_polvio
++;
656 if (ipsec4_in_reject(m
, inp
))
658 #endif /*FAST_IPSEC*/
660 * Check the minimum TTL for socket.
662 if (ip
->ip_ttl
< inp
->inp_ip_minttl
)
666 * Construct sockaddr format source address.
667 * Stuff source address and datagram in user buffer.
669 udp_in
.sin_port
= uh
->uh_sport
;
670 udp_in
.sin_addr
= ip
->ip_src
;
671 if ((inp
->inp_flags
& INP_CONTROLOPTS
) ||
672 (inp
->inp_socket
->so_options
& SO_TIMESTAMP
))
673 ip_savecontrol(inp
, &opts
, ip
, m
);
674 m_adj(m
, iphlen
+ sizeof(struct udphdr
));
676 lwkt_gettoken(&inp
->inp_socket
->so_rcv
.ssb_token
);
677 if (ssb_appendaddr(&inp
->inp_socket
->so_rcv
,
678 (struct sockaddr
*)&udp_in
, m
, opts
) == 0) {
679 lwkt_reltoken(&inp
->inp_socket
->so_rcv
.ssb_token
);
680 udp_stat
.udps_fullsock
++;
683 lwkt_reltoken(&inp
->inp_socket
->so_rcv
.ssb_token
);
684 sorwakeup(inp
->inp_socket
);
685 return(IPPROTO_DONE
);
690 return(IPPROTO_DONE
);
694 * subroutine of udp_input(), mainly for source code readability.
695 * caller must properly init udp_ip6 and udp_in6 beforehand.
698 udp_append(struct inpcb
*last
, struct ip
*ip
, struct mbuf
*n
, int off
,
699 struct sockaddr_in
*udp_in
)
701 struct mbuf
*opts
= NULL
;
704 KASSERT(INP_ISIPV4(last
), ("not inet inpcb"));
706 if (last
->inp_flags
& INP_CONTROLOPTS
||
707 last
->inp_socket
->so_options
& SO_TIMESTAMP
)
708 ip_savecontrol(last
, &opts
, ip
, n
);
711 lwkt_gettoken(&last
->inp_socket
->so_rcv
.ssb_token
);
712 ret
= ssb_appendaddr(&last
->inp_socket
->so_rcv
,
713 (struct sockaddr
*)udp_in
, n
, opts
);
714 lwkt_reltoken(&last
->inp_socket
->so_rcv
.ssb_token
);
719 udp_stat
.udps_fullsock
++;
721 sorwakeup(last
->inp_socket
);
726 * Notify a udp user of an asynchronous error;
727 * just wake up so that he can collect error status.
730 udp_notify(struct inpcb
*inp
, int error
)
732 inp
->inp_socket
->so_error
= error
;
733 sorwakeup(inp
->inp_socket
);
734 sowwakeup(inp
->inp_socket
);
737 struct netmsg_udp_notify
{
738 struct netmsg_base base
;
739 inp_notify_t nm_notify
;
740 struct in_addr nm_faddr
;
745 udp_notifyall_oncpu(netmsg_t msg
)
747 struct netmsg_udp_notify
*nm
= (struct netmsg_udp_notify
*)msg
;
748 int nextcpu
, cpu
= mycpuid
;
750 ASSERT_NETISR_NCPUS(cpu
);
752 in_pcbnotifyall(&udbinfo
[cpu
], nm
->nm_faddr
, nm
->nm_arg
, nm
->nm_notify
);
755 if (nextcpu
< netisr_ncpus
)
756 lwkt_forwardmsg(netisr_cpuport(nextcpu
), &nm
->base
.lmsg
);
758 lwkt_replymsg(&nm
->base
.lmsg
, 0);
762 udp_get_inpnotify(int cmd
, const struct sockaddr
*sa
,
763 struct ip
**ip0
, int *cpuid
)
765 struct in_addr faddr
;
766 struct ip
*ip
= *ip0
;
767 inp_notify_t notify
= udp_notify
;
769 faddr
= ((const struct sockaddr_in
*)sa
)->sin_addr
;
770 if (sa
->sa_family
!= AF_INET
|| faddr
.s_addr
== INADDR_ANY
)
773 if (PRC_IS_REDIRECT(cmd
)) {
775 notify
= in_rtchange
;
776 } else if (cmd
== PRC_HOSTDEAD
) {
778 } else if ((unsigned)cmd
>= PRC_NCMDS
|| inetctlerrmap
[cmd
] == 0) {
784 /* Go through all effective netisr CPUs. */
785 *cpuid
= netisr_ncpus
;
787 const struct udphdr
*uh
;
789 uh
= (const struct udphdr
*)
790 ((caddr_t
)ip
+ (ip
->ip_hl
<< 2));
791 *cpuid
= udp_addrcpu(faddr
.s_addr
, uh
->uh_dport
,
792 ip
->ip_src
.s_addr
, uh
->uh_sport
);
801 udp_ctlinput(netmsg_t msg
)
803 struct sockaddr
*sa
= msg
->ctlinput
.nm_arg
;
804 struct ip
*ip
= msg
->ctlinput
.nm_extra
;
805 int cmd
= msg
->ctlinput
.nm_cmd
, cpuid
;
807 struct in_addr faddr
;
809 ASSERT_NETISR_NCPUS(mycpuid
);
811 notify
= udp_get_inpnotify(cmd
, sa
, &ip
, &cpuid
);
815 faddr
= ((struct sockaddr_in
*)sa
)->sin_addr
;
817 const struct udphdr
*uh
;
820 if (cpuid
!= mycpuid
)
823 uh
= (const struct udphdr
*)((caddr_t
)ip
+ (ip
->ip_hl
<< 2));
824 inp
= in_pcblookup_hash(&udbinfo
[mycpuid
], faddr
, uh
->uh_dport
,
825 ip
->ip_src
, uh
->uh_sport
, 0, NULL
);
826 if (inp
!= NULL
&& inp
->inp_socket
!= NULL
)
827 notify(inp
, inetctlerrmap
[cmd
]);
828 } else if (msg
->ctlinput
.nm_direct
) {
829 if (cpuid
!= netisr_ncpus
&& cpuid
!= mycpuid
)
832 in_pcbnotifyall(&udbinfo
[mycpuid
], faddr
, inetctlerrmap
[cmd
],
835 struct netmsg_udp_notify
*nm
;
838 nm
= kmalloc(sizeof(*nm
), M_LWKTMSG
, M_INTWAIT
);
839 netmsg_init(&nm
->base
, NULL
, &netisr_afree_rport
,
840 0, udp_notifyall_oncpu
);
841 nm
->nm_faddr
= faddr
;
842 nm
->nm_arg
= inetctlerrmap
[cmd
];
843 nm
->nm_notify
= notify
;
844 lwkt_sendmsg(netisr_cpuport(0), &nm
->base
.lmsg
);
847 lwkt_replymsg(&msg
->lmsg
, 0);
850 SYSCTL_PROC(_net_inet_udp
, UDPCTL_PCBLIST
, pcblist
, CTLFLAG_RD
, udbinfo
, 0,
851 in_pcblist_ncpus
, "S,xinpcb", "List of active UDP sockets");
854 udp_getcred(SYSCTL_HANDLER_ARGS
)
856 struct sockaddr_in addrs
[2];
857 struct ucred cred0
, *cred
= NULL
;
859 int error
, cpu
, origcpu
;
861 error
= priv_check(req
->td
, PRIV_ROOT
);
864 error
= SYSCTL_IN(req
, addrs
, sizeof addrs
);
869 cpu
= udp_addrcpu(addrs
[1].sin_addr
.s_addr
, addrs
[1].sin_port
,
870 addrs
[0].sin_addr
.s_addr
, addrs
[0].sin_port
);
872 lwkt_migratecpu(cpu
);
874 inp
= in_pcblookup_hash(&udbinfo
[cpu
],
875 addrs
[1].sin_addr
, addrs
[1].sin_port
,
876 addrs
[0].sin_addr
, addrs
[0].sin_port
, TRUE
, NULL
);
877 if (inp
== NULL
|| inp
->inp_socket
== NULL
) {
879 } else if (inp
->inp_socket
->so_cred
!= NULL
) {
880 cred0
= *(inp
->inp_socket
->so_cred
);
884 lwkt_migratecpu(origcpu
);
889 return SYSCTL_OUT(req
, cred
, sizeof(struct ucred
));
891 SYSCTL_PROC(_net_inet_udp
, OID_AUTO
, getcred
, CTLTYPE_OPAQUE
|CTLFLAG_RW
,
892 0, 0, udp_getcred
, "S,ucred", "Get the ucred of a UDP connection");
895 udp_send_redispatch(netmsg_t msg
)
897 struct mbuf
*m
= msg
->send
.nm_m
;
898 int pru_flags
= msg
->send
.nm_flags
;
899 struct inpcb
*inp
= msg
->send
.base
.nm_so
->so_pcb
;
900 struct mbuf
*m_opt
= msg
->send
.nm_control
; /* XXX save ipopt */
901 int flags
= msg
->send
.nm_priv
; /* ip_output flags */
904 logudp(redisp_ipout_beg
, inp
);
907 * - Don't use inp route cache. It should only be used in the
909 * - Access to inp_moptions should be safe, since multicast UDP
910 * datagrams are redispatched to netisr0 and inp_moptions is
911 * changed only in netisr0.
913 error
= ip_output(m
, m_opt
, NULL
, flags
, inp
->inp_moptions
, inp
);
914 if ((pru_flags
& PRUS_NOREPLY
) == 0)
915 lwkt_replymsg(&msg
->send
.base
.lmsg
, error
);
918 /* Free saved ip options, if any */
922 logudp(redisp_ipout_end
, inp
);
926 udp_send(netmsg_t msg
)
928 struct socket
*so
= msg
->send
.base
.nm_so
;
929 struct mbuf
*m
= msg
->send
.nm_m
;
930 struct sockaddr
*dstaddr
= msg
->send
.nm_addr
;
931 int pru_flags
= msg
->send
.nm_flags
;
932 struct inpcb
*inp
= so
->so_pcb
;
933 struct thread
*td
= msg
->send
.nm_td
;
938 int len
= m
->m_pkthdr
.len
;
939 struct sockaddr_in
*sin
; /* really is initialized before use */
942 KKASSERT(msg
->send
.nm_control
== NULL
);
944 logudp(send_beg
, inp
);
951 if (len
+ sizeof(struct udpiphdr
) > IP_MAXPACKET
) {
956 if (inp
->inp_lport
== 0) { /* unbound socket */
959 error
= in_pcbbind(inp
, NULL
, td
);
964 * Need to call udp_send again, after this inpcb is
965 * inserted into wildcard hash table.
967 msg
->send
.base
.lmsg
.ms_flags
|= MSGF_UDP_SEND
;
968 forwarded
= udp_inswildcardhash(inp
, &msg
->send
.base
, 0);
971 * The message is further forwarded, so we are
974 logudp(send_inswildcard
, inp
);
979 if (dstaddr
!= NULL
) { /* destination address specified */
980 if (inp
->inp_faddr
.s_addr
!= INADDR_ANY
) {
981 /* already connected */
985 sin
= (struct sockaddr_in
*)dstaddr
;
986 if (!prison_remote_ip(td
, (struct sockaddr
*)&sin
)) {
987 error
= EAFNOSUPPORT
; /* IPv6 only jail */
991 if (inp
->inp_faddr
.s_addr
== INADDR_ANY
) {
992 /* no destination specified and not already connected */
1000 * Calculate data length and get a mbuf
1001 * for UDP and IP headers.
1003 M_PREPEND(m
, sizeof(struct udpiphdr
), M_NOWAIT
);
1010 * Fill in mbuf with extended UDP header
1011 * and addresses and length put into network format.
1013 ui
= mtod(m
, struct udpiphdr
*);
1014 bzero(ui
->ui_x1
, sizeof ui
->ui_x1
); /* XXX still needed? */
1015 ui
->ui_pr
= IPPROTO_UDP
;
1018 * Set destination address.
1020 if (dstaddr
!= NULL
) { /* use specified destination */
1021 ui
->ui_dst
= sin
->sin_addr
;
1022 ui
->ui_dport
= sin
->sin_port
;
1023 } else { /* use connected destination */
1024 ui
->ui_dst
= inp
->inp_faddr
;
1025 ui
->ui_dport
= inp
->inp_fport
;
1029 * Set source address.
1031 if (inp
->inp_laddr
.s_addr
== INADDR_ANY
||
1032 IN_MULTICAST(ntohl(inp
->inp_laddr
.s_addr
))) {
1033 struct sockaddr_in
*if_sin
;
1035 if (dstaddr
== NULL
) {
1037 * connect() had (or should have) failed because
1038 * the interface had no IP address, but the
1039 * application proceeded to call send() anyways.
1045 /* Look up outgoing interface. */
1046 error
= in_pcbladdr_find(inp
, dstaddr
, &if_sin
, td
, 1);
1049 ui
->ui_src
= if_sin
->sin_addr
; /* use address of interface */
1051 ui
->ui_src
= inp
->inp_laddr
; /* use non-null bound address */
1053 ui
->ui_sport
= inp
->inp_lport
;
1054 KASSERT(inp
->inp_lport
!= 0, ("inp lport should have been bound"));
1057 * Release the original thread, since it is no longer used
1059 if (pru_flags
& PRUS_HELDTD
) {
1061 pru_flags
&= ~PRUS_HELDTD
;
1064 * Free the dest address, since it is no longer needed
1066 if (pru_flags
& PRUS_FREEADDR
) {
1067 kfree(dstaddr
, M_SONAME
);
1068 pru_flags
&= ~PRUS_FREEADDR
;
1071 ui
->ui_ulen
= htons((u_short
)len
+ sizeof(struct udphdr
));
1074 * Set up checksum and output datagram.
1077 ui
->ui_sum
= in_pseudo(ui
->ui_src
.s_addr
, ui
->ui_dst
.s_addr
,
1078 htons((u_short
)len
+ sizeof(struct udphdr
) + IPPROTO_UDP
));
1079 m
->m_pkthdr
.csum_flags
= CSUM_UDP
;
1080 m
->m_pkthdr
.csum_data
= offsetof(struct udphdr
, uh_sum
);
1081 m
->m_pkthdr
.csum_thlen
= sizeof(struct udphdr
);
1085 ((struct ip
*)ui
)->ip_len
= sizeof(struct udpiphdr
) + len
;
1086 ((struct ip
*)ui
)->ip_ttl
= inp
->inp_ip_ttl
; /* XXX */
1087 ((struct ip
*)ui
)->ip_tos
= inp
->inp_ip_tos
; /* XXX */
1088 udp_stat
.udps_opackets
++;
1090 flags
= IP_DEBUGROUTE
|
1091 (inp
->inp_socket
->so_options
& (SO_DONTROUTE
| SO_BROADCAST
));
1092 if (pru_flags
& PRUS_DONTROUTE
)
1093 flags
|= SO_DONTROUTE
;
1095 if (inp
->inp_flags
& INP_CONNECTED
) {
1097 * For connected socket, this datagram has already
1098 * been in the correct netisr; no need to rehash.
1100 KASSERT(inp
->inp_flags
& INP_HASH
, ("inpcb has no hash"));
1101 m_sethash(m
, inp
->inp_hashval
);
1105 hash
= udp_addrhash(ui
->ui_dst
.s_addr
, ui
->ui_dport
,
1106 ui
->ui_src
.s_addr
, ui
->ui_sport
);
1109 cpu
= netisr_hashcpu(hash
);
1110 if (cpu
!= mycpuid
) {
1111 struct mbuf
*m_opt
= NULL
;
1112 struct netmsg_pru_send
*smsg
;
1113 struct lwkt_port
*port
= netisr_cpuport(cpu
);
1116 * Not on the CPU that matches this UDP datagram hash;
1117 * redispatch to the correct CPU to do the ip_output().
1119 if (inp
->inp_options
!= NULL
) {
1121 * If there are ip options, then save a copy,
1122 * since accessing inp_options on other CPUs'
1125 * XXX optimize this?
1127 m_opt
= m_copym(inp
->inp_options
, 0, M_COPYALL
,
1130 if ((pru_flags
& PRUS_NOREPLY
) == 0) {
1132 * Change some parts of the original netmsg and
1133 * forward it to the target netisr.
1135 * NOTE: so_port MUST NOT be checked in the target
1139 smsg
->nm_priv
= flags
; /* ip_output flags */
1141 smsg
->nm_control
= m_opt
; /* XXX save ipopt */
1142 smsg
->base
.lmsg
.ms_flags
|= MSGF_IGNSOPORT
;
1143 smsg
->base
.nm_dispatch
= udp_send_redispatch
;
1144 lwkt_forwardmsg(port
, &smsg
->base
.lmsg
);
1147 * Recreate the netmsg, since the original mbuf
1148 * could have been changed. And send it to the
1151 * NOTE: so_port MUST NOT be checked in the target
1154 smsg
= &m
->m_hdr
.mh_sndmsg
;
1155 netmsg_init(&smsg
->base
, so
, &netisr_apanic_rport
,
1156 MSGF_IGNSOPORT
, udp_send_redispatch
);
1157 smsg
->nm_priv
= flags
; /* ip_output flags */
1158 smsg
->nm_flags
= pru_flags
;
1160 smsg
->nm_control
= m_opt
; /* XXX save ipopt */
1161 lwkt_sendmsg(port
, &smsg
->base
.lmsg
);
1164 /* This UDP datagram is redispatched; done */
1165 logudp(send_redisp
, inp
);
1170 logudp(send_ipout
, inp
);
1171 error
= ip_output(m
, inp
->inp_options
, &inp
->inp_route
, flags
,
1172 inp
->inp_moptions
, inp
);
1179 if (pru_flags
& PRUS_HELDTD
)
1181 if (pru_flags
& PRUS_FREEADDR
)
1182 kfree(dstaddr
, M_SONAME
);
1183 if ((pru_flags
& PRUS_NOREPLY
) == 0)
1184 lwkt_replymsg(&msg
->send
.base
.lmsg
, error
);
1186 logudp(send_end
, inp
);
1189 u_long udp_sendspace
= 9216; /* really max datagram size */
1190 /* 40 1K datagrams */
1191 SYSCTL_INT(_net_inet_udp
, UDPCTL_MAXDGRAM
, maxdgram
, CTLFLAG_RW
,
1192 &udp_sendspace
, 0, "Maximum outgoing UDP datagram size");
1194 u_long udp_recvspace
= 40 * (1024 +
1196 sizeof(struct sockaddr_in6
)
1198 sizeof(struct sockaddr_in
)
1201 SYSCTL_INT(_net_inet_udp
, UDPCTL_RECVSPACE
, recvspace
, CTLFLAG_RW
,
1202 &udp_recvspace
, 0, "Maximum incoming UDP datagram size");
1205 * This should never happen, since UDP socket does not support
1206 * connection acception (SO_ACCEPTCONN, i.e. listen(2)).
1209 udp_abort(netmsg_t msg __unused
)
1211 panic("udp_abort is called");
1215 udp_preattach(struct socket
*so
, int proto __unused
, struct pru_attach_info
*ai
)
1217 return soreserve(so
, udp_sendspace
, udp_recvspace
, ai
->sb_rlimit
);
1221 udp_attach(netmsg_t msg
)
1223 struct socket
*so
= msg
->attach
.base
.nm_so
;
1224 struct pru_attach_info
*ai
= msg
->attach
.nm_ai
;
1228 KASSERT(so
->so_pcb
== NULL
, ("udp socket attached"));
1231 error
= udp_preattach(so
, 0 /* don't care */, ai
);
1235 /* Post attach; do nothing */
1238 error
= in_pcballoc(so
, &udbinfo
[mycpuid
]);
1243 inp
->inp_flags
|= INP_DIRECT_DETACH
;
1244 inp
->inp_ip_ttl
= ip_defttl
;
1247 lwkt_replymsg(&msg
->attach
.base
.lmsg
, error
);
1251 udp_inswildcard_replymsg(netmsg_t msg
)
1253 lwkt_msg_t lmsg
= &msg
->lmsg
;
1255 if (lmsg
->ms_flags
& MSGF_UDP_SEND
) {
1257 /* msg is replied by udp_send() */
1259 lwkt_replymsg(lmsg
, lmsg
->ms_error
);
1264 udp_soreuseport_dispatch(netmsg_t msg
)
1266 /* This inpcb has already been in the wildcard hash. */
1267 in_pcblink_flags(msg
->base
.nm_so
->so_pcb
, &udbinfo
[mycpuid
], 0);
1268 udp_inswildcard_replymsg(msg
);
1272 udp_sosetport(struct lwkt_msg
*msg
, lwkt_port_t port
)
1274 sosetport(((struct netmsg_base
*)msg
)->nm_so
, port
);
1278 udp_inswildcardhash_oncpu(struct inpcb
*inp
, struct netmsg_base
*msg
)
1282 KASSERT(inp
->inp_pcbinfo
== &udbinfo
[mycpuid
],
1283 ("not on owner cpu"));
1285 in_pcbinswildcardhash(inp
);
1286 for (cpu
= 0; cpu
< netisr_ncpus
; ++cpu
) {
1287 if (cpu
== mycpuid
) {
1289 * This inpcb has been inserted by the above
1290 * in_pcbinswildcardhash().
1294 in_pcbinswildcardhash_oncpu(inp
, &udbinfo
[cpu
]);
1297 /* NOTE: inp_lgrpindex is _not_ assigned in jail. */
1298 if ((inp
->inp_socket
->so_options
& SO_REUSEPORT
) &&
1299 inp
->inp_lgrpindex
>= 0) {
1301 * For SO_REUSEPORT socket, redistribute it based on its
1302 * local group index.
1304 cpu
= inp
->inp_lgrpindex
% netisr_ncpus
;
1305 if (cpu
!= mycpuid
) {
1306 struct lwkt_port
*port
= netisr_cpuport(cpu
);
1307 lwkt_msg_t lmsg
= &msg
->lmsg
;
1310 * We are moving the protocol processing port the
1311 * socket is on, we have to unlink here and re-link
1312 * on the target cpu (this inpcb is still left in
1313 * the wildcard hash).
1315 in_pcbunlink_flags(inp
, &udbinfo
[mycpuid
], 0);
1316 msg
->nm_dispatch
= udp_soreuseport_dispatch
;
1319 * See the related comment in tcp_usrreq.c
1322 lwkt_setmsg_receipt(lmsg
, udp_sosetport
);
1323 lwkt_forwardmsg(port
, lmsg
);
1324 return TRUE
; /* forwarded */
1331 udp_inswildcardhash_dispatch(netmsg_t msg
)
1333 struct inpcb
*inp
= msg
->base
.nm_so
->so_pcb
;
1334 boolean_t forwarded
;
1336 KASSERT(inp
->inp_lport
!= 0, ("local port not set yet"));
1337 KASSERT(udp_lportcpu(inp
->inp_lport
) == mycpuid
, ("not target cpu"));
1339 in_pcblink(inp
, &udbinfo
[mycpuid
]);
1341 forwarded
= udp_inswildcardhash_oncpu(inp
, &msg
->base
);
1343 /* The message is further forwarded, so we are done here. */
1346 udp_inswildcard_replymsg(msg
);
1350 udp_inswildcardhash(struct inpcb
*inp
, struct netmsg_base
*msg
, int error
)
1352 lwkt_msg_t lmsg
= &msg
->lmsg
;
1355 ASSERT_INP_NOTINHASH(inp
);
1357 /* This inpcb could no longer be directly detached */
1358 inp
->inp_flags
&= ~INP_DIRECT_DETACH
;
1361 * Always clear the route cache, so we don't need to
1362 * worry about any owner CPU changes later.
1364 in_pcbresetroute(inp
);
1366 KASSERT(inp
->inp_lport
!= 0, ("local port not set yet"));
1367 cpu
= udp_lportcpu(inp
->inp_lport
);
1369 lmsg
->ms_error
= error
;
1370 if (cpu
!= mycpuid
) {
1371 struct lwkt_port
*port
= netisr_cpuport(cpu
);
1374 * We are moving the protocol processing port the socket
1375 * is on, we have to unlink here and re-link on the
1378 in_pcbunlink(inp
, &udbinfo
[mycpuid
]);
1379 msg
->nm_dispatch
= udp_inswildcardhash_dispatch
;
1381 /* See the related comment in tcp_usrreq.c tcp_connect() */
1382 lwkt_setmsg_receipt(lmsg
, udp_sosetport
);
1383 lwkt_forwardmsg(port
, lmsg
);
1384 return TRUE
; /* forwarded */
1387 return udp_inswildcardhash_oncpu(inp
, msg
);
1391 udp_bind(netmsg_t msg
)
1393 struct socket
*so
= msg
->bind
.base
.nm_so
;
1399 struct sockaddr
*nam
= msg
->bind
.nm_nam
;
1400 struct thread
*td
= msg
->bind
.nm_td
;
1401 struct sockaddr_in
*sin
;
1406 * Check "already bound" here (in_pcbbind() does the same
1407 * check though), so we don't forward a connected/bound
1408 * socket randomly which would panic in the following
1411 if (inp
->inp_lport
!= 0 ||
1412 inp
->inp_laddr
.s_addr
!= INADDR_ANY
) {
1413 error
= EINVAL
; /* already bound */
1417 if (nam
->sa_len
!= sizeof(*sin
)) {
1421 sin
= (struct sockaddr_in
*)nam
;
1423 cpu
= udp_lportcpu(sin
->sin_port
);
1424 port
= netisr_cpuport(cpu
);
1427 * See the related comment in tcp_usrreq.c tcp_usr_bind().
1428 * The exception is that we use local port based netisr
1429 * to serialize in_pcbbind().
1431 if (&curthread
->td_msgport
!= port
) {
1432 lwkt_msg_t lmsg
= &msg
->bind
.base
.lmsg
;
1434 KASSERT((msg
->bind
.nm_flags
& PRUB_RELINK
) == 0,
1435 ("already asked to relink"));
1437 in_pcbunlink(so
->so_pcb
, &udbinfo
[mycpuid
]);
1438 msg
->bind
.nm_flags
|= PRUB_RELINK
;
1441 * See the related comment in tcp_usrreq.c
1444 lwkt_setmsg_receipt(lmsg
, udp_sosetport
);
1445 lwkt_forwardmsg(port
, lmsg
);
1446 /* msg invalid now */
1449 KASSERT(so
->so_port
== port
, ("so_port is not netisr%d", cpu
));
1451 if (msg
->bind
.nm_flags
& PRUB_RELINK
) {
1452 msg
->bind
.nm_flags
&= ~PRUB_RELINK
;
1453 in_pcblink(so
->so_pcb
, &udbinfo
[mycpuid
]);
1455 KASSERT(inp
->inp_pcbinfo
== &udbinfo
[cpu
],
1456 ("pcbinfo is not udbinfo%d", cpu
));
1458 error
= in_pcbbind(inp
, nam
, td
);
1460 boolean_t forwarded
;
1462 if (sin
->sin_addr
.s_addr
!= INADDR_ANY
)
1463 inp
->inp_flags
|= INP_WASBOUND_NOTANY
;
1465 forwarded
= udp_inswildcardhash(inp
,
1466 &msg
->bind
.base
, 0);
1469 * The message is further forwarded, so
1479 lwkt_replymsg(&msg
->bind
.base
.lmsg
, error
);
1483 udp_preconnect(struct socket
*so
, const struct sockaddr
*nam __unused
,
1484 struct thread
*td __unused
)
1486 sosetstate(so
, SS_ISCONNECTED
); /* XXX */
1491 udp_connect(netmsg_t msg
)
1493 struct socket
*so
= msg
->connect
.base
.nm_so
;
1494 struct sockaddr
*nam
= msg
->connect
.nm_nam
;
1495 struct thread
*td
= msg
->connect
.nm_td
;
1497 struct sockaddr_in
*sin
= (struct sockaddr_in
*)nam
;
1498 struct sockaddr_in
*if_sin
;
1499 struct lwkt_port
*port
;
1503 KKASSERT(msg
->connect
.nm_m
== NULL
);
1511 if (msg
->connect
.nm_flags
& PRUC_RECONNECT
) {
1512 msg
->connect
.nm_flags
&= ~PRUC_RECONNECT
;
1513 in_pcblink(inp
, &udbinfo
[mycpuid
]);
1516 if (inp
->inp_faddr
.s_addr
!= INADDR_ANY
) {
1523 * Bind if we have to
1525 if (inp
->inp_lport
== 0) {
1526 error
= in_pcbbind(inp
, NULL
, td
);
1532 * Calculate the correct protocol processing thread. The connect
1533 * operation must run there.
1535 error
= in_pcbladdr(inp
, nam
, &if_sin
, td
);
1538 if (!prison_remote_ip(td
, nam
)) {
1539 error
= EAFNOSUPPORT
; /* IPv6 only jail */
1543 hash
= udp_addrhash(sin
->sin_addr
.s_addr
, sin
->sin_port
,
1544 inp
->inp_laddr
.s_addr
!= INADDR_ANY
?
1545 inp
->inp_laddr
.s_addr
: if_sin
->sin_addr
.s_addr
, inp
->inp_lport
);
1546 port
= netisr_hashport(hash
);
1547 if (port
!= &curthread
->td_msgport
) {
1548 lwkt_msg_t lmsg
= &msg
->connect
.base
.lmsg
;
1549 int nm_flags
= PRUC_RECONNECT
;
1552 * in_pcbladdr() may have allocated a route entry for us
1553 * on the current CPU, but we need a route entry on the
1554 * inpcb's owner CPU, so free it here.
1556 in_pcbresetroute(inp
);
1558 if (inp
->inp_flags
& INP_WILDCARD
) {
1560 * Remove this inpcb from the wildcard hash before
1561 * the socket's msgport changes.
1563 udp_remwildcardhash(inp
);
1566 if (so
->so_orig_port
== NULL
) {
1568 * First time change protocol processing port.
1569 * Save the current port for synchronization upon
1572 so
->so_orig_port
= &curthread
->td_msgport
;
1575 * We have changed protocol processing port more
1576 * than once. We could not do direct detach
1577 * anymore, because we lose the track of the
1578 * original protocol processing ports to perform
1579 * synchronization upon udp_detach. This should
1582 inp
->inp_flags
&= ~INP_DIRECT_DETACH
;
1586 * We are moving the protocol processing port the socket
1587 * is on, we have to unlink here and re-link on the
1590 in_pcbunlink(inp
, &udbinfo
[mycpuid
]);
1591 msg
->connect
.nm_flags
|= nm_flags
;
1593 /* See the related comment in tcp_usrreq.c tcp_connect() */
1594 lwkt_setmsg_receipt(lmsg
, udp_sosetport
);
1595 lwkt_forwardmsg(port
, lmsg
);
1596 /* msg invalid now */
1599 error
= udp_connect_oncpu(inp
, sin
, if_sin
, hash
);
1601 if (msg
->connect
.nm_flags
& PRUC_HELDTD
)
1603 if (error
&& (msg
->connect
.nm_flags
& PRUC_ASYNC
)) {
1604 if (inp
->inp_lport
== 0) {
1606 * As long as we have the local port, it is fine
1607 * for connect to fail, e.g. disconnect.
1609 so
->so_error
= error
;
1611 soclrstate(so
, SS_ISCONNECTED
);
1613 * Wake up callers blocked on this socket to make sure
1614 * that they can see this error.
1617 * sodisconnected() can't be used here, which bricks
1618 * sending and receiving.
1620 wakeup(&so
->so_timeo
);
1624 if (error
&& inp
!= NULL
&& inp
->inp_lport
!= 0 &&
1625 (inp
->inp_flags
& INP_WILDCARD
) == 0) {
1626 boolean_t forwarded
;
1628 /* Connect failed; put it to wildcard hash. */
1629 forwarded
= udp_inswildcardhash(inp
, &msg
->connect
.base
,
1633 * The message is further forwarded, so we are done
1639 lwkt_replymsg(&msg
->connect
.base
.lmsg
, error
);
1643 udp_remwildcardhash(struct inpcb
*inp
)
1647 KASSERT(inp
->inp_pcbinfo
== &udbinfo
[mycpuid
],
1648 ("not on owner cpu"));
1650 for (cpu
= 0; cpu
< netisr_ncpus
; ++cpu
) {
1651 if (cpu
== mycpuid
) {
1653 * This inpcb will be removed by the later
1654 * in_pcbremwildcardhash().
1658 in_pcbremwildcardhash_oncpu(inp
, &udbinfo
[cpu
]);
1660 in_pcbremwildcardhash(inp
);
1664 udp_connect_oncpu(struct inpcb
*inp
, struct sockaddr_in
*sin
,
1665 struct sockaddr_in
*if_sin
, uint16_t hash
)
1667 struct socket
*so
= inp
->inp_socket
;
1670 oinp
= in_pcblookup_hash(inp
->inp_pcbinfo
,
1671 sin
->sin_addr
, sin
->sin_port
,
1672 inp
->inp_laddr
.s_addr
!= INADDR_ANY
?
1673 inp
->inp_laddr
: if_sin
->sin_addr
, inp
->inp_lport
, FALSE
, NULL
);
1678 * No more errors can occur, finish adjusting the socket
1679 * and change the processing port to reflect the connected
1680 * socket. Once set we can no longer safely mess with the
1684 if (inp
->inp_flags
& INP_WILDCARD
)
1685 udp_remwildcardhash(inp
);
1687 if (inp
->inp_laddr
.s_addr
== INADDR_ANY
)
1688 inp
->inp_laddr
= if_sin
->sin_addr
;
1689 inp
->inp_faddr
= sin
->sin_addr
;
1690 inp
->inp_fport
= sin
->sin_port
;
1691 in_pcbinsconnhash(inp
);
1693 inp
->inp_flags
|= INP_HASH
;
1694 inp
->inp_hashval
= hash
;
1702 udp_detach2(struct socket
*so
)
1704 in_pcbdetach(so
->so_pcb
);
1710 udp_detach_final_dispatch(netmsg_t msg
)
1712 udp_detach2(msg
->base
.nm_so
);
1716 udp_detach_oncpu_dispatch(netmsg_t msg
)
1718 struct netmsg_base
*clomsg
= &msg
->base
;
1719 struct socket
*so
= clomsg
->nm_so
;
1720 struct inpcb
*inp
= so
->so_pcb
;
1721 struct thread
*td
= curthread
;
1722 int nextcpu
, cpuid
= mycpuid
;
1724 KASSERT(td
->td_type
== TD_TYPE_NETISR
, ("not in netisr"));
1726 if (inp
->inp_flags
& INP_WILDCARD
) {
1728 * This inp will be removed on the inp's
1729 * owner CPU later, so don't do it now.
1731 if (&td
->td_msgport
!= so
->so_port
)
1732 in_pcbremwildcardhash_oncpu(inp
, &udbinfo
[cpuid
]);
1737 * Free and clear multicast socket option,
1738 * which is only accessed in netisr0.
1740 ip_freemoptions(inp
->inp_moptions
);
1741 inp
->inp_moptions
= NULL
;
1744 nextcpu
= cpuid
+ 1;
1745 if (nextcpu
< netisr_ncpus
) {
1746 lwkt_forwardmsg(netisr_cpuport(nextcpu
), &clomsg
->lmsg
);
1749 * No one could see this inpcb now; destroy this
1750 * inpcb in its owner netisr.
1752 netmsg_init(clomsg
, so
, &netisr_apanic_rport
, 0,
1753 udp_detach_final_dispatch
);
1754 lwkt_sendmsg(so
->so_port
, &clomsg
->lmsg
);
1759 udp_detach_syncorig_dispatch(netmsg_t msg
)
1761 struct netmsg_base
*clomsg
= &msg
->base
;
1762 struct socket
*so
= clomsg
->nm_so
;
1765 * Original protocol processing port is synchronized;
1766 * destroy this inpcb in its owner netisr.
1768 netmsg_init(clomsg
, so
, &netisr_apanic_rport
, 0,
1769 udp_detach_final_dispatch
);
1770 lwkt_sendmsg(so
->so_port
, &clomsg
->lmsg
);
1774 udp_detach(netmsg_t msg
)
1776 struct socket
*so
= msg
->detach
.base
.nm_so
;
1777 struct netmsg_base
*clomsg
;
1782 lwkt_replymsg(&msg
->detach
.base
.lmsg
, EINVAL
);
1787 * Reply EJUSTRETURN ASAP, we will call sodiscard() and
1790 lwkt_replymsg(&msg
->detach
.base
.lmsg
, EJUSTRETURN
);
1792 if (netisr_ncpus
== 1) {
1793 /* Only one CPU, detach the inpcb directly. */
1799 * Remove this inpcb from the inpcb list first, so that
1800 * no one could find this inpcb from the inpcb list.
1805 * Remove this inpcb from the local port hash directly
1806 * here, so that its bound local port could be recycled
1809 in_pcbremporthash(inp
);
1811 if (inp
->inp_flags
& INP_DIRECT_DETACH
) {
1813 * Direct detaching is allowed
1815 KASSERT((inp
->inp_flags
& INP_WILDCARD
) == 0,
1816 ("in the wildcardhash"));
1817 KASSERT(inp
->inp_moptions
== NULL
, ("has mcast options"));
1818 if (so
->so_orig_port
== NULL
) {
1822 * Protocol processing port changed once, so
1823 * we need to make sure that there are nothing
1824 * left on the original protocol processing
1825 * port before we destroy this socket and inpcb.
1826 * This is more lightweight than going through
1827 * all UDP processing netisrs.
1829 clomsg
= &so
->so_clomsg
;
1830 netmsg_init(clomsg
, so
, &netisr_apanic_rport
,
1831 MSGF_IGNSOPORT
, udp_detach_syncorig_dispatch
);
1832 lwkt_sendmsg(so
->so_orig_port
, &clomsg
->lmsg
);
1838 * Go through netisrs which process UDP to make sure
1839 * no one could find this inpcb anymore.
1841 clomsg
= &so
->so_clomsg
;
1842 netmsg_init(clomsg
, so
, &netisr_apanic_rport
, MSGF_IGNSOPORT
,
1843 udp_detach_oncpu_dispatch
);
1844 lwkt_sendmsg(netisr_cpuport(0), &clomsg
->lmsg
);
1848 udp_disconnect(netmsg_t msg
)
1850 struct socket
*so
= msg
->disconnect
.base
.nm_so
;
1852 boolean_t forwarded
;
1860 if (inp
->inp_faddr
.s_addr
== INADDR_ANY
) {
1865 soclrstate(so
, SS_ISCONNECTED
); /* XXX */
1867 in_pcbdisconnect(inp
);
1868 inp
->inp_flags
&= ~INP_HASH
;
1871 * Follow traditional BSD behavior and retain the local port
1872 * binding. But, fix the old misbehavior of overwriting any
1873 * previously bound local address.
1875 if (!(inp
->inp_flags
& INP_WASBOUND_NOTANY
))
1876 inp
->inp_laddr
.s_addr
= INADDR_ANY
;
1878 if (so
->so_state
& SS_ISCLOSING
) {
1880 * If this socket is being closed, there is no need
1881 * to put this socket back into wildcard hash table.
1887 forwarded
= udp_inswildcardhash(inp
, &msg
->disconnect
.base
, 0);
1890 * The message is further forwarded, so we are done
1896 lwkt_replymsg(&msg
->disconnect
.base
.lmsg
, error
);
1900 udp_shutdown(netmsg_t msg
)
1902 struct socket
*so
= msg
->shutdown
.base
.nm_so
;
1913 lwkt_replymsg(&msg
->shutdown
.base
.lmsg
, error
);
1916 struct pr_usrreqs udp_usrreqs
= {
1917 .pru_abort
= udp_abort
,
1918 .pru_accept
= pr_generic_notsupp
,
1919 .pru_attach
= udp_attach
,
1920 .pru_bind
= udp_bind
,
1921 .pru_connect
= udp_connect
,
1922 .pru_connect2
= pr_generic_notsupp
,
1923 .pru_control
= in_control_dispatch
,
1924 .pru_detach
= udp_detach
,
1925 .pru_disconnect
= udp_disconnect
,
1926 .pru_listen
= pr_generic_notsupp
,
1927 .pru_peeraddr
= in_setpeeraddr_dispatch
,
1928 .pru_rcvd
= pr_generic_notsupp
,
1929 .pru_rcvoob
= pr_generic_notsupp
,
1930 .pru_send
= udp_send
,
1931 .pru_sense
= pru_sense_null
,
1932 .pru_shutdown
= udp_shutdown
,
1933 .pru_sockaddr
= in_setsockaddr_dispatch
,
1934 .pru_sosend
= sosendudp
,
1935 .pru_soreceive
= soreceive
,
1936 .pru_preconnect
= udp_preconnect
,
1937 .pru_preattach
= udp_preattach