2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
6 * PACKET - implements raw packet sockets.
9 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10 * Alan Cox, <gw4pts@gw4pts.ampr.org>
13 * Alan Cox : verify_area() now used correctly
14 * Alan Cox : new skbuff lists, look ma no backlogs!
15 * Alan Cox : tidied skbuff lists.
16 * Alan Cox : Now uses generic datagram routines I
17 * added. Also fixed the peek/read crash
18 * from all old Linux datagram code.
19 * Alan Cox : Uses the improved datagram code.
20 * Alan Cox : Added NULL's for socket options.
21 * Alan Cox : Re-commented the code.
22 * Alan Cox : Use new kernel side addressing
23 * Rob Janssen : Correct MTU usage.
24 * Dave Platt : Counter leaks caused by incorrect
25 * interrupt locking and some slightly
26 * dubious gcc output. Can you read
27 * compiler: it said _VOLATILE_
28 * Richard Kooijman : Timestamp fixes.
29 * Alan Cox : New buffers. Use sk->mac.raw.
30 * Alan Cox : sendmsg/recvmsg support.
31 * Alan Cox : Protocol setting support
32 * Alexey Kuznetsov : Untied from IPv4 stack.
33 * Cyrus Durgin : Fixed kerneld for kmod.
34 * Michal Ostrowski : Module initialization cleanup.
35 * Ulises Alonso : Frame number limit removal and
36 * packet_set_ring memory leak.
37 * Eric Biederman : Allow for > 8 byte hardware addresses.
38 * The convention is that longer addresses
39 * will simply extend the hardware address
40 * byte arrays at the end of sockaddr_ll
42 * Johann Baudy : Added TX RING.
44 * This program is free software; you can redistribute it and/or
45 * modify it under the terms of the GNU General Public License
46 * as published by the Free Software Foundation; either version
47 * 2 of the License, or (at your option) any later version.
51 #include <linux/types.h>
53 #include <linux/capability.h>
54 #include <linux/fcntl.h>
55 #include <linux/socket.h>
57 #include <linux/inet.h>
58 #include <linux/netdevice.h>
59 #include <linux/if_packet.h>
60 #include <linux/wireless.h>
61 #include <linux/kernel.h>
62 #include <linux/kmod.h>
63 #include <linux/slab.h>
64 #include <linux/vmalloc.h>
65 #include <net/net_namespace.h>
67 #include <net/protocol.h>
68 #include <linux/skbuff.h>
70 #include <linux/errno.h>
71 #include <linux/timer.h>
72 #include <asm/system.h>
73 #include <asm/uaccess.h>
74 #include <asm/ioctls.h>
76 #include <asm/cacheflush.h>
78 #include <linux/proc_fs.h>
79 #include <linux/seq_file.h>
80 #include <linux/poll.h>
81 #include <linux/module.h>
82 #include <linux/init.h>
83 #include <linux/mutex.h>
84 #include <linux/if_vlan.h>
85 #include <linux/virtio_net.h>
86 #include <linux/errqueue.h>
87 #include <linux/net_tstamp.h>
90 #include <net/inet_common.h>
95 - if device has no dev->hard_header routine, it adds and removes ll header
96 inside itself. In this case ll header is invisible outside of device,
97 but higher levels still should reserve dev->hard_header_len.
98 Some devices are enough clever to reallocate skb, when header
99 will not fit to reserved space (tunnel), another ones are silly
101 - packet socket receives packets with pulled ll header,
102 so that SOCK_RAW should push it back.
107 Incoming, dev->hard_header!=NULL
108 mac_header -> ll header
111 Outgoing, dev->hard_header!=NULL
112 mac_header -> ll header
115 Incoming, dev->hard_header==NULL
116 mac_header -> UNKNOWN position. It is very likely, that it points to ll
117 header. PPP makes it, that is wrong, because introduce
118 assymetry between rx and tx paths.
121 Outgoing, dev->hard_header==NULL
122 mac_header -> data. ll header is still not built!
126 If dev->hard_header==NULL we are unlikely to restore sensible ll header.
132 dev->hard_header != NULL
133 mac_header -> ll header
136 dev->hard_header == NULL (ll header is added by device, we cannot control it)
140 We should set nh.raw on output to correct posistion,
141 packet classifier depends on it.
144 /* Private packet socket structures. */
146 struct packet_mclist
{
147 struct packet_mclist
*next
;
152 unsigned char addr
[MAX_ADDR_LEN
];
154 /* identical to struct packet_mreq except it has
155 * a longer address field.
157 struct packet_mreq_max
{
159 unsigned short mr_type
;
160 unsigned short mr_alen
;
161 unsigned char mr_address
[MAX_ADDR_LEN
];
164 static int packet_set_ring(struct sock
*sk
, struct tpacket_req
*req
,
165 int closing
, int tx_ring
);
171 struct packet_ring_buffer
{
174 unsigned int frames_per_block
;
175 unsigned int frame_size
;
176 unsigned int frame_max
;
178 unsigned int pg_vec_order
;
179 unsigned int pg_vec_pages
;
180 unsigned int pg_vec_len
;
186 static int tpacket_snd(struct packet_sock
*po
, struct msghdr
*msg
);
188 static void packet_flush_mclist(struct sock
*sk
);
190 struct packet_fanout
;
192 /* struct sock has to be the first member of packet_sock */
194 struct packet_fanout
*fanout
;
195 struct tpacket_stats stats
;
196 struct packet_ring_buffer rx_ring
;
197 struct packet_ring_buffer tx_ring
;
199 spinlock_t bind_lock
;
200 struct mutex pg_vec_lock
;
201 unsigned int running
:1, /* prot_hook is attached*/
205 int ifindex
; /* bound device */
207 struct packet_mclist
*mclist
;
209 enum tpacket_versions tp_version
;
210 unsigned int tp_hdrlen
;
211 unsigned int tp_reserve
;
212 unsigned int tp_loss
:1;
213 unsigned int tp_tstamp
;
214 struct packet_type prot_hook ____cacheline_aligned_in_smp
;
217 #define PACKET_FANOUT_MAX 256
219 struct packet_fanout
{
223 unsigned int num_members
;
228 struct list_head list
;
229 struct sock
*arr
[PACKET_FANOUT_MAX
];
232 struct packet_type prot_hook ____cacheline_aligned_in_smp
;
235 struct packet_skb_cb
{
236 unsigned int origlen
;
238 struct sockaddr_pkt pkt
;
239 struct sockaddr_ll ll
;
243 #define PACKET_SKB_CB(__skb) ((struct packet_skb_cb *)((__skb)->cb))
245 static inline struct packet_sock
*pkt_sk(struct sock
*sk
)
247 return (struct packet_sock
*)sk
;
250 static void __fanout_unlink(struct sock
*sk
, struct packet_sock
*po
);
251 static void __fanout_link(struct sock
*sk
, struct packet_sock
*po
);
253 /* register_prot_hook must be invoked with the po->bind_lock held,
254 * or from a context in which asynchronous accesses to the packet
255 * socket is not possible (packet_create()).
257 static void register_prot_hook(struct sock
*sk
)
259 struct packet_sock
*po
= pkt_sk(sk
);
262 __fanout_link(sk
, po
);
264 dev_add_pack(&po
->prot_hook
);
270 /* {,__}unregister_prot_hook() must be invoked with the po->bind_lock
271 * held. If the sync parameter is true, we will temporarily drop
272 * the po->bind_lock and do a synchronize_net to make sure no
273 * asynchronous packet processing paths still refer to the elements
274 * of po->prot_hook. If the sync parameter is false, it is the
275 * callers responsibility to take care of this.
277 static void __unregister_prot_hook(struct sock
*sk
, bool sync
)
279 struct packet_sock
*po
= pkt_sk(sk
);
283 __fanout_unlink(sk
, po
);
285 __dev_remove_pack(&po
->prot_hook
);
289 spin_unlock(&po
->bind_lock
);
291 spin_lock(&po
->bind_lock
);
295 static void unregister_prot_hook(struct sock
*sk
, bool sync
)
297 struct packet_sock
*po
= pkt_sk(sk
);
300 __unregister_prot_hook(sk
, sync
);
303 static inline __pure
struct page
*pgv_to_page(void *addr
)
305 if (is_vmalloc_addr(addr
))
306 return vmalloc_to_page(addr
);
307 return virt_to_page(addr
);
310 static void __packet_set_status(struct packet_sock
*po
, void *frame
, int status
)
313 struct tpacket_hdr
*h1
;
314 struct tpacket2_hdr
*h2
;
319 switch (po
->tp_version
) {
321 h
.h1
->tp_status
= status
;
322 flush_dcache_page(pgv_to_page(&h
.h1
->tp_status
));
325 h
.h2
->tp_status
= status
;
326 flush_dcache_page(pgv_to_page(&h
.h2
->tp_status
));
329 pr_err("TPACKET version not supported\n");
336 static int __packet_get_status(struct packet_sock
*po
, void *frame
)
339 struct tpacket_hdr
*h1
;
340 struct tpacket2_hdr
*h2
;
347 switch (po
->tp_version
) {
349 flush_dcache_page(pgv_to_page(&h
.h1
->tp_status
));
350 return h
.h1
->tp_status
;
352 flush_dcache_page(pgv_to_page(&h
.h2
->tp_status
));
353 return h
.h2
->tp_status
;
355 pr_err("TPACKET version not supported\n");
361 static void *packet_lookup_frame(struct packet_sock
*po
,
362 struct packet_ring_buffer
*rb
,
363 unsigned int position
,
366 unsigned int pg_vec_pos
, frame_offset
;
368 struct tpacket_hdr
*h1
;
369 struct tpacket2_hdr
*h2
;
373 pg_vec_pos
= position
/ rb
->frames_per_block
;
374 frame_offset
= position
% rb
->frames_per_block
;
376 h
.raw
= rb
->pg_vec
[pg_vec_pos
].buffer
+
377 (frame_offset
* rb
->frame_size
);
379 if (status
!= __packet_get_status(po
, h
.raw
))
385 static inline void *packet_current_frame(struct packet_sock
*po
,
386 struct packet_ring_buffer
*rb
,
389 return packet_lookup_frame(po
, rb
, rb
->head
, status
);
392 static inline void *packet_previous_frame(struct packet_sock
*po
,
393 struct packet_ring_buffer
*rb
,
396 unsigned int previous
= rb
->head
? rb
->head
- 1 : rb
->frame_max
;
397 return packet_lookup_frame(po
, rb
, previous
, status
);
400 static inline void packet_increment_head(struct packet_ring_buffer
*buff
)
402 buff
->head
= buff
->head
!= buff
->frame_max
? buff
->head
+1 : 0;
405 static void packet_sock_destruct(struct sock
*sk
)
407 skb_queue_purge(&sk
->sk_error_queue
);
409 WARN_ON(atomic_read(&sk
->sk_rmem_alloc
));
410 WARN_ON(atomic_read(&sk
->sk_wmem_alloc
));
412 if (!sock_flag(sk
, SOCK_DEAD
)) {
413 pr_err("Attempt to release alive packet socket: %p\n", sk
);
417 sk_refcnt_debug_dec(sk
);
420 static int fanout_rr_next(struct packet_fanout
*f
, unsigned int num
)
422 int x
= atomic_read(&f
->rr_cur
) + 1;
430 static struct sock
*fanout_demux_hash(struct packet_fanout
*f
, struct sk_buff
*skb
, unsigned int num
)
432 u32 idx
, hash
= skb
->rxhash
;
434 idx
= ((u64
)hash
* num
) >> 32;
439 static struct sock
*fanout_demux_lb(struct packet_fanout
*f
, struct sk_buff
*skb
, unsigned int num
)
443 cur
= atomic_read(&f
->rr_cur
);
444 while ((old
= atomic_cmpxchg(&f
->rr_cur
, cur
,
445 fanout_rr_next(f
, num
))) != cur
)
450 static struct sock
*fanout_demux_cpu(struct packet_fanout
*f
, struct sk_buff
*skb
, unsigned int num
)
452 unsigned int cpu
= smp_processor_id();
454 return f
->arr
[cpu
% num
];
457 static struct sk_buff
*fanout_check_defrag(struct sk_buff
*skb
)
460 const struct iphdr
*iph
;
463 if (skb
->protocol
!= htons(ETH_P_IP
))
466 if (!pskb_may_pull(skb
, sizeof(struct iphdr
)))
470 if (iph
->ihl
< 5 || iph
->version
!= 4)
472 if (!pskb_may_pull(skb
, iph
->ihl
*4))
475 len
= ntohs(iph
->tot_len
);
476 if (skb
->len
< len
|| len
< (iph
->ihl
* 4))
479 if (ip_is_fragment(ip_hdr(skb
))) {
480 skb
= skb_share_check(skb
, GFP_ATOMIC
);
482 if (pskb_trim_rcsum(skb
, len
))
484 memset(IPCB(skb
), 0, sizeof(struct inet_skb_parm
));
485 if (ip_defrag(skb
, IP_DEFRAG_AF_PACKET
))
494 static int packet_rcv_fanout(struct sk_buff
*skb
, struct net_device
*dev
,
495 struct packet_type
*pt
, struct net_device
*orig_dev
)
497 struct packet_fanout
*f
= pt
->af_packet_priv
;
498 unsigned int num
= f
->num_members
;
499 struct packet_sock
*po
;
502 if (!net_eq(dev_net(dev
), read_pnet(&f
->net
)) ||
509 case PACKET_FANOUT_HASH
:
512 skb
= fanout_check_defrag(skb
);
517 sk
= fanout_demux_hash(f
, skb
, num
);
519 case PACKET_FANOUT_LB
:
520 sk
= fanout_demux_lb(f
, skb
, num
);
522 case PACKET_FANOUT_CPU
:
523 sk
= fanout_demux_cpu(f
, skb
, num
);
529 return po
->prot_hook
.func(skb
, dev
, &po
->prot_hook
, orig_dev
);
532 static DEFINE_MUTEX(fanout_mutex
);
533 static LIST_HEAD(fanout_list
);
535 static void __fanout_link(struct sock
*sk
, struct packet_sock
*po
)
537 struct packet_fanout
*f
= po
->fanout
;
540 f
->arr
[f
->num_members
] = sk
;
543 spin_unlock(&f
->lock
);
546 static void __fanout_unlink(struct sock
*sk
, struct packet_sock
*po
)
548 struct packet_fanout
*f
= po
->fanout
;
552 for (i
= 0; i
< f
->num_members
; i
++) {
556 BUG_ON(i
>= f
->num_members
);
557 f
->arr
[i
] = f
->arr
[f
->num_members
- 1];
559 spin_unlock(&f
->lock
);
562 static int fanout_add(struct sock
*sk
, u16 id
, u16 type_flags
)
564 struct packet_sock
*po
= pkt_sk(sk
);
565 struct packet_fanout
*f
, *match
;
566 u8 type
= type_flags
& 0xff;
567 u8 defrag
= (type_flags
& PACKET_FANOUT_FLAG_DEFRAG
) ? 1 : 0;
571 case PACKET_FANOUT_HASH
:
572 case PACKET_FANOUT_LB
:
573 case PACKET_FANOUT_CPU
:
585 mutex_lock(&fanout_mutex
);
587 list_for_each_entry(f
, &fanout_list
, list
) {
589 read_pnet(&f
->net
) == sock_net(sk
)) {
595 if (match
&& match
->defrag
!= defrag
)
599 match
= kzalloc(sizeof(*match
), GFP_KERNEL
);
602 write_pnet(&match
->net
, sock_net(sk
));
605 match
->defrag
= defrag
;
606 atomic_set(&match
->rr_cur
, 0);
607 INIT_LIST_HEAD(&match
->list
);
608 spin_lock_init(&match
->lock
);
609 atomic_set(&match
->sk_ref
, 0);
610 match
->prot_hook
.type
= po
->prot_hook
.type
;
611 match
->prot_hook
.dev
= po
->prot_hook
.dev
;
612 match
->prot_hook
.func
= packet_rcv_fanout
;
613 match
->prot_hook
.af_packet_priv
= match
;
614 dev_add_pack(&match
->prot_hook
);
615 list_add(&match
->list
, &fanout_list
);
618 if (match
->type
== type
&&
619 match
->prot_hook
.type
== po
->prot_hook
.type
&&
620 match
->prot_hook
.dev
== po
->prot_hook
.dev
) {
622 if (atomic_read(&match
->sk_ref
) < PACKET_FANOUT_MAX
) {
623 __dev_remove_pack(&po
->prot_hook
);
625 atomic_inc(&match
->sk_ref
);
626 __fanout_link(sk
, po
);
631 mutex_unlock(&fanout_mutex
);
635 static void fanout_release(struct sock
*sk
)
637 struct packet_sock
*po
= pkt_sk(sk
);
638 struct packet_fanout
*f
;
646 mutex_lock(&fanout_mutex
);
647 if (atomic_dec_and_test(&f
->sk_ref
)) {
649 dev_remove_pack(&f
->prot_hook
);
652 mutex_unlock(&fanout_mutex
);
655 static const struct proto_ops packet_ops
;
657 static const struct proto_ops packet_ops_spkt
;
659 static int packet_rcv_spkt(struct sk_buff
*skb
, struct net_device
*dev
,
660 struct packet_type
*pt
, struct net_device
*orig_dev
)
663 struct sockaddr_pkt
*spkt
;
666 * When we registered the protocol we saved the socket in the data
667 * field for just this event.
670 sk
= pt
->af_packet_priv
;
673 * Yank back the headers [hope the device set this
674 * right or kerboom...]
676 * Incoming packets have ll header pulled,
679 * For outgoing ones skb->data == skb_mac_header(skb)
680 * so that this procedure is noop.
683 if (skb
->pkt_type
== PACKET_LOOPBACK
)
686 if (!net_eq(dev_net(dev
), sock_net(sk
)))
689 skb
= skb_share_check(skb
, GFP_ATOMIC
);
693 /* drop any routing info */
696 /* drop conntrack reference */
699 spkt
= &PACKET_SKB_CB(skb
)->sa
.pkt
;
701 skb_push(skb
, skb
->data
- skb_mac_header(skb
));
704 * The SOCK_PACKET socket receives _all_ frames.
707 spkt
->spkt_family
= dev
->type
;
708 strlcpy(spkt
->spkt_device
, dev
->name
, sizeof(spkt
->spkt_device
));
709 spkt
->spkt_protocol
= skb
->protocol
;
712 * Charge the memory to the socket. This is done specifically
713 * to prevent sockets using all the memory up.
716 if (sock_queue_rcv_skb(sk
, skb
) == 0)
727 * Output a raw packet to a device layer. This bypasses all the other
728 * protocol layers and you must therefore supply it with a complete frame
731 static int packet_sendmsg_spkt(struct kiocb
*iocb
, struct socket
*sock
,
732 struct msghdr
*msg
, size_t len
)
734 struct sock
*sk
= sock
->sk
;
735 struct sockaddr_pkt
*saddr
= (struct sockaddr_pkt
*)msg
->msg_name
;
736 struct sk_buff
*skb
= NULL
;
737 struct net_device
*dev
;
742 * Get and verify the address.
746 if (msg
->msg_namelen
< sizeof(struct sockaddr
))
748 if (msg
->msg_namelen
== sizeof(struct sockaddr_pkt
))
749 proto
= saddr
->spkt_protocol
;
751 return -ENOTCONN
; /* SOCK_PACKET must be sent giving an address */
754 * Find the device first to size check it
757 saddr
->spkt_device
[13] = 0;
760 dev
= dev_get_by_name_rcu(sock_net(sk
), saddr
->spkt_device
);
766 if (!(dev
->flags
& IFF_UP
))
770 * You may not queue a frame bigger than the mtu. This is the lowest level
771 * raw protocol and you must do your own fragmentation at this level.
775 if (len
> dev
->mtu
+ dev
->hard_header_len
+ VLAN_HLEN
)
779 size_t reserved
= LL_RESERVED_SPACE(dev
);
780 unsigned int hhlen
= dev
->header_ops
? dev
->hard_header_len
: 0;
783 skb
= sock_wmalloc(sk
, len
+ reserved
, 0, GFP_KERNEL
);
786 /* FIXME: Save some space for broken drivers that write a hard
787 * header at transmission time by themselves. PPP is the notable
788 * one here. This should really be fixed at the driver level.
790 skb_reserve(skb
, reserved
);
791 skb_reset_network_header(skb
);
793 /* Try to align data part correctly */
798 skb_reset_network_header(skb
);
800 err
= memcpy_fromiovec(skb_put(skb
, len
), msg
->msg_iov
, len
);
806 if (len
> (dev
->mtu
+ dev
->hard_header_len
)) {
807 /* Earlier code assumed this would be a VLAN pkt,
808 * double-check this now that we have the actual
812 skb_reset_mac_header(skb
);
814 if (ehdr
->h_proto
!= htons(ETH_P_8021Q
)) {
820 skb
->protocol
= proto
;
822 skb
->priority
= sk
->sk_priority
;
823 skb
->mark
= sk
->sk_mark
;
824 err
= sock_tx_timestamp(sk
, &skb_shinfo(skb
)->tx_flags
);
839 static inline unsigned int run_filter(const struct sk_buff
*skb
,
840 const struct sock
*sk
,
843 struct sk_filter
*filter
;
846 filter
= rcu_dereference(sk
->sk_filter
);
848 res
= SK_RUN_FILTER(filter
, skb
);
855 * This function makes lazy skb cloning in hope that most of packets
856 * are discarded by BPF.
858 * Note tricky part: we DO mangle shared skb! skb->data, skb->len
859 * and skb->cb are mangled. It works because (and until) packets
860 * falling here are owned by current CPU. Output packets are cloned
861 * by dev_queue_xmit_nit(), input packets are processed by net_bh
862 * sequencially, so that if we return skb to original state on exit,
863 * we will not harm anyone.
866 static int packet_rcv(struct sk_buff
*skb
, struct net_device
*dev
,
867 struct packet_type
*pt
, struct net_device
*orig_dev
)
870 struct sockaddr_ll
*sll
;
871 struct packet_sock
*po
;
872 u8
*skb_head
= skb
->data
;
873 int skb_len
= skb
->len
;
874 unsigned int snaplen
, res
;
876 if (skb
->pkt_type
== PACKET_LOOPBACK
)
879 sk
= pt
->af_packet_priv
;
882 if (!net_eq(dev_net(dev
), sock_net(sk
)))
887 if (dev
->header_ops
) {
888 /* The device has an explicit notion of ll header,
889 * exported to higher levels.
891 * Otherwise, the device hides details of its frame
892 * structure, so that corresponding packet head is
893 * never delivered to user.
895 if (sk
->sk_type
!= SOCK_DGRAM
)
896 skb_push(skb
, skb
->data
- skb_mac_header(skb
));
897 else if (skb
->pkt_type
== PACKET_OUTGOING
) {
898 /* Special case: outgoing packets have ll header at head */
899 skb_pull(skb
, skb_network_offset(skb
));
905 res
= run_filter(skb
, sk
, snaplen
);
911 if (atomic_read(&sk
->sk_rmem_alloc
) + skb
->truesize
>=
912 (unsigned)sk
->sk_rcvbuf
)
915 if (skb_shared(skb
)) {
916 struct sk_buff
*nskb
= skb_clone(skb
, GFP_ATOMIC
);
920 if (skb_head
!= skb
->data
) {
921 skb
->data
= skb_head
;
928 BUILD_BUG_ON(sizeof(*PACKET_SKB_CB(skb
)) + MAX_ADDR_LEN
- 8 >
931 sll
= &PACKET_SKB_CB(skb
)->sa
.ll
;
932 sll
->sll_family
= AF_PACKET
;
933 sll
->sll_hatype
= dev
->type
;
934 sll
->sll_protocol
= skb
->protocol
;
935 sll
->sll_pkttype
= skb
->pkt_type
;
936 if (unlikely(po
->origdev
))
937 sll
->sll_ifindex
= orig_dev
->ifindex
;
939 sll
->sll_ifindex
= dev
->ifindex
;
941 sll
->sll_halen
= dev_parse_header(skb
, sll
->sll_addr
);
943 PACKET_SKB_CB(skb
)->origlen
= skb
->len
;
945 if (pskb_trim(skb
, snaplen
))
948 skb_set_owner_r(skb
, sk
);
952 /* drop conntrack reference */
955 spin_lock(&sk
->sk_receive_queue
.lock
);
956 po
->stats
.tp_packets
++;
957 skb
->dropcount
= atomic_read(&sk
->sk_drops
);
958 __skb_queue_tail(&sk
->sk_receive_queue
, skb
);
959 spin_unlock(&sk
->sk_receive_queue
.lock
);
960 sk
->sk_data_ready(sk
, skb
->len
);
964 po
->stats
.tp_drops
= atomic_inc_return(&sk
->sk_drops
);
967 if (skb_head
!= skb
->data
&& skb_shared(skb
)) {
968 skb
->data
= skb_head
;
976 static int tpacket_rcv(struct sk_buff
*skb
, struct net_device
*dev
,
977 struct packet_type
*pt
, struct net_device
*orig_dev
)
980 struct packet_sock
*po
;
981 struct sockaddr_ll
*sll
;
983 struct tpacket_hdr
*h1
;
984 struct tpacket2_hdr
*h2
;
987 u8
*skb_head
= skb
->data
;
988 int skb_len
= skb
->len
;
989 unsigned int snaplen
, res
;
990 unsigned long status
= TP_STATUS_LOSING
|TP_STATUS_USER
;
991 unsigned short macoff
, netoff
, hdrlen
;
992 struct sk_buff
*copy_skb
= NULL
;
995 struct skb_shared_hwtstamps
*shhwtstamps
= skb_hwtstamps(skb
);
997 if (skb
->pkt_type
== PACKET_LOOPBACK
)
1000 sk
= pt
->af_packet_priv
;
1003 if (!net_eq(dev_net(dev
), sock_net(sk
)))
1006 if (dev
->header_ops
) {
1007 if (sk
->sk_type
!= SOCK_DGRAM
)
1008 skb_push(skb
, skb
->data
- skb_mac_header(skb
));
1009 else if (skb
->pkt_type
== PACKET_OUTGOING
) {
1010 /* Special case: outgoing packets have ll header at head */
1011 skb_pull(skb
, skb_network_offset(skb
));
1015 if (skb
->ip_summed
== CHECKSUM_PARTIAL
)
1016 status
|= TP_STATUS_CSUMNOTREADY
;
1020 res
= run_filter(skb
, sk
, snaplen
);
1022 goto drop_n_restore
;
1026 if (sk
->sk_type
== SOCK_DGRAM
) {
1027 macoff
= netoff
= TPACKET_ALIGN(po
->tp_hdrlen
) + 16 +
1030 unsigned maclen
= skb_network_offset(skb
);
1031 netoff
= TPACKET_ALIGN(po
->tp_hdrlen
+
1032 (maclen
< 16 ? 16 : maclen
)) +
1034 macoff
= netoff
- maclen
;
1037 if (macoff
+ snaplen
> po
->rx_ring
.frame_size
) {
1038 if (po
->copy_thresh
&&
1039 atomic_read(&sk
->sk_rmem_alloc
) + skb
->truesize
<
1040 (unsigned)sk
->sk_rcvbuf
) {
1041 if (skb_shared(skb
)) {
1042 copy_skb
= skb_clone(skb
, GFP_ATOMIC
);
1044 copy_skb
= skb_get(skb
);
1045 skb_head
= skb
->data
;
1048 skb_set_owner_r(copy_skb
, sk
);
1050 snaplen
= po
->rx_ring
.frame_size
- macoff
;
1051 if ((int)snaplen
< 0)
1055 spin_lock(&sk
->sk_receive_queue
.lock
);
1056 h
.raw
= packet_current_frame(po
, &po
->rx_ring
, TP_STATUS_KERNEL
);
1059 packet_increment_head(&po
->rx_ring
);
1060 po
->stats
.tp_packets
++;
1062 status
|= TP_STATUS_COPY
;
1063 __skb_queue_tail(&sk
->sk_receive_queue
, copy_skb
);
1065 if (!po
->stats
.tp_drops
)
1066 status
&= ~TP_STATUS_LOSING
;
1067 spin_unlock(&sk
->sk_receive_queue
.lock
);
1069 skb_copy_bits(skb
, 0, h
.raw
+ macoff
, snaplen
);
1071 switch (po
->tp_version
) {
1073 h
.h1
->tp_len
= skb
->len
;
1074 h
.h1
->tp_snaplen
= snaplen
;
1075 h
.h1
->tp_mac
= macoff
;
1076 h
.h1
->tp_net
= netoff
;
1077 if ((po
->tp_tstamp
& SOF_TIMESTAMPING_SYS_HARDWARE
)
1078 && shhwtstamps
->syststamp
.tv64
)
1079 tv
= ktime_to_timeval(shhwtstamps
->syststamp
);
1080 else if ((po
->tp_tstamp
& SOF_TIMESTAMPING_RAW_HARDWARE
)
1081 && shhwtstamps
->hwtstamp
.tv64
)
1082 tv
= ktime_to_timeval(shhwtstamps
->hwtstamp
);
1083 else if (skb
->tstamp
.tv64
)
1084 tv
= ktime_to_timeval(skb
->tstamp
);
1086 do_gettimeofday(&tv
);
1087 h
.h1
->tp_sec
= tv
.tv_sec
;
1088 h
.h1
->tp_usec
= tv
.tv_usec
;
1089 hdrlen
= sizeof(*h
.h1
);
1092 h
.h2
->tp_len
= skb
->len
;
1093 h
.h2
->tp_snaplen
= snaplen
;
1094 h
.h2
->tp_mac
= macoff
;
1095 h
.h2
->tp_net
= netoff
;
1096 if ((po
->tp_tstamp
& SOF_TIMESTAMPING_SYS_HARDWARE
)
1097 && shhwtstamps
->syststamp
.tv64
)
1098 ts
= ktime_to_timespec(shhwtstamps
->syststamp
);
1099 else if ((po
->tp_tstamp
& SOF_TIMESTAMPING_RAW_HARDWARE
)
1100 && shhwtstamps
->hwtstamp
.tv64
)
1101 ts
= ktime_to_timespec(shhwtstamps
->hwtstamp
);
1102 else if (skb
->tstamp
.tv64
)
1103 ts
= ktime_to_timespec(skb
->tstamp
);
1105 getnstimeofday(&ts
);
1106 h
.h2
->tp_sec
= ts
.tv_sec
;
1107 h
.h2
->tp_nsec
= ts
.tv_nsec
;
1108 if (vlan_tx_tag_present(skb
)) {
1109 h
.h2
->tp_vlan_tci
= vlan_tx_tag_get(skb
);
1110 status
|= TP_STATUS_VLAN_VALID
;
1112 h
.h2
->tp_vlan_tci
= 0;
1114 h
.h2
->tp_padding
= 0;
1115 hdrlen
= sizeof(*h
.h2
);
1121 sll
= h
.raw
+ TPACKET_ALIGN(hdrlen
);
1122 sll
->sll_halen
= dev_parse_header(skb
, sll
->sll_addr
);
1123 sll
->sll_family
= AF_PACKET
;
1124 sll
->sll_hatype
= dev
->type
;
1125 sll
->sll_protocol
= skb
->protocol
;
1126 sll
->sll_pkttype
= skb
->pkt_type
;
1127 if (unlikely(po
->origdev
))
1128 sll
->sll_ifindex
= orig_dev
->ifindex
;
1130 sll
->sll_ifindex
= dev
->ifindex
;
1132 __packet_set_status(po
, h
.raw
, status
);
1134 #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
1138 end
= (u8
*)PAGE_ALIGN((unsigned long)h
.raw
+ macoff
+ snaplen
);
1139 for (start
= h
.raw
; start
< end
; start
+= PAGE_SIZE
)
1140 flush_dcache_page(pgv_to_page(start
));
1144 sk
->sk_data_ready(sk
, 0);
1147 if (skb_head
!= skb
->data
&& skb_shared(skb
)) {
1148 skb
->data
= skb_head
;
1156 po
->stats
.tp_drops
++;
1157 spin_unlock(&sk
->sk_receive_queue
.lock
);
1159 sk
->sk_data_ready(sk
, 0);
1160 kfree_skb(copy_skb
);
1161 goto drop_n_restore
;
1164 static void tpacket_destruct_skb(struct sk_buff
*skb
)
1166 struct packet_sock
*po
= pkt_sk(skb
->sk
);
1169 BUG_ON(skb
== NULL
);
1171 if (likely(po
->tx_ring
.pg_vec
)) {
1172 ph
= skb_shinfo(skb
)->destructor_arg
;
1173 BUG_ON(__packet_get_status(po
, ph
) != TP_STATUS_SENDING
);
1174 BUG_ON(atomic_read(&po
->tx_ring
.pending
) == 0);
1175 atomic_dec(&po
->tx_ring
.pending
);
1176 __packet_set_status(po
, ph
, TP_STATUS_AVAILABLE
);
1182 static int tpacket_fill_skb(struct packet_sock
*po
, struct sk_buff
*skb
,
1183 void *frame
, struct net_device
*dev
, int size_max
,
1184 __be16 proto
, unsigned char *addr
)
1187 struct tpacket_hdr
*h1
;
1188 struct tpacket2_hdr
*h2
;
1191 int to_write
, offset
, len
, tp_len
, nr_frags
, len_max
;
1192 struct socket
*sock
= po
->sk
.sk_socket
;
1199 skb
->protocol
= proto
;
1201 skb
->priority
= po
->sk
.sk_priority
;
1202 skb
->mark
= po
->sk
.sk_mark
;
1203 skb_shinfo(skb
)->destructor_arg
= ph
.raw
;
1205 switch (po
->tp_version
) {
1207 tp_len
= ph
.h2
->tp_len
;
1210 tp_len
= ph
.h1
->tp_len
;
1213 if (unlikely(tp_len
> size_max
)) {
1214 pr_err("packet size is too long (%d > %d)\n", tp_len
, size_max
);
1218 skb_reserve(skb
, LL_RESERVED_SPACE(dev
));
1219 skb_reset_network_header(skb
);
1221 data
= ph
.raw
+ po
->tp_hdrlen
- sizeof(struct sockaddr_ll
);
1224 if (sock
->type
== SOCK_DGRAM
) {
1225 err
= dev_hard_header(skb
, dev
, ntohs(proto
), addr
,
1227 if (unlikely(err
< 0))
1229 } else if (dev
->hard_header_len
) {
1230 /* net device doesn't like empty head */
1231 if (unlikely(tp_len
<= dev
->hard_header_len
)) {
1232 pr_err("packet size is too short (%d < %d)\n",
1233 tp_len
, dev
->hard_header_len
);
1237 skb_push(skb
, dev
->hard_header_len
);
1238 err
= skb_store_bits(skb
, 0, data
,
1239 dev
->hard_header_len
);
1243 data
+= dev
->hard_header_len
;
1244 to_write
-= dev
->hard_header_len
;
1248 offset
= offset_in_page(data
);
1249 len_max
= PAGE_SIZE
- offset
;
1250 len
= ((to_write
> len_max
) ? len_max
: to_write
);
1252 skb
->data_len
= to_write
;
1253 skb
->len
+= to_write
;
1254 skb
->truesize
+= to_write
;
1255 atomic_add(to_write
, &po
->sk
.sk_wmem_alloc
);
1257 while (likely(to_write
)) {
1258 nr_frags
= skb_shinfo(skb
)->nr_frags
;
1260 if (unlikely(nr_frags
>= MAX_SKB_FRAGS
)) {
1261 pr_err("Packet exceed the number of skb frags(%lu)\n",
1266 page
= pgv_to_page(data
);
1268 flush_dcache_page(page
);
1270 skb_fill_page_desc(skb
, nr_frags
, page
, offset
, len
);
1273 len_max
= PAGE_SIZE
;
1274 len
= ((to_write
> len_max
) ? len_max
: to_write
);
1280 static int tpacket_snd(struct packet_sock
*po
, struct msghdr
*msg
)
1282 struct sk_buff
*skb
;
1283 struct net_device
*dev
;
1285 bool need_rls_dev
= false;
1286 int err
, reserve
= 0;
1288 struct sockaddr_ll
*saddr
= (struct sockaddr_ll
*)msg
->msg_name
;
1289 int tp_len
, size_max
;
1290 unsigned char *addr
;
1294 mutex_lock(&po
->pg_vec_lock
);
1297 if (saddr
== NULL
) {
1298 dev
= po
->prot_hook
.dev
;
1303 if (msg
->msg_namelen
< sizeof(struct sockaddr_ll
))
1305 if (msg
->msg_namelen
< (saddr
->sll_halen
1306 + offsetof(struct sockaddr_ll
,
1309 proto
= saddr
->sll_protocol
;
1310 addr
= saddr
->sll_addr
;
1311 dev
= dev_get_by_index(sock_net(&po
->sk
), saddr
->sll_ifindex
);
1312 need_rls_dev
= true;
1316 if (unlikely(dev
== NULL
))
1319 reserve
= dev
->hard_header_len
;
1322 if (unlikely(!(dev
->flags
& IFF_UP
)))
1325 size_max
= po
->tx_ring
.frame_size
1326 - (po
->tp_hdrlen
- sizeof(struct sockaddr_ll
));
1328 if (size_max
> dev
->mtu
+ reserve
)
1329 size_max
= dev
->mtu
+ reserve
;
1332 ph
= packet_current_frame(po
, &po
->tx_ring
,
1333 TP_STATUS_SEND_REQUEST
);
1335 if (unlikely(ph
== NULL
)) {
1340 status
= TP_STATUS_SEND_REQUEST
;
1341 skb
= sock_alloc_send_skb(&po
->sk
,
1342 LL_ALLOCATED_SPACE(dev
)
1343 + sizeof(struct sockaddr_ll
),
1346 if (unlikely(skb
== NULL
))
1349 tp_len
= tpacket_fill_skb(po
, skb
, ph
, dev
, size_max
, proto
,
1352 if (unlikely(tp_len
< 0)) {
1354 __packet_set_status(po
, ph
,
1355 TP_STATUS_AVAILABLE
);
1356 packet_increment_head(&po
->tx_ring
);
1360 status
= TP_STATUS_WRONG_FORMAT
;
1366 skb
->destructor
= tpacket_destruct_skb
;
1367 __packet_set_status(po
, ph
, TP_STATUS_SENDING
);
1368 atomic_inc(&po
->tx_ring
.pending
);
1370 status
= TP_STATUS_SEND_REQUEST
;
1371 err
= dev_queue_xmit(skb
);
1372 if (unlikely(err
> 0)) {
1373 err
= net_xmit_errno(err
);
1374 if (err
&& __packet_get_status(po
, ph
) ==
1375 TP_STATUS_AVAILABLE
) {
1376 /* skb was destructed already */
1381 * skb was dropped but not destructed yet;
1382 * let's treat it like congestion or err < 0
1386 packet_increment_head(&po
->tx_ring
);
1388 } while (likely((ph
!= NULL
) ||
1389 ((!(msg
->msg_flags
& MSG_DONTWAIT
)) &&
1390 (atomic_read(&po
->tx_ring
.pending
))))
1397 __packet_set_status(po
, ph
, status
);
1403 mutex_unlock(&po
->pg_vec_lock
);
1407 static inline struct sk_buff
*packet_alloc_skb(struct sock
*sk
, size_t prepad
,
1408 size_t reserve
, size_t len
,
1409 size_t linear
, int noblock
,
1412 struct sk_buff
*skb
;
1414 /* Under a page? Don't bother with paged skb. */
1415 if (prepad
+ len
< PAGE_SIZE
|| !linear
)
1418 skb
= sock_alloc_send_pskb(sk
, prepad
+ linear
, len
- linear
, noblock
,
1423 skb_reserve(skb
, reserve
);
1424 skb_put(skb
, linear
);
1425 skb
->data_len
= len
- linear
;
1426 skb
->len
+= len
- linear
;
1431 static int packet_snd(struct socket
*sock
,
1432 struct msghdr
*msg
, size_t len
)
1434 struct sock
*sk
= sock
->sk
;
1435 struct sockaddr_ll
*saddr
= (struct sockaddr_ll
*)msg
->msg_name
;
1436 struct sk_buff
*skb
;
1437 struct net_device
*dev
;
1439 bool need_rls_dev
= false;
1440 unsigned char *addr
;
1441 int err
, reserve
= 0;
1442 struct virtio_net_hdr vnet_hdr
= { 0 };
1445 struct packet_sock
*po
= pkt_sk(sk
);
1446 unsigned short gso_type
= 0;
1449 * Get and verify the address.
1452 if (saddr
== NULL
) {
1453 dev
= po
->prot_hook
.dev
;
1458 if (msg
->msg_namelen
< sizeof(struct sockaddr_ll
))
1460 if (msg
->msg_namelen
< (saddr
->sll_halen
+ offsetof(struct sockaddr_ll
, sll_addr
)))
1462 proto
= saddr
->sll_protocol
;
1463 addr
= saddr
->sll_addr
;
1464 dev
= dev_get_by_index(sock_net(sk
), saddr
->sll_ifindex
);
1465 need_rls_dev
= true;
1471 if (sock
->type
== SOCK_RAW
)
1472 reserve
= dev
->hard_header_len
;
1475 if (!(dev
->flags
& IFF_UP
))
1478 if (po
->has_vnet_hdr
) {
1479 vnet_hdr_len
= sizeof(vnet_hdr
);
1482 if (len
< vnet_hdr_len
)
1485 len
-= vnet_hdr_len
;
1487 err
= memcpy_fromiovec((void *)&vnet_hdr
, msg
->msg_iov
,
1492 if ((vnet_hdr
.flags
& VIRTIO_NET_HDR_F_NEEDS_CSUM
) &&
1493 (vnet_hdr
.csum_start
+ vnet_hdr
.csum_offset
+ 2 >
1495 vnet_hdr
.hdr_len
= vnet_hdr
.csum_start
+
1496 vnet_hdr
.csum_offset
+ 2;
1499 if (vnet_hdr
.hdr_len
> len
)
1502 if (vnet_hdr
.gso_type
!= VIRTIO_NET_HDR_GSO_NONE
) {
1503 switch (vnet_hdr
.gso_type
& ~VIRTIO_NET_HDR_GSO_ECN
) {
1504 case VIRTIO_NET_HDR_GSO_TCPV4
:
1505 gso_type
= SKB_GSO_TCPV4
;
1507 case VIRTIO_NET_HDR_GSO_TCPV6
:
1508 gso_type
= SKB_GSO_TCPV6
;
1510 case VIRTIO_NET_HDR_GSO_UDP
:
1511 gso_type
= SKB_GSO_UDP
;
1517 if (vnet_hdr
.gso_type
& VIRTIO_NET_HDR_GSO_ECN
)
1518 gso_type
|= SKB_GSO_TCP_ECN
;
1520 if (vnet_hdr
.gso_size
== 0)
1527 if (!gso_type
&& (len
> dev
->mtu
+ reserve
+ VLAN_HLEN
))
1531 skb
= packet_alloc_skb(sk
, LL_ALLOCATED_SPACE(dev
),
1532 LL_RESERVED_SPACE(dev
), len
, vnet_hdr
.hdr_len
,
1533 msg
->msg_flags
& MSG_DONTWAIT
, &err
);
1537 skb_set_network_header(skb
, reserve
);
1540 if (sock
->type
== SOCK_DGRAM
&&
1541 (offset
= dev_hard_header(skb
, dev
, ntohs(proto
), addr
, NULL
, len
)) < 0)
1544 /* Returns -EFAULT on error */
1545 err
= skb_copy_datagram_from_iovec(skb
, offset
, msg
->msg_iov
, 0, len
);
1548 err
= sock_tx_timestamp(sk
, &skb_shinfo(skb
)->tx_flags
);
1552 if (!gso_type
&& (len
> dev
->mtu
+ reserve
)) {
1553 /* Earlier code assumed this would be a VLAN pkt,
1554 * double-check this now that we have the actual
1557 struct ethhdr
*ehdr
;
1558 skb_reset_mac_header(skb
);
1559 ehdr
= eth_hdr(skb
);
1560 if (ehdr
->h_proto
!= htons(ETH_P_8021Q
)) {
1566 skb
->protocol
= proto
;
1568 skb
->priority
= sk
->sk_priority
;
1569 skb
->mark
= sk
->sk_mark
;
1571 if (po
->has_vnet_hdr
) {
1572 if (vnet_hdr
.flags
& VIRTIO_NET_HDR_F_NEEDS_CSUM
) {
1573 if (!skb_partial_csum_set(skb
, vnet_hdr
.csum_start
,
1574 vnet_hdr
.csum_offset
)) {
1580 skb_shinfo(skb
)->gso_size
= vnet_hdr
.gso_size
;
1581 skb_shinfo(skb
)->gso_type
= gso_type
;
1583 /* Header must be checked, and gso_segs computed. */
1584 skb_shinfo(skb
)->gso_type
|= SKB_GSO_DODGY
;
1585 skb_shinfo(skb
)->gso_segs
= 0;
1587 len
+= vnet_hdr_len
;
1594 err
= dev_queue_xmit(skb
);
1595 if (err
> 0 && (err
= net_xmit_errno(err
)) != 0)
1606 if (dev
&& need_rls_dev
)
1612 static int packet_sendmsg(struct kiocb
*iocb
, struct socket
*sock
,
1613 struct msghdr
*msg
, size_t len
)
1615 struct sock
*sk
= sock
->sk
;
1616 struct packet_sock
*po
= pkt_sk(sk
);
1617 if (po
->tx_ring
.pg_vec
)
1618 return tpacket_snd(po
, msg
);
1620 return packet_snd(sock
, msg
, len
);
1624 * Close a PACKET socket. This is fairly simple. We immediately go
1625 * to 'closed' state and remove our protocol entry in the device list.
1628 static int packet_release(struct socket
*sock
)
1630 struct sock
*sk
= sock
->sk
;
1631 struct packet_sock
*po
;
1633 struct tpacket_req req
;
1641 spin_lock_bh(&net
->packet
.sklist_lock
);
1642 sk_del_node_init_rcu(sk
);
1643 sock_prot_inuse_add(net
, sk
->sk_prot
, -1);
1644 spin_unlock_bh(&net
->packet
.sklist_lock
);
1646 spin_lock(&po
->bind_lock
);
1647 unregister_prot_hook(sk
, false);
1648 if (po
->prot_hook
.dev
) {
1649 dev_put(po
->prot_hook
.dev
);
1650 po
->prot_hook
.dev
= NULL
;
1652 spin_unlock(&po
->bind_lock
);
1654 packet_flush_mclist(sk
);
1656 memset(&req
, 0, sizeof(req
));
1658 if (po
->rx_ring
.pg_vec
)
1659 packet_set_ring(sk
, &req
, 1, 0);
1661 if (po
->tx_ring
.pg_vec
)
1662 packet_set_ring(sk
, &req
, 1, 1);
1668 * Now the socket is dead. No more input will appear.
1675 skb_queue_purge(&sk
->sk_receive_queue
);
1676 sk_refcnt_debug_release(sk
);
1683 * Attach a packet hook.
1686 static int packet_do_bind(struct sock
*sk
, struct net_device
*dev
, __be16 protocol
)
1688 struct packet_sock
*po
= pkt_sk(sk
);
1695 spin_lock(&po
->bind_lock
);
1696 unregister_prot_hook(sk
, true);
1698 po
->prot_hook
.type
= protocol
;
1699 if (po
->prot_hook
.dev
)
1700 dev_put(po
->prot_hook
.dev
);
1701 po
->prot_hook
.dev
= dev
;
1703 po
->ifindex
= dev
? dev
->ifindex
: 0;
1708 if (!dev
|| (dev
->flags
& IFF_UP
)) {
1709 register_prot_hook(sk
);
1711 sk
->sk_err
= ENETDOWN
;
1712 if (!sock_flag(sk
, SOCK_DEAD
))
1713 sk
->sk_error_report(sk
);
1717 spin_unlock(&po
->bind_lock
);
1723 * Bind a packet socket to a device
1726 static int packet_bind_spkt(struct socket
*sock
, struct sockaddr
*uaddr
,
1729 struct sock
*sk
= sock
->sk
;
1731 struct net_device
*dev
;
1738 if (addr_len
!= sizeof(struct sockaddr
))
1740 strlcpy(name
, uaddr
->sa_data
, sizeof(name
));
1742 dev
= dev_get_by_name(sock_net(sk
), name
);
1744 err
= packet_do_bind(sk
, dev
, pkt_sk(sk
)->num
);
1748 static int packet_bind(struct socket
*sock
, struct sockaddr
*uaddr
, int addr_len
)
1750 struct sockaddr_ll
*sll
= (struct sockaddr_ll
*)uaddr
;
1751 struct sock
*sk
= sock
->sk
;
1752 struct net_device
*dev
= NULL
;
1760 if (addr_len
< sizeof(struct sockaddr_ll
))
1762 if (sll
->sll_family
!= AF_PACKET
)
1765 if (sll
->sll_ifindex
) {
1767 dev
= dev_get_by_index(sock_net(sk
), sll
->sll_ifindex
);
1771 err
= packet_do_bind(sk
, dev
, sll
->sll_protocol
? : pkt_sk(sk
)->num
);
1777 static struct proto packet_proto
= {
1779 .owner
= THIS_MODULE
,
1780 .obj_size
= sizeof(struct packet_sock
),
1784 * Create a packet of type SOCK_PACKET.
1787 static int packet_create(struct net
*net
, struct socket
*sock
, int protocol
,
1791 struct packet_sock
*po
;
1792 __be16 proto
= (__force __be16
)protocol
; /* weird, but documented */
1795 if (!capable(CAP_NET_RAW
))
1797 if (sock
->type
!= SOCK_DGRAM
&& sock
->type
!= SOCK_RAW
&&
1798 sock
->type
!= SOCK_PACKET
)
1799 return -ESOCKTNOSUPPORT
;
1801 sock
->state
= SS_UNCONNECTED
;
1804 sk
= sk_alloc(net
, PF_PACKET
, GFP_KERNEL
, &packet_proto
);
1808 sock
->ops
= &packet_ops
;
1809 if (sock
->type
== SOCK_PACKET
)
1810 sock
->ops
= &packet_ops_spkt
;
1812 sock_init_data(sock
, sk
);
1815 sk
->sk_family
= PF_PACKET
;
1818 sk
->sk_destruct
= packet_sock_destruct
;
1819 sk_refcnt_debug_inc(sk
);
1822 * Attach a protocol block
1825 spin_lock_init(&po
->bind_lock
);
1826 mutex_init(&po
->pg_vec_lock
);
1827 po
->prot_hook
.func
= packet_rcv
;
1829 if (sock
->type
== SOCK_PACKET
)
1830 po
->prot_hook
.func
= packet_rcv_spkt
;
1832 po
->prot_hook
.af_packet_priv
= sk
;
1835 po
->prot_hook
.type
= proto
;
1836 register_prot_hook(sk
);
1839 spin_lock_bh(&net
->packet
.sklist_lock
);
1840 sk_add_node_rcu(sk
, &net
->packet
.sklist
);
1841 sock_prot_inuse_add(net
, &packet_proto
, 1);
1842 spin_unlock_bh(&net
->packet
.sklist_lock
);
1849 static int packet_recv_error(struct sock
*sk
, struct msghdr
*msg
, int len
)
1851 struct sock_exterr_skb
*serr
;
1852 struct sk_buff
*skb
, *skb2
;
1856 skb
= skb_dequeue(&sk
->sk_error_queue
);
1862 msg
->msg_flags
|= MSG_TRUNC
;
1865 err
= skb_copy_datagram_iovec(skb
, 0, msg
->msg_iov
, copied
);
1869 sock_recv_timestamp(msg
, sk
, skb
);
1871 serr
= SKB_EXT_ERR(skb
);
1872 put_cmsg(msg
, SOL_PACKET
, PACKET_TX_TIMESTAMP
,
1873 sizeof(serr
->ee
), &serr
->ee
);
1875 msg
->msg_flags
|= MSG_ERRQUEUE
;
1878 /* Reset and regenerate socket error */
1879 spin_lock_bh(&sk
->sk_error_queue
.lock
);
1881 if ((skb2
= skb_peek(&sk
->sk_error_queue
)) != NULL
) {
1882 sk
->sk_err
= SKB_EXT_ERR(skb2
)->ee
.ee_errno
;
1883 spin_unlock_bh(&sk
->sk_error_queue
.lock
);
1884 sk
->sk_error_report(sk
);
1886 spin_unlock_bh(&sk
->sk_error_queue
.lock
);
1895 * Pull a packet from our receive queue and hand it to the user.
1896 * If necessary we block.
1899 static int packet_recvmsg(struct kiocb
*iocb
, struct socket
*sock
,
1900 struct msghdr
*msg
, size_t len
, int flags
)
1902 struct sock
*sk
= sock
->sk
;
1903 struct sk_buff
*skb
;
1905 struct sockaddr_ll
*sll
;
1906 int vnet_hdr_len
= 0;
1909 if (flags
& ~(MSG_PEEK
|MSG_DONTWAIT
|MSG_TRUNC
|MSG_CMSG_COMPAT
|MSG_ERRQUEUE
))
1913 /* What error should we return now? EUNATTACH? */
1914 if (pkt_sk(sk
)->ifindex
< 0)
1918 if (flags
& MSG_ERRQUEUE
) {
1919 err
= packet_recv_error(sk
, msg
, len
);
1924 * Call the generic datagram receiver. This handles all sorts
1925 * of horrible races and re-entrancy so we can forget about it
1926 * in the protocol layers.
1928 * Now it will return ENETDOWN, if device have just gone down,
1929 * but then it will block.
1932 skb
= skb_recv_datagram(sk
, flags
, flags
& MSG_DONTWAIT
, &err
);
1935 * An error occurred so return it. Because skb_recv_datagram()
1936 * handles the blocking we don't see and worry about blocking
1943 if (pkt_sk(sk
)->has_vnet_hdr
) {
1944 struct virtio_net_hdr vnet_hdr
= { 0 };
1947 vnet_hdr_len
= sizeof(vnet_hdr
);
1948 if (len
< vnet_hdr_len
)
1951 len
-= vnet_hdr_len
;
1953 if (skb_is_gso(skb
)) {
1954 struct skb_shared_info
*sinfo
= skb_shinfo(skb
);
1956 /* This is a hint as to how much should be linear. */
1957 vnet_hdr
.hdr_len
= skb_headlen(skb
);
1958 vnet_hdr
.gso_size
= sinfo
->gso_size
;
1959 if (sinfo
->gso_type
& SKB_GSO_TCPV4
)
1960 vnet_hdr
.gso_type
= VIRTIO_NET_HDR_GSO_TCPV4
;
1961 else if (sinfo
->gso_type
& SKB_GSO_TCPV6
)
1962 vnet_hdr
.gso_type
= VIRTIO_NET_HDR_GSO_TCPV6
;
1963 else if (sinfo
->gso_type
& SKB_GSO_UDP
)
1964 vnet_hdr
.gso_type
= VIRTIO_NET_HDR_GSO_UDP
;
1965 else if (sinfo
->gso_type
& SKB_GSO_FCOE
)
1969 if (sinfo
->gso_type
& SKB_GSO_TCP_ECN
)
1970 vnet_hdr
.gso_type
|= VIRTIO_NET_HDR_GSO_ECN
;
1972 vnet_hdr
.gso_type
= VIRTIO_NET_HDR_GSO_NONE
;
1974 if (skb
->ip_summed
== CHECKSUM_PARTIAL
) {
1975 vnet_hdr
.flags
= VIRTIO_NET_HDR_F_NEEDS_CSUM
;
1976 vnet_hdr
.csum_start
= skb_checksum_start_offset(skb
);
1977 vnet_hdr
.csum_offset
= skb
->csum_offset
;
1978 } else if (skb
->ip_summed
== CHECKSUM_UNNECESSARY
) {
1979 vnet_hdr
.flags
= VIRTIO_NET_HDR_F_DATA_VALID
;
1980 } /* else everything is zero */
1982 err
= memcpy_toiovec(msg
->msg_iov
, (void *)&vnet_hdr
,
1989 * If the address length field is there to be filled in, we fill
1993 sll
= &PACKET_SKB_CB(skb
)->sa
.ll
;
1994 if (sock
->type
== SOCK_PACKET
)
1995 msg
->msg_namelen
= sizeof(struct sockaddr_pkt
);
1997 msg
->msg_namelen
= sll
->sll_halen
+ offsetof(struct sockaddr_ll
, sll_addr
);
2000 * You lose any data beyond the buffer you gave. If it worries a
2001 * user program they can ask the device for its MTU anyway.
2007 msg
->msg_flags
|= MSG_TRUNC
;
2010 err
= skb_copy_datagram_iovec(skb
, 0, msg
->msg_iov
, copied
);
2014 sock_recv_ts_and_drops(msg
, sk
, skb
);
2017 memcpy(msg
->msg_name
, &PACKET_SKB_CB(skb
)->sa
,
2020 if (pkt_sk(sk
)->auxdata
) {
2021 struct tpacket_auxdata aux
;
2023 aux
.tp_status
= TP_STATUS_USER
;
2024 if (skb
->ip_summed
== CHECKSUM_PARTIAL
)
2025 aux
.tp_status
|= TP_STATUS_CSUMNOTREADY
;
2026 aux
.tp_len
= PACKET_SKB_CB(skb
)->origlen
;
2027 aux
.tp_snaplen
= skb
->len
;
2029 aux
.tp_net
= skb_network_offset(skb
);
2030 if (vlan_tx_tag_present(skb
)) {
2031 aux
.tp_vlan_tci
= vlan_tx_tag_get(skb
);
2032 aux
.tp_status
|= TP_STATUS_VLAN_VALID
;
2034 aux
.tp_vlan_tci
= 0;
2037 put_cmsg(msg
, SOL_PACKET
, PACKET_AUXDATA
, sizeof(aux
), &aux
);
2041 * Free or return the buffer as appropriate. Again this
2042 * hides all the races and re-entrancy issues from us.
2044 err
= vnet_hdr_len
+ ((flags
&MSG_TRUNC
) ? skb
->len
: copied
);
2047 skb_free_datagram(sk
, skb
);
2052 static int packet_getname_spkt(struct socket
*sock
, struct sockaddr
*uaddr
,
2053 int *uaddr_len
, int peer
)
2055 struct net_device
*dev
;
2056 struct sock
*sk
= sock
->sk
;
2061 uaddr
->sa_family
= AF_PACKET
;
2063 dev
= dev_get_by_index_rcu(sock_net(sk
), pkt_sk(sk
)->ifindex
);
2065 strncpy(uaddr
->sa_data
, dev
->name
, 14);
2067 memset(uaddr
->sa_data
, 0, 14);
2069 *uaddr_len
= sizeof(*uaddr
);
2074 static int packet_getname(struct socket
*sock
, struct sockaddr
*uaddr
,
2075 int *uaddr_len
, int peer
)
2077 struct net_device
*dev
;
2078 struct sock
*sk
= sock
->sk
;
2079 struct packet_sock
*po
= pkt_sk(sk
);
2080 DECLARE_SOCKADDR(struct sockaddr_ll
*, sll
, uaddr
);
2085 sll
->sll_family
= AF_PACKET
;
2086 sll
->sll_ifindex
= po
->ifindex
;
2087 sll
->sll_protocol
= po
->num
;
2088 sll
->sll_pkttype
= 0;
2090 dev
= dev_get_by_index_rcu(sock_net(sk
), po
->ifindex
);
2092 sll
->sll_hatype
= dev
->type
;
2093 sll
->sll_halen
= dev
->addr_len
;
2094 memcpy(sll
->sll_addr
, dev
->dev_addr
, dev
->addr_len
);
2096 sll
->sll_hatype
= 0; /* Bad: we have no ARPHRD_UNSPEC */
2100 *uaddr_len
= offsetof(struct sockaddr_ll
, sll_addr
) + sll
->sll_halen
;
2105 static int packet_dev_mc(struct net_device
*dev
, struct packet_mclist
*i
,
2109 case PACKET_MR_MULTICAST
:
2110 if (i
->alen
!= dev
->addr_len
)
2113 return dev_mc_add(dev
, i
->addr
);
2115 return dev_mc_del(dev
, i
->addr
);
2117 case PACKET_MR_PROMISC
:
2118 return dev_set_promiscuity(dev
, what
);
2120 case PACKET_MR_ALLMULTI
:
2121 return dev_set_allmulti(dev
, what
);
2123 case PACKET_MR_UNICAST
:
2124 if (i
->alen
!= dev
->addr_len
)
2127 return dev_uc_add(dev
, i
->addr
);
2129 return dev_uc_del(dev
, i
->addr
);
2137 static void packet_dev_mclist(struct net_device
*dev
, struct packet_mclist
*i
, int what
)
2139 for ( ; i
; i
= i
->next
) {
2140 if (i
->ifindex
== dev
->ifindex
)
2141 packet_dev_mc(dev
, i
, what
);
2145 static int packet_mc_add(struct sock
*sk
, struct packet_mreq_max
*mreq
)
2147 struct packet_sock
*po
= pkt_sk(sk
);
2148 struct packet_mclist
*ml
, *i
;
2149 struct net_device
*dev
;
2155 dev
= __dev_get_by_index(sock_net(sk
), mreq
->mr_ifindex
);
2160 if (mreq
->mr_alen
> dev
->addr_len
)
2164 i
= kmalloc(sizeof(*i
), GFP_KERNEL
);
2169 for (ml
= po
->mclist
; ml
; ml
= ml
->next
) {
2170 if (ml
->ifindex
== mreq
->mr_ifindex
&&
2171 ml
->type
== mreq
->mr_type
&&
2172 ml
->alen
== mreq
->mr_alen
&&
2173 memcmp(ml
->addr
, mreq
->mr_address
, ml
->alen
) == 0) {
2175 /* Free the new element ... */
2181 i
->type
= mreq
->mr_type
;
2182 i
->ifindex
= mreq
->mr_ifindex
;
2183 i
->alen
= mreq
->mr_alen
;
2184 memcpy(i
->addr
, mreq
->mr_address
, i
->alen
);
2186 i
->next
= po
->mclist
;
2188 err
= packet_dev_mc(dev
, i
, 1);
2190 po
->mclist
= i
->next
;
2199 static int packet_mc_drop(struct sock
*sk
, struct packet_mreq_max
*mreq
)
2201 struct packet_mclist
*ml
, **mlp
;
2205 for (mlp
= &pkt_sk(sk
)->mclist
; (ml
= *mlp
) != NULL
; mlp
= &ml
->next
) {
2206 if (ml
->ifindex
== mreq
->mr_ifindex
&&
2207 ml
->type
== mreq
->mr_type
&&
2208 ml
->alen
== mreq
->mr_alen
&&
2209 memcmp(ml
->addr
, mreq
->mr_address
, ml
->alen
) == 0) {
2210 if (--ml
->count
== 0) {
2211 struct net_device
*dev
;
2213 dev
= __dev_get_by_index(sock_net(sk
), ml
->ifindex
);
2215 packet_dev_mc(dev
, ml
, -1);
2223 return -EADDRNOTAVAIL
;
2226 static void packet_flush_mclist(struct sock
*sk
)
2228 struct packet_sock
*po
= pkt_sk(sk
);
2229 struct packet_mclist
*ml
;
2235 while ((ml
= po
->mclist
) != NULL
) {
2236 struct net_device
*dev
;
2238 po
->mclist
= ml
->next
;
2239 dev
= __dev_get_by_index(sock_net(sk
), ml
->ifindex
);
2241 packet_dev_mc(dev
, ml
, -1);
2248 packet_setsockopt(struct socket
*sock
, int level
, int optname
, char __user
*optval
, unsigned int optlen
)
2250 struct sock
*sk
= sock
->sk
;
2251 struct packet_sock
*po
= pkt_sk(sk
);
2254 if (level
!= SOL_PACKET
)
2255 return -ENOPROTOOPT
;
2258 case PACKET_ADD_MEMBERSHIP
:
2259 case PACKET_DROP_MEMBERSHIP
:
2261 struct packet_mreq_max mreq
;
2263 memset(&mreq
, 0, sizeof(mreq
));
2264 if (len
< sizeof(struct packet_mreq
))
2266 if (len
> sizeof(mreq
))
2268 if (copy_from_user(&mreq
, optval
, len
))
2270 if (len
< (mreq
.mr_alen
+ offsetof(struct packet_mreq
, mr_address
)))
2272 if (optname
== PACKET_ADD_MEMBERSHIP
)
2273 ret
= packet_mc_add(sk
, &mreq
);
2275 ret
= packet_mc_drop(sk
, &mreq
);
2279 case PACKET_RX_RING
:
2280 case PACKET_TX_RING
:
2282 struct tpacket_req req
;
2284 if (optlen
< sizeof(req
))
2286 if (pkt_sk(sk
)->has_vnet_hdr
)
2288 if (copy_from_user(&req
, optval
, sizeof(req
)))
2290 return packet_set_ring(sk
, &req
, 0, optname
== PACKET_TX_RING
);
2292 case PACKET_COPY_THRESH
:
2296 if (optlen
!= sizeof(val
))
2298 if (copy_from_user(&val
, optval
, sizeof(val
)))
2301 pkt_sk(sk
)->copy_thresh
= val
;
2304 case PACKET_VERSION
:
2308 if (optlen
!= sizeof(val
))
2310 if (po
->rx_ring
.pg_vec
|| po
->tx_ring
.pg_vec
)
2312 if (copy_from_user(&val
, optval
, sizeof(val
)))
2317 po
->tp_version
= val
;
2323 case PACKET_RESERVE
:
2327 if (optlen
!= sizeof(val
))
2329 if (po
->rx_ring
.pg_vec
|| po
->tx_ring
.pg_vec
)
2331 if (copy_from_user(&val
, optval
, sizeof(val
)))
2333 po
->tp_reserve
= val
;
2340 if (optlen
!= sizeof(val
))
2342 if (po
->rx_ring
.pg_vec
|| po
->tx_ring
.pg_vec
)
2344 if (copy_from_user(&val
, optval
, sizeof(val
)))
2346 po
->tp_loss
= !!val
;
2349 case PACKET_AUXDATA
:
2353 if (optlen
< sizeof(val
))
2355 if (copy_from_user(&val
, optval
, sizeof(val
)))
2358 po
->auxdata
= !!val
;
2361 case PACKET_ORIGDEV
:
2365 if (optlen
< sizeof(val
))
2367 if (copy_from_user(&val
, optval
, sizeof(val
)))
2370 po
->origdev
= !!val
;
2373 case PACKET_VNET_HDR
:
2377 if (sock
->type
!= SOCK_RAW
)
2379 if (po
->rx_ring
.pg_vec
|| po
->tx_ring
.pg_vec
)
2381 if (optlen
< sizeof(val
))
2383 if (copy_from_user(&val
, optval
, sizeof(val
)))
2386 po
->has_vnet_hdr
= !!val
;
2389 case PACKET_TIMESTAMP
:
2393 if (optlen
!= sizeof(val
))
2395 if (copy_from_user(&val
, optval
, sizeof(val
)))
2398 po
->tp_tstamp
= val
;
2405 if (optlen
!= sizeof(val
))
2407 if (copy_from_user(&val
, optval
, sizeof(val
)))
2410 return fanout_add(sk
, val
& 0xffff, val
>> 16);
2413 return -ENOPROTOOPT
;
2417 static int packet_getsockopt(struct socket
*sock
, int level
, int optname
,
2418 char __user
*optval
, int __user
*optlen
)
2422 struct sock
*sk
= sock
->sk
;
2423 struct packet_sock
*po
= pkt_sk(sk
);
2425 struct tpacket_stats st
;
2427 if (level
!= SOL_PACKET
)
2428 return -ENOPROTOOPT
;
2430 if (get_user(len
, optlen
))
2437 case PACKET_STATISTICS
:
2438 if (len
> sizeof(struct tpacket_stats
))
2439 len
= sizeof(struct tpacket_stats
);
2440 spin_lock_bh(&sk
->sk_receive_queue
.lock
);
2442 memset(&po
->stats
, 0, sizeof(st
));
2443 spin_unlock_bh(&sk
->sk_receive_queue
.lock
);
2444 st
.tp_packets
+= st
.tp_drops
;
2448 case PACKET_AUXDATA
:
2449 if (len
> sizeof(int))
2455 case PACKET_ORIGDEV
:
2456 if (len
> sizeof(int))
2462 case PACKET_VNET_HDR
:
2463 if (len
> sizeof(int))
2465 val
= po
->has_vnet_hdr
;
2469 case PACKET_VERSION
:
2470 if (len
> sizeof(int))
2472 val
= po
->tp_version
;
2476 if (len
> sizeof(int))
2478 if (copy_from_user(&val
, optval
, len
))
2482 val
= sizeof(struct tpacket_hdr
);
2485 val
= sizeof(struct tpacket2_hdr
);
2492 case PACKET_RESERVE
:
2493 if (len
> sizeof(unsigned int))
2494 len
= sizeof(unsigned int);
2495 val
= po
->tp_reserve
;
2499 if (len
> sizeof(unsigned int))
2500 len
= sizeof(unsigned int);
2504 case PACKET_TIMESTAMP
:
2505 if (len
> sizeof(int))
2507 val
= po
->tp_tstamp
;
2511 if (len
> sizeof(int))
2514 ((u32
)po
->fanout
->id
|
2515 ((u32
)po
->fanout
->type
<< 16)) :
2520 return -ENOPROTOOPT
;
2523 if (put_user(len
, optlen
))
2525 if (copy_to_user(optval
, data
, len
))
2531 static int packet_notifier(struct notifier_block
*this, unsigned long msg
, void *data
)
2534 struct hlist_node
*node
;
2535 struct net_device
*dev
= data
;
2536 struct net
*net
= dev_net(dev
);
2539 sk_for_each_rcu(sk
, node
, &net
->packet
.sklist
) {
2540 struct packet_sock
*po
= pkt_sk(sk
);
2543 case NETDEV_UNREGISTER
:
2545 packet_dev_mclist(dev
, po
->mclist
, -1);
2549 if (dev
->ifindex
== po
->ifindex
) {
2550 spin_lock(&po
->bind_lock
);
2552 __unregister_prot_hook(sk
, false);
2553 sk
->sk_err
= ENETDOWN
;
2554 if (!sock_flag(sk
, SOCK_DEAD
))
2555 sk
->sk_error_report(sk
);
2557 if (msg
== NETDEV_UNREGISTER
) {
2559 if (po
->prot_hook
.dev
)
2560 dev_put(po
->prot_hook
.dev
);
2561 po
->prot_hook
.dev
= NULL
;
2563 spin_unlock(&po
->bind_lock
);
2567 if (dev
->ifindex
== po
->ifindex
) {
2568 spin_lock(&po
->bind_lock
);
2570 register_prot_hook(sk
);
2571 spin_unlock(&po
->bind_lock
);
2581 static int packet_ioctl(struct socket
*sock
, unsigned int cmd
,
2584 struct sock
*sk
= sock
->sk
;
2589 int amount
= sk_wmem_alloc_get(sk
);
2591 return put_user(amount
, (int __user
*)arg
);
2595 struct sk_buff
*skb
;
2598 spin_lock_bh(&sk
->sk_receive_queue
.lock
);
2599 skb
= skb_peek(&sk
->sk_receive_queue
);
2602 spin_unlock_bh(&sk
->sk_receive_queue
.lock
);
2603 return put_user(amount
, (int __user
*)arg
);
2606 return sock_get_timestamp(sk
, (struct timeval __user
*)arg
);
2608 return sock_get_timestampns(sk
, (struct timespec __user
*)arg
);
2618 case SIOCGIFBRDADDR
:
2619 case SIOCSIFBRDADDR
:
2620 case SIOCGIFNETMASK
:
2621 case SIOCSIFNETMASK
:
2622 case SIOCGIFDSTADDR
:
2623 case SIOCSIFDSTADDR
:
2625 return inet_dgram_ops
.ioctl(sock
, cmd
, arg
);
2629 return -ENOIOCTLCMD
;
2634 static unsigned int packet_poll(struct file
*file
, struct socket
*sock
,
2637 struct sock
*sk
= sock
->sk
;
2638 struct packet_sock
*po
= pkt_sk(sk
);
2639 unsigned int mask
= datagram_poll(file
, sock
, wait
);
2641 spin_lock_bh(&sk
->sk_receive_queue
.lock
);
2642 if (po
->rx_ring
.pg_vec
) {
2643 if (!packet_previous_frame(po
, &po
->rx_ring
, TP_STATUS_KERNEL
))
2644 mask
|= POLLIN
| POLLRDNORM
;
2646 spin_unlock_bh(&sk
->sk_receive_queue
.lock
);
2647 spin_lock_bh(&sk
->sk_write_queue
.lock
);
2648 if (po
->tx_ring
.pg_vec
) {
2649 if (packet_current_frame(po
, &po
->tx_ring
, TP_STATUS_AVAILABLE
))
2650 mask
|= POLLOUT
| POLLWRNORM
;
2652 spin_unlock_bh(&sk
->sk_write_queue
.lock
);
2657 /* Dirty? Well, I still did not learn better way to account
2661 static void packet_mm_open(struct vm_area_struct
*vma
)
2663 struct file
*file
= vma
->vm_file
;
2664 struct socket
*sock
= file
->private_data
;
2665 struct sock
*sk
= sock
->sk
;
2668 atomic_inc(&pkt_sk(sk
)->mapped
);
2671 static void packet_mm_close(struct vm_area_struct
*vma
)
2673 struct file
*file
= vma
->vm_file
;
2674 struct socket
*sock
= file
->private_data
;
2675 struct sock
*sk
= sock
->sk
;
2678 atomic_dec(&pkt_sk(sk
)->mapped
);
2681 static const struct vm_operations_struct packet_mmap_ops
= {
2682 .open
= packet_mm_open
,
2683 .close
= packet_mm_close
,
2686 static void free_pg_vec(struct pgv
*pg_vec
, unsigned int order
,
2691 for (i
= 0; i
< len
; i
++) {
2692 if (likely(pg_vec
[i
].buffer
)) {
2693 if (is_vmalloc_addr(pg_vec
[i
].buffer
))
2694 vfree(pg_vec
[i
].buffer
);
2696 free_pages((unsigned long)pg_vec
[i
].buffer
,
2698 pg_vec
[i
].buffer
= NULL
;
2704 static inline char *alloc_one_pg_vec_page(unsigned long order
)
2706 char *buffer
= NULL
;
2707 gfp_t gfp_flags
= GFP_KERNEL
| __GFP_COMP
|
2708 __GFP_ZERO
| __GFP_NOWARN
| __GFP_NORETRY
;
2710 buffer
= (char *) __get_free_pages(gfp_flags
, order
);
2716 * __get_free_pages failed, fall back to vmalloc
2718 buffer
= vzalloc((1 << order
) * PAGE_SIZE
);
2724 * vmalloc failed, lets dig into swap here
2726 gfp_flags
&= ~__GFP_NORETRY
;
2727 buffer
= (char *)__get_free_pages(gfp_flags
, order
);
2732 * complete and utter failure
2737 static struct pgv
*alloc_pg_vec(struct tpacket_req
*req
, int order
)
2739 unsigned int block_nr
= req
->tp_block_nr
;
2743 pg_vec
= kcalloc(block_nr
, sizeof(struct pgv
), GFP_KERNEL
);
2744 if (unlikely(!pg_vec
))
2747 for (i
= 0; i
< block_nr
; i
++) {
2748 pg_vec
[i
].buffer
= alloc_one_pg_vec_page(order
);
2749 if (unlikely(!pg_vec
[i
].buffer
))
2750 goto out_free_pgvec
;
2757 free_pg_vec(pg_vec
, order
, block_nr
);
2762 static int packet_set_ring(struct sock
*sk
, struct tpacket_req
*req
,
2763 int closing
, int tx_ring
)
2765 struct pgv
*pg_vec
= NULL
;
2766 struct packet_sock
*po
= pkt_sk(sk
);
2767 int was_running
, order
= 0;
2768 struct packet_ring_buffer
*rb
;
2769 struct sk_buff_head
*rb_queue
;
2773 rb
= tx_ring
? &po
->tx_ring
: &po
->rx_ring
;
2774 rb_queue
= tx_ring
? &sk
->sk_write_queue
: &sk
->sk_receive_queue
;
2778 if (atomic_read(&po
->mapped
))
2780 if (atomic_read(&rb
->pending
))
2784 if (req
->tp_block_nr
) {
2785 /* Sanity tests and some calculations */
2787 if (unlikely(rb
->pg_vec
))
2790 switch (po
->tp_version
) {
2792 po
->tp_hdrlen
= TPACKET_HDRLEN
;
2795 po
->tp_hdrlen
= TPACKET2_HDRLEN
;
2800 if (unlikely((int)req
->tp_block_size
<= 0))
2802 if (unlikely(req
->tp_block_size
& (PAGE_SIZE
- 1)))
2804 if (unlikely(req
->tp_frame_size
< po
->tp_hdrlen
+
2807 if (unlikely(req
->tp_frame_size
& (TPACKET_ALIGNMENT
- 1)))
2810 rb
->frames_per_block
= req
->tp_block_size
/req
->tp_frame_size
;
2811 if (unlikely(rb
->frames_per_block
<= 0))
2813 if (unlikely((rb
->frames_per_block
* req
->tp_block_nr
) !=
2818 order
= get_order(req
->tp_block_size
);
2819 pg_vec
= alloc_pg_vec(req
, order
);
2820 if (unlikely(!pg_vec
))
2826 if (unlikely(req
->tp_frame_nr
))
2832 /* Detach socket from network */
2833 spin_lock(&po
->bind_lock
);
2834 was_running
= po
->running
;
2838 __unregister_prot_hook(sk
, false);
2840 spin_unlock(&po
->bind_lock
);
2845 mutex_lock(&po
->pg_vec_lock
);
2846 if (closing
|| atomic_read(&po
->mapped
) == 0) {
2848 spin_lock_bh(&rb_queue
->lock
);
2849 swap(rb
->pg_vec
, pg_vec
);
2850 rb
->frame_max
= (req
->tp_frame_nr
- 1);
2852 rb
->frame_size
= req
->tp_frame_size
;
2853 spin_unlock_bh(&rb_queue
->lock
);
2855 swap(rb
->pg_vec_order
, order
);
2856 swap(rb
->pg_vec_len
, req
->tp_block_nr
);
2858 rb
->pg_vec_pages
= req
->tp_block_size
/PAGE_SIZE
;
2859 po
->prot_hook
.func
= (po
->rx_ring
.pg_vec
) ?
2860 tpacket_rcv
: packet_rcv
;
2861 skb_queue_purge(rb_queue
);
2862 if (atomic_read(&po
->mapped
))
2863 pr_err("packet_mmap: vma is busy: %d\n",
2864 atomic_read(&po
->mapped
));
2866 mutex_unlock(&po
->pg_vec_lock
);
2868 spin_lock(&po
->bind_lock
);
2871 register_prot_hook(sk
);
2873 spin_unlock(&po
->bind_lock
);
2878 free_pg_vec(pg_vec
, order
, req
->tp_block_nr
);
2883 static int packet_mmap(struct file
*file
, struct socket
*sock
,
2884 struct vm_area_struct
*vma
)
2886 struct sock
*sk
= sock
->sk
;
2887 struct packet_sock
*po
= pkt_sk(sk
);
2888 unsigned long size
, expected_size
;
2889 struct packet_ring_buffer
*rb
;
2890 unsigned long start
;
2897 mutex_lock(&po
->pg_vec_lock
);
2900 for (rb
= &po
->rx_ring
; rb
<= &po
->tx_ring
; rb
++) {
2902 expected_size
+= rb
->pg_vec_len
2908 if (expected_size
== 0)
2911 size
= vma
->vm_end
- vma
->vm_start
;
2912 if (size
!= expected_size
)
2915 start
= vma
->vm_start
;
2916 for (rb
= &po
->rx_ring
; rb
<= &po
->tx_ring
; rb
++) {
2917 if (rb
->pg_vec
== NULL
)
2920 for (i
= 0; i
< rb
->pg_vec_len
; i
++) {
2922 void *kaddr
= rb
->pg_vec
[i
].buffer
;
2925 for (pg_num
= 0; pg_num
< rb
->pg_vec_pages
; pg_num
++) {
2926 page
= pgv_to_page(kaddr
);
2927 err
= vm_insert_page(vma
, start
, page
);
2936 atomic_inc(&po
->mapped
);
2937 vma
->vm_ops
= &packet_mmap_ops
;
2941 mutex_unlock(&po
->pg_vec_lock
);
2945 static const struct proto_ops packet_ops_spkt
= {
2946 .family
= PF_PACKET
,
2947 .owner
= THIS_MODULE
,
2948 .release
= packet_release
,
2949 .bind
= packet_bind_spkt
,
2950 .connect
= sock_no_connect
,
2951 .socketpair
= sock_no_socketpair
,
2952 .accept
= sock_no_accept
,
2953 .getname
= packet_getname_spkt
,
2954 .poll
= datagram_poll
,
2955 .ioctl
= packet_ioctl
,
2956 .listen
= sock_no_listen
,
2957 .shutdown
= sock_no_shutdown
,
2958 .setsockopt
= sock_no_setsockopt
,
2959 .getsockopt
= sock_no_getsockopt
,
2960 .sendmsg
= packet_sendmsg_spkt
,
2961 .recvmsg
= packet_recvmsg
,
2962 .mmap
= sock_no_mmap
,
2963 .sendpage
= sock_no_sendpage
,
2966 static const struct proto_ops packet_ops
= {
2967 .family
= PF_PACKET
,
2968 .owner
= THIS_MODULE
,
2969 .release
= packet_release
,
2970 .bind
= packet_bind
,
2971 .connect
= sock_no_connect
,
2972 .socketpair
= sock_no_socketpair
,
2973 .accept
= sock_no_accept
,
2974 .getname
= packet_getname
,
2975 .poll
= packet_poll
,
2976 .ioctl
= packet_ioctl
,
2977 .listen
= sock_no_listen
,
2978 .shutdown
= sock_no_shutdown
,
2979 .setsockopt
= packet_setsockopt
,
2980 .getsockopt
= packet_getsockopt
,
2981 .sendmsg
= packet_sendmsg
,
2982 .recvmsg
= packet_recvmsg
,
2983 .mmap
= packet_mmap
,
2984 .sendpage
= sock_no_sendpage
,
2987 static const struct net_proto_family packet_family_ops
= {
2988 .family
= PF_PACKET
,
2989 .create
= packet_create
,
2990 .owner
= THIS_MODULE
,
2993 static struct notifier_block packet_netdev_notifier
= {
2994 .notifier_call
= packet_notifier
,
2997 #ifdef CONFIG_PROC_FS
2999 static void *packet_seq_start(struct seq_file
*seq
, loff_t
*pos
)
3002 struct net
*net
= seq_file_net(seq
);
3005 return seq_hlist_start_head_rcu(&net
->packet
.sklist
, *pos
);
3008 static void *packet_seq_next(struct seq_file
*seq
, void *v
, loff_t
*pos
)
3010 struct net
*net
= seq_file_net(seq
);
3011 return seq_hlist_next_rcu(v
, &net
->packet
.sklist
, pos
);
3014 static void packet_seq_stop(struct seq_file
*seq
, void *v
)
3020 static int packet_seq_show(struct seq_file
*seq
, void *v
)
3022 if (v
== SEQ_START_TOKEN
)
3023 seq_puts(seq
, "sk RefCnt Type Proto Iface R Rmem User Inode\n");
3025 struct sock
*s
= sk_entry(v
);
3026 const struct packet_sock
*po
= pkt_sk(s
);
3029 "%pK %-6d %-4d %04x %-5d %1d %-6u %-6u %-6lu\n",
3031 atomic_read(&s
->sk_refcnt
),
3036 atomic_read(&s
->sk_rmem_alloc
),
3044 static const struct seq_operations packet_seq_ops
= {
3045 .start
= packet_seq_start
,
3046 .next
= packet_seq_next
,
3047 .stop
= packet_seq_stop
,
3048 .show
= packet_seq_show
,
3051 static int packet_seq_open(struct inode
*inode
, struct file
*file
)
3053 return seq_open_net(inode
, file
, &packet_seq_ops
,
3054 sizeof(struct seq_net_private
));
3057 static const struct file_operations packet_seq_fops
= {
3058 .owner
= THIS_MODULE
,
3059 .open
= packet_seq_open
,
3061 .llseek
= seq_lseek
,
3062 .release
= seq_release_net
,
3067 static int __net_init
packet_net_init(struct net
*net
)
3069 spin_lock_init(&net
->packet
.sklist_lock
);
3070 INIT_HLIST_HEAD(&net
->packet
.sklist
);
3072 if (!proc_net_fops_create(net
, "packet", 0, &packet_seq_fops
))
3078 static void __net_exit
packet_net_exit(struct net
*net
)
3080 proc_net_remove(net
, "packet");
3083 static struct pernet_operations packet_net_ops
= {
3084 .init
= packet_net_init
,
3085 .exit
= packet_net_exit
,
3089 static void __exit
packet_exit(void)
3091 unregister_netdevice_notifier(&packet_netdev_notifier
);
3092 unregister_pernet_subsys(&packet_net_ops
);
3093 sock_unregister(PF_PACKET
);
3094 proto_unregister(&packet_proto
);
3097 static int __init
packet_init(void)
3099 int rc
= proto_register(&packet_proto
, 0);
3104 sock_register(&packet_family_ops
);
3105 register_pernet_subsys(&packet_net_ops
);
3106 register_netdevice_notifier(&packet_netdev_notifier
);
3111 module_init(packet_init
);
3112 module_exit(packet_exit
);
3113 MODULE_LICENSE("GPL");
3114 MODULE_ALIAS_NETPROTO(PF_PACKET
);