?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
net: Convert GRO SKB handling to list_head.
[linux-2.6/btrfs-unstable.git] / drivers / net / geneve.c
blob3e94375b9b01cb90494e62bc9883dc84dc76c0f5
1 /*
2 * GENEVE: Generic Network Virtualization Encapsulation
3 *
4 * Copyright (c) 2015 Red Hat, Inc.
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 */
10
11 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
12
13 #include <linux/kernel.h>
14 #include <linux/module.h>
15 #include <linux/etherdevice.h>
16 #include <linux/hash.h>
17 #include <net/dst_metadata.h>
18 #include <net/gro_cells.h>
19 #include <net/rtnetlink.h>
20 #include <net/geneve.h>
21 #include <net/protocol.h>
22
23 #define GENEVE_NETDEV_VER "0.6"
24
25 #define GENEVE_UDP_PORT 6081
26
27 #define GENEVE_N_VID (1u << 24)
28 #define GENEVE_VID_MASK (GENEVE_N_VID - 1)
29
30 #define VNI_HASH_BITS 10
31 #define VNI_HASH_SIZE (1<<VNI_HASH_BITS)
32
33 static bool log_ecn_error = true;
34 module_param(log_ecn_error, bool, 0644);
35 MODULE_PARM_DESC(log_ecn_error, "Log packets received with corrupted ECN");
36
37 #define GENEVE_VER 0
38 #define GENEVE_BASE_HLEN (sizeof(struct udphdr) + sizeof(struct genevehdr))
39 #define GENEVE_IPV4_HLEN (ETH_HLEN + sizeof(struct iphdr) + GENEVE_BASE_HLEN)
40 #define GENEVE_IPV6_HLEN (ETH_HLEN + sizeof(struct ipv6hdr) + GENEVE_BASE_HLEN)
41
42 /* per-network namespace private data for this module */
43 struct geneve_net {
44 struct list_head geneve_list;
45 struct list_head sock_list;
46 };
47
48 static unsigned int geneve_net_id;
49
50 struct geneve_dev_node {
51 struct hlist_node hlist;
52 struct geneve_dev *geneve;
53 };
54
55 /* Pseudo network device */
56 struct geneve_dev {
57 struct geneve_dev_node hlist4; /* vni hash table for IPv4 socket */
58 #if IS_ENABLED(CONFIG_IPV6)
59 struct geneve_dev_node hlist6; /* vni hash table for IPv6 socket */
60 #endif
61 struct net *net; /* netns for packet i/o */
62 struct net_device *dev; /* netdev for geneve tunnel */
63 struct ip_tunnel_info info;
64 struct geneve_sock __rcu *sock4; /* IPv4 socket used for geneve tunnel */
65 #if IS_ENABLED(CONFIG_IPV6)
66 struct geneve_sock __rcu *sock6; /* IPv6 socket used for geneve tunnel */
67 #endif
68 struct list_head next; /* geneve's per namespace list */
69 struct gro_cells gro_cells;
70 bool collect_md;
71 bool use_udp6_rx_checksums;
72 };
73
74 struct geneve_sock {
75 bool collect_md;
76 struct list_head list;
77 struct socket *sock;
78 struct rcu_head rcu;
79 int refcnt;
80 struct hlist_head vni_list[VNI_HASH_SIZE];
81 };
82
83 static inline __u32 geneve_net_vni_hash(u8 vni[3])
84 {
85 __u32 vnid;
86
87 vnid = (vni[0] << 16) | (vni[1] << 8) | vni[2];
88 return hash_32(vnid, VNI_HASH_BITS);
89 }
90
91 static __be64 vni_to_tunnel_id(const __u8 *vni)
92 {
93 #ifdef __BIG_ENDIAN
94 return (vni[0] << 16) | (vni[1] << 8) | vni[2];
95 #else
96 return (__force __be64)(((__force u64)vni[0] << 40) |
97 ((__force u64)vni[1] << 48) |
98 ((__force u64)vni[2] << 56));
99 #endif
100 }
101
102 /* Convert 64 bit tunnel ID to 24 bit VNI. */
103 static void tunnel_id_to_vni(__be64 tun_id, __u8 *vni)
104 {
105 #ifdef __BIG_ENDIAN
106 vni[0] = (__force __u8)(tun_id >> 16);
107 vni[1] = (__force __u8)(tun_id >> 8);
108 vni[2] = (__force __u8)tun_id;
109 #else
110 vni[0] = (__force __u8)((__force u64)tun_id >> 40);
111 vni[1] = (__force __u8)((__force u64)tun_id >> 48);
112 vni[2] = (__force __u8)((__force u64)tun_id >> 56);
113 #endif
114 }
115
116 static bool eq_tun_id_and_vni(u8 *tun_id, u8 *vni)
117 {
118 return !memcmp(vni, &tun_id[5], 3);
119 }
120
121 static sa_family_t geneve_get_sk_family(struct geneve_sock *gs)
122 {
123 return gs->sock->sk->sk_family;
124 }
125
126 static struct geneve_dev *geneve_lookup(struct geneve_sock *gs,
127 __be32 addr, u8 vni[])
128 {
129 struct hlist_head *vni_list_head;
130 struct geneve_dev_node *node;
131 __u32 hash;
132
133 /* Find the device for this VNI */
134 hash = geneve_net_vni_hash(vni);
135 vni_list_head = &gs->vni_list[hash];
136 hlist_for_each_entry_rcu(node, vni_list_head, hlist) {
137 if (eq_tun_id_and_vni((u8 *)&node->geneve->info.key.tun_id, vni) &&
138 addr == node->geneve->info.key.u.ipv4.dst)
139 return node->geneve;
140 }
141 return NULL;
142 }
143
144 #if IS_ENABLED(CONFIG_IPV6)
145 static struct geneve_dev *geneve6_lookup(struct geneve_sock *gs,
146 struct in6_addr addr6, u8 vni[])
147 {
148 struct hlist_head *vni_list_head;
149 struct geneve_dev_node *node;
150 __u32 hash;
151
152 /* Find the device for this VNI */
153 hash = geneve_net_vni_hash(vni);
154 vni_list_head = &gs->vni_list[hash];
155 hlist_for_each_entry_rcu(node, vni_list_head, hlist) {
156 if (eq_tun_id_and_vni((u8 *)&node->geneve->info.key.tun_id, vni) &&
157 ipv6_addr_equal(&addr6, &node->geneve->info.key.u.ipv6.dst))
158 return node->geneve;
159 }
160 return NULL;
161 }
162 #endif
163
164 static inline struct genevehdr *geneve_hdr(const struct sk_buff *skb)
165 {
166 return (struct genevehdr *)(udp_hdr(skb) + 1);
167 }
168
169 static struct geneve_dev *geneve_lookup_skb(struct geneve_sock *gs,
170 struct sk_buff *skb)
171 {
172 static u8 zero_vni[3];
173 u8 *vni;
174
175 if (geneve_get_sk_family(gs) == AF_INET) {
176 struct iphdr *iph;
177 __be32 addr;
178
179 iph = ip_hdr(skb); /* outer IP header... */
180
181 if (gs->collect_md) {
182 vni = zero_vni;
183 addr = 0;
184 } else {
185 vni = geneve_hdr(skb)->vni;
186 addr = iph->saddr;
187 }
188
189 return geneve_lookup(gs, addr, vni);
190 #if IS_ENABLED(CONFIG_IPV6)
191 } else if (geneve_get_sk_family(gs) == AF_INET6) {
192 static struct in6_addr zero_addr6;
193 struct ipv6hdr *ip6h;
194 struct in6_addr addr6;
195
196 ip6h = ipv6_hdr(skb); /* outer IPv6 header... */
197
198 if (gs->collect_md) {
199 vni = zero_vni;
200 addr6 = zero_addr6;
201 } else {
202 vni = geneve_hdr(skb)->vni;
203 addr6 = ip6h->saddr;
204 }
205
206 return geneve6_lookup(gs, addr6, vni);
207 #endif
208 }
209 return NULL;
210 }
211
212 /* geneve receive/decap routine */
213 static void geneve_rx(struct geneve_dev *geneve, struct geneve_sock *gs,
214 struct sk_buff *skb)
215 {
216 struct genevehdr *gnvh = geneve_hdr(skb);
217 struct metadata_dst *tun_dst = NULL;
218 struct pcpu_sw_netstats *stats;
219 unsigned int len;
220 int err = 0;
221 void *oiph;
222
223 if (ip_tunnel_collect_metadata() || gs->collect_md) {
224 __be16 flags;
225
226 flags = TUNNEL_KEY | TUNNEL_GENEVE_OPT |
227 (gnvh->oam ? TUNNEL_OAM : 0) |
228 (gnvh->critical ? TUNNEL_CRIT_OPT : 0);
229
230 tun_dst = udp_tun_rx_dst(skb, geneve_get_sk_family(gs), flags,
231 vni_to_tunnel_id(gnvh->vni),
232 gnvh->opt_len * 4);
233 if (!tun_dst) {
234 geneve->dev->stats.rx_dropped++;
235 goto drop;
236 }
237 /* Update tunnel dst according to Geneve options. */
238 ip_tunnel_info_opts_set(&tun_dst->u.tun_info,
239 gnvh->options, gnvh->opt_len * 4);
240 } else {
241 /* Drop packets w/ critical options,
242 * since we don't support any...
243 */
244 if (gnvh->critical) {
245 geneve->dev->stats.rx_frame_errors++;
246 geneve->dev->stats.rx_errors++;
247 goto drop;
248 }
249 }
250
251 skb_reset_mac_header(skb);
252 skb->protocol = eth_type_trans(skb, geneve->dev);
253 skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
254
255 if (tun_dst)
256 skb_dst_set(skb, &tun_dst->dst);
257
258 /* Ignore packet loops (and multicast echo) */
259 if (ether_addr_equal(eth_hdr(skb)->h_source, geneve->dev->dev_addr)) {
260 geneve->dev->stats.rx_errors++;
261 goto drop;
262 }
263
264 oiph = skb_network_header(skb);
265 skb_reset_network_header(skb);
266
267 if (geneve_get_sk_family(gs) == AF_INET)
268 err = IP_ECN_decapsulate(oiph, skb);
269 #if IS_ENABLED(CONFIG_IPV6)
270 else
271 err = IP6_ECN_decapsulate(oiph, skb);
272 #endif
273
274 if (unlikely(err)) {
275 if (log_ecn_error) {
276 if (geneve_get_sk_family(gs) == AF_INET)
277 net_info_ratelimited("non-ECT from %pI4 "
278 "with TOS=%#x\n",
279 &((struct iphdr *)oiph)->saddr,
280 ((struct iphdr *)oiph)->tos);
281 #if IS_ENABLED(CONFIG_IPV6)
282 else
283 net_info_ratelimited("non-ECT from %pI6\n",
284 &((struct ipv6hdr *)oiph)->saddr);
285 #endif
286 }
287 if (err > 1) {
288 ++geneve->dev->stats.rx_frame_errors;
289 ++geneve->dev->stats.rx_errors;
290 goto drop;
291 }
292 }
293
294 len = skb->len;
295 err = gro_cells_receive(&geneve->gro_cells, skb);
296 if (likely(err == NET_RX_SUCCESS)) {
297 stats = this_cpu_ptr(geneve->dev->tstats);
298 u64_stats_update_begin(&stats->syncp);
299 stats->rx_packets++;
300 stats->rx_bytes += len;
301 u64_stats_update_end(&stats->syncp);
302 }
303 return;
304 drop:
305 /* Consume bad packet */
306 kfree_skb(skb);
307 }
308
309 /* Setup stats when device is created */
310 static int geneve_init(struct net_device *dev)
311 {
312 struct geneve_dev *geneve = netdev_priv(dev);
313 int err;
314
315 dev->tstats = netdev_alloc_pcpu_stats(struct pcpu_sw_netstats);
316 if (!dev->tstats)
317 return -ENOMEM;
318
319 err = gro_cells_init(&geneve->gro_cells, dev);
320 if (err) {
321 free_percpu(dev->tstats);
322 return err;
323 }
324
325 err = dst_cache_init(&geneve->info.dst_cache, GFP_KERNEL);
326 if (err) {
327 free_percpu(dev->tstats);
328 gro_cells_destroy(&geneve->gro_cells);
329 return err;
330 }
331 return 0;
332 }
333
334 static void geneve_uninit(struct net_device *dev)
335 {
336 struct geneve_dev *geneve = netdev_priv(dev);
337
338 dst_cache_destroy(&geneve->info.dst_cache);
339 gro_cells_destroy(&geneve->gro_cells);
340 free_percpu(dev->tstats);
341 }
342
343 /* Callback from net/ipv4/udp.c to receive packets */
344 static int geneve_udp_encap_recv(struct sock *sk, struct sk_buff *skb)
345 {
346 struct genevehdr *geneveh;
347 struct geneve_dev *geneve;
348 struct geneve_sock *gs;
349 int opts_len;
350
351 /* Need UDP and Geneve header to be present */
352 if (unlikely(!pskb_may_pull(skb, GENEVE_BASE_HLEN)))
353 goto drop;
354
355 /* Return packets with reserved bits set */
356 geneveh = geneve_hdr(skb);
357 if (unlikely(geneveh->ver != GENEVE_VER))
358 goto drop;
359
360 if (unlikely(geneveh->proto_type != htons(ETH_P_TEB)))
361 goto drop;
362
363 gs = rcu_dereference_sk_user_data(sk);
364 if (!gs)
365 goto drop;
366
367 geneve = geneve_lookup_skb(gs, skb);
368 if (!geneve)
369 goto drop;
370
371 opts_len = geneveh->opt_len * 4;
372 if (iptunnel_pull_header(skb, GENEVE_BASE_HLEN + opts_len,
373 htons(ETH_P_TEB),
374 !net_eq(geneve->net, dev_net(geneve->dev)))) {
375 geneve->dev->stats.rx_dropped++;
376 goto drop;
377 }
378
379 geneve_rx(geneve, gs, skb);
380 return 0;
381
382 drop:
383 /* Consume bad packet */
384 kfree_skb(skb);
385 return 0;
386 }
387
388 static struct socket *geneve_create_sock(struct net *net, bool ipv6,
389 __be16 port, bool ipv6_rx_csum)
390 {
391 struct socket *sock;
392 struct udp_port_cfg udp_conf;
393 int err;
394
395 memset(&udp_conf, 0, sizeof(udp_conf));
396
397 if (ipv6) {
398 udp_conf.family = AF_INET6;
399 udp_conf.ipv6_v6only = 1;
400 udp_conf.use_udp6_rx_checksums = ipv6_rx_csum;
401 } else {
402 udp_conf.family = AF_INET;
403 udp_conf.local_ip.s_addr = htonl(INADDR_ANY);
404 }
405
406 udp_conf.local_udp_port = port;
407
408 /* Open UDP socket */
409 err = udp_sock_create(net, &udp_conf, &sock);
410 if (err < 0)
411 return ERR_PTR(err);
412
413 return sock;
414 }
415
416 static int geneve_hlen(struct genevehdr *gh)
417 {
418 return sizeof(*gh) + gh->opt_len * 4;
419 }
420
421 static struct sk_buff *geneve_gro_receive(struct sock *sk,
422 struct list_head *head,
423 struct sk_buff *skb)
424 {
425 struct sk_buff *pp = NULL;
426 struct sk_buff *p;
427 struct genevehdr *gh, *gh2;
428 unsigned int hlen, gh_len, off_gnv;
429 const struct packet_offload *ptype;
430 __be16 type;
431 int flush = 1;
432
433 off_gnv = skb_gro_offset(skb);
434 hlen = off_gnv + sizeof(*gh);
435 gh = skb_gro_header_fast(skb, off_gnv);
436 if (skb_gro_header_hard(skb, hlen)) {
437 gh = skb_gro_header_slow(skb, hlen, off_gnv);
438 if (unlikely(!gh))
439 goto out;
440 }
441
442 if (gh->ver != GENEVE_VER || gh->oam)
443 goto out;
444 gh_len = geneve_hlen(gh);
445
446 hlen = off_gnv + gh_len;
447 if (skb_gro_header_hard(skb, hlen)) {
448 gh = skb_gro_header_slow(skb, hlen, off_gnv);
449 if (unlikely(!gh))
450 goto out;
451 }
452
453 list_for_each_entry(p, head, list) {
454 if (!NAPI_GRO_CB(p)->same_flow)
455 continue;
456
457 gh2 = (struct genevehdr *)(p->data + off_gnv);
458 if (gh->opt_len != gh2->opt_len ||
459 memcmp(gh, gh2, gh_len)) {
460 NAPI_GRO_CB(p)->same_flow = 0;
461 continue;
462 }
463 }
464
465 type = gh->proto_type;
466
467 rcu_read_lock();
468 ptype = gro_find_receive_by_type(type);
469 if (!ptype)
470 goto out_unlock;
471
472 skb_gro_pull(skb, gh_len);
473 skb_gro_postpull_rcsum(skb, gh, gh_len);
474 pp = call_gro_receive(ptype->callbacks.gro_receive, head, skb);
475 flush = 0;
476
477 out_unlock:
478 rcu_read_unlock();
479 out:
480 NAPI_GRO_CB(skb)->flush |= flush;
481
482 return pp;
483 }
484
485 static int geneve_gro_complete(struct sock *sk, struct sk_buff *skb,
486 int nhoff)
487 {
488 struct genevehdr *gh;
489 struct packet_offload *ptype;
490 __be16 type;
491 int gh_len;
492 int err = -ENOSYS;
493
494 gh = (struct genevehdr *)(skb->data + nhoff);
495 gh_len = geneve_hlen(gh);
496 type = gh->proto_type;
497
498 rcu_read_lock();
499 ptype = gro_find_complete_by_type(type);
500 if (ptype)
501 err = ptype->callbacks.gro_complete(skb, nhoff + gh_len);
502
503 rcu_read_unlock();
504
505 skb_set_inner_mac_header(skb, nhoff + gh_len);
506
507 return err;
508 }
509
510 /* Create new listen socket if needed */
511 static struct geneve_sock *geneve_socket_create(struct net *net, __be16 port,
512 bool ipv6, bool ipv6_rx_csum)
513 {
514 struct geneve_net *gn = net_generic(net, geneve_net_id);
515 struct geneve_sock *gs;
516 struct socket *sock;
517 struct udp_tunnel_sock_cfg tunnel_cfg;
518 int h;
519
520 gs = kzalloc(sizeof(*gs), GFP_KERNEL);
521 if (!gs)
522 return ERR_PTR(-ENOMEM);
523
524 sock = geneve_create_sock(net, ipv6, port, ipv6_rx_csum);
525 if (IS_ERR(sock)) {
526 kfree(gs);
527 return ERR_CAST(sock);
528 }
529
530 gs->sock = sock;
531 gs->refcnt = 1;
532 for (h = 0; h < VNI_HASH_SIZE; ++h)
533 INIT_HLIST_HEAD(&gs->vni_list[h]);
534
535 /* Initialize the geneve udp offloads structure */
536 udp_tunnel_notify_add_rx_port(gs->sock, UDP_TUNNEL_TYPE_GENEVE);
537
538 /* Mark socket as an encapsulation socket */
539 memset(&tunnel_cfg, 0, sizeof(tunnel_cfg));
540 tunnel_cfg.sk_user_data = gs;
541 tunnel_cfg.encap_type = 1;
542 tunnel_cfg.gro_receive = geneve_gro_receive;
543 tunnel_cfg.gro_complete = geneve_gro_complete;
544 tunnel_cfg.encap_rcv = geneve_udp_encap_recv;
545 tunnel_cfg.encap_destroy = NULL;
546 setup_udp_tunnel_sock(net, sock, &tunnel_cfg);
547 list_add(&gs->list, &gn->sock_list);
548 return gs;
549 }
550
551 static void __geneve_sock_release(struct geneve_sock *gs)
552 {
553 if (!gs || --gs->refcnt)
554 return;
555
556 list_del(&gs->list);
557 udp_tunnel_notify_del_rx_port(gs->sock, UDP_TUNNEL_TYPE_GENEVE);
558 udp_tunnel_sock_release(gs->sock);
559 kfree_rcu(gs, rcu);
560 }
561
562 static void geneve_sock_release(struct geneve_dev *geneve)
563 {
564 struct geneve_sock *gs4 = rtnl_dereference(geneve->sock4);
565 #if IS_ENABLED(CONFIG_IPV6)
566 struct geneve_sock *gs6 = rtnl_dereference(geneve->sock6);
567
568 rcu_assign_pointer(geneve->sock6, NULL);
569 #endif
570
571 rcu_assign_pointer(geneve->sock4, NULL);
572 synchronize_net();
573
574 __geneve_sock_release(gs4);
575 #if IS_ENABLED(CONFIG_IPV6)
576 __geneve_sock_release(gs6);
577 #endif
578 }
579
580 static struct geneve_sock *geneve_find_sock(struct geneve_net *gn,
581 sa_family_t family,
582 __be16 dst_port)
583 {
584 struct geneve_sock *gs;
585
586 list_for_each_entry(gs, &gn->sock_list, list) {
587 if (inet_sk(gs->sock->sk)->inet_sport == dst_port &&
588 geneve_get_sk_family(gs) == family) {
589 return gs;
590 }
591 }
592 return NULL;
593 }
594
595 static int geneve_sock_add(struct geneve_dev *geneve, bool ipv6)
596 {
597 struct net *net = geneve->net;
598 struct geneve_net *gn = net_generic(net, geneve_net_id);
599 struct geneve_dev_node *node;
600 struct geneve_sock *gs;
601 __u8 vni[3];
602 __u32 hash;
603
604 gs = geneve_find_sock(gn, ipv6 ? AF_INET6 : AF_INET, geneve->info.key.tp_dst);
605 if (gs) {
606 gs->refcnt++;
607 goto out;
608 }
609
610 gs = geneve_socket_create(net, geneve->info.key.tp_dst, ipv6,
611 geneve->use_udp6_rx_checksums);
612 if (IS_ERR(gs))
613 return PTR_ERR(gs);
614
615 out:
616 gs->collect_md = geneve->collect_md;
617 #if IS_ENABLED(CONFIG_IPV6)
618 if (ipv6) {
619 rcu_assign_pointer(geneve->sock6, gs);
620 node = &geneve->hlist6;
621 } else
622 #endif
623 {
624 rcu_assign_pointer(geneve->sock4, gs);
625 node = &geneve->hlist4;
626 }
627 node->geneve = geneve;
628
629 tunnel_id_to_vni(geneve->info.key.tun_id, vni);
630 hash = geneve_net_vni_hash(vni);
631 hlist_add_head_rcu(&node->hlist, &gs->vni_list[hash]);
632 return 0;
633 }
634
635 static int geneve_open(struct net_device *dev)
636 {
637 struct geneve_dev *geneve = netdev_priv(dev);
638 bool ipv6 = !!(geneve->info.mode & IP_TUNNEL_INFO_IPV6);
639 bool metadata = geneve->collect_md;
640 int ret = 0;
641
642 #if IS_ENABLED(CONFIG_IPV6)
643 if (ipv6 || metadata)
644 ret = geneve_sock_add(geneve, true);
645 #endif
646 if (!ret && (!ipv6 || metadata))
647 ret = geneve_sock_add(geneve, false);
648 if (ret < 0)
649 geneve_sock_release(geneve);
650
651 return ret;
652 }
653
654 static int geneve_stop(struct net_device *dev)
655 {
656 struct geneve_dev *geneve = netdev_priv(dev);
657
658 hlist_del_init_rcu(&geneve->hlist4.hlist);
659 #if IS_ENABLED(CONFIG_IPV6)
660 hlist_del_init_rcu(&geneve->hlist6.hlist);
661 #endif
662 geneve_sock_release(geneve);
663 return 0;
664 }
665
666 static void geneve_build_header(struct genevehdr *geneveh,
667 const struct ip_tunnel_info *info)
668 {
669 geneveh->ver = GENEVE_VER;
670 geneveh->opt_len = info->options_len / 4;
671 geneveh->oam = !!(info->key.tun_flags & TUNNEL_OAM);
672 geneveh->critical = !!(info->key.tun_flags & TUNNEL_CRIT_OPT);
673 geneveh->rsvd1 = 0;
674 tunnel_id_to_vni(info->key.tun_id, geneveh->vni);
675 geneveh->proto_type = htons(ETH_P_TEB);
676 geneveh->rsvd2 = 0;
677
678 ip_tunnel_info_opts_get(geneveh->options, info);
679 }
680
681 static int geneve_build_skb(struct dst_entry *dst, struct sk_buff *skb,
682 const struct ip_tunnel_info *info,
683 bool xnet, int ip_hdr_len)
684 {
685 bool udp_sum = !!(info->key.tun_flags & TUNNEL_CSUM);
686 struct genevehdr *gnvh;
687 int min_headroom;
688 int err;
689
690 skb_reset_mac_header(skb);
691 skb_scrub_packet(skb, xnet);
692
693 min_headroom = LL_RESERVED_SPACE(dst->dev) + dst->header_len +
694 GENEVE_BASE_HLEN + info->options_len + ip_hdr_len;
695 err = skb_cow_head(skb, min_headroom);
696 if (unlikely(err))
697 goto free_dst;
698
699 err = udp_tunnel_handle_offloads(skb, udp_sum);
700 if (err)
701 goto free_dst;
702
703 gnvh = __skb_push(skb, sizeof(*gnvh) + info->options_len);
704 geneve_build_header(gnvh, info);
705 skb_set_inner_protocol(skb, htons(ETH_P_TEB));
706 return 0;
707
708 free_dst:
709 dst_release(dst);
710 return err;
711 }
712
713 static struct rtable *geneve_get_v4_rt(struct sk_buff *skb,
714 struct net_device *dev,
715 struct geneve_sock *gs4,
716 struct flowi4 *fl4,
717 const struct ip_tunnel_info *info)
718 {
719 bool use_cache = ip_tunnel_dst_cache_usable(skb, info);
720 struct geneve_dev *geneve = netdev_priv(dev);
721 struct dst_cache *dst_cache;
722 struct rtable *rt = NULL;
723 __u8 tos;
724
725 if (!gs4)
726 return ERR_PTR(-EIO);
727
728 memset(fl4, 0, sizeof(*fl4));
729 fl4->flowi4_mark = skb->mark;
730 fl4->flowi4_proto = IPPROTO_UDP;
731 fl4->daddr = info->key.u.ipv4.dst;
732 fl4->saddr = info->key.u.ipv4.src;
733
734 tos = info->key.tos;
735 if ((tos == 1) && !geneve->collect_md) {
736 tos = ip_tunnel_get_dsfield(ip_hdr(skb), skb);
737 use_cache = false;
738 }
739 fl4->flowi4_tos = RT_TOS(tos);
740
741 dst_cache = (struct dst_cache *)&info->dst_cache;
742 if (use_cache) {
743 rt = dst_cache_get_ip4(dst_cache, &fl4->saddr);
744 if (rt)
745 return rt;
746 }
747 rt = ip_route_output_key(geneve->net, fl4);
748 if (IS_ERR(rt)) {
749 netdev_dbg(dev, "no route to %pI4\n", &fl4->daddr);
750 return ERR_PTR(-ENETUNREACH);
751 }
752 if (rt->dst.dev == dev) { /* is this necessary? */
753 netdev_dbg(dev, "circular route to %pI4\n", &fl4->daddr);
754 ip_rt_put(rt);
755 return ERR_PTR(-ELOOP);
756 }
757 if (use_cache)
758 dst_cache_set_ip4(dst_cache, &rt->dst, fl4->saddr);
759 return rt;
760 }
761
762 #if IS_ENABLED(CONFIG_IPV6)
763 static struct dst_entry *geneve_get_v6_dst(struct sk_buff *skb,
764 struct net_device *dev,
765 struct geneve_sock *gs6,
766 struct flowi6 *fl6,
767 const struct ip_tunnel_info *info)
768 {
769 bool use_cache = ip_tunnel_dst_cache_usable(skb, info);
770 struct geneve_dev *geneve = netdev_priv(dev);
771 struct dst_entry *dst = NULL;
772 struct dst_cache *dst_cache;
773 __u8 prio;
774
775 if (!gs6)
776 return ERR_PTR(-EIO);
777
778 memset(fl6, 0, sizeof(*fl6));
779 fl6->flowi6_mark = skb->mark;
780 fl6->flowi6_proto = IPPROTO_UDP;
781 fl6->daddr = info->key.u.ipv6.dst;
782 fl6->saddr = info->key.u.ipv6.src;
783 prio = info->key.tos;
784 if ((prio == 1) && !geneve->collect_md) {
785 prio = ip_tunnel_get_dsfield(ip_hdr(skb), skb);
786 use_cache = false;
787 }
788
789 fl6->flowlabel = ip6_make_flowinfo(RT_TOS(prio),
790 info->key.label);
791 dst_cache = (struct dst_cache *)&info->dst_cache;
792 if (use_cache) {
793 dst = dst_cache_get_ip6(dst_cache, &fl6->saddr);
794 if (dst)
795 return dst;
796 }
797 if (ipv6_stub->ipv6_dst_lookup(geneve->net, gs6->sock->sk, &dst, fl6)) {
798 netdev_dbg(dev, "no route to %pI6\n", &fl6->daddr);
799 return ERR_PTR(-ENETUNREACH);
800 }
801 if (dst->dev == dev) { /* is this necessary? */
802 netdev_dbg(dev, "circular route to %pI6\n", &fl6->daddr);
803 dst_release(dst);
804 return ERR_PTR(-ELOOP);
805 }
806
807 if (use_cache)
808 dst_cache_set_ip6(dst_cache, dst, &fl6->saddr);
809 return dst;
810 }
811 #endif
812
813 static int geneve_xmit_skb(struct sk_buff *skb, struct net_device *dev,
814 struct geneve_dev *geneve,
815 const struct ip_tunnel_info *info)
816 {
817 bool xnet = !net_eq(geneve->net, dev_net(geneve->dev));
818 struct geneve_sock *gs4 = rcu_dereference(geneve->sock4);
819 const struct ip_tunnel_key *key = &info->key;
820 struct rtable *rt;
821 struct flowi4 fl4;
822 __u8 tos, ttl;
823 __be16 sport;
824 __be16 df;
825 int err;
826
827 rt = geneve_get_v4_rt(skb, dev, gs4, &fl4, info);
828 if (IS_ERR(rt))
829 return PTR_ERR(rt);
830
831 if (skb_dst(skb)) {
832 int mtu = dst_mtu(&rt->dst) - GENEVE_IPV4_HLEN -
833 info->options_len;
834
835 skb_dst_update_pmtu(skb, mtu);
836 }
837
838 sport = udp_flow_src_port(geneve->net, skb, 1, USHRT_MAX, true);
839 if (geneve->collect_md) {
840 tos = ip_tunnel_ecn_encap(key->tos, ip_hdr(skb), skb);
841 ttl = key->ttl;
842 } else {
843 tos = ip_tunnel_ecn_encap(fl4.flowi4_tos, ip_hdr(skb), skb);
844 ttl = key->ttl ? : ip4_dst_hoplimit(&rt->dst);
845 }
846 df = key->tun_flags & TUNNEL_DONT_FRAGMENT ? htons(IP_DF) : 0;
847
848 err = geneve_build_skb(&rt->dst, skb, info, xnet, sizeof(struct iphdr));
849 if (unlikely(err))
850 return err;
851
852 udp_tunnel_xmit_skb(rt, gs4->sock->sk, skb, fl4.saddr, fl4.daddr,
853 tos, ttl, df, sport, geneve->info.key.tp_dst,
854 !net_eq(geneve->net, dev_net(geneve->dev)),
855 !(info->key.tun_flags & TUNNEL_CSUM));
856 return 0;
857 }
858
859 #if IS_ENABLED(CONFIG_IPV6)
860 static int geneve6_xmit_skb(struct sk_buff *skb, struct net_device *dev,
861 struct geneve_dev *geneve,
862 const struct ip_tunnel_info *info)
863 {
864 bool xnet = !net_eq(geneve->net, dev_net(geneve->dev));
865 struct geneve_sock *gs6 = rcu_dereference(geneve->sock6);
866 const struct ip_tunnel_key *key = &info->key;
867 struct dst_entry *dst = NULL;
868 struct flowi6 fl6;
869 __u8 prio, ttl;
870 __be16 sport;
871 int err;
872
873 dst = geneve_get_v6_dst(skb, dev, gs6, &fl6, info);
874 if (IS_ERR(dst))
875 return PTR_ERR(dst);
876
877 if (skb_dst(skb)) {
878 int mtu = dst_mtu(dst) - GENEVE_IPV6_HLEN - info->options_len;
879
880 skb_dst_update_pmtu(skb, mtu);
881 }
882
883 sport = udp_flow_src_port(geneve->net, skb, 1, USHRT_MAX, true);
884 if (geneve->collect_md) {
885 prio = ip_tunnel_ecn_encap(key->tos, ip_hdr(skb), skb);
886 ttl = key->ttl;
887 } else {
888 prio = ip_tunnel_ecn_encap(ip6_tclass(fl6.flowlabel),
889 ip_hdr(skb), skb);
890 ttl = key->ttl ? : ip6_dst_hoplimit(dst);
891 }
892 err = geneve_build_skb(dst, skb, info, xnet, sizeof(struct ipv6hdr));
893 if (unlikely(err))
894 return err;
895
896 udp_tunnel6_xmit_skb(dst, gs6->sock->sk, skb, dev,
897 &fl6.saddr, &fl6.daddr, prio, ttl,
898 info->key.label, sport, geneve->info.key.tp_dst,
899 !(info->key.tun_flags & TUNNEL_CSUM));
900 return 0;
901 }
902 #endif
903
904 static netdev_tx_t geneve_xmit(struct sk_buff *skb, struct net_device *dev)
905 {
906 struct geneve_dev *geneve = netdev_priv(dev);
907 struct ip_tunnel_info *info = NULL;
908 int err;
909
910 if (geneve->collect_md) {
911 info = skb_tunnel_info(skb);
912 if (unlikely(!info || !(info->mode & IP_TUNNEL_INFO_TX))) {
913 err = -EINVAL;
914 netdev_dbg(dev, "no tunnel metadata\n");
915 goto tx_error;
916 }
917 } else {
918 info = &geneve->info;
919 }
920
921 rcu_read_lock();
922 #if IS_ENABLED(CONFIG_IPV6)
923 if (info->mode & IP_TUNNEL_INFO_IPV6)
924 err = geneve6_xmit_skb(skb, dev, geneve, info);
925 else
926 #endif
927 err = geneve_xmit_skb(skb, dev, geneve, info);
928 rcu_read_unlock();
929
930 if (likely(!err))
931 return NETDEV_TX_OK;
932 tx_error:
933 dev_kfree_skb(skb);
934
935 if (err == -ELOOP)
936 dev->stats.collisions++;
937 else if (err == -ENETUNREACH)
938 dev->stats.tx_carrier_errors++;
939
940 dev->stats.tx_errors++;
941 return NETDEV_TX_OK;
942 }
943
944 static int geneve_change_mtu(struct net_device *dev, int new_mtu)
945 {
946 if (new_mtu > dev->max_mtu)
947 new_mtu = dev->max_mtu;
948 else if (new_mtu < dev->min_mtu)
949 new_mtu = dev->min_mtu;
950
951 dev->mtu = new_mtu;
952 return 0;
953 }
954
955 static int geneve_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb)
956 {
957 struct ip_tunnel_info *info = skb_tunnel_info(skb);
958 struct geneve_dev *geneve = netdev_priv(dev);
959
960 if (ip_tunnel_info_af(info) == AF_INET) {
961 struct rtable *rt;
962 struct flowi4 fl4;
963 struct geneve_sock *gs4 = rcu_dereference(geneve->sock4);
964
965 rt = geneve_get_v4_rt(skb, dev, gs4, &fl4, info);
966 if (IS_ERR(rt))
967 return PTR_ERR(rt);
968
969 ip_rt_put(rt);
970 info->key.u.ipv4.src = fl4.saddr;
971 #if IS_ENABLED(CONFIG_IPV6)
972 } else if (ip_tunnel_info_af(info) == AF_INET6) {
973 struct dst_entry *dst;
974 struct flowi6 fl6;
975 struct geneve_sock *gs6 = rcu_dereference(geneve->sock6);
976
977 dst = geneve_get_v6_dst(skb, dev, gs6, &fl6, info);
978 if (IS_ERR(dst))
979 return PTR_ERR(dst);
980
981 dst_release(dst);
982 info->key.u.ipv6.src = fl6.saddr;
983 #endif
984 } else {
985 return -EINVAL;
986 }
987
988 info->key.tp_src = udp_flow_src_port(geneve->net, skb,
989 1, USHRT_MAX, true);
990 info->key.tp_dst = geneve->info.key.tp_dst;
991 return 0;
992 }
993
994 static const struct net_device_ops geneve_netdev_ops = {
995 .ndo_init = geneve_init,
996 .ndo_uninit = geneve_uninit,
997 .ndo_open = geneve_open,
998 .ndo_stop = geneve_stop,
999 .ndo_start_xmit = geneve_xmit,
1000 .ndo_get_stats64 = ip_tunnel_get_stats64,
1001 .ndo_change_mtu = geneve_change_mtu,
1002 .ndo_validate_addr = eth_validate_addr,
1003 .ndo_set_mac_address = eth_mac_addr,
1004 .ndo_fill_metadata_dst = geneve_fill_metadata_dst,
1005 };
1006
1007 static void geneve_get_drvinfo(struct net_device *dev,
1008 struct ethtool_drvinfo *drvinfo)
1009 {
1010 strlcpy(drvinfo->version, GENEVE_NETDEV_VER, sizeof(drvinfo->version));
1011 strlcpy(drvinfo->driver, "geneve", sizeof(drvinfo->driver));
1012 }
1013
1014 static const struct ethtool_ops geneve_ethtool_ops = {
1015 .get_drvinfo = geneve_get_drvinfo,
1016 .get_link = ethtool_op_get_link,
1017 };
1018
1019 /* Info for udev, that this is a virtual tunnel endpoint */
1020 static struct device_type geneve_type = {
1021 .name = "geneve",
1022 };
1023
1024 /* Calls the ndo_udp_tunnel_add of the caller in order to
1025 * supply the listening GENEVE udp ports. Callers are expected
1026 * to implement the ndo_udp_tunnel_add.
1027 */
1028 static void geneve_offload_rx_ports(struct net_device *dev, bool push)
1029 {
1030 struct net *net = dev_net(dev);
1031 struct geneve_net *gn = net_generic(net, geneve_net_id);
1032 struct geneve_sock *gs;
1033
1034 rcu_read_lock();
1035 list_for_each_entry_rcu(gs, &gn->sock_list, list) {
1036 if (push) {
1037 udp_tunnel_push_rx_port(dev, gs->sock,
1038 UDP_TUNNEL_TYPE_GENEVE);
1039 } else {
1040 udp_tunnel_drop_rx_port(dev, gs->sock,
1041 UDP_TUNNEL_TYPE_GENEVE);
1042 }
1043 }
1044 rcu_read_unlock();
1045 }
1046
1047 /* Initialize the device structure. */
1048 static void geneve_setup(struct net_device *dev)
1049 {
1050 ether_setup(dev);
1051
1052 dev->netdev_ops = &geneve_netdev_ops;
1053 dev->ethtool_ops = &geneve_ethtool_ops;
1054 dev->needs_free_netdev = true;
1055
1056 SET_NETDEV_DEVTYPE(dev, &geneve_type);
1057
1058 dev->features |= NETIF_F_LLTX;
1059 dev->features |= NETIF_F_SG | NETIF_F_HW_CSUM;
1060 dev->features |= NETIF_F_RXCSUM;
1061 dev->features |= NETIF_F_GSO_SOFTWARE;
1062
1063 dev->hw_features |= NETIF_F_SG | NETIF_F_HW_CSUM | NETIF_F_RXCSUM;
1064 dev->hw_features |= NETIF_F_GSO_SOFTWARE;
1065
1066 /* MTU range: 68 - (something less than 65535) */
1067 dev->min_mtu = ETH_MIN_MTU;
1068 /* The max_mtu calculation does not take account of GENEVE
1069 * options, to avoid excluding potentially valid
1070 * configurations. This will be further reduced by IPvX hdr size.
1071 */
1072 dev->max_mtu = IP_MAX_MTU - GENEVE_BASE_HLEN - dev->hard_header_len;
1073
1074 netif_keep_dst(dev);
1075 dev->priv_flags &= ~IFF_TX_SKB_SHARING;
1076 dev->priv_flags |= IFF_LIVE_ADDR_CHANGE | IFF_NO_QUEUE;
1077 eth_hw_addr_random(dev);
1078 }
1079
1080 static const struct nla_policy geneve_policy[IFLA_GENEVE_MAX + 1] = {
1081 [IFLA_GENEVE_ID] = { .type = NLA_U32 },
1082 [IFLA_GENEVE_REMOTE] = { .len = FIELD_SIZEOF(struct iphdr, daddr) },
1083 [IFLA_GENEVE_REMOTE6] = { .len = sizeof(struct in6_addr) },
1084 [IFLA_GENEVE_TTL] = { .type = NLA_U8 },
1085 [IFLA_GENEVE_TOS] = { .type = NLA_U8 },
1086 [IFLA_GENEVE_LABEL] = { .type = NLA_U32 },
1087 [IFLA_GENEVE_PORT] = { .type = NLA_U16 },
1088 [IFLA_GENEVE_COLLECT_METADATA] = { .type = NLA_FLAG },
1089 [IFLA_GENEVE_UDP_CSUM] = { .type = NLA_U8 },
1090 [IFLA_GENEVE_UDP_ZERO_CSUM6_TX] = { .type = NLA_U8 },
1091 [IFLA_GENEVE_UDP_ZERO_CSUM6_RX] = { .type = NLA_U8 },
1092 };
1093
1094 static int geneve_validate(struct nlattr *tb[], struct nlattr *data[],
1095 struct netlink_ext_ack *extack)
1096 {
1097 if (tb[IFLA_ADDRESS]) {
1098 if (nla_len(tb[IFLA_ADDRESS]) != ETH_ALEN) {
1099 NL_SET_ERR_MSG_ATTR(extack, tb[IFLA_ADDRESS],
1100 "Provided link layer address is not Ethernet");
1101 return -EINVAL;
1102 }
1103
1104 if (!is_valid_ether_addr(nla_data(tb[IFLA_ADDRESS]))) {
1105 NL_SET_ERR_MSG_ATTR(extack, tb[IFLA_ADDRESS],
1106 "Provided Ethernet address is not unicast");
1107 return -EADDRNOTAVAIL;
1108 }
1109 }
1110
1111 if (!data) {
1112 NL_SET_ERR_MSG(extack,
1113 "Not enough attributes provided to perform the operation");
1114 return -EINVAL;
1115 }
1116
1117 if (data[IFLA_GENEVE_ID]) {
1118 __u32 vni = nla_get_u32(data[IFLA_GENEVE_ID]);
1119
1120 if (vni >= GENEVE_N_VID) {
1121 NL_SET_ERR_MSG_ATTR(extack, data[IFLA_GENEVE_ID],
1122 "Geneve ID must be lower than 16777216");
1123 return -ERANGE;
1124 }
1125 }
1126
1127 return 0;
1128 }
1129
1130 static struct geneve_dev *geneve_find_dev(struct geneve_net *gn,
1131 const struct ip_tunnel_info *info,
1132 bool *tun_on_same_port,
1133 bool *tun_collect_md)
1134 {
1135 struct geneve_dev *geneve, *t = NULL;
1136
1137 *tun_on_same_port = false;
1138 *tun_collect_md = false;
1139 list_for_each_entry(geneve, &gn->geneve_list, next) {
1140 if (info->key.tp_dst == geneve->info.key.tp_dst) {
1141 *tun_collect_md = geneve->collect_md;
1142 *tun_on_same_port = true;
1143 }
1144 if (info->key.tun_id == geneve->info.key.tun_id &&
1145 info->key.tp_dst == geneve->info.key.tp_dst &&
1146 !memcmp(&info->key.u, &geneve->info.key.u, sizeof(info->key.u)))
1147 t = geneve;
1148 }
1149 return t;
1150 }
1151
1152 static bool is_tnl_info_zero(const struct ip_tunnel_info *info)
1153 {
1154 return !(info->key.tun_id || info->key.tun_flags || info->key.tos ||
1155 info->key.ttl || info->key.label || info->key.tp_src ||
1156 memchr_inv(&info->key.u, 0, sizeof(info->key.u)));
1157 }
1158
1159 static bool geneve_dst_addr_equal(struct ip_tunnel_info *a,
1160 struct ip_tunnel_info *b)
1161 {
1162 if (ip_tunnel_info_af(a) == AF_INET)
1163 return a->key.u.ipv4.dst == b->key.u.ipv4.dst;
1164 else
1165 return ipv6_addr_equal(&a->key.u.ipv6.dst, &b->key.u.ipv6.dst);
1166 }
1167
1168 static int geneve_configure(struct net *net, struct net_device *dev,
1169 struct netlink_ext_ack *extack,
1170 const struct ip_tunnel_info *info,
1171 bool metadata, bool ipv6_rx_csum)
1172 {
1173 struct geneve_net *gn = net_generic(net, geneve_net_id);
1174 struct geneve_dev *t, *geneve = netdev_priv(dev);
1175 bool tun_collect_md, tun_on_same_port;
1176 int err, encap_len;
1177
1178 if (metadata && !is_tnl_info_zero(info)) {
1179 NL_SET_ERR_MSG(extack,
1180 "Device is externally controlled, so attributes (VNI, Port, and so on) must not be specified");
1181 return -EINVAL;
1182 }
1183
1184 geneve->net = net;
1185 geneve->dev = dev;
1186
1187 t = geneve_find_dev(gn, info, &tun_on_same_port, &tun_collect_md);
1188 if (t)
1189 return -EBUSY;
1190
1191 /* make enough headroom for basic scenario */
1192 encap_len = GENEVE_BASE_HLEN + ETH_HLEN;
1193 if (!metadata && ip_tunnel_info_af(info) == AF_INET) {
1194 encap_len += sizeof(struct iphdr);
1195 dev->max_mtu -= sizeof(struct iphdr);
1196 } else {
1197 encap_len += sizeof(struct ipv6hdr);
1198 dev->max_mtu -= sizeof(struct ipv6hdr);
1199 }
1200 dev->needed_headroom = encap_len + ETH_HLEN;
1201
1202 if (metadata) {
1203 if (tun_on_same_port) {
1204 NL_SET_ERR_MSG(extack,
1205 "There can be only one externally controlled device on a destination port");
1206 return -EPERM;
1207 }
1208 } else {
1209 if (tun_collect_md) {
1210 NL_SET_ERR_MSG(extack,
1211 "There already exists an externally controlled device on this destination port");
1212 return -EPERM;
1213 }
1214 }
1215
1216 dst_cache_reset(&geneve->info.dst_cache);
1217 geneve->info = *info;
1218 geneve->collect_md = metadata;
1219 geneve->use_udp6_rx_checksums = ipv6_rx_csum;
1220
1221 err = register_netdevice(dev);
1222 if (err)
1223 return err;
1224
1225 list_add(&geneve->next, &gn->geneve_list);
1226 return 0;
1227 }
1228
1229 static void init_tnl_info(struct ip_tunnel_info *info, __u16 dst_port)
1230 {
1231 memset(info, 0, sizeof(*info));
1232 info->key.tp_dst = htons(dst_port);
1233 }
1234
1235 static int geneve_nl2info(struct nlattr *tb[], struct nlattr *data[],
1236 struct netlink_ext_ack *extack,
1237 struct ip_tunnel_info *info, bool *metadata,
1238 bool *use_udp6_rx_checksums, bool changelink)
1239 {
1240 int attrtype;
1241
1242 if (data[IFLA_GENEVE_REMOTE] && data[IFLA_GENEVE_REMOTE6]) {
1243 NL_SET_ERR_MSG(extack,
1244 "Cannot specify both IPv4 and IPv6 Remote addresses");
1245 return -EINVAL;
1246 }
1247
1248 if (data[IFLA_GENEVE_REMOTE]) {
1249 if (changelink && (ip_tunnel_info_af(info) == AF_INET6)) {
1250 attrtype = IFLA_GENEVE_REMOTE;
1251 goto change_notsup;
1252 }
1253
1254 info->key.u.ipv4.dst =
1255 nla_get_in_addr(data[IFLA_GENEVE_REMOTE]);
1256
1257 if (IN_MULTICAST(ntohl(info->key.u.ipv4.dst))) {
1258 NL_SET_ERR_MSG_ATTR(extack, data[IFLA_GENEVE_REMOTE],
1259 "Remote IPv4 address cannot be Multicast");
1260 return -EINVAL;
1261 }
1262 }
1263
1264 if (data[IFLA_GENEVE_REMOTE6]) {
1265 #if IS_ENABLED(CONFIG_IPV6)
1266 if (changelink && (ip_tunnel_info_af(info) == AF_INET)) {
1267 attrtype = IFLA_GENEVE_REMOTE6;
1268 goto change_notsup;
1269 }
1270
1271 info->mode = IP_TUNNEL_INFO_IPV6;
1272 info->key.u.ipv6.dst =
1273 nla_get_in6_addr(data[IFLA_GENEVE_REMOTE6]);
1274
1275 if (ipv6_addr_type(&info->key.u.ipv6.dst) &
1276 IPV6_ADDR_LINKLOCAL) {
1277 NL_SET_ERR_MSG_ATTR(extack, data[IFLA_GENEVE_REMOTE6],
1278 "Remote IPv6 address cannot be link-local");
1279 return -EINVAL;
1280 }
1281 if (ipv6_addr_is_multicast(&info->key.u.ipv6.dst)) {
1282 NL_SET_ERR_MSG_ATTR(extack, data[IFLA_GENEVE_REMOTE6],
1283 "Remote IPv6 address cannot be Multicast");
1284 return -EINVAL;
1285 }
1286 info->key.tun_flags |= TUNNEL_CSUM;
1287 *use_udp6_rx_checksums = true;
1288 #else
1289 NL_SET_ERR_MSG_ATTR(extack, data[IFLA_GENEVE_REMOTE6],
1290 "IPv6 support not enabled in the kernel");
1291 return -EPFNOSUPPORT;
1292 #endif
1293 }
1294
1295 if (data[IFLA_GENEVE_ID]) {
1296 __u32 vni;
1297 __u8 tvni[3];
1298 __be64 tunid;
1299
1300 vni = nla_get_u32(data[IFLA_GENEVE_ID]);
1301 tvni[0] = (vni & 0x00ff0000) >> 16;
1302 tvni[1] = (vni & 0x0000ff00) >> 8;
1303 tvni[2] = vni & 0x000000ff;
1304
1305 tunid = vni_to_tunnel_id(tvni);
1306 if (changelink && (tunid != info->key.tun_id)) {
1307 attrtype = IFLA_GENEVE_ID;
1308 goto change_notsup;
1309 }
1310 info->key.tun_id = tunid;
1311 }
1312
1313 if (data[IFLA_GENEVE_TTL])
1314 info->key.ttl = nla_get_u8(data[IFLA_GENEVE_TTL]);
1315
1316 if (data[IFLA_GENEVE_TOS])
1317 info->key.tos = nla_get_u8(data[IFLA_GENEVE_TOS]);
1318
1319 if (data[IFLA_GENEVE_LABEL]) {
1320 info->key.label = nla_get_be32(data[IFLA_GENEVE_LABEL]) &
1321 IPV6_FLOWLABEL_MASK;
1322 if (info->key.label && (!(info->mode & IP_TUNNEL_INFO_IPV6))) {
1323 NL_SET_ERR_MSG_ATTR(extack, data[IFLA_GENEVE_LABEL],
1324 "Label attribute only applies for IPv6 Geneve devices");
1325 return -EINVAL;
1326 }
1327 }
1328
1329 if (data[IFLA_GENEVE_PORT]) {
1330 if (changelink) {
1331 attrtype = IFLA_GENEVE_PORT;
1332 goto change_notsup;
1333 }
1334 info->key.tp_dst = nla_get_be16(data[IFLA_GENEVE_PORT]);
1335 }
1336
1337 if (data[IFLA_GENEVE_COLLECT_METADATA]) {
1338 if (changelink) {
1339 attrtype = IFLA_GENEVE_COLLECT_METADATA;
1340 goto change_notsup;
1341 }
1342 *metadata = true;
1343 }
1344
1345 if (data[IFLA_GENEVE_UDP_CSUM]) {
1346 if (changelink) {
1347 attrtype = IFLA_GENEVE_UDP_CSUM;
1348 goto change_notsup;
1349 }
1350 if (nla_get_u8(data[IFLA_GENEVE_UDP_CSUM]))
1351 info->key.tun_flags |= TUNNEL_CSUM;
1352 }
1353
1354 if (data[IFLA_GENEVE_UDP_ZERO_CSUM6_TX]) {
1355 #if IS_ENABLED(CONFIG_IPV6)
1356 if (changelink) {
1357 attrtype = IFLA_GENEVE_UDP_ZERO_CSUM6_TX;
1358 goto change_notsup;
1359 }
1360 if (nla_get_u8(data[IFLA_GENEVE_UDP_ZERO_CSUM6_TX]))
1361 info->key.tun_flags &= ~TUNNEL_CSUM;
1362 #else
1363 NL_SET_ERR_MSG_ATTR(extack, data[IFLA_GENEVE_UDP_ZERO_CSUM6_TX],
1364 "IPv6 support not enabled in the kernel");
1365 return -EPFNOSUPPORT;
1366 #endif
1367 }
1368
1369 if (data[IFLA_GENEVE_UDP_ZERO_CSUM6_RX]) {
1370 #if IS_ENABLED(CONFIG_IPV6)
1371 if (changelink) {
1372 attrtype = IFLA_GENEVE_UDP_ZERO_CSUM6_RX;
1373 goto change_notsup;
1374 }
1375 if (nla_get_u8(data[IFLA_GENEVE_UDP_ZERO_CSUM6_RX]))
1376 *use_udp6_rx_checksums = false;
1377 #else
1378 NL_SET_ERR_MSG_ATTR(extack, data[IFLA_GENEVE_UDP_ZERO_CSUM6_RX],
1379 "IPv6 support not enabled in the kernel");
1380 return -EPFNOSUPPORT;
1381 #endif
1382 }
1383
1384 return 0;
1385 change_notsup:
1386 NL_SET_ERR_MSG_ATTR(extack, data[attrtype],
1387 "Changing VNI, Port, endpoint IP address family, external, and UDP checksum attributes are not supported");
1388 return -EOPNOTSUPP;
1389 }
1390
1391 static void geneve_link_config(struct net_device *dev,
1392 struct ip_tunnel_info *info, struct nlattr *tb[])
1393 {
1394 struct geneve_dev *geneve = netdev_priv(dev);
1395 int ldev_mtu = 0;
1396
1397 if (tb[IFLA_MTU]) {
1398 geneve_change_mtu(dev, nla_get_u32(tb[IFLA_MTU]));
1399 return;
1400 }
1401
1402 switch (ip_tunnel_info_af(info)) {
1403 case AF_INET: {
1404 struct flowi4 fl4 = { .daddr = info->key.u.ipv4.dst };
1405 struct rtable *rt = ip_route_output_key(geneve->net, &fl4);
1406
1407 if (!IS_ERR(rt) && rt->dst.dev) {
1408 ldev_mtu = rt->dst.dev->mtu - GENEVE_IPV4_HLEN;
1409 ip_rt_put(rt);
1410 }
1411 break;
1412 }
1413 #if IS_ENABLED(CONFIG_IPV6)
1414 case AF_INET6: {
1415 struct rt6_info *rt = rt6_lookup(geneve->net,
1416 &info->key.u.ipv6.dst, NULL, 0,
1417 NULL, 0);
1418
1419 if (rt && rt->dst.dev)
1420 ldev_mtu = rt->dst.dev->mtu - GENEVE_IPV6_HLEN;
1421 ip6_rt_put(rt);
1422 break;
1423 }
1424 #endif
1425 }
1426
1427 if (ldev_mtu <= 0)
1428 return;
1429
1430 geneve_change_mtu(dev, ldev_mtu - info->options_len);
1431 }
1432
1433 static int geneve_newlink(struct net *net, struct net_device *dev,
1434 struct nlattr *tb[], struct nlattr *data[],
1435 struct netlink_ext_ack *extack)
1436 {
1437 bool use_udp6_rx_checksums = false;
1438 struct ip_tunnel_info info;
1439 bool metadata = false;
1440 int err;
1441
1442 init_tnl_info(&info, GENEVE_UDP_PORT);
1443 err = geneve_nl2info(tb, data, extack, &info, &metadata,
1444 &use_udp6_rx_checksums, false);
1445 if (err)
1446 return err;
1447
1448 err = geneve_configure(net, dev, extack, &info, metadata,
1449 use_udp6_rx_checksums);
1450 if (err)
1451 return err;
1452
1453 geneve_link_config(dev, &info, tb);
1454
1455 return 0;
1456 }
1457
1458 /* Quiesces the geneve device data path for both TX and RX.
1459 *
1460 * On transmit geneve checks for non-NULL geneve_sock before it proceeds.
1461 * So, if we set that socket to NULL under RCU and wait for synchronize_net()
1462 * to complete for the existing set of in-flight packets to be transmitted,
1463 * then we would have quiesced the transmit data path. All the future packets
1464 * will get dropped until we unquiesce the data path.
1465 *
1466 * On receive geneve dereference the geneve_sock stashed in the socket. So,
1467 * if we set that to NULL under RCU and wait for synchronize_net() to
1468 * complete, then we would have quiesced the receive data path.
1469 */
1470 static void geneve_quiesce(struct geneve_dev *geneve, struct geneve_sock **gs4,
1471 struct geneve_sock **gs6)
1472 {
1473 *gs4 = rtnl_dereference(geneve->sock4);
1474 rcu_assign_pointer(geneve->sock4, NULL);
1475 if (*gs4)
1476 rcu_assign_sk_user_data((*gs4)->sock->sk, NULL);
1477 #if IS_ENABLED(CONFIG_IPV6)
1478 *gs6 = rtnl_dereference(geneve->sock6);
1479 rcu_assign_pointer(geneve->sock6, NULL);
1480 if (*gs6)
1481 rcu_assign_sk_user_data((*gs6)->sock->sk, NULL);
1482 #else
1483 *gs6 = NULL;
1484 #endif
1485 synchronize_net();
1486 }
1487
1488 /* Resumes the geneve device data path for both TX and RX. */
1489 static void geneve_unquiesce(struct geneve_dev *geneve, struct geneve_sock *gs4,
1490 struct geneve_sock __maybe_unused *gs6)
1491 {
1492 rcu_assign_pointer(geneve->sock4, gs4);
1493 if (gs4)
1494 rcu_assign_sk_user_data(gs4->sock->sk, gs4);
1495 #if IS_ENABLED(CONFIG_IPV6)
1496 rcu_assign_pointer(geneve->sock6, gs6);
1497 if (gs6)
1498 rcu_assign_sk_user_data(gs6->sock->sk, gs6);
1499 #endif
1500 synchronize_net();
1501 }
1502
1503 static int geneve_changelink(struct net_device *dev, struct nlattr *tb[],
1504 struct nlattr *data[],
1505 struct netlink_ext_ack *extack)
1506 {
1507 struct geneve_dev *geneve = netdev_priv(dev);
1508 struct geneve_sock *gs4, *gs6;
1509 struct ip_tunnel_info info;
1510 bool metadata;
1511 bool use_udp6_rx_checksums;
1512 int err;
1513
1514 /* If the geneve device is configured for metadata (or externally
1515 * controlled, for example, OVS), then nothing can be changed.
1516 */
1517 if (geneve->collect_md)
1518 return -EOPNOTSUPP;
1519
1520 /* Start with the existing info. */
1521 memcpy(&info, &geneve->info, sizeof(info));
1522 metadata = geneve->collect_md;
1523 use_udp6_rx_checksums = geneve->use_udp6_rx_checksums;
1524 err = geneve_nl2info(tb, data, extack, &info, &metadata,
1525 &use_udp6_rx_checksums, true);
1526 if (err)
1527 return err;
1528
1529 if (!geneve_dst_addr_equal(&geneve->info, &info)) {
1530 dst_cache_reset(&info.dst_cache);
1531 geneve_link_config(dev, &info, tb);
1532 }
1533
1534 geneve_quiesce(geneve, &gs4, &gs6);
1535 geneve->info = info;
1536 geneve->collect_md = metadata;
1537 geneve->use_udp6_rx_checksums = use_udp6_rx_checksums;
1538 geneve_unquiesce(geneve, gs4, gs6);
1539
1540 return 0;
1541 }
1542
1543 static void geneve_dellink(struct net_device *dev, struct list_head *head)
1544 {
1545 struct geneve_dev *geneve = netdev_priv(dev);
1546
1547 list_del(&geneve->next);
1548 unregister_netdevice_queue(dev, head);
1549 }
1550
1551 static size_t geneve_get_size(const struct net_device *dev)
1552 {
1553 return nla_total_size(sizeof(__u32)) + /* IFLA_GENEVE_ID */
1554 nla_total_size(sizeof(struct in6_addr)) + /* IFLA_GENEVE_REMOTE{6} */
1555 nla_total_size(sizeof(__u8)) + /* IFLA_GENEVE_TTL */
1556 nla_total_size(sizeof(__u8)) + /* IFLA_GENEVE_TOS */
1557 nla_total_size(sizeof(__be32)) + /* IFLA_GENEVE_LABEL */
1558 nla_total_size(sizeof(__be16)) + /* IFLA_GENEVE_PORT */
1559 nla_total_size(0) + /* IFLA_GENEVE_COLLECT_METADATA */
1560 nla_total_size(sizeof(__u8)) + /* IFLA_GENEVE_UDP_CSUM */
1561 nla_total_size(sizeof(__u8)) + /* IFLA_GENEVE_UDP_ZERO_CSUM6_TX */
1562 nla_total_size(sizeof(__u8)) + /* IFLA_GENEVE_UDP_ZERO_CSUM6_RX */
1563 0;
1564 }
1565
1566 static int geneve_fill_info(struct sk_buff *skb, const struct net_device *dev)
1567 {
1568 struct geneve_dev *geneve = netdev_priv(dev);
1569 struct ip_tunnel_info *info = &geneve->info;
1570 bool metadata = geneve->collect_md;
1571 __u8 tmp_vni[3];
1572 __u32 vni;
1573
1574 tunnel_id_to_vni(info->key.tun_id, tmp_vni);
1575 vni = (tmp_vni[0] << 16) | (tmp_vni[1] << 8) | tmp_vni[2];
1576 if (nla_put_u32(skb, IFLA_GENEVE_ID, vni))
1577 goto nla_put_failure;
1578
1579 if (!metadata && ip_tunnel_info_af(info) == AF_INET) {
1580 if (nla_put_in_addr(skb, IFLA_GENEVE_REMOTE,
1581 info->key.u.ipv4.dst))
1582 goto nla_put_failure;
1583 if (nla_put_u8(skb, IFLA_GENEVE_UDP_CSUM,
1584 !!(info->key.tun_flags & TUNNEL_CSUM)))
1585 goto nla_put_failure;
1586
1587 #if IS_ENABLED(CONFIG_IPV6)
1588 } else if (!metadata) {
1589 if (nla_put_in6_addr(skb, IFLA_GENEVE_REMOTE6,
1590 &info->key.u.ipv6.dst))
1591 goto nla_put_failure;
1592 if (nla_put_u8(skb, IFLA_GENEVE_UDP_ZERO_CSUM6_TX,
1593 !(info->key.tun_flags & TUNNEL_CSUM)))
1594 goto nla_put_failure;
1595 #endif
1596 }
1597
1598 if (nla_put_u8(skb, IFLA_GENEVE_TTL, info->key.ttl) ||
1599 nla_put_u8(skb, IFLA_GENEVE_TOS, info->key.tos) ||
1600 nla_put_be32(skb, IFLA_GENEVE_LABEL, info->key.label))
1601 goto nla_put_failure;
1602
1603 if (nla_put_be16(skb, IFLA_GENEVE_PORT, info->key.tp_dst))
1604 goto nla_put_failure;
1605
1606 if (metadata && nla_put_flag(skb, IFLA_GENEVE_COLLECT_METADATA))
1607 goto nla_put_failure;
1608
1609 #if IS_ENABLED(CONFIG_IPV6)
1610 if (nla_put_u8(skb, IFLA_GENEVE_UDP_ZERO_CSUM6_RX,
1611 !geneve->use_udp6_rx_checksums))
1612 goto nla_put_failure;
1613 #endif
1614
1615 return 0;
1616
1617 nla_put_failure:
1618 return -EMSGSIZE;
1619 }
1620
1621 static struct rtnl_link_ops geneve_link_ops __read_mostly = {
1622 .kind = "geneve",
1623 .maxtype = IFLA_GENEVE_MAX,
1624 .policy = geneve_policy,
1625 .priv_size = sizeof(struct geneve_dev),
1626 .setup = geneve_setup,
1627 .validate = geneve_validate,
1628 .newlink = geneve_newlink,
1629 .changelink = geneve_changelink,
1630 .dellink = geneve_dellink,
1631 .get_size = geneve_get_size,
1632 .fill_info = geneve_fill_info,
1633 };
1634
1635 struct net_device *geneve_dev_create_fb(struct net *net, const char *name,
1636 u8 name_assign_type, u16 dst_port)
1637 {
1638 struct nlattr *tb[IFLA_MAX + 1];
1639 struct ip_tunnel_info info;
1640 struct net_device *dev;
1641 LIST_HEAD(list_kill);
1642 int err;
1643
1644 memset(tb, 0, sizeof(tb));
1645 dev = rtnl_create_link(net, name, name_assign_type,
1646 &geneve_link_ops, tb);
1647 if (IS_ERR(dev))
1648 return dev;
1649
1650 init_tnl_info(&info, dst_port);
1651 err = geneve_configure(net, dev, NULL, &info, true, true);
1652 if (err) {
1653 free_netdev(dev);
1654 return ERR_PTR(err);
1655 }
1656
1657 /* openvswitch users expect packet sizes to be unrestricted,
1658 * so set the largest MTU we can.
1659 */
1660 err = geneve_change_mtu(dev, IP_MAX_MTU);
1661 if (err)
1662 goto err;
1663
1664 err = rtnl_configure_link(dev, NULL);
1665 if (err < 0)
1666 goto err;
1667
1668 return dev;
1669 err:
1670 geneve_dellink(dev, &list_kill);
1671 unregister_netdevice_many(&list_kill);
1672 return ERR_PTR(err);
1673 }
1674 EXPORT_SYMBOL_GPL(geneve_dev_create_fb);
1675
1676 static int geneve_netdevice_event(struct notifier_block *unused,
1677 unsigned long event, void *ptr)
1678 {
1679 struct net_device *dev = netdev_notifier_info_to_dev(ptr);
1680
1681 if (event == NETDEV_UDP_TUNNEL_PUSH_INFO ||
1682 event == NETDEV_UDP_TUNNEL_DROP_INFO) {
1683 geneve_offload_rx_ports(dev, event == NETDEV_UDP_TUNNEL_PUSH_INFO);
1684 } else if (event == NETDEV_UNREGISTER) {
1685 geneve_offload_rx_ports(dev, false);
1686 } else if (event == NETDEV_REGISTER) {
1687 geneve_offload_rx_ports(dev, true);
1688 }
1689
1690 return NOTIFY_DONE;
1691 }
1692
1693 static struct notifier_block geneve_notifier_block __read_mostly = {
1694 .notifier_call = geneve_netdevice_event,
1695 };
1696
1697 static __net_init int geneve_init_net(struct net *net)
1698 {
1699 struct geneve_net *gn = net_generic(net, geneve_net_id);
1700
1701 INIT_LIST_HEAD(&gn->geneve_list);
1702 INIT_LIST_HEAD(&gn->sock_list);
1703 return 0;
1704 }
1705
1706 static void geneve_destroy_tunnels(struct net *net, struct list_head *head)
1707 {
1708 struct geneve_net *gn = net_generic(net, geneve_net_id);
1709 struct geneve_dev *geneve, *next;
1710 struct net_device *dev, *aux;
1711
1712 /* gather any geneve devices that were moved into this ns */
1713 for_each_netdev_safe(net, dev, aux)
1714 if (dev->rtnl_link_ops == &geneve_link_ops)
1715 unregister_netdevice_queue(dev, head);
1716
1717 /* now gather any other geneve devices that were created in this ns */
1718 list_for_each_entry_safe(geneve, next, &gn->geneve_list, next) {
1719 /* If geneve->dev is in the same netns, it was already added
1720 * to the list by the previous loop.
1721 */
1722 if (!net_eq(dev_net(geneve->dev), net))
1723 unregister_netdevice_queue(geneve->dev, head);
1724 }
1725
1726 WARN_ON_ONCE(!list_empty(&gn->sock_list));
1727 }
1728
1729 static void __net_exit geneve_exit_batch_net(struct list_head *net_list)
1730 {
1731 struct net *net;
1732 LIST_HEAD(list);
1733
1734 rtnl_lock();
1735 list_for_each_entry(net, net_list, exit_list)
1736 geneve_destroy_tunnels(net, &list);
1737
1738 /* unregister the devices gathered above */
1739 unregister_netdevice_many(&list);
1740 rtnl_unlock();
1741 }
1742
1743 static struct pernet_operations geneve_net_ops = {
1744 .init = geneve_init_net,
1745 .exit_batch = geneve_exit_batch_net,
1746 .id = &geneve_net_id,
1747 .size = sizeof(struct geneve_net),
1748 };
1749
1750 static int __init geneve_init_module(void)
1751 {
1752 int rc;
1753
1754 rc = register_pernet_subsys(&geneve_net_ops);
1755 if (rc)
1756 goto out1;
1757
1758 rc = register_netdevice_notifier(&geneve_notifier_block);
1759 if (rc)
1760 goto out2;
1761
1762 rc = rtnl_link_register(&geneve_link_ops);
1763 if (rc)
1764 goto out3;
1765
1766 return 0;
1767 out3:
1768 unregister_netdevice_notifier(&geneve_notifier_block);
1769 out2:
1770 unregister_pernet_subsys(&geneve_net_ops);
1771 out1:
1772 return rc;
1773 }
1774 late_initcall(geneve_init_module);
1775
1776 static void __exit geneve_cleanup_module(void)
1777 {
1778 rtnl_link_unregister(&geneve_link_ops);
1779 unregister_netdevice_notifier(&geneve_notifier_block);
1780 unregister_pernet_subsys(&geneve_net_ops);
1781 }
1782 module_exit(geneve_cleanup_module);
1783
1784 MODULE_LICENSE("GPL");
1785 MODULE_VERSION(GENEVE_NETDEV_VER);
1786 MODULE_AUTHOR("John W. Linville <linville@tuxdriver.com>");
1787 MODULE_DESCRIPTION("Interface driver for GENEVE encapsulated traffic");
1788 MODULE_ALIAS_RTNL_LINK("geneve");
(deprecated) Btrfs devel tree