?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
mlxsw: spectrum_router: Extract mlxsw_sp_fi_is_gateway()
[linux-2.6/btrfs-unstable.git] / drivers / net / ethernet / mellanox / mlxsw / spectrum_router.c
blob04e188772109b6149aceb6b319628c7e4d8a7ca0
1 /*
2 * drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
3 * Copyright (c) 2016-2017 Mellanox Technologies. All rights reserved.
4 * Copyright (c) 2016 Jiri Pirko <jiri@mellanox.com>
5 * Copyright (c) 2016 Ido Schimmel <idosch@mellanox.com>
6 * Copyright (c) 2016 Yotam Gigi <yotamg@mellanox.com>
7 * Copyright (c) 2017 Petr Machata <petrm@mellanox.com>
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions are met:
11 *
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 * 3. Neither the names of the copyright holders nor the names of its
18 * contributors may be used to endorse or promote products derived from
19 * this software without specific prior written permission.
20 *
21 * Alternatively, this software may be distributed under the terms of the
22 * GNU General Public License ("GPL") version 2 as published by the Free
23 * Software Foundation.
24 *
25 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
26 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
27 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
28 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
29 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
35 * POSSIBILITY OF SUCH DAMAGE.
36 */
37
38 #include <linux/kernel.h>
39 #include <linux/types.h>
40 #include <linux/rhashtable.h>
41 #include <linux/bitops.h>
42 #include <linux/in6.h>
43 #include <linux/notifier.h>
44 #include <linux/inetdevice.h>
45 #include <linux/netdevice.h>
46 #include <linux/if_bridge.h>
47 #include <linux/socket.h>
48 #include <linux/route.h>
49 #include <net/netevent.h>
50 #include <net/neighbour.h>
51 #include <net/arp.h>
52 #include <net/ip_fib.h>
53 #include <net/ip6_fib.h>
54 #include <net/fib_rules.h>
55 #include <net/ip_tunnels.h>
56 #include <net/l3mdev.h>
57 #include <net/addrconf.h>
58 #include <net/ndisc.h>
59 #include <net/ipv6.h>
60 #include <net/fib_notifier.h>
61
62 #include "spectrum.h"
63 #include "core.h"
64 #include "reg.h"
65 #include "spectrum_cnt.h"
66 #include "spectrum_dpipe.h"
67 #include "spectrum_ipip.h"
68 #include "spectrum_router.h"
69
70 struct mlxsw_sp_vr;
71 struct mlxsw_sp_lpm_tree;
72 struct mlxsw_sp_rif_ops;
73
74 struct mlxsw_sp_router {
75 struct mlxsw_sp *mlxsw_sp;
76 struct mlxsw_sp_rif **rifs;
77 struct mlxsw_sp_vr *vrs;
78 struct rhashtable neigh_ht;
79 struct rhashtable nexthop_group_ht;
80 struct rhashtable nexthop_ht;
81 struct {
82 struct mlxsw_sp_lpm_tree *trees;
83 unsigned int tree_count;
84 } lpm;
85 struct {
86 struct delayed_work dw;
87 unsigned long interval; /* ms */
88 } neighs_update;
89 struct delayed_work nexthop_probe_dw;
90 #define MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL 5000 /* ms */
91 struct list_head nexthop_neighs_list;
92 bool aborted;
93 struct notifier_block fib_nb;
94 const struct mlxsw_sp_rif_ops **rif_ops_arr;
95 const struct mlxsw_sp_ipip_ops **ipip_ops_arr;
96 };
97
98 struct mlxsw_sp_rif {
99 struct list_head nexthop_list;
100 struct list_head neigh_list;
101 struct net_device *dev;
102 struct mlxsw_sp_fid *fid;
103 unsigned char addr[ETH_ALEN];
104 int mtu;
105 u16 rif_index;
106 u16 vr_id;
107 const struct mlxsw_sp_rif_ops *ops;
108 struct mlxsw_sp *mlxsw_sp;
109
110 unsigned int counter_ingress;
111 bool counter_ingress_valid;
112 unsigned int counter_egress;
113 bool counter_egress_valid;
114 };
115
116 struct mlxsw_sp_rif_params {
117 struct net_device *dev;
118 union {
119 u16 system_port;
120 u16 lag_id;
121 };
122 u16 vid;
123 bool lag;
124 };
125
126 struct mlxsw_sp_rif_subport {
127 struct mlxsw_sp_rif common;
128 union {
129 u16 system_port;
130 u16 lag_id;
131 };
132 u16 vid;
133 bool lag;
134 };
135
136 struct mlxsw_sp_rif_ipip_lb {
137 struct mlxsw_sp_rif common;
138 struct mlxsw_sp_rif_ipip_lb_config lb_config;
139 u16 ul_vr_id; /* Reserved for Spectrum-2. */
140 };
141
142 struct mlxsw_sp_rif_params_ipip_lb {
143 struct mlxsw_sp_rif_params common;
144 struct mlxsw_sp_rif_ipip_lb_config lb_config;
145 };
146
147 struct mlxsw_sp_rif_ops {
148 enum mlxsw_sp_rif_type type;
149 size_t rif_size;
150
151 void (*setup)(struct mlxsw_sp_rif *rif,
152 const struct mlxsw_sp_rif_params *params);
153 int (*configure)(struct mlxsw_sp_rif *rif);
154 void (*deconfigure)(struct mlxsw_sp_rif *rif);
155 struct mlxsw_sp_fid * (*fid_get)(struct mlxsw_sp_rif *rif);
156 };
157
158 static unsigned int *
159 mlxsw_sp_rif_p_counter_get(struct mlxsw_sp_rif *rif,
160 enum mlxsw_sp_rif_counter_dir dir)
161 {
162 switch (dir) {
163 case MLXSW_SP_RIF_COUNTER_EGRESS:
164 return &rif->counter_egress;
165 case MLXSW_SP_RIF_COUNTER_INGRESS:
166 return &rif->counter_ingress;
167 }
168 return NULL;
169 }
170
171 static bool
172 mlxsw_sp_rif_counter_valid_get(struct mlxsw_sp_rif *rif,
173 enum mlxsw_sp_rif_counter_dir dir)
174 {
175 switch (dir) {
176 case MLXSW_SP_RIF_COUNTER_EGRESS:
177 return rif->counter_egress_valid;
178 case MLXSW_SP_RIF_COUNTER_INGRESS:
179 return rif->counter_ingress_valid;
180 }
181 return false;
182 }
183
184 static void
185 mlxsw_sp_rif_counter_valid_set(struct mlxsw_sp_rif *rif,
186 enum mlxsw_sp_rif_counter_dir dir,
187 bool valid)
188 {
189 switch (dir) {
190 case MLXSW_SP_RIF_COUNTER_EGRESS:
191 rif->counter_egress_valid = valid;
192 break;
193 case MLXSW_SP_RIF_COUNTER_INGRESS:
194 rif->counter_ingress_valid = valid;
195 break;
196 }
197 }
198
199 static int mlxsw_sp_rif_counter_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
200 unsigned int counter_index, bool enable,
201 enum mlxsw_sp_rif_counter_dir dir)
202 {
203 char ritr_pl[MLXSW_REG_RITR_LEN];
204 bool is_egress = false;
205 int err;
206
207 if (dir == MLXSW_SP_RIF_COUNTER_EGRESS)
208 is_egress = true;
209 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
210 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
211 if (err)
212 return err;
213
214 mlxsw_reg_ritr_counter_pack(ritr_pl, counter_index, enable,
215 is_egress);
216 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
217 }
218
219 int mlxsw_sp_rif_counter_value_get(struct mlxsw_sp *mlxsw_sp,
220 struct mlxsw_sp_rif *rif,
221 enum mlxsw_sp_rif_counter_dir dir, u64 *cnt)
222 {
223 char ricnt_pl[MLXSW_REG_RICNT_LEN];
224 unsigned int *p_counter_index;
225 bool valid;
226 int err;
227
228 valid = mlxsw_sp_rif_counter_valid_get(rif, dir);
229 if (!valid)
230 return -EINVAL;
231
232 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
233 if (!p_counter_index)
234 return -EINVAL;
235 mlxsw_reg_ricnt_pack(ricnt_pl, *p_counter_index,
236 MLXSW_REG_RICNT_OPCODE_NOP);
237 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl);
238 if (err)
239 return err;
240 *cnt = mlxsw_reg_ricnt_good_unicast_packets_get(ricnt_pl);
241 return 0;
242 }
243
244 static int mlxsw_sp_rif_counter_clear(struct mlxsw_sp *mlxsw_sp,
245 unsigned int counter_index)
246 {
247 char ricnt_pl[MLXSW_REG_RICNT_LEN];
248
249 mlxsw_reg_ricnt_pack(ricnt_pl, counter_index,
250 MLXSW_REG_RICNT_OPCODE_CLEAR);
251 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl);
252 }
253
254 int mlxsw_sp_rif_counter_alloc(struct mlxsw_sp *mlxsw_sp,
255 struct mlxsw_sp_rif *rif,
256 enum mlxsw_sp_rif_counter_dir dir)
257 {
258 unsigned int *p_counter_index;
259 int err;
260
261 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
262 if (!p_counter_index)
263 return -EINVAL;
264 err = mlxsw_sp_counter_alloc(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
265 p_counter_index);
266 if (err)
267 return err;
268
269 err = mlxsw_sp_rif_counter_clear(mlxsw_sp, *p_counter_index);
270 if (err)
271 goto err_counter_clear;
272
273 err = mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index,
274 *p_counter_index, true, dir);
275 if (err)
276 goto err_counter_edit;
277 mlxsw_sp_rif_counter_valid_set(rif, dir, true);
278 return 0;
279
280 err_counter_edit:
281 err_counter_clear:
282 mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
283 *p_counter_index);
284 return err;
285 }
286
287 void mlxsw_sp_rif_counter_free(struct mlxsw_sp *mlxsw_sp,
288 struct mlxsw_sp_rif *rif,
289 enum mlxsw_sp_rif_counter_dir dir)
290 {
291 unsigned int *p_counter_index;
292
293 if (!mlxsw_sp_rif_counter_valid_get(rif, dir))
294 return;
295
296 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
297 if (WARN_ON(!p_counter_index))
298 return;
299 mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index,
300 *p_counter_index, false, dir);
301 mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
302 *p_counter_index);
303 mlxsw_sp_rif_counter_valid_set(rif, dir, false);
304 }
305
306 static void mlxsw_sp_rif_counters_alloc(struct mlxsw_sp_rif *rif)
307 {
308 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
309 struct devlink *devlink;
310
311 devlink = priv_to_devlink(mlxsw_sp->core);
312 if (!devlink_dpipe_table_counter_enabled(devlink,
313 MLXSW_SP_DPIPE_TABLE_NAME_ERIF))
314 return;
315 mlxsw_sp_rif_counter_alloc(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS);
316 }
317
318 static void mlxsw_sp_rif_counters_free(struct mlxsw_sp_rif *rif)
319 {
320 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
321
322 mlxsw_sp_rif_counter_free(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS);
323 }
324
325 static struct mlxsw_sp_rif *
326 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
327 const struct net_device *dev);
328
329 #define MLXSW_SP_PREFIX_COUNT (sizeof(struct in6_addr) * BITS_PER_BYTE + 1)
330
331 struct mlxsw_sp_prefix_usage {
332 DECLARE_BITMAP(b, MLXSW_SP_PREFIX_COUNT);
333 };
334
335 #define mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) \
336 for_each_set_bit(prefix, (prefix_usage)->b, MLXSW_SP_PREFIX_COUNT)
337
338 static bool
339 mlxsw_sp_prefix_usage_eq(struct mlxsw_sp_prefix_usage *prefix_usage1,
340 struct mlxsw_sp_prefix_usage *prefix_usage2)
341 {
342 return !memcmp(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
343 }
344
345 static bool
346 mlxsw_sp_prefix_usage_none(struct mlxsw_sp_prefix_usage *prefix_usage)
347 {
348 struct mlxsw_sp_prefix_usage prefix_usage_none = {{ 0 } };
349
350 return mlxsw_sp_prefix_usage_eq(prefix_usage, &prefix_usage_none);
351 }
352
353 static void
354 mlxsw_sp_prefix_usage_cpy(struct mlxsw_sp_prefix_usage *prefix_usage1,
355 struct mlxsw_sp_prefix_usage *prefix_usage2)
356 {
357 memcpy(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
358 }
359
360 static void
361 mlxsw_sp_prefix_usage_set(struct mlxsw_sp_prefix_usage *prefix_usage,
362 unsigned char prefix_len)
363 {
364 set_bit(prefix_len, prefix_usage->b);
365 }
366
367 static void
368 mlxsw_sp_prefix_usage_clear(struct mlxsw_sp_prefix_usage *prefix_usage,
369 unsigned char prefix_len)
370 {
371 clear_bit(prefix_len, prefix_usage->b);
372 }
373
374 struct mlxsw_sp_fib_key {
375 unsigned char addr[sizeof(struct in6_addr)];
376 unsigned char prefix_len;
377 };
378
379 enum mlxsw_sp_fib_entry_type {
380 MLXSW_SP_FIB_ENTRY_TYPE_REMOTE,
381 MLXSW_SP_FIB_ENTRY_TYPE_LOCAL,
382 MLXSW_SP_FIB_ENTRY_TYPE_TRAP,
383 };
384
385 struct mlxsw_sp_nexthop_group;
386 struct mlxsw_sp_fib;
387
388 struct mlxsw_sp_fib_node {
389 struct list_head entry_list;
390 struct list_head list;
391 struct rhash_head ht_node;
392 struct mlxsw_sp_fib *fib;
393 struct mlxsw_sp_fib_key key;
394 };
395
396 struct mlxsw_sp_fib_entry {
397 struct list_head list;
398 struct mlxsw_sp_fib_node *fib_node;
399 enum mlxsw_sp_fib_entry_type type;
400 struct list_head nexthop_group_node;
401 struct mlxsw_sp_nexthop_group *nh_group;
402 };
403
404 struct mlxsw_sp_fib4_entry {
405 struct mlxsw_sp_fib_entry common;
406 u32 tb_id;
407 u32 prio;
408 u8 tos;
409 u8 type;
410 };
411
412 struct mlxsw_sp_fib6_entry {
413 struct mlxsw_sp_fib_entry common;
414 struct list_head rt6_list;
415 unsigned int nrt6;
416 };
417
418 struct mlxsw_sp_rt6 {
419 struct list_head list;
420 struct rt6_info *rt;
421 };
422
423 struct mlxsw_sp_lpm_tree {
424 u8 id; /* tree ID */
425 unsigned int ref_count;
426 enum mlxsw_sp_l3proto proto;
427 struct mlxsw_sp_prefix_usage prefix_usage;
428 };
429
430 struct mlxsw_sp_fib {
431 struct rhashtable ht;
432 struct list_head node_list;
433 struct mlxsw_sp_vr *vr;
434 struct mlxsw_sp_lpm_tree *lpm_tree;
435 unsigned long prefix_ref_count[MLXSW_SP_PREFIX_COUNT];
436 struct mlxsw_sp_prefix_usage prefix_usage;
437 enum mlxsw_sp_l3proto proto;
438 };
439
440 struct mlxsw_sp_vr {
441 u16 id; /* virtual router ID */
442 u32 tb_id; /* kernel fib table id */
443 unsigned int rif_count;
444 struct mlxsw_sp_fib *fib4;
445 struct mlxsw_sp_fib *fib6;
446 };
447
448 static const struct rhashtable_params mlxsw_sp_fib_ht_params;
449
450 static struct mlxsw_sp_fib *mlxsw_sp_fib_create(struct mlxsw_sp_vr *vr,
451 enum mlxsw_sp_l3proto proto)
452 {
453 struct mlxsw_sp_fib *fib;
454 int err;
455
456 fib = kzalloc(sizeof(*fib), GFP_KERNEL);
457 if (!fib)
458 return ERR_PTR(-ENOMEM);
459 err = rhashtable_init(&fib->ht, &mlxsw_sp_fib_ht_params);
460 if (err)
461 goto err_rhashtable_init;
462 INIT_LIST_HEAD(&fib->node_list);
463 fib->proto = proto;
464 fib->vr = vr;
465 return fib;
466
467 err_rhashtable_init:
468 kfree(fib);
469 return ERR_PTR(err);
470 }
471
472 static void mlxsw_sp_fib_destroy(struct mlxsw_sp_fib *fib)
473 {
474 WARN_ON(!list_empty(&fib->node_list));
475 WARN_ON(fib->lpm_tree);
476 rhashtable_destroy(&fib->ht);
477 kfree(fib);
478 }
479
480 static struct mlxsw_sp_lpm_tree *
481 mlxsw_sp_lpm_tree_find_unused(struct mlxsw_sp *mlxsw_sp)
482 {
483 static struct mlxsw_sp_lpm_tree *lpm_tree;
484 int i;
485
486 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
487 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
488 if (lpm_tree->ref_count == 0)
489 return lpm_tree;
490 }
491 return NULL;
492 }
493
494 static int mlxsw_sp_lpm_tree_alloc(struct mlxsw_sp *mlxsw_sp,
495 struct mlxsw_sp_lpm_tree *lpm_tree)
496 {
497 char ralta_pl[MLXSW_REG_RALTA_LEN];
498
499 mlxsw_reg_ralta_pack(ralta_pl, true,
500 (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
501 lpm_tree->id);
502 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
503 }
504
505 static void mlxsw_sp_lpm_tree_free(struct mlxsw_sp *mlxsw_sp,
506 struct mlxsw_sp_lpm_tree *lpm_tree)
507 {
508 char ralta_pl[MLXSW_REG_RALTA_LEN];
509
510 mlxsw_reg_ralta_pack(ralta_pl, false,
511 (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
512 lpm_tree->id);
513 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
514 }
515
516 static int
517 mlxsw_sp_lpm_tree_left_struct_set(struct mlxsw_sp *mlxsw_sp,
518 struct mlxsw_sp_prefix_usage *prefix_usage,
519 struct mlxsw_sp_lpm_tree *lpm_tree)
520 {
521 char ralst_pl[MLXSW_REG_RALST_LEN];
522 u8 root_bin = 0;
523 u8 prefix;
524 u8 last_prefix = MLXSW_REG_RALST_BIN_NO_CHILD;
525
526 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage)
527 root_bin = prefix;
528
529 mlxsw_reg_ralst_pack(ralst_pl, root_bin, lpm_tree->id);
530 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) {
531 if (prefix == 0)
532 continue;
533 mlxsw_reg_ralst_bin_pack(ralst_pl, prefix, last_prefix,
534 MLXSW_REG_RALST_BIN_NO_CHILD);
535 last_prefix = prefix;
536 }
537 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
538 }
539
540 static struct mlxsw_sp_lpm_tree *
541 mlxsw_sp_lpm_tree_create(struct mlxsw_sp *mlxsw_sp,
542 struct mlxsw_sp_prefix_usage *prefix_usage,
543 enum mlxsw_sp_l3proto proto)
544 {
545 struct mlxsw_sp_lpm_tree *lpm_tree;
546 int err;
547
548 lpm_tree = mlxsw_sp_lpm_tree_find_unused(mlxsw_sp);
549 if (!lpm_tree)
550 return ERR_PTR(-EBUSY);
551 lpm_tree->proto = proto;
552 err = mlxsw_sp_lpm_tree_alloc(mlxsw_sp, lpm_tree);
553 if (err)
554 return ERR_PTR(err);
555
556 err = mlxsw_sp_lpm_tree_left_struct_set(mlxsw_sp, prefix_usage,
557 lpm_tree);
558 if (err)
559 goto err_left_struct_set;
560 memcpy(&lpm_tree->prefix_usage, prefix_usage,
561 sizeof(lpm_tree->prefix_usage));
562 return lpm_tree;
563
564 err_left_struct_set:
565 mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
566 return ERR_PTR(err);
567 }
568
569 static void mlxsw_sp_lpm_tree_destroy(struct mlxsw_sp *mlxsw_sp,
570 struct mlxsw_sp_lpm_tree *lpm_tree)
571 {
572 mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
573 }
574
575 static struct mlxsw_sp_lpm_tree *
576 mlxsw_sp_lpm_tree_get(struct mlxsw_sp *mlxsw_sp,
577 struct mlxsw_sp_prefix_usage *prefix_usage,
578 enum mlxsw_sp_l3proto proto)
579 {
580 struct mlxsw_sp_lpm_tree *lpm_tree;
581 int i;
582
583 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
584 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
585 if (lpm_tree->ref_count != 0 &&
586 lpm_tree->proto == proto &&
587 mlxsw_sp_prefix_usage_eq(&lpm_tree->prefix_usage,
588 prefix_usage))
589 return lpm_tree;
590 }
591 return mlxsw_sp_lpm_tree_create(mlxsw_sp, prefix_usage, proto);
592 }
593
594 static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree)
595 {
596 lpm_tree->ref_count++;
597 }
598
599 static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp,
600 struct mlxsw_sp_lpm_tree *lpm_tree)
601 {
602 if (--lpm_tree->ref_count == 0)
603 mlxsw_sp_lpm_tree_destroy(mlxsw_sp, lpm_tree);
604 }
605
606 #define MLXSW_SP_LPM_TREE_MIN 1 /* tree 0 is reserved */
607
608 static int mlxsw_sp_lpm_init(struct mlxsw_sp *mlxsw_sp)
609 {
610 struct mlxsw_sp_lpm_tree *lpm_tree;
611 u64 max_trees;
612 int i;
613
614 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_LPM_TREES))
615 return -EIO;
616
617 max_trees = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_LPM_TREES);
618 mlxsw_sp->router->lpm.tree_count = max_trees - MLXSW_SP_LPM_TREE_MIN;
619 mlxsw_sp->router->lpm.trees = kcalloc(mlxsw_sp->router->lpm.tree_count,
620 sizeof(struct mlxsw_sp_lpm_tree),
621 GFP_KERNEL);
622 if (!mlxsw_sp->router->lpm.trees)
623 return -ENOMEM;
624
625 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
626 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
627 lpm_tree->id = i + MLXSW_SP_LPM_TREE_MIN;
628 }
629
630 return 0;
631 }
632
633 static void mlxsw_sp_lpm_fini(struct mlxsw_sp *mlxsw_sp)
634 {
635 kfree(mlxsw_sp->router->lpm.trees);
636 }
637
638 static bool mlxsw_sp_vr_is_used(const struct mlxsw_sp_vr *vr)
639 {
640 return !!vr->fib4 || !!vr->fib6;
641 }
642
643 static struct mlxsw_sp_vr *mlxsw_sp_vr_find_unused(struct mlxsw_sp *mlxsw_sp)
644 {
645 struct mlxsw_sp_vr *vr;
646 int i;
647
648 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
649 vr = &mlxsw_sp->router->vrs[i];
650 if (!mlxsw_sp_vr_is_used(vr))
651 return vr;
652 }
653 return NULL;
654 }
655
656 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp,
657 const struct mlxsw_sp_fib *fib, u8 tree_id)
658 {
659 char raltb_pl[MLXSW_REG_RALTB_LEN];
660
661 mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id,
662 (enum mlxsw_reg_ralxx_protocol) fib->proto,
663 tree_id);
664 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
665 }
666
667 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp,
668 const struct mlxsw_sp_fib *fib)
669 {
670 char raltb_pl[MLXSW_REG_RALTB_LEN];
671
672 /* Bind to tree 0 which is default */
673 mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id,
674 (enum mlxsw_reg_ralxx_protocol) fib->proto, 0);
675 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
676 }
677
678 static u32 mlxsw_sp_fix_tb_id(u32 tb_id)
679 {
680 /* For our purpose, squash main and local table into one */
681 if (tb_id == RT_TABLE_LOCAL)
682 tb_id = RT_TABLE_MAIN;
683 return tb_id;
684 }
685
686 static struct mlxsw_sp_vr *mlxsw_sp_vr_find(struct mlxsw_sp *mlxsw_sp,
687 u32 tb_id)
688 {
689 struct mlxsw_sp_vr *vr;
690 int i;
691
692 tb_id = mlxsw_sp_fix_tb_id(tb_id);
693
694 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
695 vr = &mlxsw_sp->router->vrs[i];
696 if (mlxsw_sp_vr_is_used(vr) && vr->tb_id == tb_id)
697 return vr;
698 }
699 return NULL;
700 }
701
702 static struct mlxsw_sp_fib *mlxsw_sp_vr_fib(const struct mlxsw_sp_vr *vr,
703 enum mlxsw_sp_l3proto proto)
704 {
705 switch (proto) {
706 case MLXSW_SP_L3_PROTO_IPV4:
707 return vr->fib4;
708 case MLXSW_SP_L3_PROTO_IPV6:
709 return vr->fib6;
710 }
711 return NULL;
712 }
713
714 static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp,
715 u32 tb_id)
716 {
717 struct mlxsw_sp_vr *vr;
718 int err;
719
720 vr = mlxsw_sp_vr_find_unused(mlxsw_sp);
721 if (!vr)
722 return ERR_PTR(-EBUSY);
723 vr->fib4 = mlxsw_sp_fib_create(vr, MLXSW_SP_L3_PROTO_IPV4);
724 if (IS_ERR(vr->fib4))
725 return ERR_CAST(vr->fib4);
726 vr->fib6 = mlxsw_sp_fib_create(vr, MLXSW_SP_L3_PROTO_IPV6);
727 if (IS_ERR(vr->fib6)) {
728 err = PTR_ERR(vr->fib6);
729 goto err_fib6_create;
730 }
731 vr->tb_id = tb_id;
732 return vr;
733
734 err_fib6_create:
735 mlxsw_sp_fib_destroy(vr->fib4);
736 vr->fib4 = NULL;
737 return ERR_PTR(err);
738 }
739
740 static void mlxsw_sp_vr_destroy(struct mlxsw_sp_vr *vr)
741 {
742 mlxsw_sp_fib_destroy(vr->fib6);
743 vr->fib6 = NULL;
744 mlxsw_sp_fib_destroy(vr->fib4);
745 vr->fib4 = NULL;
746 }
747
748 static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id)
749 {
750 struct mlxsw_sp_vr *vr;
751
752 tb_id = mlxsw_sp_fix_tb_id(tb_id);
753 vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id);
754 if (!vr)
755 vr = mlxsw_sp_vr_create(mlxsw_sp, tb_id);
756 return vr;
757 }
758
759 static void mlxsw_sp_vr_put(struct mlxsw_sp_vr *vr)
760 {
761 if (!vr->rif_count && list_empty(&vr->fib4->node_list) &&
762 list_empty(&vr->fib6->node_list))
763 mlxsw_sp_vr_destroy(vr);
764 }
765
766 static bool
767 mlxsw_sp_vr_lpm_tree_should_replace(struct mlxsw_sp_vr *vr,
768 enum mlxsw_sp_l3proto proto, u8 tree_id)
769 {
770 struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
771
772 if (!mlxsw_sp_vr_is_used(vr))
773 return false;
774 if (fib->lpm_tree && fib->lpm_tree->id == tree_id)
775 return true;
776 return false;
777 }
778
779 static int mlxsw_sp_vr_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp,
780 struct mlxsw_sp_fib *fib,
781 struct mlxsw_sp_lpm_tree *new_tree)
782 {
783 struct mlxsw_sp_lpm_tree *old_tree = fib->lpm_tree;
784 int err;
785
786 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, new_tree->id);
787 if (err)
788 return err;
789 fib->lpm_tree = new_tree;
790 mlxsw_sp_lpm_tree_hold(new_tree);
791 mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree);
792 return 0;
793 }
794
795 static int mlxsw_sp_vrs_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp,
796 struct mlxsw_sp_fib *fib,
797 struct mlxsw_sp_lpm_tree *new_tree)
798 {
799 struct mlxsw_sp_lpm_tree *old_tree = fib->lpm_tree;
800 enum mlxsw_sp_l3proto proto = fib->proto;
801 u8 old_id, new_id = new_tree->id;
802 struct mlxsw_sp_vr *vr;
803 int i, err;
804
805 if (!old_tree)
806 goto no_replace;
807 old_id = old_tree->id;
808
809 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
810 vr = &mlxsw_sp->router->vrs[i];
811 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, old_id))
812 continue;
813 err = mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp,
814 mlxsw_sp_vr_fib(vr, proto),
815 new_tree);
816 if (err)
817 goto err_tree_replace;
818 }
819
820 return 0;
821
822 err_tree_replace:
823 for (i--; i >= 0; i--) {
824 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, new_id))
825 continue;
826 mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp,
827 mlxsw_sp_vr_fib(vr, proto),
828 old_tree);
829 }
830 return err;
831
832 no_replace:
833 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, new_tree->id);
834 if (err)
835 return err;
836 fib->lpm_tree = new_tree;
837 mlxsw_sp_lpm_tree_hold(new_tree);
838 return 0;
839 }
840
841 static void
842 mlxsw_sp_vrs_prefixes(struct mlxsw_sp *mlxsw_sp,
843 enum mlxsw_sp_l3proto proto,
844 struct mlxsw_sp_prefix_usage *req_prefix_usage)
845 {
846 int i;
847
848 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
849 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
850 struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
851 unsigned char prefix;
852
853 if (!mlxsw_sp_vr_is_used(vr))
854 continue;
855 mlxsw_sp_prefix_usage_for_each(prefix, &fib->prefix_usage)
856 mlxsw_sp_prefix_usage_set(req_prefix_usage, prefix);
857 }
858 }
859
860 static int mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp)
861 {
862 struct mlxsw_sp_vr *vr;
863 u64 max_vrs;
864 int i;
865
866 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_VRS))
867 return -EIO;
868
869 max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS);
870 mlxsw_sp->router->vrs = kcalloc(max_vrs, sizeof(struct mlxsw_sp_vr),
871 GFP_KERNEL);
872 if (!mlxsw_sp->router->vrs)
873 return -ENOMEM;
874
875 for (i = 0; i < max_vrs; i++) {
876 vr = &mlxsw_sp->router->vrs[i];
877 vr->id = i;
878 }
879
880 return 0;
881 }
882
883 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp);
884
885 static void mlxsw_sp_vrs_fini(struct mlxsw_sp *mlxsw_sp)
886 {
887 /* At this stage we're guaranteed not to have new incoming
888 * FIB notifications and the work queue is free from FIBs
889 * sitting on top of mlxsw netdevs. However, we can still
890 * have other FIBs queued. Flush the queue before flushing
891 * the device's tables. No need for locks, as we're the only
892 * writer.
893 */
894 mlxsw_core_flush_owq();
895 mlxsw_sp_router_fib_flush(mlxsw_sp);
896 kfree(mlxsw_sp->router->vrs);
897 }
898
899 static struct net_device *
900 __mlxsw_sp_ipip_netdev_ul_dev_get(const struct net_device *ol_dev)
901 {
902 struct ip_tunnel *tun = netdev_priv(ol_dev);
903 struct net *net = dev_net(ol_dev);
904
905 return __dev_get_by_index(net, tun->parms.link);
906 }
907
908 static u32 mlxsw_sp_ipip_dev_ul_tb_id(const struct net_device *ol_dev)
909 {
910 struct net_device *d = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev);
911
912 if (d)
913 return l3mdev_fib_table(d) ? : RT_TABLE_MAIN;
914 else
915 return l3mdev_fib_table(ol_dev) ? : RT_TABLE_MAIN;
916 }
917
918 struct mlxsw_sp_neigh_key {
919 struct neighbour *n;
920 };
921
922 struct mlxsw_sp_neigh_entry {
923 struct list_head rif_list_node;
924 struct rhash_head ht_node;
925 struct mlxsw_sp_neigh_key key;
926 u16 rif;
927 bool connected;
928 unsigned char ha[ETH_ALEN];
929 struct list_head nexthop_list; /* list of nexthops using
930 * this neigh entry
931 */
932 struct list_head nexthop_neighs_list_node;
933 unsigned int counter_index;
934 bool counter_valid;
935 };
936
937 static const struct rhashtable_params mlxsw_sp_neigh_ht_params = {
938 .key_offset = offsetof(struct mlxsw_sp_neigh_entry, key),
939 .head_offset = offsetof(struct mlxsw_sp_neigh_entry, ht_node),
940 .key_len = sizeof(struct mlxsw_sp_neigh_key),
941 };
942
943 struct mlxsw_sp_neigh_entry *
944 mlxsw_sp_rif_neigh_next(struct mlxsw_sp_rif *rif,
945 struct mlxsw_sp_neigh_entry *neigh_entry)
946 {
947 if (!neigh_entry) {
948 if (list_empty(&rif->neigh_list))
949 return NULL;
950 else
951 return list_first_entry(&rif->neigh_list,
952 typeof(*neigh_entry),
953 rif_list_node);
954 }
955 if (neigh_entry->rif_list_node.next == &rif->neigh_list)
956 return NULL;
957 return list_next_entry(neigh_entry, rif_list_node);
958 }
959
960 int mlxsw_sp_neigh_entry_type(struct mlxsw_sp_neigh_entry *neigh_entry)
961 {
962 return neigh_entry->key.n->tbl->family;
963 }
964
965 unsigned char *
966 mlxsw_sp_neigh_entry_ha(struct mlxsw_sp_neigh_entry *neigh_entry)
967 {
968 return neigh_entry->ha;
969 }
970
971 u32 mlxsw_sp_neigh4_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry)
972 {
973 struct neighbour *n;
974
975 n = neigh_entry->key.n;
976 return ntohl(*((__be32 *) n->primary_key));
977 }
978
979 struct in6_addr *
980 mlxsw_sp_neigh6_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry)
981 {
982 struct neighbour *n;
983
984 n = neigh_entry->key.n;
985 return (struct in6_addr *) &n->primary_key;
986 }
987
988 int mlxsw_sp_neigh_counter_get(struct mlxsw_sp *mlxsw_sp,
989 struct mlxsw_sp_neigh_entry *neigh_entry,
990 u64 *p_counter)
991 {
992 if (!neigh_entry->counter_valid)
993 return -EINVAL;
994
995 return mlxsw_sp_flow_counter_get(mlxsw_sp, neigh_entry->counter_index,
996 p_counter, NULL);
997 }
998
999 static struct mlxsw_sp_neigh_entry *
1000 mlxsw_sp_neigh_entry_alloc(struct mlxsw_sp *mlxsw_sp, struct neighbour *n,
1001 u16 rif)
1002 {
1003 struct mlxsw_sp_neigh_entry *neigh_entry;
1004
1005 neigh_entry = kzalloc(sizeof(*neigh_entry), GFP_KERNEL);
1006 if (!neigh_entry)
1007 return NULL;
1008
1009 neigh_entry->key.n = n;
1010 neigh_entry->rif = rif;
1011 INIT_LIST_HEAD(&neigh_entry->nexthop_list);
1012
1013 return neigh_entry;
1014 }
1015
1016 static void mlxsw_sp_neigh_entry_free(struct mlxsw_sp_neigh_entry *neigh_entry)
1017 {
1018 kfree(neigh_entry);
1019 }
1020
1021 static int
1022 mlxsw_sp_neigh_entry_insert(struct mlxsw_sp *mlxsw_sp,
1023 struct mlxsw_sp_neigh_entry *neigh_entry)
1024 {
1025 return rhashtable_insert_fast(&mlxsw_sp->router->neigh_ht,
1026 &neigh_entry->ht_node,
1027 mlxsw_sp_neigh_ht_params);
1028 }
1029
1030 static void
1031 mlxsw_sp_neigh_entry_remove(struct mlxsw_sp *mlxsw_sp,
1032 struct mlxsw_sp_neigh_entry *neigh_entry)
1033 {
1034 rhashtable_remove_fast(&mlxsw_sp->router->neigh_ht,
1035 &neigh_entry->ht_node,
1036 mlxsw_sp_neigh_ht_params);
1037 }
1038
1039 static bool
1040 mlxsw_sp_neigh_counter_should_alloc(struct mlxsw_sp *mlxsw_sp,
1041 struct mlxsw_sp_neigh_entry *neigh_entry)
1042 {
1043 struct devlink *devlink;
1044 const char *table_name;
1045
1046 switch (mlxsw_sp_neigh_entry_type(neigh_entry)) {
1047 case AF_INET:
1048 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST4;
1049 break;
1050 case AF_INET6:
1051 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST6;
1052 break;
1053 default:
1054 WARN_ON(1);
1055 return false;
1056 }
1057
1058 devlink = priv_to_devlink(mlxsw_sp->core);
1059 return devlink_dpipe_table_counter_enabled(devlink, table_name);
1060 }
1061
1062 static void
1063 mlxsw_sp_neigh_counter_alloc(struct mlxsw_sp *mlxsw_sp,
1064 struct mlxsw_sp_neigh_entry *neigh_entry)
1065 {
1066 if (!mlxsw_sp_neigh_counter_should_alloc(mlxsw_sp, neigh_entry))
1067 return;
1068
1069 if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &neigh_entry->counter_index))
1070 return;
1071
1072 neigh_entry->counter_valid = true;
1073 }
1074
1075 static void
1076 mlxsw_sp_neigh_counter_free(struct mlxsw_sp *mlxsw_sp,
1077 struct mlxsw_sp_neigh_entry *neigh_entry)
1078 {
1079 if (!neigh_entry->counter_valid)
1080 return;
1081 mlxsw_sp_flow_counter_free(mlxsw_sp,
1082 neigh_entry->counter_index);
1083 neigh_entry->counter_valid = false;
1084 }
1085
1086 static struct mlxsw_sp_neigh_entry *
1087 mlxsw_sp_neigh_entry_create(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
1088 {
1089 struct mlxsw_sp_neigh_entry *neigh_entry;
1090 struct mlxsw_sp_rif *rif;
1091 int err;
1092
1093 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, n->dev);
1094 if (!rif)
1095 return ERR_PTR(-EINVAL);
1096
1097 neigh_entry = mlxsw_sp_neigh_entry_alloc(mlxsw_sp, n, rif->rif_index);
1098 if (!neigh_entry)
1099 return ERR_PTR(-ENOMEM);
1100
1101 err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
1102 if (err)
1103 goto err_neigh_entry_insert;
1104
1105 mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry);
1106 list_add(&neigh_entry->rif_list_node, &rif->neigh_list);
1107
1108 return neigh_entry;
1109
1110 err_neigh_entry_insert:
1111 mlxsw_sp_neigh_entry_free(neigh_entry);
1112 return ERR_PTR(err);
1113 }
1114
1115 static void
1116 mlxsw_sp_neigh_entry_destroy(struct mlxsw_sp *mlxsw_sp,
1117 struct mlxsw_sp_neigh_entry *neigh_entry)
1118 {
1119 list_del(&neigh_entry->rif_list_node);
1120 mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry);
1121 mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry);
1122 mlxsw_sp_neigh_entry_free(neigh_entry);
1123 }
1124
1125 static struct mlxsw_sp_neigh_entry *
1126 mlxsw_sp_neigh_entry_lookup(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
1127 {
1128 struct mlxsw_sp_neigh_key key;
1129
1130 key.n = n;
1131 return rhashtable_lookup_fast(&mlxsw_sp->router->neigh_ht,
1132 &key, mlxsw_sp_neigh_ht_params);
1133 }
1134
1135 static void
1136 mlxsw_sp_router_neighs_update_interval_init(struct mlxsw_sp *mlxsw_sp)
1137 {
1138 unsigned long interval;
1139
1140 #if IS_ENABLED(CONFIG_IPV6)
1141 interval = min_t(unsigned long,
1142 NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME),
1143 NEIGH_VAR(&nd_tbl.parms, DELAY_PROBE_TIME));
1144 #else
1145 interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME);
1146 #endif
1147 mlxsw_sp->router->neighs_update.interval = jiffies_to_msecs(interval);
1148 }
1149
1150 static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp,
1151 char *rauhtd_pl,
1152 int ent_index)
1153 {
1154 struct net_device *dev;
1155 struct neighbour *n;
1156 __be32 dipn;
1157 u32 dip;
1158 u16 rif;
1159
1160 mlxsw_reg_rauhtd_ent_ipv4_unpack(rauhtd_pl, ent_index, &rif, &dip);
1161
1162 if (!mlxsw_sp->router->rifs[rif]) {
1163 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
1164 return;
1165 }
1166
1167 dipn = htonl(dip);
1168 dev = mlxsw_sp->router->rifs[rif]->dev;
1169 n = neigh_lookup(&arp_tbl, &dipn, dev);
1170 if (!n) {
1171 netdev_err(dev, "Failed to find matching neighbour for IP=%pI4h\n",
1172 &dip);
1173 return;
1174 }
1175
1176 netdev_dbg(dev, "Updating neighbour with IP=%pI4h\n", &dip);
1177 neigh_event_send(n, NULL);
1178 neigh_release(n);
1179 }
1180
1181 #if IS_ENABLED(CONFIG_IPV6)
1182 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp,
1183 char *rauhtd_pl,
1184 int rec_index)
1185 {
1186 struct net_device *dev;
1187 struct neighbour *n;
1188 struct in6_addr dip;
1189 u16 rif;
1190
1191 mlxsw_reg_rauhtd_ent_ipv6_unpack(rauhtd_pl, rec_index, &rif,
1192 (char *) &dip);
1193
1194 if (!mlxsw_sp->router->rifs[rif]) {
1195 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
1196 return;
1197 }
1198
1199 dev = mlxsw_sp->router->rifs[rif]->dev;
1200 n = neigh_lookup(&nd_tbl, &dip, dev);
1201 if (!n) {
1202 netdev_err(dev, "Failed to find matching neighbour for IP=%pI6c\n",
1203 &dip);
1204 return;
1205 }
1206
1207 netdev_dbg(dev, "Updating neighbour with IP=%pI6c\n", &dip);
1208 neigh_event_send(n, NULL);
1209 neigh_release(n);
1210 }
1211 #else
1212 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp,
1213 char *rauhtd_pl,
1214 int rec_index)
1215 {
1216 }
1217 #endif
1218
1219 static void mlxsw_sp_router_neigh_rec_ipv4_process(struct mlxsw_sp *mlxsw_sp,
1220 char *rauhtd_pl,
1221 int rec_index)
1222 {
1223 u8 num_entries;
1224 int i;
1225
1226 num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
1227 rec_index);
1228 /* Hardware starts counting at 0, so add 1. */
1229 num_entries++;
1230
1231 /* Each record consists of several neighbour entries. */
1232 for (i = 0; i < num_entries; i++) {
1233 int ent_index;
1234
1235 ent_index = rec_index * MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC + i;
1236 mlxsw_sp_router_neigh_ent_ipv4_process(mlxsw_sp, rauhtd_pl,
1237 ent_index);
1238 }
1239
1240 }
1241
1242 static void mlxsw_sp_router_neigh_rec_ipv6_process(struct mlxsw_sp *mlxsw_sp,
1243 char *rauhtd_pl,
1244 int rec_index)
1245 {
1246 /* One record contains one entry. */
1247 mlxsw_sp_router_neigh_ent_ipv6_process(mlxsw_sp, rauhtd_pl,
1248 rec_index);
1249 }
1250
1251 static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp,
1252 char *rauhtd_pl, int rec_index)
1253 {
1254 switch (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, rec_index)) {
1255 case MLXSW_REG_RAUHTD_TYPE_IPV4:
1256 mlxsw_sp_router_neigh_rec_ipv4_process(mlxsw_sp, rauhtd_pl,
1257 rec_index);
1258 break;
1259 case MLXSW_REG_RAUHTD_TYPE_IPV6:
1260 mlxsw_sp_router_neigh_rec_ipv6_process(mlxsw_sp, rauhtd_pl,
1261 rec_index);
1262 break;
1263 }
1264 }
1265
1266 static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl)
1267 {
1268 u8 num_rec, last_rec_index, num_entries;
1269
1270 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
1271 last_rec_index = num_rec - 1;
1272
1273 if (num_rec < MLXSW_REG_RAUHTD_REC_MAX_NUM)
1274 return false;
1275 if (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, last_rec_index) ==
1276 MLXSW_REG_RAUHTD_TYPE_IPV6)
1277 return true;
1278
1279 num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
1280 last_rec_index);
1281 if (++num_entries == MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC)
1282 return true;
1283 return false;
1284 }
1285
1286 static int
1287 __mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp,
1288 char *rauhtd_pl,
1289 enum mlxsw_reg_rauhtd_type type)
1290 {
1291 int i, num_rec;
1292 int err;
1293
1294 /* Make sure the neighbour's netdev isn't removed in the
1295 * process.
1296 */
1297 rtnl_lock();
1298 do {
1299 mlxsw_reg_rauhtd_pack(rauhtd_pl, type);
1300 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(rauhtd),
1301 rauhtd_pl);
1302 if (err) {
1303 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to dump neighbour talbe\n");
1304 break;
1305 }
1306 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
1307 for (i = 0; i < num_rec; i++)
1308 mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl,
1309 i);
1310 } while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl));
1311 rtnl_unlock();
1312
1313 return err;
1314 }
1315
1316 static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp)
1317 {
1318 enum mlxsw_reg_rauhtd_type type;
1319 char *rauhtd_pl;
1320 int err;
1321
1322 rauhtd_pl = kmalloc(MLXSW_REG_RAUHTD_LEN, GFP_KERNEL);
1323 if (!rauhtd_pl)
1324 return -ENOMEM;
1325
1326 type = MLXSW_REG_RAUHTD_TYPE_IPV4;
1327 err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type);
1328 if (err)
1329 goto out;
1330
1331 type = MLXSW_REG_RAUHTD_TYPE_IPV6;
1332 err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type);
1333 out:
1334 kfree(rauhtd_pl);
1335 return err;
1336 }
1337
1338 static void mlxsw_sp_router_neighs_update_nh(struct mlxsw_sp *mlxsw_sp)
1339 {
1340 struct mlxsw_sp_neigh_entry *neigh_entry;
1341
1342 /* Take RTNL mutex here to prevent lists from changes */
1343 rtnl_lock();
1344 list_for_each_entry(neigh_entry, &mlxsw_sp->router->nexthop_neighs_list,
1345 nexthop_neighs_list_node)
1346 /* If this neigh have nexthops, make the kernel think this neigh
1347 * is active regardless of the traffic.
1348 */
1349 neigh_event_send(neigh_entry->key.n, NULL);
1350 rtnl_unlock();
1351 }
1352
1353 static void
1354 mlxsw_sp_router_neighs_update_work_schedule(struct mlxsw_sp *mlxsw_sp)
1355 {
1356 unsigned long interval = mlxsw_sp->router->neighs_update.interval;
1357
1358 mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw,
1359 msecs_to_jiffies(interval));
1360 }
1361
1362 static void mlxsw_sp_router_neighs_update_work(struct work_struct *work)
1363 {
1364 struct mlxsw_sp_router *router;
1365 int err;
1366
1367 router = container_of(work, struct mlxsw_sp_router,
1368 neighs_update.dw.work);
1369 err = mlxsw_sp_router_neighs_update_rauhtd(router->mlxsw_sp);
1370 if (err)
1371 dev_err(router->mlxsw_sp->bus_info->dev, "Could not update kernel for neigh activity");
1372
1373 mlxsw_sp_router_neighs_update_nh(router->mlxsw_sp);
1374
1375 mlxsw_sp_router_neighs_update_work_schedule(router->mlxsw_sp);
1376 }
1377
1378 static void mlxsw_sp_router_probe_unresolved_nexthops(struct work_struct *work)
1379 {
1380 struct mlxsw_sp_neigh_entry *neigh_entry;
1381 struct mlxsw_sp_router *router;
1382
1383 router = container_of(work, struct mlxsw_sp_router,
1384 nexthop_probe_dw.work);
1385 /* Iterate over nexthop neighbours, find those who are unresolved and
1386 * send arp on them. This solves the chicken-egg problem when
1387 * the nexthop wouldn't get offloaded until the neighbor is resolved
1388 * but it wouldn't get resolved ever in case traffic is flowing in HW
1389 * using different nexthop.
1390 *
1391 * Take RTNL mutex here to prevent lists from changes.
1392 */
1393 rtnl_lock();
1394 list_for_each_entry(neigh_entry, &router->nexthop_neighs_list,
1395 nexthop_neighs_list_node)
1396 if (!neigh_entry->connected)
1397 neigh_event_send(neigh_entry->key.n, NULL);
1398 rtnl_unlock();
1399
1400 mlxsw_core_schedule_dw(&router->nexthop_probe_dw,
1401 MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL);
1402 }
1403
1404 static void
1405 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
1406 struct mlxsw_sp_neigh_entry *neigh_entry,
1407 bool removing);
1408
1409 static enum mlxsw_reg_rauht_op mlxsw_sp_rauht_op(bool adding)
1410 {
1411 return adding ? MLXSW_REG_RAUHT_OP_WRITE_ADD :
1412 MLXSW_REG_RAUHT_OP_WRITE_DELETE;
1413 }
1414
1415 static void
1416 mlxsw_sp_router_neigh_entry_op4(struct mlxsw_sp *mlxsw_sp,
1417 struct mlxsw_sp_neigh_entry *neigh_entry,
1418 enum mlxsw_reg_rauht_op op)
1419 {
1420 struct neighbour *n = neigh_entry->key.n;
1421 u32 dip = ntohl(*((__be32 *) n->primary_key));
1422 char rauht_pl[MLXSW_REG_RAUHT_LEN];
1423
1424 mlxsw_reg_rauht_pack4(rauht_pl, op, neigh_entry->rif, neigh_entry->ha,
1425 dip);
1426 if (neigh_entry->counter_valid)
1427 mlxsw_reg_rauht_pack_counter(rauht_pl,
1428 neigh_entry->counter_index);
1429 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
1430 }
1431
1432 static void
1433 mlxsw_sp_router_neigh_entry_op6(struct mlxsw_sp *mlxsw_sp,
1434 struct mlxsw_sp_neigh_entry *neigh_entry,
1435 enum mlxsw_reg_rauht_op op)
1436 {
1437 struct neighbour *n = neigh_entry->key.n;
1438 char rauht_pl[MLXSW_REG_RAUHT_LEN];
1439 const char *dip = n->primary_key;
1440
1441 mlxsw_reg_rauht_pack6(rauht_pl, op, neigh_entry->rif, neigh_entry->ha,
1442 dip);
1443 if (neigh_entry->counter_valid)
1444 mlxsw_reg_rauht_pack_counter(rauht_pl,
1445 neigh_entry->counter_index);
1446 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
1447 }
1448
1449 bool mlxsw_sp_neigh_ipv6_ignore(struct mlxsw_sp_neigh_entry *neigh_entry)
1450 {
1451 struct neighbour *n = neigh_entry->key.n;
1452
1453 /* Packets with a link-local destination address are trapped
1454 * after LPM lookup and never reach the neighbour table, so
1455 * there is no need to program such neighbours to the device.
1456 */
1457 if (ipv6_addr_type((struct in6_addr *) &n->primary_key) &
1458 IPV6_ADDR_LINKLOCAL)
1459 return true;
1460 return false;
1461 }
1462
1463 static void
1464 mlxsw_sp_neigh_entry_update(struct mlxsw_sp *mlxsw_sp,
1465 struct mlxsw_sp_neigh_entry *neigh_entry,
1466 bool adding)
1467 {
1468 if (!adding && !neigh_entry->connected)
1469 return;
1470 neigh_entry->connected = adding;
1471 if (neigh_entry->key.n->tbl->family == AF_INET) {
1472 mlxsw_sp_router_neigh_entry_op4(mlxsw_sp, neigh_entry,
1473 mlxsw_sp_rauht_op(adding));
1474 } else if (neigh_entry->key.n->tbl->family == AF_INET6) {
1475 if (mlxsw_sp_neigh_ipv6_ignore(neigh_entry))
1476 return;
1477 mlxsw_sp_router_neigh_entry_op6(mlxsw_sp, neigh_entry,
1478 mlxsw_sp_rauht_op(adding));
1479 } else {
1480 WARN_ON_ONCE(1);
1481 }
1482 }
1483
1484 void
1485 mlxsw_sp_neigh_entry_counter_update(struct mlxsw_sp *mlxsw_sp,
1486 struct mlxsw_sp_neigh_entry *neigh_entry,
1487 bool adding)
1488 {
1489 if (adding)
1490 mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry);
1491 else
1492 mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry);
1493 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, true);
1494 }
1495
1496 struct mlxsw_sp_neigh_event_work {
1497 struct work_struct work;
1498 struct mlxsw_sp *mlxsw_sp;
1499 struct neighbour *n;
1500 };
1501
1502 static void mlxsw_sp_router_neigh_event_work(struct work_struct *work)
1503 {
1504 struct mlxsw_sp_neigh_event_work *neigh_work =
1505 container_of(work, struct mlxsw_sp_neigh_event_work, work);
1506 struct mlxsw_sp *mlxsw_sp = neigh_work->mlxsw_sp;
1507 struct mlxsw_sp_neigh_entry *neigh_entry;
1508 struct neighbour *n = neigh_work->n;
1509 unsigned char ha[ETH_ALEN];
1510 bool entry_connected;
1511 u8 nud_state, dead;
1512
1513 /* If these parameters are changed after we release the lock,
1514 * then we are guaranteed to receive another event letting us
1515 * know about it.
1516 */
1517 read_lock_bh(&n->lock);
1518 memcpy(ha, n->ha, ETH_ALEN);
1519 nud_state = n->nud_state;
1520 dead = n->dead;
1521 read_unlock_bh(&n->lock);
1522
1523 rtnl_lock();
1524 entry_connected = nud_state & NUD_VALID && !dead;
1525 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
1526 if (!entry_connected && !neigh_entry)
1527 goto out;
1528 if (!neigh_entry) {
1529 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
1530 if (IS_ERR(neigh_entry))
1531 goto out;
1532 }
1533
1534 memcpy(neigh_entry->ha, ha, ETH_ALEN);
1535 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, entry_connected);
1536 mlxsw_sp_nexthop_neigh_update(mlxsw_sp, neigh_entry, !entry_connected);
1537
1538 if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
1539 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
1540
1541 out:
1542 rtnl_unlock();
1543 neigh_release(n);
1544 kfree(neigh_work);
1545 }
1546
1547 int mlxsw_sp_router_netevent_event(struct notifier_block *unused,
1548 unsigned long event, void *ptr)
1549 {
1550 struct mlxsw_sp_neigh_event_work *neigh_work;
1551 struct mlxsw_sp_port *mlxsw_sp_port;
1552 struct mlxsw_sp *mlxsw_sp;
1553 unsigned long interval;
1554 struct neigh_parms *p;
1555 struct neighbour *n;
1556
1557 switch (event) {
1558 case NETEVENT_DELAY_PROBE_TIME_UPDATE:
1559 p = ptr;
1560
1561 /* We don't care about changes in the default table. */
1562 if (!p->dev || (p->tbl->family != AF_INET &&
1563 p->tbl->family != AF_INET6))
1564 return NOTIFY_DONE;
1565
1566 /* We are in atomic context and can't take RTNL mutex,
1567 * so use RCU variant to walk the device chain.
1568 */
1569 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(p->dev);
1570 if (!mlxsw_sp_port)
1571 return NOTIFY_DONE;
1572
1573 mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1574 interval = jiffies_to_msecs(NEIGH_VAR(p, DELAY_PROBE_TIME));
1575 mlxsw_sp->router->neighs_update.interval = interval;
1576
1577 mlxsw_sp_port_dev_put(mlxsw_sp_port);
1578 break;
1579 case NETEVENT_NEIGH_UPDATE:
1580 n = ptr;
1581
1582 if (n->tbl->family != AF_INET && n->tbl->family != AF_INET6)
1583 return NOTIFY_DONE;
1584
1585 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(n->dev);
1586 if (!mlxsw_sp_port)
1587 return NOTIFY_DONE;
1588
1589 neigh_work = kzalloc(sizeof(*neigh_work), GFP_ATOMIC);
1590 if (!neigh_work) {
1591 mlxsw_sp_port_dev_put(mlxsw_sp_port);
1592 return NOTIFY_BAD;
1593 }
1594
1595 INIT_WORK(&neigh_work->work, mlxsw_sp_router_neigh_event_work);
1596 neigh_work->mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1597 neigh_work->n = n;
1598
1599 /* Take a reference to ensure the neighbour won't be
1600 * destructed until we drop the reference in delayed
1601 * work.
1602 */
1603 neigh_clone(n);
1604 mlxsw_core_schedule_work(&neigh_work->work);
1605 mlxsw_sp_port_dev_put(mlxsw_sp_port);
1606 break;
1607 }
1608
1609 return NOTIFY_DONE;
1610 }
1611
1612 static int mlxsw_sp_neigh_init(struct mlxsw_sp *mlxsw_sp)
1613 {
1614 int err;
1615
1616 err = rhashtable_init(&mlxsw_sp->router->neigh_ht,
1617 &mlxsw_sp_neigh_ht_params);
1618 if (err)
1619 return err;
1620
1621 /* Initialize the polling interval according to the default
1622 * table.
1623 */
1624 mlxsw_sp_router_neighs_update_interval_init(mlxsw_sp);
1625
1626 /* Create the delayed works for the activity_update */
1627 INIT_DELAYED_WORK(&mlxsw_sp->router->neighs_update.dw,
1628 mlxsw_sp_router_neighs_update_work);
1629 INIT_DELAYED_WORK(&mlxsw_sp->router->nexthop_probe_dw,
1630 mlxsw_sp_router_probe_unresolved_nexthops);
1631 mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw, 0);
1632 mlxsw_core_schedule_dw(&mlxsw_sp->router->nexthop_probe_dw, 0);
1633 return 0;
1634 }
1635
1636 static void mlxsw_sp_neigh_fini(struct mlxsw_sp *mlxsw_sp)
1637 {
1638 cancel_delayed_work_sync(&mlxsw_sp->router->neighs_update.dw);
1639 cancel_delayed_work_sync(&mlxsw_sp->router->nexthop_probe_dw);
1640 rhashtable_destroy(&mlxsw_sp->router->neigh_ht);
1641 }
1642
1643 static void mlxsw_sp_neigh_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
1644 struct mlxsw_sp_rif *rif)
1645 {
1646 struct mlxsw_sp_neigh_entry *neigh_entry, *tmp;
1647
1648 list_for_each_entry_safe(neigh_entry, tmp, &rif->neigh_list,
1649 rif_list_node) {
1650 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, false);
1651 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
1652 }
1653 }
1654
1655 struct mlxsw_sp_nexthop_key {
1656 struct fib_nh *fib_nh;
1657 };
1658
1659 struct mlxsw_sp_nexthop {
1660 struct list_head neigh_list_node; /* member of neigh entry list */
1661 struct list_head rif_list_node;
1662 struct mlxsw_sp_nexthop_group *nh_grp; /* pointer back to the group
1663 * this belongs to
1664 */
1665 struct rhash_head ht_node;
1666 struct mlxsw_sp_nexthop_key key;
1667 unsigned char gw_addr[sizeof(struct in6_addr)];
1668 int ifindex;
1669 struct mlxsw_sp_rif *rif;
1670 u8 should_offload:1, /* set indicates this neigh is connected and
1671 * should be put to KVD linear area of this group.
1672 */
1673 offloaded:1, /* set in case the neigh is actually put into
1674 * KVD linear area of this group.
1675 */
1676 update:1; /* set indicates that MAC of this neigh should be
1677 * updated in HW
1678 */
1679 struct mlxsw_sp_neigh_entry *neigh_entry;
1680 };
1681
1682 struct mlxsw_sp_nexthop_group {
1683 void *priv;
1684 struct rhash_head ht_node;
1685 struct list_head fib_list; /* list of fib entries that use this group */
1686 struct neigh_table *neigh_tbl;
1687 u8 adj_index_valid:1,
1688 gateway:1; /* routes using the group use a gateway */
1689 u32 adj_index;
1690 u16 ecmp_size;
1691 u16 count;
1692 struct mlxsw_sp_nexthop nexthops[0];
1693 #define nh_rif nexthops[0].rif
1694 };
1695
1696 static struct fib_info *
1697 mlxsw_sp_nexthop4_group_fi(const struct mlxsw_sp_nexthop_group *nh_grp)
1698 {
1699 return nh_grp->priv;
1700 }
1701
1702 struct mlxsw_sp_nexthop_group_cmp_arg {
1703 enum mlxsw_sp_l3proto proto;
1704 union {
1705 struct fib_info *fi;
1706 struct mlxsw_sp_fib6_entry *fib6_entry;
1707 };
1708 };
1709
1710 static bool
1711 mlxsw_sp_nexthop6_group_has_nexthop(const struct mlxsw_sp_nexthop_group *nh_grp,
1712 const struct in6_addr *gw, int ifindex)
1713 {
1714 int i;
1715
1716 for (i = 0; i < nh_grp->count; i++) {
1717 const struct mlxsw_sp_nexthop *nh;
1718
1719 nh = &nh_grp->nexthops[i];
1720 if (nh->ifindex == ifindex &&
1721 ipv6_addr_equal(gw, (struct in6_addr *) nh->gw_addr))
1722 return true;
1723 }
1724
1725 return false;
1726 }
1727
1728 static bool
1729 mlxsw_sp_nexthop6_group_cmp(const struct mlxsw_sp_nexthop_group *nh_grp,
1730 const struct mlxsw_sp_fib6_entry *fib6_entry)
1731 {
1732 struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
1733
1734 if (nh_grp->count != fib6_entry->nrt6)
1735 return false;
1736
1737 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
1738 struct in6_addr *gw;
1739 int ifindex;
1740
1741 ifindex = mlxsw_sp_rt6->rt->dst.dev->ifindex;
1742 gw = &mlxsw_sp_rt6->rt->rt6i_gateway;
1743 if (!mlxsw_sp_nexthop6_group_has_nexthop(nh_grp, gw, ifindex))
1744 return false;
1745 }
1746
1747 return true;
1748 }
1749
1750 static int
1751 mlxsw_sp_nexthop_group_cmp(struct rhashtable_compare_arg *arg, const void *ptr)
1752 {
1753 const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = arg->key;
1754 const struct mlxsw_sp_nexthop_group *nh_grp = ptr;
1755
1756 switch (cmp_arg->proto) {
1757 case MLXSW_SP_L3_PROTO_IPV4:
1758 return cmp_arg->fi != mlxsw_sp_nexthop4_group_fi(nh_grp);
1759 case MLXSW_SP_L3_PROTO_IPV6:
1760 return !mlxsw_sp_nexthop6_group_cmp(nh_grp,
1761 cmp_arg->fib6_entry);
1762 default:
1763 WARN_ON(1);
1764 return 1;
1765 }
1766 }
1767
1768 static int
1769 mlxsw_sp_nexthop_group_type(const struct mlxsw_sp_nexthop_group *nh_grp)
1770 {
1771 return nh_grp->neigh_tbl->family;
1772 }
1773
1774 static u32 mlxsw_sp_nexthop_group_hash_obj(const void *data, u32 len, u32 seed)
1775 {
1776 const struct mlxsw_sp_nexthop_group *nh_grp = data;
1777 const struct mlxsw_sp_nexthop *nh;
1778 struct fib_info *fi;
1779 unsigned int val;
1780 int i;
1781
1782 switch (mlxsw_sp_nexthop_group_type(nh_grp)) {
1783 case AF_INET:
1784 fi = mlxsw_sp_nexthop4_group_fi(nh_grp);
1785 return jhash(&fi, sizeof(fi), seed);
1786 case AF_INET6:
1787 val = nh_grp->count;
1788 for (i = 0; i < nh_grp->count; i++) {
1789 nh = &nh_grp->nexthops[i];
1790 val ^= nh->ifindex;
1791 }
1792 return jhash(&val, sizeof(val), seed);
1793 default:
1794 WARN_ON(1);
1795 return 0;
1796 }
1797 }
1798
1799 static u32
1800 mlxsw_sp_nexthop6_group_hash(struct mlxsw_sp_fib6_entry *fib6_entry, u32 seed)
1801 {
1802 unsigned int val = fib6_entry->nrt6;
1803 struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
1804 struct net_device *dev;
1805
1806 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
1807 dev = mlxsw_sp_rt6->rt->dst.dev;
1808 val ^= dev->ifindex;
1809 }
1810
1811 return jhash(&val, sizeof(val), seed);
1812 }
1813
1814 static u32
1815 mlxsw_sp_nexthop_group_hash(const void *data, u32 len, u32 seed)
1816 {
1817 const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = data;
1818
1819 switch (cmp_arg->proto) {
1820 case MLXSW_SP_L3_PROTO_IPV4:
1821 return jhash(&cmp_arg->fi, sizeof(cmp_arg->fi), seed);
1822 case MLXSW_SP_L3_PROTO_IPV6:
1823 return mlxsw_sp_nexthop6_group_hash(cmp_arg->fib6_entry, seed);
1824 default:
1825 WARN_ON(1);
1826 return 0;
1827 }
1828 }
1829
1830 static const struct rhashtable_params mlxsw_sp_nexthop_group_ht_params = {
1831 .head_offset = offsetof(struct mlxsw_sp_nexthop_group, ht_node),
1832 .hashfn = mlxsw_sp_nexthop_group_hash,
1833 .obj_hashfn = mlxsw_sp_nexthop_group_hash_obj,
1834 .obj_cmpfn = mlxsw_sp_nexthop_group_cmp,
1835 };
1836
1837 static int mlxsw_sp_nexthop_group_insert(struct mlxsw_sp *mlxsw_sp,
1838 struct mlxsw_sp_nexthop_group *nh_grp)
1839 {
1840 if (mlxsw_sp_nexthop_group_type(nh_grp) == AF_INET6 &&
1841 !nh_grp->gateway)
1842 return 0;
1843
1844 return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_group_ht,
1845 &nh_grp->ht_node,
1846 mlxsw_sp_nexthop_group_ht_params);
1847 }
1848
1849 static void mlxsw_sp_nexthop_group_remove(struct mlxsw_sp *mlxsw_sp,
1850 struct mlxsw_sp_nexthop_group *nh_grp)
1851 {
1852 if (mlxsw_sp_nexthop_group_type(nh_grp) == AF_INET6 &&
1853 !nh_grp->gateway)
1854 return;
1855
1856 rhashtable_remove_fast(&mlxsw_sp->router->nexthop_group_ht,
1857 &nh_grp->ht_node,
1858 mlxsw_sp_nexthop_group_ht_params);
1859 }
1860
1861 static struct mlxsw_sp_nexthop_group *
1862 mlxsw_sp_nexthop4_group_lookup(struct mlxsw_sp *mlxsw_sp,
1863 struct fib_info *fi)
1864 {
1865 struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg;
1866
1867 cmp_arg.proto = MLXSW_SP_L3_PROTO_IPV4;
1868 cmp_arg.fi = fi;
1869 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht,
1870 &cmp_arg,
1871 mlxsw_sp_nexthop_group_ht_params);
1872 }
1873
1874 static struct mlxsw_sp_nexthop_group *
1875 mlxsw_sp_nexthop6_group_lookup(struct mlxsw_sp *mlxsw_sp,
1876 struct mlxsw_sp_fib6_entry *fib6_entry)
1877 {
1878 struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg;
1879
1880 cmp_arg.proto = MLXSW_SP_L3_PROTO_IPV6;
1881 cmp_arg.fib6_entry = fib6_entry;
1882 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht,
1883 &cmp_arg,
1884 mlxsw_sp_nexthop_group_ht_params);
1885 }
1886
1887 static const struct rhashtable_params mlxsw_sp_nexthop_ht_params = {
1888 .key_offset = offsetof(struct mlxsw_sp_nexthop, key),
1889 .head_offset = offsetof(struct mlxsw_sp_nexthop, ht_node),
1890 .key_len = sizeof(struct mlxsw_sp_nexthop_key),
1891 };
1892
1893 static int mlxsw_sp_nexthop_insert(struct mlxsw_sp *mlxsw_sp,
1894 struct mlxsw_sp_nexthop *nh)
1895 {
1896 return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_ht,
1897 &nh->ht_node, mlxsw_sp_nexthop_ht_params);
1898 }
1899
1900 static void mlxsw_sp_nexthop_remove(struct mlxsw_sp *mlxsw_sp,
1901 struct mlxsw_sp_nexthop *nh)
1902 {
1903 rhashtable_remove_fast(&mlxsw_sp->router->nexthop_ht, &nh->ht_node,
1904 mlxsw_sp_nexthop_ht_params);
1905 }
1906
1907 static struct mlxsw_sp_nexthop *
1908 mlxsw_sp_nexthop_lookup(struct mlxsw_sp *mlxsw_sp,
1909 struct mlxsw_sp_nexthop_key key)
1910 {
1911 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_ht, &key,
1912 mlxsw_sp_nexthop_ht_params);
1913 }
1914
1915 static int mlxsw_sp_adj_index_mass_update_vr(struct mlxsw_sp *mlxsw_sp,
1916 const struct mlxsw_sp_fib *fib,
1917 u32 adj_index, u16 ecmp_size,
1918 u32 new_adj_index,
1919 u16 new_ecmp_size)
1920 {
1921 char raleu_pl[MLXSW_REG_RALEU_LEN];
1922
1923 mlxsw_reg_raleu_pack(raleu_pl,
1924 (enum mlxsw_reg_ralxx_protocol) fib->proto,
1925 fib->vr->id, adj_index, ecmp_size, new_adj_index,
1926 new_ecmp_size);
1927 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raleu), raleu_pl);
1928 }
1929
1930 static int mlxsw_sp_adj_index_mass_update(struct mlxsw_sp *mlxsw_sp,
1931 struct mlxsw_sp_nexthop_group *nh_grp,
1932 u32 old_adj_index, u16 old_ecmp_size)
1933 {
1934 struct mlxsw_sp_fib_entry *fib_entry;
1935 struct mlxsw_sp_fib *fib = NULL;
1936 int err;
1937
1938 list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
1939 if (fib == fib_entry->fib_node->fib)
1940 continue;
1941 fib = fib_entry->fib_node->fib;
1942 err = mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, fib,
1943 old_adj_index,
1944 old_ecmp_size,
1945 nh_grp->adj_index,
1946 nh_grp->ecmp_size);
1947 if (err)
1948 return err;
1949 }
1950 return 0;
1951 }
1952
1953 static int mlxsw_sp_nexthop_mac_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index,
1954 struct mlxsw_sp_nexthop *nh)
1955 {
1956 struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
1957 char ratr_pl[MLXSW_REG_RATR_LEN];
1958
1959 mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY,
1960 true, MLXSW_REG_RATR_TYPE_ETHERNET,
1961 adj_index, neigh_entry->rif);
1962 mlxsw_reg_ratr_eth_entry_pack(ratr_pl, neigh_entry->ha);
1963 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl);
1964 }
1965
1966 static int
1967 mlxsw_sp_nexthop_group_mac_update(struct mlxsw_sp *mlxsw_sp,
1968 struct mlxsw_sp_nexthop_group *nh_grp,
1969 bool reallocate)
1970 {
1971 u32 adj_index = nh_grp->adj_index; /* base */
1972 struct mlxsw_sp_nexthop *nh;
1973 int i;
1974 int err;
1975
1976 for (i = 0; i < nh_grp->count; i++) {
1977 nh = &nh_grp->nexthops[i];
1978
1979 if (!nh->should_offload) {
1980 nh->offloaded = 0;
1981 continue;
1982 }
1983
1984 if (nh->update || reallocate) {
1985 err = mlxsw_sp_nexthop_mac_update(mlxsw_sp,
1986 adj_index, nh);
1987 if (err)
1988 return err;
1989 nh->update = 0;
1990 nh->offloaded = 1;
1991 }
1992 adj_index++;
1993 }
1994 return 0;
1995 }
1996
1997 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
1998 struct mlxsw_sp_fib_entry *fib_entry);
1999
2000 static bool
2001 mlxsw_sp_fib_node_entry_is_first(const struct mlxsw_sp_fib_node *fib_node,
2002 const struct mlxsw_sp_fib_entry *fib_entry);
2003
2004 static int
2005 mlxsw_sp_nexthop_fib_entries_update(struct mlxsw_sp *mlxsw_sp,
2006 struct mlxsw_sp_nexthop_group *nh_grp)
2007 {
2008 struct mlxsw_sp_fib_entry *fib_entry;
2009 int err;
2010
2011 list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
2012 if (!mlxsw_sp_fib_node_entry_is_first(fib_entry->fib_node,
2013 fib_entry))
2014 continue;
2015 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
2016 if (err)
2017 return err;
2018 }
2019 return 0;
2020 }
2021
2022 static void
2023 mlxsw_sp_fib_entry_offload_refresh(struct mlxsw_sp_fib_entry *fib_entry,
2024 enum mlxsw_reg_ralue_op op, int err);
2025
2026 static void
2027 mlxsw_sp_nexthop_fib_entries_refresh(struct mlxsw_sp_nexthop_group *nh_grp)
2028 {
2029 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_WRITE;
2030 struct mlxsw_sp_fib_entry *fib_entry;
2031
2032 list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
2033 if (!mlxsw_sp_fib_node_entry_is_first(fib_entry->fib_node,
2034 fib_entry))
2035 continue;
2036 mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, 0);
2037 }
2038 }
2039
2040 static void
2041 mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp,
2042 struct mlxsw_sp_nexthop_group *nh_grp)
2043 {
2044 struct mlxsw_sp_nexthop *nh;
2045 bool offload_change = false;
2046 u32 adj_index;
2047 u16 ecmp_size = 0;
2048 bool old_adj_index_valid;
2049 u32 old_adj_index;
2050 u16 old_ecmp_size;
2051 int i;
2052 int err;
2053
2054 if (!nh_grp->gateway) {
2055 mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
2056 return;
2057 }
2058
2059 for (i = 0; i < nh_grp->count; i++) {
2060 nh = &nh_grp->nexthops[i];
2061
2062 if (nh->should_offload != nh->offloaded) {
2063 offload_change = true;
2064 if (nh->should_offload)
2065 nh->update = 1;
2066 }
2067 if (nh->should_offload)
2068 ecmp_size++;
2069 }
2070 if (!offload_change) {
2071 /* Nothing was added or removed, so no need to reallocate. Just
2072 * update MAC on existing adjacency indexes.
2073 */
2074 err = mlxsw_sp_nexthop_group_mac_update(mlxsw_sp, nh_grp,
2075 false);
2076 if (err) {
2077 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
2078 goto set_trap;
2079 }
2080 return;
2081 }
2082 if (!ecmp_size)
2083 /* No neigh of this group is connected so we just set
2084 * the trap and let everthing flow through kernel.
2085 */
2086 goto set_trap;
2087
2088 err = mlxsw_sp_kvdl_alloc(mlxsw_sp, ecmp_size, &adj_index);
2089 if (err) {
2090 /* We ran out of KVD linear space, just set the
2091 * trap and let everything flow through kernel.
2092 */
2093 dev_warn(mlxsw_sp->bus_info->dev, "Failed to allocate KVD linear area for nexthop group.\n");
2094 goto set_trap;
2095 }
2096 old_adj_index_valid = nh_grp->adj_index_valid;
2097 old_adj_index = nh_grp->adj_index;
2098 old_ecmp_size = nh_grp->ecmp_size;
2099 nh_grp->adj_index_valid = 1;
2100 nh_grp->adj_index = adj_index;
2101 nh_grp->ecmp_size = ecmp_size;
2102 err = mlxsw_sp_nexthop_group_mac_update(mlxsw_sp, nh_grp, true);
2103 if (err) {
2104 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
2105 goto set_trap;
2106 }
2107
2108 if (!old_adj_index_valid) {
2109 /* The trap was set for fib entries, so we have to call
2110 * fib entry update to unset it and use adjacency index.
2111 */
2112 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
2113 if (err) {
2114 dev_warn(mlxsw_sp->bus_info->dev, "Failed to add adjacency index to fib entries.\n");
2115 goto set_trap;
2116 }
2117 return;
2118 }
2119
2120 err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, nh_grp,
2121 old_adj_index, old_ecmp_size);
2122 mlxsw_sp_kvdl_free(mlxsw_sp, old_adj_index);
2123 if (err) {
2124 dev_warn(mlxsw_sp->bus_info->dev, "Failed to mass-update adjacency index for nexthop group.\n");
2125 goto set_trap;
2126 }
2127
2128 /* Offload state within the group changed, so update the flags. */
2129 mlxsw_sp_nexthop_fib_entries_refresh(nh_grp);
2130
2131 return;
2132
2133 set_trap:
2134 old_adj_index_valid = nh_grp->adj_index_valid;
2135 nh_grp->adj_index_valid = 0;
2136 for (i = 0; i < nh_grp->count; i++) {
2137 nh = &nh_grp->nexthops[i];
2138 nh->offloaded = 0;
2139 }
2140 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
2141 if (err)
2142 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set traps for fib entries.\n");
2143 if (old_adj_index_valid)
2144 mlxsw_sp_kvdl_free(mlxsw_sp, nh_grp->adj_index);
2145 }
2146
2147 static void __mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp_nexthop *nh,
2148 bool removing)
2149 {
2150 if (!removing)
2151 nh->should_offload = 1;
2152 else if (nh->offloaded)
2153 nh->should_offload = 0;
2154 nh->update = 1;
2155 }
2156
2157 static void
2158 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
2159 struct mlxsw_sp_neigh_entry *neigh_entry,
2160 bool removing)
2161 {
2162 struct mlxsw_sp_nexthop *nh;
2163
2164 list_for_each_entry(nh, &neigh_entry->nexthop_list,
2165 neigh_list_node) {
2166 __mlxsw_sp_nexthop_neigh_update(nh, removing);
2167 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
2168 }
2169 }
2170
2171 static void mlxsw_sp_nexthop_rif_init(struct mlxsw_sp_nexthop *nh,
2172 struct mlxsw_sp_rif *rif)
2173 {
2174 if (nh->rif)
2175 return;
2176
2177 nh->rif = rif;
2178 list_add(&nh->rif_list_node, &rif->nexthop_list);
2179 }
2180
2181 static void mlxsw_sp_nexthop_rif_fini(struct mlxsw_sp_nexthop *nh)
2182 {
2183 if (!nh->rif)
2184 return;
2185
2186 list_del(&nh->rif_list_node);
2187 nh->rif = NULL;
2188 }
2189
2190 static int mlxsw_sp_nexthop_neigh_init(struct mlxsw_sp *mlxsw_sp,
2191 struct mlxsw_sp_nexthop *nh)
2192 {
2193 struct mlxsw_sp_neigh_entry *neigh_entry;
2194 struct neighbour *n;
2195 u8 nud_state, dead;
2196 int err;
2197
2198 if (!nh->nh_grp->gateway || nh->neigh_entry)
2199 return 0;
2200
2201 /* Take a reference of neigh here ensuring that neigh would
2202 * not be destructed before the nexthop entry is finished.
2203 * The reference is taken either in neigh_lookup() or
2204 * in neigh_create() in case n is not found.
2205 */
2206 n = neigh_lookup(nh->nh_grp->neigh_tbl, &nh->gw_addr, nh->rif->dev);
2207 if (!n) {
2208 n = neigh_create(nh->nh_grp->neigh_tbl, &nh->gw_addr,
2209 nh->rif->dev);
2210 if (IS_ERR(n))
2211 return PTR_ERR(n);
2212 neigh_event_send(n, NULL);
2213 }
2214 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
2215 if (!neigh_entry) {
2216 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
2217 if (IS_ERR(neigh_entry)) {
2218 err = -EINVAL;
2219 goto err_neigh_entry_create;
2220 }
2221 }
2222
2223 /* If that is the first nexthop connected to that neigh, add to
2224 * nexthop_neighs_list
2225 */
2226 if (list_empty(&neigh_entry->nexthop_list))
2227 list_add_tail(&neigh_entry->nexthop_neighs_list_node,
2228 &mlxsw_sp->router->nexthop_neighs_list);
2229
2230 nh->neigh_entry = neigh_entry;
2231 list_add_tail(&nh->neigh_list_node, &neigh_entry->nexthop_list);
2232 read_lock_bh(&n->lock);
2233 nud_state = n->nud_state;
2234 dead = n->dead;
2235 read_unlock_bh(&n->lock);
2236 __mlxsw_sp_nexthop_neigh_update(nh, !(nud_state & NUD_VALID && !dead));
2237
2238 return 0;
2239
2240 err_neigh_entry_create:
2241 neigh_release(n);
2242 return err;
2243 }
2244
2245 static void mlxsw_sp_nexthop_neigh_fini(struct mlxsw_sp *mlxsw_sp,
2246 struct mlxsw_sp_nexthop *nh)
2247 {
2248 struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
2249 struct neighbour *n;
2250
2251 if (!neigh_entry)
2252 return;
2253 n = neigh_entry->key.n;
2254
2255 __mlxsw_sp_nexthop_neigh_update(nh, true);
2256 list_del(&nh->neigh_list_node);
2257 nh->neigh_entry = NULL;
2258
2259 /* If that is the last nexthop connected to that neigh, remove from
2260 * nexthop_neighs_list
2261 */
2262 if (list_empty(&neigh_entry->nexthop_list))
2263 list_del(&neigh_entry->nexthop_neighs_list_node);
2264
2265 if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
2266 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
2267
2268 neigh_release(n);
2269 }
2270
2271 static bool mlxsw_sp_netdev_ipip_type(const struct mlxsw_sp *mlxsw_sp,
2272 const struct net_device *dev,
2273 enum mlxsw_sp_ipip_type *p_type)
2274 {
2275 struct mlxsw_sp_router *router = mlxsw_sp->router;
2276 const struct mlxsw_sp_ipip_ops *ipip_ops;
2277 enum mlxsw_sp_ipip_type ipipt;
2278
2279 for (ipipt = 0; ipipt < MLXSW_SP_IPIP_TYPE_MAX; ++ipipt) {
2280 ipip_ops = router->ipip_ops_arr[ipipt];
2281 if (dev->type == ipip_ops->dev_type) {
2282 if (p_type)
2283 *p_type = ipipt;
2284 return true;
2285 }
2286 }
2287 return false;
2288 }
2289
2290 static int mlxsw_sp_nexthop4_init(struct mlxsw_sp *mlxsw_sp,
2291 struct mlxsw_sp_nexthop_group *nh_grp,
2292 struct mlxsw_sp_nexthop *nh,
2293 struct fib_nh *fib_nh)
2294 {
2295 struct net_device *dev = fib_nh->nh_dev;
2296 struct in_device *in_dev;
2297 struct mlxsw_sp_rif *rif;
2298 int err;
2299
2300 nh->nh_grp = nh_grp;
2301 nh->key.fib_nh = fib_nh;
2302 memcpy(&nh->gw_addr, &fib_nh->nh_gw, sizeof(fib_nh->nh_gw));
2303 err = mlxsw_sp_nexthop_insert(mlxsw_sp, nh);
2304 if (err)
2305 return err;
2306
2307 if (!dev)
2308 return 0;
2309
2310 in_dev = __in_dev_get_rtnl(dev);
2311 if (in_dev && IN_DEV_IGNORE_ROUTES_WITH_LINKDOWN(in_dev) &&
2312 fib_nh->nh_flags & RTNH_F_LINKDOWN)
2313 return 0;
2314
2315 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
2316 if (!rif)
2317 return 0;
2318 mlxsw_sp_nexthop_rif_init(nh, rif);
2319
2320 err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
2321 if (err)
2322 goto err_nexthop_neigh_init;
2323
2324 return 0;
2325
2326 err_nexthop_neigh_init:
2327 mlxsw_sp_nexthop_rif_fini(nh);
2328 mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
2329 return err;
2330 }
2331
2332 static void mlxsw_sp_nexthop4_fini(struct mlxsw_sp *mlxsw_sp,
2333 struct mlxsw_sp_nexthop *nh)
2334 {
2335 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
2336 mlxsw_sp_nexthop_rif_fini(nh);
2337 mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
2338 }
2339
2340 static void mlxsw_sp_nexthop4_event(struct mlxsw_sp *mlxsw_sp,
2341 unsigned long event, struct fib_nh *fib_nh)
2342 {
2343 struct mlxsw_sp_nexthop_key key;
2344 struct mlxsw_sp_nexthop *nh;
2345 struct mlxsw_sp_rif *rif;
2346
2347 if (mlxsw_sp->router->aborted)
2348 return;
2349
2350 key.fib_nh = fib_nh;
2351 nh = mlxsw_sp_nexthop_lookup(mlxsw_sp, key);
2352 if (WARN_ON_ONCE(!nh))
2353 return;
2354
2355 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, fib_nh->nh_dev);
2356 if (!rif)
2357 return;
2358
2359 switch (event) {
2360 case FIB_EVENT_NH_ADD:
2361 mlxsw_sp_nexthop_rif_init(nh, rif);
2362 mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
2363 break;
2364 case FIB_EVENT_NH_DEL:
2365 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
2366 mlxsw_sp_nexthop_rif_fini(nh);
2367 break;
2368 }
2369
2370 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
2371 }
2372
2373 static void mlxsw_sp_nexthop_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
2374 struct mlxsw_sp_rif *rif)
2375 {
2376 struct mlxsw_sp_nexthop *nh, *tmp;
2377
2378 list_for_each_entry_safe(nh, tmp, &rif->nexthop_list, rif_list_node) {
2379 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
2380 mlxsw_sp_nexthop_rif_fini(nh);
2381 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
2382 }
2383 }
2384
2385 static bool mlxsw_sp_fi_is_gateway(const struct mlxsw_sp *mlxsw_sp,
2386 const struct fib_info *fi)
2387 {
2388 return fi->fib_nh->nh_scope == RT_SCOPE_LINK;
2389 }
2390
2391 static struct mlxsw_sp_nexthop_group *
2392 mlxsw_sp_nexthop4_group_create(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi)
2393 {
2394 struct mlxsw_sp_nexthop_group *nh_grp;
2395 struct mlxsw_sp_nexthop *nh;
2396 struct fib_nh *fib_nh;
2397 size_t alloc_size;
2398 int i;
2399 int err;
2400
2401 alloc_size = sizeof(*nh_grp) +
2402 fi->fib_nhs * sizeof(struct mlxsw_sp_nexthop);
2403 nh_grp = kzalloc(alloc_size, GFP_KERNEL);
2404 if (!nh_grp)
2405 return ERR_PTR(-ENOMEM);
2406 nh_grp->priv = fi;
2407 INIT_LIST_HEAD(&nh_grp->fib_list);
2408 nh_grp->neigh_tbl = &arp_tbl;
2409
2410 nh_grp->gateway = mlxsw_sp_fi_is_gateway(mlxsw_sp, fi);
2411 nh_grp->count = fi->fib_nhs;
2412 fib_info_hold(fi);
2413 for (i = 0; i < nh_grp->count; i++) {
2414 nh = &nh_grp->nexthops[i];
2415 fib_nh = &fi->fib_nh[i];
2416 err = mlxsw_sp_nexthop4_init(mlxsw_sp, nh_grp, nh, fib_nh);
2417 if (err)
2418 goto err_nexthop4_init;
2419 }
2420 err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
2421 if (err)
2422 goto err_nexthop_group_insert;
2423 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
2424 return nh_grp;
2425
2426 err_nexthop_group_insert:
2427 err_nexthop4_init:
2428 for (i--; i >= 0; i--) {
2429 nh = &nh_grp->nexthops[i];
2430 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh);
2431 }
2432 fib_info_put(fi);
2433 kfree(nh_grp);
2434 return ERR_PTR(err);
2435 }
2436
2437 static void
2438 mlxsw_sp_nexthop4_group_destroy(struct mlxsw_sp *mlxsw_sp,
2439 struct mlxsw_sp_nexthop_group *nh_grp)
2440 {
2441 struct mlxsw_sp_nexthop *nh;
2442 int i;
2443
2444 mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
2445 for (i = 0; i < nh_grp->count; i++) {
2446 nh = &nh_grp->nexthops[i];
2447 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh);
2448 }
2449 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
2450 WARN_ON_ONCE(nh_grp->adj_index_valid);
2451 fib_info_put(mlxsw_sp_nexthop4_group_fi(nh_grp));
2452 kfree(nh_grp);
2453 }
2454
2455 static int mlxsw_sp_nexthop4_group_get(struct mlxsw_sp *mlxsw_sp,
2456 struct mlxsw_sp_fib_entry *fib_entry,
2457 struct fib_info *fi)
2458 {
2459 struct mlxsw_sp_nexthop_group *nh_grp;
2460
2461 nh_grp = mlxsw_sp_nexthop4_group_lookup(mlxsw_sp, fi);
2462 if (!nh_grp) {
2463 nh_grp = mlxsw_sp_nexthop4_group_create(mlxsw_sp, fi);
2464 if (IS_ERR(nh_grp))
2465 return PTR_ERR(nh_grp);
2466 }
2467 list_add_tail(&fib_entry->nexthop_group_node, &nh_grp->fib_list);
2468 fib_entry->nh_group = nh_grp;
2469 return 0;
2470 }
2471
2472 static void mlxsw_sp_nexthop4_group_put(struct mlxsw_sp *mlxsw_sp,
2473 struct mlxsw_sp_fib_entry *fib_entry)
2474 {
2475 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
2476
2477 list_del(&fib_entry->nexthop_group_node);
2478 if (!list_empty(&nh_grp->fib_list))
2479 return;
2480 mlxsw_sp_nexthop4_group_destroy(mlxsw_sp, nh_grp);
2481 }
2482
2483 static bool
2484 mlxsw_sp_fib4_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry)
2485 {
2486 struct mlxsw_sp_fib4_entry *fib4_entry;
2487
2488 fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry,
2489 common);
2490 return !fib4_entry->tos;
2491 }
2492
2493 static bool
2494 mlxsw_sp_fib_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry)
2495 {
2496 struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group;
2497
2498 switch (fib_entry->fib_node->fib->proto) {
2499 case MLXSW_SP_L3_PROTO_IPV4:
2500 if (!mlxsw_sp_fib4_entry_should_offload(fib_entry))
2501 return false;
2502 break;
2503 case MLXSW_SP_L3_PROTO_IPV6:
2504 break;
2505 }
2506
2507 switch (fib_entry->type) {
2508 case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
2509 return !!nh_group->adj_index_valid;
2510 case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
2511 return !!nh_group->nh_rif;
2512 default:
2513 return false;
2514 }
2515 }
2516
2517 static struct mlxsw_sp_nexthop *
2518 mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp,
2519 const struct mlxsw_sp_rt6 *mlxsw_sp_rt6)
2520 {
2521 int i;
2522
2523 for (i = 0; i < nh_grp->count; i++) {
2524 struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i];
2525 struct rt6_info *rt = mlxsw_sp_rt6->rt;
2526
2527 if (nh->rif && nh->rif->dev == rt->dst.dev &&
2528 ipv6_addr_equal((const struct in6_addr *) &nh->gw_addr,
2529 &rt->rt6i_gateway))
2530 return nh;
2531 continue;
2532 }
2533
2534 return NULL;
2535 }
2536
2537 static void
2538 mlxsw_sp_fib4_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry)
2539 {
2540 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
2541 int i;
2542
2543 if (fib_entry->type == MLXSW_SP_FIB_ENTRY_TYPE_LOCAL) {
2544 nh_grp->nexthops->key.fib_nh->nh_flags |= RTNH_F_OFFLOAD;
2545 return;
2546 }
2547
2548 for (i = 0; i < nh_grp->count; i++) {
2549 struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i];
2550
2551 if (nh->offloaded)
2552 nh->key.fib_nh->nh_flags |= RTNH_F_OFFLOAD;
2553 else
2554 nh->key.fib_nh->nh_flags &= ~RTNH_F_OFFLOAD;
2555 }
2556 }
2557
2558 static void
2559 mlxsw_sp_fib4_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry)
2560 {
2561 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
2562 int i;
2563
2564 for (i = 0; i < nh_grp->count; i++) {
2565 struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i];
2566
2567 nh->key.fib_nh->nh_flags &= ~RTNH_F_OFFLOAD;
2568 }
2569 }
2570
2571 static void
2572 mlxsw_sp_fib6_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry)
2573 {
2574 struct mlxsw_sp_fib6_entry *fib6_entry;
2575 struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
2576
2577 fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry,
2578 common);
2579
2580 if (fib_entry->type == MLXSW_SP_FIB_ENTRY_TYPE_LOCAL) {
2581 list_first_entry(&fib6_entry->rt6_list, struct mlxsw_sp_rt6,
2582 list)->rt->rt6i_nh_flags |= RTNH_F_OFFLOAD;
2583 return;
2584 }
2585
2586 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
2587 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
2588 struct mlxsw_sp_nexthop *nh;
2589
2590 nh = mlxsw_sp_rt6_nexthop(nh_grp, mlxsw_sp_rt6);
2591 if (nh && nh->offloaded)
2592 mlxsw_sp_rt6->rt->rt6i_nh_flags |= RTNH_F_OFFLOAD;
2593 else
2594 mlxsw_sp_rt6->rt->rt6i_nh_flags &= ~RTNH_F_OFFLOAD;
2595 }
2596 }
2597
2598 static void
2599 mlxsw_sp_fib6_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry)
2600 {
2601 struct mlxsw_sp_fib6_entry *fib6_entry;
2602 struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
2603
2604 fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry,
2605 common);
2606 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
2607 struct rt6_info *rt = mlxsw_sp_rt6->rt;
2608
2609 rt->rt6i_nh_flags &= ~RTNH_F_OFFLOAD;
2610 }
2611 }
2612
2613 static void mlxsw_sp_fib_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry)
2614 {
2615 switch (fib_entry->fib_node->fib->proto) {
2616 case MLXSW_SP_L3_PROTO_IPV4:
2617 mlxsw_sp_fib4_entry_offload_set(fib_entry);
2618 break;
2619 case MLXSW_SP_L3_PROTO_IPV6:
2620 mlxsw_sp_fib6_entry_offload_set(fib_entry);
2621 break;
2622 }
2623 }
2624
2625 static void
2626 mlxsw_sp_fib_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry)
2627 {
2628 switch (fib_entry->fib_node->fib->proto) {
2629 case MLXSW_SP_L3_PROTO_IPV4:
2630 mlxsw_sp_fib4_entry_offload_unset(fib_entry);
2631 break;
2632 case MLXSW_SP_L3_PROTO_IPV6:
2633 mlxsw_sp_fib6_entry_offload_unset(fib_entry);
2634 break;
2635 }
2636 }
2637
2638 static void
2639 mlxsw_sp_fib_entry_offload_refresh(struct mlxsw_sp_fib_entry *fib_entry,
2640 enum mlxsw_reg_ralue_op op, int err)
2641 {
2642 switch (op) {
2643 case MLXSW_REG_RALUE_OP_WRITE_DELETE:
2644 return mlxsw_sp_fib_entry_offload_unset(fib_entry);
2645 case MLXSW_REG_RALUE_OP_WRITE_WRITE:
2646 if (err)
2647 return;
2648 if (mlxsw_sp_fib_entry_should_offload(fib_entry))
2649 mlxsw_sp_fib_entry_offload_set(fib_entry);
2650 else if (!mlxsw_sp_fib_entry_should_offload(fib_entry))
2651 mlxsw_sp_fib_entry_offload_unset(fib_entry);
2652 return;
2653 default:
2654 return;
2655 }
2656 }
2657
2658 static void
2659 mlxsw_sp_fib_entry_ralue_pack(char *ralue_pl,
2660 const struct mlxsw_sp_fib_entry *fib_entry,
2661 enum mlxsw_reg_ralue_op op)
2662 {
2663 struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
2664 enum mlxsw_reg_ralxx_protocol proto;
2665 u32 *p_dip;
2666
2667 proto = (enum mlxsw_reg_ralxx_protocol) fib->proto;
2668
2669 switch (fib->proto) {
2670 case MLXSW_SP_L3_PROTO_IPV4:
2671 p_dip = (u32 *) fib_entry->fib_node->key.addr;
2672 mlxsw_reg_ralue_pack4(ralue_pl, proto, op, fib->vr->id,
2673 fib_entry->fib_node->key.prefix_len,
2674 *p_dip);
2675 break;
2676 case MLXSW_SP_L3_PROTO_IPV6:
2677 mlxsw_reg_ralue_pack6(ralue_pl, proto, op, fib->vr->id,
2678 fib_entry->fib_node->key.prefix_len,
2679 fib_entry->fib_node->key.addr);
2680 break;
2681 }
2682 }
2683
2684 static int mlxsw_sp_fib_entry_op_remote(struct mlxsw_sp *mlxsw_sp,
2685 struct mlxsw_sp_fib_entry *fib_entry,
2686 enum mlxsw_reg_ralue_op op)
2687 {
2688 char ralue_pl[MLXSW_REG_RALUE_LEN];
2689 enum mlxsw_reg_ralue_trap_action trap_action;
2690 u16 trap_id = 0;
2691 u32 adjacency_index = 0;
2692 u16 ecmp_size = 0;
2693
2694 /* In case the nexthop group adjacency index is valid, use it
2695 * with provided ECMP size. Otherwise, setup trap and pass
2696 * traffic to kernel.
2697 */
2698 if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
2699 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
2700 adjacency_index = fib_entry->nh_group->adj_index;
2701 ecmp_size = fib_entry->nh_group->ecmp_size;
2702 } else {
2703 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
2704 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
2705 }
2706
2707 mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
2708 mlxsw_reg_ralue_act_remote_pack(ralue_pl, trap_action, trap_id,
2709 adjacency_index, ecmp_size);
2710 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
2711 }
2712
2713 static int mlxsw_sp_fib_entry_op_local(struct mlxsw_sp *mlxsw_sp,
2714 struct mlxsw_sp_fib_entry *fib_entry,
2715 enum mlxsw_reg_ralue_op op)
2716 {
2717 struct mlxsw_sp_rif *rif = fib_entry->nh_group->nh_rif;
2718 enum mlxsw_reg_ralue_trap_action trap_action;
2719 char ralue_pl[MLXSW_REG_RALUE_LEN];
2720 u16 trap_id = 0;
2721 u16 rif_index = 0;
2722
2723 if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
2724 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
2725 rif_index = rif->rif_index;
2726 } else {
2727 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
2728 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
2729 }
2730
2731 mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
2732 mlxsw_reg_ralue_act_local_pack(ralue_pl, trap_action, trap_id,
2733 rif_index);
2734 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
2735 }
2736
2737 static int mlxsw_sp_fib_entry_op_trap(struct mlxsw_sp *mlxsw_sp,
2738 struct mlxsw_sp_fib_entry *fib_entry,
2739 enum mlxsw_reg_ralue_op op)
2740 {
2741 char ralue_pl[MLXSW_REG_RALUE_LEN];
2742
2743 mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
2744 mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
2745 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
2746 }
2747
2748 static int __mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
2749 struct mlxsw_sp_fib_entry *fib_entry,
2750 enum mlxsw_reg_ralue_op op)
2751 {
2752 switch (fib_entry->type) {
2753 case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
2754 return mlxsw_sp_fib_entry_op_remote(mlxsw_sp, fib_entry, op);
2755 case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
2756 return mlxsw_sp_fib_entry_op_local(mlxsw_sp, fib_entry, op);
2757 case MLXSW_SP_FIB_ENTRY_TYPE_TRAP:
2758 return mlxsw_sp_fib_entry_op_trap(mlxsw_sp, fib_entry, op);
2759 }
2760 return -EINVAL;
2761 }
2762
2763 static int mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
2764 struct mlxsw_sp_fib_entry *fib_entry,
2765 enum mlxsw_reg_ralue_op op)
2766 {
2767 int err = __mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry, op);
2768
2769 mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, err);
2770
2771 return err;
2772 }
2773
2774 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
2775 struct mlxsw_sp_fib_entry *fib_entry)
2776 {
2777 return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
2778 MLXSW_REG_RALUE_OP_WRITE_WRITE);
2779 }
2780
2781 static int mlxsw_sp_fib_entry_del(struct mlxsw_sp *mlxsw_sp,
2782 struct mlxsw_sp_fib_entry *fib_entry)
2783 {
2784 return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
2785 MLXSW_REG_RALUE_OP_WRITE_DELETE);
2786 }
2787
2788 static int
2789 mlxsw_sp_fib4_entry_type_set(struct mlxsw_sp *mlxsw_sp,
2790 const struct fib_entry_notifier_info *fen_info,
2791 struct mlxsw_sp_fib_entry *fib_entry)
2792 {
2793 struct fib_info *fi = fen_info->fi;
2794
2795 switch (fen_info->type) {
2796 case RTN_BROADCAST: /* fall through */
2797 case RTN_LOCAL:
2798 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
2799 return 0;
2800 case RTN_UNREACHABLE: /* fall through */
2801 case RTN_BLACKHOLE: /* fall through */
2802 case RTN_PROHIBIT:
2803 /* Packets hitting these routes need to be trapped, but
2804 * can do so with a lower priority than packets directed
2805 * at the host, so use action type local instead of trap.
2806 */
2807 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
2808 return 0;
2809 case RTN_UNICAST:
2810 if (mlxsw_sp_fi_is_gateway(mlxsw_sp, fi))
2811 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
2812 else
2813 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
2814 return 0;
2815 default:
2816 return -EINVAL;
2817 }
2818 }
2819
2820 static struct mlxsw_sp_fib4_entry *
2821 mlxsw_sp_fib4_entry_create(struct mlxsw_sp *mlxsw_sp,
2822 struct mlxsw_sp_fib_node *fib_node,
2823 const struct fib_entry_notifier_info *fen_info)
2824 {
2825 struct mlxsw_sp_fib4_entry *fib4_entry;
2826 struct mlxsw_sp_fib_entry *fib_entry;
2827 int err;
2828
2829 fib4_entry = kzalloc(sizeof(*fib4_entry), GFP_KERNEL);
2830 if (!fib4_entry)
2831 return ERR_PTR(-ENOMEM);
2832 fib_entry = &fib4_entry->common;
2833
2834 err = mlxsw_sp_fib4_entry_type_set(mlxsw_sp, fen_info, fib_entry);
2835 if (err)
2836 goto err_fib4_entry_type_set;
2837
2838 err = mlxsw_sp_nexthop4_group_get(mlxsw_sp, fib_entry, fen_info->fi);
2839 if (err)
2840 goto err_nexthop4_group_get;
2841
2842 fib4_entry->prio = fen_info->fi->fib_priority;
2843 fib4_entry->tb_id = fen_info->tb_id;
2844 fib4_entry->type = fen_info->type;
2845 fib4_entry->tos = fen_info->tos;
2846
2847 fib_entry->fib_node = fib_node;
2848
2849 return fib4_entry;
2850
2851 err_nexthop4_group_get:
2852 err_fib4_entry_type_set:
2853 kfree(fib4_entry);
2854 return ERR_PTR(err);
2855 }
2856
2857 static void mlxsw_sp_fib4_entry_destroy(struct mlxsw_sp *mlxsw_sp,
2858 struct mlxsw_sp_fib4_entry *fib4_entry)
2859 {
2860 mlxsw_sp_nexthop4_group_put(mlxsw_sp, &fib4_entry->common);
2861 kfree(fib4_entry);
2862 }
2863
2864 static struct mlxsw_sp_fib_node *
2865 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr,
2866 size_t addr_len, unsigned char prefix_len);
2867
2868 static struct mlxsw_sp_fib4_entry *
2869 mlxsw_sp_fib4_entry_lookup(struct mlxsw_sp *mlxsw_sp,
2870 const struct fib_entry_notifier_info *fen_info)
2871 {
2872 struct mlxsw_sp_fib4_entry *fib4_entry;
2873 struct mlxsw_sp_fib_node *fib_node;
2874 struct mlxsw_sp_fib *fib;
2875 struct mlxsw_sp_vr *vr;
2876
2877 vr = mlxsw_sp_vr_find(mlxsw_sp, fen_info->tb_id);
2878 if (!vr)
2879 return NULL;
2880 fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV4);
2881
2882 fib_node = mlxsw_sp_fib_node_lookup(fib, &fen_info->dst,
2883 sizeof(fen_info->dst),
2884 fen_info->dst_len);
2885 if (!fib_node)
2886 return NULL;
2887
2888 list_for_each_entry(fib4_entry, &fib_node->entry_list, common.list) {
2889 if (fib4_entry->tb_id == fen_info->tb_id &&
2890 fib4_entry->tos == fen_info->tos &&
2891 fib4_entry->type == fen_info->type &&
2892 mlxsw_sp_nexthop4_group_fi(fib4_entry->common.nh_group) ==
2893 fen_info->fi) {
2894 return fib4_entry;
2895 }
2896 }
2897
2898 return NULL;
2899 }
2900
2901 static const struct rhashtable_params mlxsw_sp_fib_ht_params = {
2902 .key_offset = offsetof(struct mlxsw_sp_fib_node, key),
2903 .head_offset = offsetof(struct mlxsw_sp_fib_node, ht_node),
2904 .key_len = sizeof(struct mlxsw_sp_fib_key),
2905 .automatic_shrinking = true,
2906 };
2907
2908 static int mlxsw_sp_fib_node_insert(struct mlxsw_sp_fib *fib,
2909 struct mlxsw_sp_fib_node *fib_node)
2910 {
2911 return rhashtable_insert_fast(&fib->ht, &fib_node->ht_node,
2912 mlxsw_sp_fib_ht_params);
2913 }
2914
2915 static void mlxsw_sp_fib_node_remove(struct mlxsw_sp_fib *fib,
2916 struct mlxsw_sp_fib_node *fib_node)
2917 {
2918 rhashtable_remove_fast(&fib->ht, &fib_node->ht_node,
2919 mlxsw_sp_fib_ht_params);
2920 }
2921
2922 static struct mlxsw_sp_fib_node *
2923 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr,
2924 size_t addr_len, unsigned char prefix_len)
2925 {
2926 struct mlxsw_sp_fib_key key;
2927
2928 memset(&key, 0, sizeof(key));
2929 memcpy(key.addr, addr, addr_len);
2930 key.prefix_len = prefix_len;
2931 return rhashtable_lookup_fast(&fib->ht, &key, mlxsw_sp_fib_ht_params);
2932 }
2933
2934 static struct mlxsw_sp_fib_node *
2935 mlxsw_sp_fib_node_create(struct mlxsw_sp_fib *fib, const void *addr,
2936 size_t addr_len, unsigned char prefix_len)
2937 {
2938 struct mlxsw_sp_fib_node *fib_node;
2939
2940 fib_node = kzalloc(sizeof(*fib_node), GFP_KERNEL);
2941 if (!fib_node)
2942 return NULL;
2943
2944 INIT_LIST_HEAD(&fib_node->entry_list);
2945 list_add(&fib_node->list, &fib->node_list);
2946 memcpy(fib_node->key.addr, addr, addr_len);
2947 fib_node->key.prefix_len = prefix_len;
2948
2949 return fib_node;
2950 }
2951
2952 static void mlxsw_sp_fib_node_destroy(struct mlxsw_sp_fib_node *fib_node)
2953 {
2954 list_del(&fib_node->list);
2955 WARN_ON(!list_empty(&fib_node->entry_list));
2956 kfree(fib_node);
2957 }
2958
2959 static bool
2960 mlxsw_sp_fib_node_entry_is_first(const struct mlxsw_sp_fib_node *fib_node,
2961 const struct mlxsw_sp_fib_entry *fib_entry)
2962 {
2963 return list_first_entry(&fib_node->entry_list,
2964 struct mlxsw_sp_fib_entry, list) == fib_entry;
2965 }
2966
2967 static int mlxsw_sp_fib_lpm_tree_link(struct mlxsw_sp *mlxsw_sp,
2968 struct mlxsw_sp_fib *fib,
2969 struct mlxsw_sp_fib_node *fib_node)
2970 {
2971 struct mlxsw_sp_prefix_usage req_prefix_usage = {{ 0 } };
2972 struct mlxsw_sp_lpm_tree *lpm_tree;
2973 int err;
2974
2975 /* Since the tree is shared between all virtual routers we must
2976 * make sure it contains all the required prefix lengths. This
2977 * can be computed by either adding the new prefix length to the
2978 * existing prefix usage of a bound tree, or by aggregating the
2979 * prefix lengths across all virtual routers and adding the new
2980 * one as well.
2981 */
2982 if (fib->lpm_tree)
2983 mlxsw_sp_prefix_usage_cpy(&req_prefix_usage,
2984 &fib->lpm_tree->prefix_usage);
2985 else
2986 mlxsw_sp_vrs_prefixes(mlxsw_sp, fib->proto, &req_prefix_usage);
2987 mlxsw_sp_prefix_usage_set(&req_prefix_usage, fib_node->key.prefix_len);
2988
2989 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
2990 fib->proto);
2991 if (IS_ERR(lpm_tree))
2992 return PTR_ERR(lpm_tree);
2993
2994 if (fib->lpm_tree && fib->lpm_tree->id == lpm_tree->id)
2995 return 0;
2996
2997 err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree);
2998 if (err)
2999 return err;
3000
3001 return 0;
3002 }
3003
3004 static void mlxsw_sp_fib_lpm_tree_unlink(struct mlxsw_sp *mlxsw_sp,
3005 struct mlxsw_sp_fib *fib)
3006 {
3007 struct mlxsw_sp_prefix_usage req_prefix_usage = {{ 0 } };
3008 struct mlxsw_sp_lpm_tree *lpm_tree;
3009
3010 /* Aggregate prefix lengths across all virtual routers to make
3011 * sure we only have used prefix lengths in the LPM tree.
3012 */
3013 mlxsw_sp_vrs_prefixes(mlxsw_sp, fib->proto, &req_prefix_usage);
3014 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
3015 fib->proto);
3016 if (IS_ERR(lpm_tree))
3017 goto err_tree_get;
3018 mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree);
3019
3020 err_tree_get:
3021 if (!mlxsw_sp_prefix_usage_none(&fib->prefix_usage))
3022 return;
3023 mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, fib);
3024 mlxsw_sp_lpm_tree_put(mlxsw_sp, fib->lpm_tree);
3025 fib->lpm_tree = NULL;
3026 }
3027
3028 static void mlxsw_sp_fib_node_prefix_inc(struct mlxsw_sp_fib_node *fib_node)
3029 {
3030 unsigned char prefix_len = fib_node->key.prefix_len;
3031 struct mlxsw_sp_fib *fib = fib_node->fib;
3032
3033 if (fib->prefix_ref_count[prefix_len]++ == 0)
3034 mlxsw_sp_prefix_usage_set(&fib->prefix_usage, prefix_len);
3035 }
3036
3037 static void mlxsw_sp_fib_node_prefix_dec(struct mlxsw_sp_fib_node *fib_node)
3038 {
3039 unsigned char prefix_len = fib_node->key.prefix_len;
3040 struct mlxsw_sp_fib *fib = fib_node->fib;
3041
3042 if (--fib->prefix_ref_count[prefix_len] == 0)
3043 mlxsw_sp_prefix_usage_clear(&fib->prefix_usage, prefix_len);
3044 }
3045
3046 static int mlxsw_sp_fib_node_init(struct mlxsw_sp *mlxsw_sp,
3047 struct mlxsw_sp_fib_node *fib_node,
3048 struct mlxsw_sp_fib *fib)
3049 {
3050 int err;
3051
3052 err = mlxsw_sp_fib_node_insert(fib, fib_node);
3053 if (err)
3054 return err;
3055 fib_node->fib = fib;
3056
3057 err = mlxsw_sp_fib_lpm_tree_link(mlxsw_sp, fib, fib_node);
3058 if (err)
3059 goto err_fib_lpm_tree_link;
3060
3061 mlxsw_sp_fib_node_prefix_inc(fib_node);
3062
3063 return 0;
3064
3065 err_fib_lpm_tree_link:
3066 fib_node->fib = NULL;
3067 mlxsw_sp_fib_node_remove(fib, fib_node);
3068 return err;
3069 }
3070
3071 static void mlxsw_sp_fib_node_fini(struct mlxsw_sp *mlxsw_sp,
3072 struct mlxsw_sp_fib_node *fib_node)
3073 {
3074 struct mlxsw_sp_fib *fib = fib_node->fib;
3075
3076 mlxsw_sp_fib_node_prefix_dec(fib_node);
3077 mlxsw_sp_fib_lpm_tree_unlink(mlxsw_sp, fib);
3078 fib_node->fib = NULL;
3079 mlxsw_sp_fib_node_remove(fib, fib_node);
3080 }
3081
3082 static struct mlxsw_sp_fib_node *
3083 mlxsw_sp_fib_node_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, const void *addr,
3084 size_t addr_len, unsigned char prefix_len,
3085 enum mlxsw_sp_l3proto proto)
3086 {
3087 struct mlxsw_sp_fib_node *fib_node;
3088 struct mlxsw_sp_fib *fib;
3089 struct mlxsw_sp_vr *vr;
3090 int err;
3091
3092 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id);
3093 if (IS_ERR(vr))
3094 return ERR_CAST(vr);
3095 fib = mlxsw_sp_vr_fib(vr, proto);
3096
3097 fib_node = mlxsw_sp_fib_node_lookup(fib, addr, addr_len, prefix_len);
3098 if (fib_node)
3099 return fib_node;
3100
3101 fib_node = mlxsw_sp_fib_node_create(fib, addr, addr_len, prefix_len);
3102 if (!fib_node) {
3103 err = -ENOMEM;
3104 goto err_fib_node_create;
3105 }
3106
3107 err = mlxsw_sp_fib_node_init(mlxsw_sp, fib_node, fib);
3108 if (err)
3109 goto err_fib_node_init;
3110
3111 return fib_node;
3112
3113 err_fib_node_init:
3114 mlxsw_sp_fib_node_destroy(fib_node);
3115 err_fib_node_create:
3116 mlxsw_sp_vr_put(vr);
3117 return ERR_PTR(err);
3118 }
3119
3120 static void mlxsw_sp_fib_node_put(struct mlxsw_sp *mlxsw_sp,
3121 struct mlxsw_sp_fib_node *fib_node)
3122 {
3123 struct mlxsw_sp_vr *vr = fib_node->fib->vr;
3124
3125 if (!list_empty(&fib_node->entry_list))
3126 return;
3127 mlxsw_sp_fib_node_fini(mlxsw_sp, fib_node);
3128 mlxsw_sp_fib_node_destroy(fib_node);
3129 mlxsw_sp_vr_put(vr);
3130 }
3131
3132 static struct mlxsw_sp_fib4_entry *
3133 mlxsw_sp_fib4_node_entry_find(const struct mlxsw_sp_fib_node *fib_node,
3134 const struct mlxsw_sp_fib4_entry *new4_entry)
3135 {
3136 struct mlxsw_sp_fib4_entry *fib4_entry;
3137
3138 list_for_each_entry(fib4_entry, &fib_node->entry_list, common.list) {
3139 if (fib4_entry->tb_id > new4_entry->tb_id)
3140 continue;
3141 if (fib4_entry->tb_id != new4_entry->tb_id)
3142 break;
3143 if (fib4_entry->tos > new4_entry->tos)
3144 continue;
3145 if (fib4_entry->prio >= new4_entry->prio ||
3146 fib4_entry->tos < new4_entry->tos)
3147 return fib4_entry;
3148 }
3149
3150 return NULL;
3151 }
3152
3153 static int
3154 mlxsw_sp_fib4_node_list_append(struct mlxsw_sp_fib4_entry *fib4_entry,
3155 struct mlxsw_sp_fib4_entry *new4_entry)
3156 {
3157 struct mlxsw_sp_fib_node *fib_node;
3158
3159 if (WARN_ON(!fib4_entry))
3160 return -EINVAL;
3161
3162 fib_node = fib4_entry->common.fib_node;
3163 list_for_each_entry_from(fib4_entry, &fib_node->entry_list,
3164 common.list) {
3165 if (fib4_entry->tb_id != new4_entry->tb_id ||
3166 fib4_entry->tos != new4_entry->tos ||
3167 fib4_entry->prio != new4_entry->prio)
3168 break;
3169 }
3170
3171 list_add_tail(&new4_entry->common.list, &fib4_entry->common.list);
3172 return 0;
3173 }
3174
3175 static int
3176 mlxsw_sp_fib4_node_list_insert(struct mlxsw_sp_fib4_entry *new4_entry,
3177 bool replace, bool append)
3178 {
3179 struct mlxsw_sp_fib_node *fib_node = new4_entry->common.fib_node;
3180 struct mlxsw_sp_fib4_entry *fib4_entry;
3181
3182 fib4_entry = mlxsw_sp_fib4_node_entry_find(fib_node, new4_entry);
3183
3184 if (append)
3185 return mlxsw_sp_fib4_node_list_append(fib4_entry, new4_entry);
3186 if (replace && WARN_ON(!fib4_entry))
3187 return -EINVAL;
3188
3189 /* Insert new entry before replaced one, so that we can later
3190 * remove the second.
3191 */
3192 if (fib4_entry) {
3193 list_add_tail(&new4_entry->common.list,
3194 &fib4_entry->common.list);
3195 } else {
3196 struct mlxsw_sp_fib4_entry *last;
3197
3198 list_for_each_entry(last, &fib_node->entry_list, common.list) {
3199 if (new4_entry->tb_id > last->tb_id)
3200 break;
3201 fib4_entry = last;
3202 }
3203
3204 if (fib4_entry)
3205 list_add(&new4_entry->common.list,
3206 &fib4_entry->common.list);
3207 else
3208 list_add(&new4_entry->common.list,
3209 &fib_node->entry_list);
3210 }
3211
3212 return 0;
3213 }
3214
3215 static void
3216 mlxsw_sp_fib4_node_list_remove(struct mlxsw_sp_fib4_entry *fib4_entry)
3217 {
3218 list_del(&fib4_entry->common.list);
3219 }
3220
3221 static int mlxsw_sp_fib_node_entry_add(struct mlxsw_sp *mlxsw_sp,
3222 struct mlxsw_sp_fib_entry *fib_entry)
3223 {
3224 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
3225
3226 if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry))
3227 return 0;
3228
3229 /* To prevent packet loss, overwrite the previously offloaded
3230 * entry.
3231 */
3232 if (!list_is_singular(&fib_node->entry_list)) {
3233 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_DELETE;
3234 struct mlxsw_sp_fib_entry *n = list_next_entry(fib_entry, list);
3235
3236 mlxsw_sp_fib_entry_offload_refresh(n, op, 0);
3237 }
3238
3239 return mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
3240 }
3241
3242 static void mlxsw_sp_fib_node_entry_del(struct mlxsw_sp *mlxsw_sp,
3243 struct mlxsw_sp_fib_entry *fib_entry)
3244 {
3245 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
3246
3247 if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry))
3248 return;
3249
3250 /* Promote the next entry by overwriting the deleted entry */
3251 if (!list_is_singular(&fib_node->entry_list)) {
3252 struct mlxsw_sp_fib_entry *n = list_next_entry(fib_entry, list);
3253 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_DELETE;
3254
3255 mlxsw_sp_fib_entry_update(mlxsw_sp, n);
3256 mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, 0);
3257 return;
3258 }
3259
3260 mlxsw_sp_fib_entry_del(mlxsw_sp, fib_entry);
3261 }
3262
3263 static int mlxsw_sp_fib4_node_entry_link(struct mlxsw_sp *mlxsw_sp,
3264 struct mlxsw_sp_fib4_entry *fib4_entry,
3265 bool replace, bool append)
3266 {
3267 int err;
3268
3269 err = mlxsw_sp_fib4_node_list_insert(fib4_entry, replace, append);
3270 if (err)
3271 return err;
3272
3273 err = mlxsw_sp_fib_node_entry_add(mlxsw_sp, &fib4_entry->common);
3274 if (err)
3275 goto err_fib_node_entry_add;
3276
3277 return 0;
3278
3279 err_fib_node_entry_add:
3280 mlxsw_sp_fib4_node_list_remove(fib4_entry);
3281 return err;
3282 }
3283
3284 static void
3285 mlxsw_sp_fib4_node_entry_unlink(struct mlxsw_sp *mlxsw_sp,
3286 struct mlxsw_sp_fib4_entry *fib4_entry)
3287 {
3288 mlxsw_sp_fib_node_entry_del(mlxsw_sp, &fib4_entry->common);
3289 mlxsw_sp_fib4_node_list_remove(fib4_entry);
3290 }
3291
3292 static void mlxsw_sp_fib4_entry_replace(struct mlxsw_sp *mlxsw_sp,
3293 struct mlxsw_sp_fib4_entry *fib4_entry,
3294 bool replace)
3295 {
3296 struct mlxsw_sp_fib_node *fib_node = fib4_entry->common.fib_node;
3297 struct mlxsw_sp_fib4_entry *replaced;
3298
3299 if (!replace)
3300 return;
3301
3302 /* We inserted the new entry before replaced one */
3303 replaced = list_next_entry(fib4_entry, common.list);
3304
3305 mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, replaced);
3306 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, replaced);
3307 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
3308 }
3309
3310 static int
3311 mlxsw_sp_router_fib4_add(struct mlxsw_sp *mlxsw_sp,
3312 const struct fib_entry_notifier_info *fen_info,
3313 bool replace, bool append)
3314 {
3315 struct mlxsw_sp_fib4_entry *fib4_entry;
3316 struct mlxsw_sp_fib_node *fib_node;
3317 int err;
3318
3319 if (mlxsw_sp->router->aborted)
3320 return 0;
3321
3322 fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, fen_info->tb_id,
3323 &fen_info->dst, sizeof(fen_info->dst),
3324 fen_info->dst_len,
3325 MLXSW_SP_L3_PROTO_IPV4);
3326 if (IS_ERR(fib_node)) {
3327 dev_warn(mlxsw_sp->bus_info->dev, "Failed to get FIB node\n");
3328 return PTR_ERR(fib_node);
3329 }
3330
3331 fib4_entry = mlxsw_sp_fib4_entry_create(mlxsw_sp, fib_node, fen_info);
3332 if (IS_ERR(fib4_entry)) {
3333 dev_warn(mlxsw_sp->bus_info->dev, "Failed to create FIB entry\n");
3334 err = PTR_ERR(fib4_entry);
3335 goto err_fib4_entry_create;
3336 }
3337
3338 err = mlxsw_sp_fib4_node_entry_link(mlxsw_sp, fib4_entry, replace,
3339 append);
3340 if (err) {
3341 dev_warn(mlxsw_sp->bus_info->dev, "Failed to link FIB entry to node\n");
3342 goto err_fib4_node_entry_link;
3343 }
3344
3345 mlxsw_sp_fib4_entry_replace(mlxsw_sp, fib4_entry, replace);
3346
3347 return 0;
3348
3349 err_fib4_node_entry_link:
3350 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
3351 err_fib4_entry_create:
3352 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
3353 return err;
3354 }
3355
3356 static void mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp,
3357 struct fib_entry_notifier_info *fen_info)
3358 {
3359 struct mlxsw_sp_fib4_entry *fib4_entry;
3360 struct mlxsw_sp_fib_node *fib_node;
3361
3362 if (mlxsw_sp->router->aborted)
3363 return;
3364
3365 fib4_entry = mlxsw_sp_fib4_entry_lookup(mlxsw_sp, fen_info);
3366 if (WARN_ON(!fib4_entry))
3367 return;
3368 fib_node = fib4_entry->common.fib_node;
3369
3370 mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib4_entry);
3371 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
3372 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
3373 }
3374
3375 static bool mlxsw_sp_fib6_rt_should_ignore(const struct rt6_info *rt)
3376 {
3377 /* Packets with link-local destination IP arriving to the router
3378 * are trapped to the CPU, so no need to program specific routes
3379 * for them.
3380 */
3381 if (ipv6_addr_type(&rt->rt6i_dst.addr) & IPV6_ADDR_LINKLOCAL)
3382 return true;
3383
3384 /* Multicast routes aren't supported, so ignore them. Neighbour
3385 * Discovery packets are specifically trapped.
3386 */
3387 if (ipv6_addr_type(&rt->rt6i_dst.addr) & IPV6_ADDR_MULTICAST)
3388 return true;
3389
3390 /* Cloned routes are irrelevant in the forwarding path. */
3391 if (rt->rt6i_flags & RTF_CACHE)
3392 return true;
3393
3394 return false;
3395 }
3396
3397 static struct mlxsw_sp_rt6 *mlxsw_sp_rt6_create(struct rt6_info *rt)
3398 {
3399 struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3400
3401 mlxsw_sp_rt6 = kzalloc(sizeof(*mlxsw_sp_rt6), GFP_KERNEL);
3402 if (!mlxsw_sp_rt6)
3403 return ERR_PTR(-ENOMEM);
3404
3405 /* In case of route replace, replaced route is deleted with
3406 * no notification. Take reference to prevent accessing freed
3407 * memory.
3408 */
3409 mlxsw_sp_rt6->rt = rt;
3410 rt6_hold(rt);
3411
3412 return mlxsw_sp_rt6;
3413 }
3414
3415 #if IS_ENABLED(CONFIG_IPV6)
3416 static void mlxsw_sp_rt6_release(struct rt6_info *rt)
3417 {
3418 rt6_release(rt);
3419 }
3420 #else
3421 static void mlxsw_sp_rt6_release(struct rt6_info *rt)
3422 {
3423 }
3424 #endif
3425
3426 static void mlxsw_sp_rt6_destroy(struct mlxsw_sp_rt6 *mlxsw_sp_rt6)
3427 {
3428 mlxsw_sp_rt6_release(mlxsw_sp_rt6->rt);
3429 kfree(mlxsw_sp_rt6);
3430 }
3431
3432 static bool mlxsw_sp_fib6_rt_can_mp(const struct rt6_info *rt)
3433 {
3434 /* RTF_CACHE routes are ignored */
3435 return (rt->rt6i_flags & (RTF_GATEWAY | RTF_ADDRCONF)) == RTF_GATEWAY;
3436 }
3437
3438 static struct rt6_info *
3439 mlxsw_sp_fib6_entry_rt(const struct mlxsw_sp_fib6_entry *fib6_entry)
3440 {
3441 return list_first_entry(&fib6_entry->rt6_list, struct mlxsw_sp_rt6,
3442 list)->rt;
3443 }
3444
3445 static struct mlxsw_sp_fib6_entry *
3446 mlxsw_sp_fib6_node_mp_entry_find(const struct mlxsw_sp_fib_node *fib_node,
3447 const struct rt6_info *nrt, bool replace)
3448 {
3449 struct mlxsw_sp_fib6_entry *fib6_entry;
3450
3451 if (!mlxsw_sp_fib6_rt_can_mp(nrt) || replace)
3452 return NULL;
3453
3454 list_for_each_entry(fib6_entry, &fib_node->entry_list, common.list) {
3455 struct rt6_info *rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
3456
3457 /* RT6_TABLE_LOCAL and RT6_TABLE_MAIN share the same
3458 * virtual router.
3459 */
3460 if (rt->rt6i_table->tb6_id > nrt->rt6i_table->tb6_id)
3461 continue;
3462 if (rt->rt6i_table->tb6_id != nrt->rt6i_table->tb6_id)
3463 break;
3464 if (rt->rt6i_metric < nrt->rt6i_metric)
3465 continue;
3466 if (rt->rt6i_metric == nrt->rt6i_metric &&
3467 mlxsw_sp_fib6_rt_can_mp(rt))
3468 return fib6_entry;
3469 if (rt->rt6i_metric > nrt->rt6i_metric)
3470 break;
3471 }
3472
3473 return NULL;
3474 }
3475
3476 static struct mlxsw_sp_rt6 *
3477 mlxsw_sp_fib6_entry_rt_find(const struct mlxsw_sp_fib6_entry *fib6_entry,
3478 const struct rt6_info *rt)
3479 {
3480 struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3481
3482 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
3483 if (mlxsw_sp_rt6->rt == rt)
3484 return mlxsw_sp_rt6;
3485 }
3486
3487 return NULL;
3488 }
3489
3490 static int mlxsw_sp_nexthop6_init(struct mlxsw_sp *mlxsw_sp,
3491 struct mlxsw_sp_nexthop_group *nh_grp,
3492 struct mlxsw_sp_nexthop *nh,
3493 const struct rt6_info *rt)
3494 {
3495 struct net_device *dev = rt->dst.dev;
3496 struct mlxsw_sp_rif *rif;
3497 int err;
3498
3499 nh->nh_grp = nh_grp;
3500 memcpy(&nh->gw_addr, &rt->rt6i_gateway, sizeof(nh->gw_addr));
3501
3502 if (!dev)
3503 return 0;
3504 nh->ifindex = dev->ifindex;
3505
3506 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
3507 if (!rif)
3508 return 0;
3509 mlxsw_sp_nexthop_rif_init(nh, rif);
3510
3511 err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
3512 if (err)
3513 goto err_nexthop_neigh_init;
3514
3515 return 0;
3516
3517 err_nexthop_neigh_init:
3518 mlxsw_sp_nexthop_rif_fini(nh);
3519 return err;
3520 }
3521
3522 static void mlxsw_sp_nexthop6_fini(struct mlxsw_sp *mlxsw_sp,
3523 struct mlxsw_sp_nexthop *nh)
3524 {
3525 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
3526 mlxsw_sp_nexthop_rif_fini(nh);
3527 }
3528
3529 static struct mlxsw_sp_nexthop_group *
3530 mlxsw_sp_nexthop6_group_create(struct mlxsw_sp *mlxsw_sp,
3531 struct mlxsw_sp_fib6_entry *fib6_entry)
3532 {
3533 struct mlxsw_sp_nexthop_group *nh_grp;
3534 struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3535 struct mlxsw_sp_nexthop *nh;
3536 size_t alloc_size;
3537 int i = 0;
3538 int err;
3539
3540 alloc_size = sizeof(*nh_grp) +
3541 fib6_entry->nrt6 * sizeof(struct mlxsw_sp_nexthop);
3542 nh_grp = kzalloc(alloc_size, GFP_KERNEL);
3543 if (!nh_grp)
3544 return ERR_PTR(-ENOMEM);
3545 INIT_LIST_HEAD(&nh_grp->fib_list);
3546 #if IS_ENABLED(CONFIG_IPV6)
3547 nh_grp->neigh_tbl = &nd_tbl;
3548 #endif
3549 mlxsw_sp_rt6 = list_first_entry(&fib6_entry->rt6_list,
3550 struct mlxsw_sp_rt6, list);
3551 nh_grp->gateway = !!(mlxsw_sp_rt6->rt->rt6i_flags & RTF_GATEWAY);
3552 nh_grp->count = fib6_entry->nrt6;
3553 for (i = 0; i < nh_grp->count; i++) {
3554 struct rt6_info *rt = mlxsw_sp_rt6->rt;
3555
3556 nh = &nh_grp->nexthops[i];
3557 err = mlxsw_sp_nexthop6_init(mlxsw_sp, nh_grp, nh, rt);
3558 if (err)
3559 goto err_nexthop6_init;
3560 mlxsw_sp_rt6 = list_next_entry(mlxsw_sp_rt6, list);
3561 }
3562
3563 err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
3564 if (err)
3565 goto err_nexthop_group_insert;
3566
3567 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
3568 return nh_grp;
3569
3570 err_nexthop_group_insert:
3571 err_nexthop6_init:
3572 for (i--; i >= 0; i--) {
3573 nh = &nh_grp->nexthops[i];
3574 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh);
3575 }
3576 kfree(nh_grp);
3577 return ERR_PTR(err);
3578 }
3579
3580 static void
3581 mlxsw_sp_nexthop6_group_destroy(struct mlxsw_sp *mlxsw_sp,
3582 struct mlxsw_sp_nexthop_group *nh_grp)
3583 {
3584 struct mlxsw_sp_nexthop *nh;
3585 int i = nh_grp->count;
3586
3587 mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
3588 for (i--; i >= 0; i--) {
3589 nh = &nh_grp->nexthops[i];
3590 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh);
3591 }
3592 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
3593 WARN_ON(nh_grp->adj_index_valid);
3594 kfree(nh_grp);
3595 }
3596
3597 static int mlxsw_sp_nexthop6_group_get(struct mlxsw_sp *mlxsw_sp,
3598 struct mlxsw_sp_fib6_entry *fib6_entry)
3599 {
3600 struct mlxsw_sp_nexthop_group *nh_grp;
3601
3602 nh_grp = mlxsw_sp_nexthop6_group_lookup(mlxsw_sp, fib6_entry);
3603 if (!nh_grp) {
3604 nh_grp = mlxsw_sp_nexthop6_group_create(mlxsw_sp, fib6_entry);
3605 if (IS_ERR(nh_grp))
3606 return PTR_ERR(nh_grp);
3607 }
3608
3609 list_add_tail(&fib6_entry->common.nexthop_group_node,
3610 &nh_grp->fib_list);
3611 fib6_entry->common.nh_group = nh_grp;
3612
3613 return 0;
3614 }
3615
3616 static void mlxsw_sp_nexthop6_group_put(struct mlxsw_sp *mlxsw_sp,
3617 struct mlxsw_sp_fib_entry *fib_entry)
3618 {
3619 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
3620
3621 list_del(&fib_entry->nexthop_group_node);
3622 if (!list_empty(&nh_grp->fib_list))
3623 return;
3624 mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, nh_grp);
3625 }
3626
3627 static int
3628 mlxsw_sp_nexthop6_group_update(struct mlxsw_sp *mlxsw_sp,
3629 struct mlxsw_sp_fib6_entry *fib6_entry)
3630 {
3631 struct mlxsw_sp_nexthop_group *old_nh_grp = fib6_entry->common.nh_group;
3632 int err;
3633
3634 fib6_entry->common.nh_group = NULL;
3635 list_del(&fib6_entry->common.nexthop_group_node);
3636
3637 err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry);
3638 if (err)
3639 goto err_nexthop6_group_get;
3640
3641 /* In case this entry is offloaded, then the adjacency index
3642 * currently associated with it in the device's table is that
3643 * of the old group. Start using the new one instead.
3644 */
3645 err = mlxsw_sp_fib_node_entry_add(mlxsw_sp, &fib6_entry->common);
3646 if (err)
3647 goto err_fib_node_entry_add;
3648
3649 if (list_empty(&old_nh_grp->fib_list))
3650 mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, old_nh_grp);
3651
3652 return 0;
3653
3654 err_fib_node_entry_add:
3655 mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common);
3656 err_nexthop6_group_get:
3657 list_add_tail(&fib6_entry->common.nexthop_group_node,
3658 &old_nh_grp->fib_list);
3659 fib6_entry->common.nh_group = old_nh_grp;
3660 return err;
3661 }
3662
3663 static int
3664 mlxsw_sp_fib6_entry_nexthop_add(struct mlxsw_sp *mlxsw_sp,
3665 struct mlxsw_sp_fib6_entry *fib6_entry,
3666 struct rt6_info *rt)
3667 {
3668 struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3669 int err;
3670
3671 mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt);
3672 if (IS_ERR(mlxsw_sp_rt6))
3673 return PTR_ERR(mlxsw_sp_rt6);
3674
3675 list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list);
3676 fib6_entry->nrt6++;
3677
3678 err = mlxsw_sp_nexthop6_group_update(mlxsw_sp, fib6_entry);
3679 if (err)
3680 goto err_nexthop6_group_update;
3681
3682 return 0;
3683
3684 err_nexthop6_group_update:
3685 fib6_entry->nrt6--;
3686 list_del(&mlxsw_sp_rt6->list);
3687 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
3688 return err;
3689 }
3690
3691 static void
3692 mlxsw_sp_fib6_entry_nexthop_del(struct mlxsw_sp *mlxsw_sp,
3693 struct mlxsw_sp_fib6_entry *fib6_entry,
3694 struct rt6_info *rt)
3695 {
3696 struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3697
3698 mlxsw_sp_rt6 = mlxsw_sp_fib6_entry_rt_find(fib6_entry, rt);
3699 if (WARN_ON(!mlxsw_sp_rt6))
3700 return;
3701
3702 fib6_entry->nrt6--;
3703 list_del(&mlxsw_sp_rt6->list);
3704 mlxsw_sp_nexthop6_group_update(mlxsw_sp, fib6_entry);
3705 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
3706 }
3707
3708 static void mlxsw_sp_fib6_entry_type_set(struct mlxsw_sp_fib_entry *fib_entry,
3709 const struct rt6_info *rt)
3710 {
3711 /* Packets hitting RTF_REJECT routes need to be discarded by the
3712 * stack. We can rely on their destination device not having a
3713 * RIF (it's the loopback device) and can thus use action type
3714 * local, which will cause them to be trapped with a lower
3715 * priority than packets that need to be locally received.
3716 */
3717 if (rt->rt6i_flags & (RTF_LOCAL | RTF_ANYCAST))
3718 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
3719 else if (rt->rt6i_flags & RTF_REJECT)
3720 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
3721 else if (rt->rt6i_flags & RTF_GATEWAY)
3722 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
3723 else
3724 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
3725 }
3726
3727 static void
3728 mlxsw_sp_fib6_entry_rt_destroy_all(struct mlxsw_sp_fib6_entry *fib6_entry)
3729 {
3730 struct mlxsw_sp_rt6 *mlxsw_sp_rt6, *tmp;
3731
3732 list_for_each_entry_safe(mlxsw_sp_rt6, tmp, &fib6_entry->rt6_list,
3733 list) {
3734 fib6_entry->nrt6--;
3735 list_del(&mlxsw_sp_rt6->list);
3736 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
3737 }
3738 }
3739
3740 static struct mlxsw_sp_fib6_entry *
3741 mlxsw_sp_fib6_entry_create(struct mlxsw_sp *mlxsw_sp,
3742 struct mlxsw_sp_fib_node *fib_node,
3743 struct rt6_info *rt)
3744 {
3745 struct mlxsw_sp_fib6_entry *fib6_entry;
3746 struct mlxsw_sp_fib_entry *fib_entry;
3747 struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3748 int err;
3749
3750 fib6_entry = kzalloc(sizeof(*fib6_entry), GFP_KERNEL);
3751 if (!fib6_entry)
3752 return ERR_PTR(-ENOMEM);
3753 fib_entry = &fib6_entry->common;
3754
3755 mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt);
3756 if (IS_ERR(mlxsw_sp_rt6)) {
3757 err = PTR_ERR(mlxsw_sp_rt6);
3758 goto err_rt6_create;
3759 }
3760
3761 mlxsw_sp_fib6_entry_type_set(fib_entry, mlxsw_sp_rt6->rt);
3762
3763 INIT_LIST_HEAD(&fib6_entry->rt6_list);
3764 list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list);
3765 fib6_entry->nrt6 = 1;
3766 err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry);
3767 if (err)
3768 goto err_nexthop6_group_get;
3769
3770 fib_entry->fib_node = fib_node;
3771
3772 return fib6_entry;
3773
3774 err_nexthop6_group_get:
3775 list_del(&mlxsw_sp_rt6->list);
3776 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
3777 err_rt6_create:
3778 kfree(fib6_entry);
3779 return ERR_PTR(err);
3780 }
3781
3782 static void mlxsw_sp_fib6_entry_destroy(struct mlxsw_sp *mlxsw_sp,
3783 struct mlxsw_sp_fib6_entry *fib6_entry)
3784 {
3785 mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common);
3786 mlxsw_sp_fib6_entry_rt_destroy_all(fib6_entry);
3787 WARN_ON(fib6_entry->nrt6);
3788 kfree(fib6_entry);
3789 }
3790
3791 static struct mlxsw_sp_fib6_entry *
3792 mlxsw_sp_fib6_node_entry_find(const struct mlxsw_sp_fib_node *fib_node,
3793 const struct rt6_info *nrt, bool replace)
3794 {
3795 struct mlxsw_sp_fib6_entry *fib6_entry, *fallback = NULL;
3796
3797 list_for_each_entry(fib6_entry, &fib_node->entry_list, common.list) {
3798 struct rt6_info *rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
3799
3800 if (rt->rt6i_table->tb6_id > nrt->rt6i_table->tb6_id)
3801 continue;
3802 if (rt->rt6i_table->tb6_id != nrt->rt6i_table->tb6_id)
3803 break;
3804 if (replace && rt->rt6i_metric == nrt->rt6i_metric) {
3805 if (mlxsw_sp_fib6_rt_can_mp(rt) ==
3806 mlxsw_sp_fib6_rt_can_mp(nrt))
3807 return fib6_entry;
3808 if (mlxsw_sp_fib6_rt_can_mp(nrt))
3809 fallback = fallback ?: fib6_entry;
3810 }
3811 if (rt->rt6i_metric > nrt->rt6i_metric)
3812 return fallback ?: fib6_entry;
3813 }
3814
3815 return fallback;
3816 }
3817
3818 static int
3819 mlxsw_sp_fib6_node_list_insert(struct mlxsw_sp_fib6_entry *new6_entry,
3820 bool replace)
3821 {
3822 struct mlxsw_sp_fib_node *fib_node = new6_entry->common.fib_node;
3823 struct rt6_info *nrt = mlxsw_sp_fib6_entry_rt(new6_entry);
3824 struct mlxsw_sp_fib6_entry *fib6_entry;
3825
3826 fib6_entry = mlxsw_sp_fib6_node_entry_find(fib_node, nrt, replace);
3827
3828 if (replace && WARN_ON(!fib6_entry))
3829 return -EINVAL;
3830
3831 if (fib6_entry) {
3832 list_add_tail(&new6_entry->common.list,
3833 &fib6_entry->common.list);
3834 } else {
3835 struct mlxsw_sp_fib6_entry *last;
3836
3837 list_for_each_entry(last, &fib_node->entry_list, common.list) {
3838 struct rt6_info *rt = mlxsw_sp_fib6_entry_rt(last);
3839
3840 if (nrt->rt6i_table->tb6_id > rt->rt6i_table->tb6_id)
3841 break;
3842 fib6_entry = last;
3843 }
3844
3845 if (fib6_entry)
3846 list_add(&new6_entry->common.list,
3847 &fib6_entry->common.list);
3848 else
3849 list_add(&new6_entry->common.list,
3850 &fib_node->entry_list);
3851 }
3852
3853 return 0;
3854 }
3855
3856 static void
3857 mlxsw_sp_fib6_node_list_remove(struct mlxsw_sp_fib6_entry *fib6_entry)
3858 {
3859 list_del(&fib6_entry->common.list);
3860 }
3861
3862 static int mlxsw_sp_fib6_node_entry_link(struct mlxsw_sp *mlxsw_sp,
3863 struct mlxsw_sp_fib6_entry *fib6_entry,
3864 bool replace)
3865 {
3866 int err;
3867
3868 err = mlxsw_sp_fib6_node_list_insert(fib6_entry, replace);
3869 if (err)
3870 return err;
3871
3872 err = mlxsw_sp_fib_node_entry_add(mlxsw_sp, &fib6_entry->common);
3873 if (err)
3874 goto err_fib_node_entry_add;
3875
3876 return 0;
3877
3878 err_fib_node_entry_add:
3879 mlxsw_sp_fib6_node_list_remove(fib6_entry);
3880 return err;
3881 }
3882
3883 static void
3884 mlxsw_sp_fib6_node_entry_unlink(struct mlxsw_sp *mlxsw_sp,
3885 struct mlxsw_sp_fib6_entry *fib6_entry)
3886 {
3887 mlxsw_sp_fib_node_entry_del(mlxsw_sp, &fib6_entry->common);
3888 mlxsw_sp_fib6_node_list_remove(fib6_entry);
3889 }
3890
3891 static struct mlxsw_sp_fib6_entry *
3892 mlxsw_sp_fib6_entry_lookup(struct mlxsw_sp *mlxsw_sp,
3893 const struct rt6_info *rt)
3894 {
3895 struct mlxsw_sp_fib6_entry *fib6_entry;
3896 struct mlxsw_sp_fib_node *fib_node;
3897 struct mlxsw_sp_fib *fib;
3898 struct mlxsw_sp_vr *vr;
3899
3900 vr = mlxsw_sp_vr_find(mlxsw_sp, rt->rt6i_table->tb6_id);
3901 if (!vr)
3902 return NULL;
3903 fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV6);
3904
3905 fib_node = mlxsw_sp_fib_node_lookup(fib, &rt->rt6i_dst.addr,
3906 sizeof(rt->rt6i_dst.addr),
3907 rt->rt6i_dst.plen);
3908 if (!fib_node)
3909 return NULL;
3910
3911 list_for_each_entry(fib6_entry, &fib_node->entry_list, common.list) {
3912 struct rt6_info *iter_rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
3913
3914 if (rt->rt6i_table->tb6_id == iter_rt->rt6i_table->tb6_id &&
3915 rt->rt6i_metric == iter_rt->rt6i_metric &&
3916 mlxsw_sp_fib6_entry_rt_find(fib6_entry, rt))
3917 return fib6_entry;
3918 }
3919
3920 return NULL;
3921 }
3922
3923 static void mlxsw_sp_fib6_entry_replace(struct mlxsw_sp *mlxsw_sp,
3924 struct mlxsw_sp_fib6_entry *fib6_entry,
3925 bool replace)
3926 {
3927 struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node;
3928 struct mlxsw_sp_fib6_entry *replaced;
3929
3930 if (!replace)
3931 return;
3932
3933 replaced = list_next_entry(fib6_entry, common.list);
3934
3935 mlxsw_sp_fib6_node_entry_unlink(mlxsw_sp, replaced);
3936 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, replaced);
3937 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
3938 }
3939
3940 static int mlxsw_sp_router_fib6_add(struct mlxsw_sp *mlxsw_sp,
3941 struct rt6_info *rt, bool replace)
3942 {
3943 struct mlxsw_sp_fib6_entry *fib6_entry;
3944 struct mlxsw_sp_fib_node *fib_node;
3945 int err;
3946
3947 if (mlxsw_sp->router->aborted)
3948 return 0;
3949
3950 if (rt->rt6i_src.plen)
3951 return -EINVAL;
3952
3953 if (mlxsw_sp_fib6_rt_should_ignore(rt))
3954 return 0;
3955
3956 fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, rt->rt6i_table->tb6_id,
3957 &rt->rt6i_dst.addr,
3958 sizeof(rt->rt6i_dst.addr),
3959 rt->rt6i_dst.plen,
3960 MLXSW_SP_L3_PROTO_IPV6);
3961 if (IS_ERR(fib_node))
3962 return PTR_ERR(fib_node);
3963
3964 /* Before creating a new entry, try to append route to an existing
3965 * multipath entry.
3966 */
3967 fib6_entry = mlxsw_sp_fib6_node_mp_entry_find(fib_node, rt, replace);
3968 if (fib6_entry) {
3969 err = mlxsw_sp_fib6_entry_nexthop_add(mlxsw_sp, fib6_entry, rt);
3970 if (err)
3971 goto err_fib6_entry_nexthop_add;
3972 return 0;
3973 }
3974
3975 fib6_entry = mlxsw_sp_fib6_entry_create(mlxsw_sp, fib_node, rt);
3976 if (IS_ERR(fib6_entry)) {
3977 err = PTR_ERR(fib6_entry);
3978 goto err_fib6_entry_create;
3979 }
3980
3981 err = mlxsw_sp_fib6_node_entry_link(mlxsw_sp, fib6_entry, replace);
3982 if (err)
3983 goto err_fib6_node_entry_link;
3984
3985 mlxsw_sp_fib6_entry_replace(mlxsw_sp, fib6_entry, replace);
3986
3987 return 0;
3988
3989 err_fib6_node_entry_link:
3990 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
3991 err_fib6_entry_create:
3992 err_fib6_entry_nexthop_add:
3993 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
3994 return err;
3995 }
3996
3997 static void mlxsw_sp_router_fib6_del(struct mlxsw_sp *mlxsw_sp,
3998 struct rt6_info *rt)
3999 {
4000 struct mlxsw_sp_fib6_entry *fib6_entry;
4001 struct mlxsw_sp_fib_node *fib_node;
4002
4003 if (mlxsw_sp->router->aborted)
4004 return;
4005
4006 if (mlxsw_sp_fib6_rt_should_ignore(rt))
4007 return;
4008
4009 fib6_entry = mlxsw_sp_fib6_entry_lookup(mlxsw_sp, rt);
4010 if (WARN_ON(!fib6_entry))
4011 return;
4012
4013 /* If route is part of a multipath entry, but not the last one
4014 * removed, then only reduce its nexthop group.
4015 */
4016 if (!list_is_singular(&fib6_entry->rt6_list)) {
4017 mlxsw_sp_fib6_entry_nexthop_del(mlxsw_sp, fib6_entry, rt);
4018 return;
4019 }
4020
4021 fib_node = fib6_entry->common.fib_node;
4022
4023 mlxsw_sp_fib6_node_entry_unlink(mlxsw_sp, fib6_entry);
4024 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
4025 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
4026 }
4027
4028 static int __mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp,
4029 enum mlxsw_reg_ralxx_protocol proto,
4030 u8 tree_id)
4031 {
4032 char ralta_pl[MLXSW_REG_RALTA_LEN];
4033 char ralst_pl[MLXSW_REG_RALST_LEN];
4034 int i, err;
4035
4036 mlxsw_reg_ralta_pack(ralta_pl, true, proto, tree_id);
4037 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
4038 if (err)
4039 return err;
4040
4041 mlxsw_reg_ralst_pack(ralst_pl, 0xff, tree_id);
4042 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
4043 if (err)
4044 return err;
4045
4046 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
4047 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
4048 char raltb_pl[MLXSW_REG_RALTB_LEN];
4049 char ralue_pl[MLXSW_REG_RALUE_LEN];
4050
4051 mlxsw_reg_raltb_pack(raltb_pl, vr->id, proto, tree_id);
4052 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb),
4053 raltb_pl);
4054 if (err)
4055 return err;
4056
4057 mlxsw_reg_ralue_pack(ralue_pl, proto,
4058 MLXSW_REG_RALUE_OP_WRITE_WRITE, vr->id, 0);
4059 mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
4060 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue),
4061 ralue_pl);
4062 if (err)
4063 return err;
4064 }
4065
4066 return 0;
4067 }
4068
4069 static int mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp)
4070 {
4071 enum mlxsw_reg_ralxx_protocol proto = MLXSW_REG_RALXX_PROTOCOL_IPV4;
4072 int err;
4073
4074 err = __mlxsw_sp_router_set_abort_trap(mlxsw_sp, proto,
4075 MLXSW_SP_LPM_TREE_MIN);
4076 if (err)
4077 return err;
4078
4079 proto = MLXSW_REG_RALXX_PROTOCOL_IPV6;
4080 return __mlxsw_sp_router_set_abort_trap(mlxsw_sp, proto,
4081 MLXSW_SP_LPM_TREE_MIN + 1);
4082 }
4083
4084 static void mlxsw_sp_fib4_node_flush(struct mlxsw_sp *mlxsw_sp,
4085 struct mlxsw_sp_fib_node *fib_node)
4086 {
4087 struct mlxsw_sp_fib4_entry *fib4_entry, *tmp;
4088
4089 list_for_each_entry_safe(fib4_entry, tmp, &fib_node->entry_list,
4090 common.list) {
4091 bool do_break = &tmp->common.list == &fib_node->entry_list;
4092
4093 mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib4_entry);
4094 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
4095 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
4096 /* Break when entry list is empty and node was freed.
4097 * Otherwise, we'll access freed memory in the next
4098 * iteration.
4099 */
4100 if (do_break)
4101 break;
4102 }
4103 }
4104
4105 static void mlxsw_sp_fib6_node_flush(struct mlxsw_sp *mlxsw_sp,
4106 struct mlxsw_sp_fib_node *fib_node)
4107 {
4108 struct mlxsw_sp_fib6_entry *fib6_entry, *tmp;
4109
4110 list_for_each_entry_safe(fib6_entry, tmp, &fib_node->entry_list,
4111 common.list) {
4112 bool do_break = &tmp->common.list == &fib_node->entry_list;
4113
4114 mlxsw_sp_fib6_node_entry_unlink(mlxsw_sp, fib6_entry);
4115 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
4116 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
4117 if (do_break)
4118 break;
4119 }
4120 }
4121
4122 static void mlxsw_sp_fib_node_flush(struct mlxsw_sp *mlxsw_sp,
4123 struct mlxsw_sp_fib_node *fib_node)
4124 {
4125 switch (fib_node->fib->proto) {
4126 case MLXSW_SP_L3_PROTO_IPV4:
4127 mlxsw_sp_fib4_node_flush(mlxsw_sp, fib_node);
4128 break;
4129 case MLXSW_SP_L3_PROTO_IPV6:
4130 mlxsw_sp_fib6_node_flush(mlxsw_sp, fib_node);
4131 break;
4132 }
4133 }
4134
4135 static void mlxsw_sp_vr_fib_flush(struct mlxsw_sp *mlxsw_sp,
4136 struct mlxsw_sp_vr *vr,
4137 enum mlxsw_sp_l3proto proto)
4138 {
4139 struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
4140 struct mlxsw_sp_fib_node *fib_node, *tmp;
4141
4142 list_for_each_entry_safe(fib_node, tmp, &fib->node_list, list) {
4143 bool do_break = &tmp->list == &fib->node_list;
4144
4145 mlxsw_sp_fib_node_flush(mlxsw_sp, fib_node);
4146 if (do_break)
4147 break;
4148 }
4149 }
4150
4151 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp)
4152 {
4153 int i;
4154
4155 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
4156 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
4157
4158 if (!mlxsw_sp_vr_is_used(vr))
4159 continue;
4160 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4);
4161
4162 /* If virtual router was only used for IPv4, then it's no
4163 * longer used.
4164 */
4165 if (!mlxsw_sp_vr_is_used(vr))
4166 continue;
4167 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6);
4168 }
4169 }
4170
4171 static void mlxsw_sp_router_fib_abort(struct mlxsw_sp *mlxsw_sp)
4172 {
4173 int err;
4174
4175 if (mlxsw_sp->router->aborted)
4176 return;
4177 dev_warn(mlxsw_sp->bus_info->dev, "FIB abort triggered. Note that FIB entries are no longer being offloaded to this device.\n");
4178 mlxsw_sp_router_fib_flush(mlxsw_sp);
4179 mlxsw_sp->router->aborted = true;
4180 err = mlxsw_sp_router_set_abort_trap(mlxsw_sp);
4181 if (err)
4182 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set abort trap.\n");
4183 }
4184
4185 struct mlxsw_sp_fib_event_work {
4186 struct work_struct work;
4187 union {
4188 struct fib6_entry_notifier_info fen6_info;
4189 struct fib_entry_notifier_info fen_info;
4190 struct fib_rule_notifier_info fr_info;
4191 struct fib_nh_notifier_info fnh_info;
4192 };
4193 struct mlxsw_sp *mlxsw_sp;
4194 unsigned long event;
4195 };
4196
4197 static void mlxsw_sp_router_fib4_event_work(struct work_struct *work)
4198 {
4199 struct mlxsw_sp_fib_event_work *fib_work =
4200 container_of(work, struct mlxsw_sp_fib_event_work, work);
4201 struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp;
4202 struct fib_rule *rule;
4203 bool replace, append;
4204 int err;
4205
4206 /* Protect internal structures from changes */
4207 rtnl_lock();
4208 switch (fib_work->event) {
4209 case FIB_EVENT_ENTRY_REPLACE: /* fall through */
4210 case FIB_EVENT_ENTRY_APPEND: /* fall through */
4211 case FIB_EVENT_ENTRY_ADD:
4212 replace = fib_work->event == FIB_EVENT_ENTRY_REPLACE;
4213 append = fib_work->event == FIB_EVENT_ENTRY_APPEND;
4214 err = mlxsw_sp_router_fib4_add(mlxsw_sp, &fib_work->fen_info,
4215 replace, append);
4216 if (err)
4217 mlxsw_sp_router_fib_abort(mlxsw_sp);
4218 fib_info_put(fib_work->fen_info.fi);
4219 break;
4220 case FIB_EVENT_ENTRY_DEL:
4221 mlxsw_sp_router_fib4_del(mlxsw_sp, &fib_work->fen_info);
4222 fib_info_put(fib_work->fen_info.fi);
4223 break;
4224 case FIB_EVENT_RULE_ADD: /* fall through */
4225 case FIB_EVENT_RULE_DEL:
4226 rule = fib_work->fr_info.rule;
4227 if (!fib4_rule_default(rule) && !rule->l3mdev)
4228 mlxsw_sp_router_fib_abort(mlxsw_sp);
4229 fib_rule_put(rule);
4230 break;
4231 case FIB_EVENT_NH_ADD: /* fall through */
4232 case FIB_EVENT_NH_DEL:
4233 mlxsw_sp_nexthop4_event(mlxsw_sp, fib_work->event,
4234 fib_work->fnh_info.fib_nh);
4235 fib_info_put(fib_work->fnh_info.fib_nh->nh_parent);
4236 break;
4237 }
4238 rtnl_unlock();
4239 kfree(fib_work);
4240 }
4241
4242 static void mlxsw_sp_router_fib6_event_work(struct work_struct *work)
4243 {
4244 struct mlxsw_sp_fib_event_work *fib_work =
4245 container_of(work, struct mlxsw_sp_fib_event_work, work);
4246 struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp;
4247 struct fib_rule *rule;
4248 bool replace;
4249 int err;
4250
4251 rtnl_lock();
4252 switch (fib_work->event) {
4253 case FIB_EVENT_ENTRY_REPLACE: /* fall through */
4254 case FIB_EVENT_ENTRY_ADD:
4255 replace = fib_work->event == FIB_EVENT_ENTRY_REPLACE;
4256 err = mlxsw_sp_router_fib6_add(mlxsw_sp,
4257 fib_work->fen6_info.rt, replace);
4258 if (err)
4259 mlxsw_sp_router_fib_abort(mlxsw_sp);
4260 mlxsw_sp_rt6_release(fib_work->fen6_info.rt);
4261 break;
4262 case FIB_EVENT_ENTRY_DEL:
4263 mlxsw_sp_router_fib6_del(mlxsw_sp, fib_work->fen6_info.rt);
4264 mlxsw_sp_rt6_release(fib_work->fen6_info.rt);
4265 break;
4266 case FIB_EVENT_RULE_ADD: /* fall through */
4267 case FIB_EVENT_RULE_DEL:
4268 rule = fib_work->fr_info.rule;
4269 if (!fib6_rule_default(rule) && !rule->l3mdev)
4270 mlxsw_sp_router_fib_abort(mlxsw_sp);
4271 fib_rule_put(rule);
4272 break;
4273 }
4274 rtnl_unlock();
4275 kfree(fib_work);
4276 }
4277
4278 static void mlxsw_sp_router_fib4_event(struct mlxsw_sp_fib_event_work *fib_work,
4279 struct fib_notifier_info *info)
4280 {
4281 switch (fib_work->event) {
4282 case FIB_EVENT_ENTRY_REPLACE: /* fall through */
4283 case FIB_EVENT_ENTRY_APPEND: /* fall through */
4284 case FIB_EVENT_ENTRY_ADD: /* fall through */
4285 case FIB_EVENT_ENTRY_DEL:
4286 memcpy(&fib_work->fen_info, info, sizeof(fib_work->fen_info));
4287 /* Take referece on fib_info to prevent it from being
4288 * freed while work is queued. Release it afterwards.
4289 */
4290 fib_info_hold(fib_work->fen_info.fi);
4291 break;
4292 case FIB_EVENT_RULE_ADD: /* fall through */
4293 case FIB_EVENT_RULE_DEL:
4294 memcpy(&fib_work->fr_info, info, sizeof(fib_work->fr_info));
4295 fib_rule_get(fib_work->fr_info.rule);
4296 break;
4297 case FIB_EVENT_NH_ADD: /* fall through */
4298 case FIB_EVENT_NH_DEL:
4299 memcpy(&fib_work->fnh_info, info, sizeof(fib_work->fnh_info));
4300 fib_info_hold(fib_work->fnh_info.fib_nh->nh_parent);
4301 break;
4302 }
4303 }
4304
4305 static void mlxsw_sp_router_fib6_event(struct mlxsw_sp_fib_event_work *fib_work,
4306 struct fib_notifier_info *info)
4307 {
4308 switch (fib_work->event) {
4309 case FIB_EVENT_ENTRY_REPLACE: /* fall through */
4310 case FIB_EVENT_ENTRY_ADD: /* fall through */
4311 case FIB_EVENT_ENTRY_DEL:
4312 memcpy(&fib_work->fen6_info, info, sizeof(fib_work->fen6_info));
4313 rt6_hold(fib_work->fen6_info.rt);
4314 break;
4315 case FIB_EVENT_RULE_ADD: /* fall through */
4316 case FIB_EVENT_RULE_DEL:
4317 memcpy(&fib_work->fr_info, info, sizeof(fib_work->fr_info));
4318 fib_rule_get(fib_work->fr_info.rule);
4319 break;
4320 }
4321 }
4322
4323 /* Called with rcu_read_lock() */
4324 static int mlxsw_sp_router_fib_event(struct notifier_block *nb,
4325 unsigned long event, void *ptr)
4326 {
4327 struct mlxsw_sp_fib_event_work *fib_work;
4328 struct fib_notifier_info *info = ptr;
4329 struct mlxsw_sp_router *router;
4330
4331 if (!net_eq(info->net, &init_net))
4332 return NOTIFY_DONE;
4333
4334 fib_work = kzalloc(sizeof(*fib_work), GFP_ATOMIC);
4335 if (WARN_ON(!fib_work))
4336 return NOTIFY_BAD;
4337
4338 router = container_of(nb, struct mlxsw_sp_router, fib_nb);
4339 fib_work->mlxsw_sp = router->mlxsw_sp;
4340 fib_work->event = event;
4341
4342 switch (info->family) {
4343 case AF_INET:
4344 INIT_WORK(&fib_work->work, mlxsw_sp_router_fib4_event_work);
4345 mlxsw_sp_router_fib4_event(fib_work, info);
4346 break;
4347 case AF_INET6:
4348 INIT_WORK(&fib_work->work, mlxsw_sp_router_fib6_event_work);
4349 mlxsw_sp_router_fib6_event(fib_work, info);
4350 break;
4351 }
4352
4353 mlxsw_core_schedule_work(&fib_work->work);
4354
4355 return NOTIFY_DONE;
4356 }
4357
4358 static struct mlxsw_sp_rif *
4359 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
4360 const struct net_device *dev)
4361 {
4362 int i;
4363
4364 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
4365 if (mlxsw_sp->router->rifs[i] &&
4366 mlxsw_sp->router->rifs[i]->dev == dev)
4367 return mlxsw_sp->router->rifs[i];
4368
4369 return NULL;
4370 }
4371
4372 static int mlxsw_sp_router_rif_disable(struct mlxsw_sp *mlxsw_sp, u16 rif)
4373 {
4374 char ritr_pl[MLXSW_REG_RITR_LEN];
4375 int err;
4376
4377 mlxsw_reg_ritr_rif_pack(ritr_pl, rif);
4378 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
4379 if (WARN_ON_ONCE(err))
4380 return err;
4381
4382 mlxsw_reg_ritr_enable_set(ritr_pl, false);
4383 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
4384 }
4385
4386 static void mlxsw_sp_router_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
4387 struct mlxsw_sp_rif *rif)
4388 {
4389 mlxsw_sp_router_rif_disable(mlxsw_sp, rif->rif_index);
4390 mlxsw_sp_nexthop_rif_gone_sync(mlxsw_sp, rif);
4391 mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif);
4392 }
4393
4394 static bool
4395 mlxsw_sp_rif_should_config(struct mlxsw_sp_rif *rif, struct net_device *dev,
4396 unsigned long event)
4397 {
4398 struct inet6_dev *inet6_dev;
4399 bool addr_list_empty = true;
4400 struct in_device *idev;
4401
4402 switch (event) {
4403 case NETDEV_UP:
4404 return rif == NULL;
4405 case NETDEV_DOWN:
4406 idev = __in_dev_get_rtnl(dev);
4407 if (idev && idev->ifa_list)
4408 addr_list_empty = false;
4409
4410 inet6_dev = __in6_dev_get(dev);
4411 if (addr_list_empty && inet6_dev &&
4412 !list_empty(&inet6_dev->addr_list))
4413 addr_list_empty = false;
4414
4415 if (rif && addr_list_empty &&
4416 !netif_is_l3_slave(rif->dev))
4417 return true;
4418 /* It is possible we already removed the RIF ourselves
4419 * if it was assigned to a netdev that is now a bridge
4420 * or LAG slave.
4421 */
4422 return false;
4423 }
4424
4425 return false;
4426 }
4427
4428 static enum mlxsw_sp_rif_type
4429 mlxsw_sp_dev_rif_type(const struct mlxsw_sp *mlxsw_sp,
4430 const struct net_device *dev)
4431 {
4432 enum mlxsw_sp_fid_type type;
4433
4434 if (mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL))
4435 return MLXSW_SP_RIF_TYPE_IPIP_LB;
4436
4437 /* Otherwise RIF type is derived from the type of the underlying FID. */
4438 if (is_vlan_dev(dev) && netif_is_bridge_master(vlan_dev_real_dev(dev)))
4439 type = MLXSW_SP_FID_TYPE_8021Q;
4440 else if (netif_is_bridge_master(dev) && br_vlan_enabled(dev))
4441 type = MLXSW_SP_FID_TYPE_8021Q;
4442 else if (netif_is_bridge_master(dev))
4443 type = MLXSW_SP_FID_TYPE_8021D;
4444 else
4445 type = MLXSW_SP_FID_TYPE_RFID;
4446
4447 return mlxsw_sp_fid_type_rif_type(mlxsw_sp, type);
4448 }
4449
4450 static int mlxsw_sp_rif_index_alloc(struct mlxsw_sp *mlxsw_sp, u16 *p_rif_index)
4451 {
4452 int i;
4453
4454 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) {
4455 if (!mlxsw_sp->router->rifs[i]) {
4456 *p_rif_index = i;
4457 return 0;
4458 }
4459 }
4460
4461 return -ENOBUFS;
4462 }
4463
4464 static struct mlxsw_sp_rif *mlxsw_sp_rif_alloc(size_t rif_size, u16 rif_index,
4465 u16 vr_id,
4466 struct net_device *l3_dev)
4467 {
4468 struct mlxsw_sp_rif *rif;
4469
4470 rif = kzalloc(rif_size, GFP_KERNEL);
4471 if (!rif)
4472 return NULL;
4473
4474 INIT_LIST_HEAD(&rif->nexthop_list);
4475 INIT_LIST_HEAD(&rif->neigh_list);
4476 ether_addr_copy(rif->addr, l3_dev->dev_addr);
4477 rif->mtu = l3_dev->mtu;
4478 rif->vr_id = vr_id;
4479 rif->dev = l3_dev;
4480 rif->rif_index = rif_index;
4481
4482 return rif;
4483 }
4484
4485 struct mlxsw_sp_rif *mlxsw_sp_rif_by_index(const struct mlxsw_sp *mlxsw_sp,
4486 u16 rif_index)
4487 {
4488 return mlxsw_sp->router->rifs[rif_index];
4489 }
4490
4491 u16 mlxsw_sp_rif_index(const struct mlxsw_sp_rif *rif)
4492 {
4493 return rif->rif_index;
4494 }
4495
4496 int mlxsw_sp_rif_dev_ifindex(const struct mlxsw_sp_rif *rif)
4497 {
4498 return rif->dev->ifindex;
4499 }
4500
4501 static struct mlxsw_sp_rif *
4502 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp,
4503 const struct mlxsw_sp_rif_params *params)
4504 {
4505 u32 tb_id = l3mdev_fib_table(params->dev);
4506 const struct mlxsw_sp_rif_ops *ops;
4507 struct mlxsw_sp_fid *fid = NULL;
4508 enum mlxsw_sp_rif_type type;
4509 struct mlxsw_sp_rif *rif;
4510 struct mlxsw_sp_vr *vr;
4511 u16 rif_index;
4512 int err;
4513
4514 type = mlxsw_sp_dev_rif_type(mlxsw_sp, params->dev);
4515 ops = mlxsw_sp->router->rif_ops_arr[type];
4516
4517 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id ? : RT_TABLE_MAIN);
4518 if (IS_ERR(vr))
4519 return ERR_CAST(vr);
4520
4521 err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index);
4522 if (err)
4523 goto err_rif_index_alloc;
4524
4525 rif = mlxsw_sp_rif_alloc(ops->rif_size, rif_index, vr->id, params->dev);
4526 if (!rif) {
4527 err = -ENOMEM;
4528 goto err_rif_alloc;
4529 }
4530 rif->mlxsw_sp = mlxsw_sp;
4531 rif->ops = ops;
4532
4533 if (ops->fid_get) {
4534 fid = ops->fid_get(rif);
4535 if (IS_ERR(fid)) {
4536 err = PTR_ERR(fid);
4537 goto err_fid_get;
4538 }
4539 rif->fid = fid;
4540 }
4541
4542 if (ops->setup)
4543 ops->setup(rif, params);
4544
4545 err = ops->configure(rif);
4546 if (err)
4547 goto err_configure;
4548
4549 mlxsw_sp_rif_counters_alloc(rif);
4550 mlxsw_sp->router->rifs[rif_index] = rif;
4551 vr->rif_count++;
4552
4553 return rif;
4554
4555 err_configure:
4556 if (fid)
4557 mlxsw_sp_fid_put(fid);
4558 err_fid_get:
4559 kfree(rif);
4560 err_rif_alloc:
4561 err_rif_index_alloc:
4562 mlxsw_sp_vr_put(vr);
4563 return ERR_PTR(err);
4564 }
4565
4566 void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif)
4567 {
4568 const struct mlxsw_sp_rif_ops *ops = rif->ops;
4569 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
4570 struct mlxsw_sp_fid *fid = rif->fid;
4571 struct mlxsw_sp_vr *vr;
4572
4573 mlxsw_sp_router_rif_gone_sync(mlxsw_sp, rif);
4574 vr = &mlxsw_sp->router->vrs[rif->vr_id];
4575
4576 vr->rif_count--;
4577 mlxsw_sp->router->rifs[rif->rif_index] = NULL;
4578 mlxsw_sp_rif_counters_free(rif);
4579 ops->deconfigure(rif);
4580 if (fid)
4581 /* Loopback RIFs are not associated with a FID. */
4582 mlxsw_sp_fid_put(fid);
4583 kfree(rif);
4584 mlxsw_sp_vr_put(vr);
4585 }
4586
4587 static void
4588 mlxsw_sp_rif_subport_params_init(struct mlxsw_sp_rif_params *params,
4589 struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
4590 {
4591 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
4592
4593 params->vid = mlxsw_sp_port_vlan->vid;
4594 params->lag = mlxsw_sp_port->lagged;
4595 if (params->lag)
4596 params->lag_id = mlxsw_sp_port->lag_id;
4597 else
4598 params->system_port = mlxsw_sp_port->local_port;
4599 }
4600
4601 static int
4602 mlxsw_sp_port_vlan_router_join(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan,
4603 struct net_device *l3_dev)
4604 {
4605 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
4606 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
4607 u16 vid = mlxsw_sp_port_vlan->vid;
4608 struct mlxsw_sp_rif *rif;
4609 struct mlxsw_sp_fid *fid;
4610 int err;
4611
4612 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
4613 if (!rif) {
4614 struct mlxsw_sp_rif_params params = {
4615 .dev = l3_dev,
4616 };
4617
4618 mlxsw_sp_rif_subport_params_init(&params, mlxsw_sp_port_vlan);
4619 rif = mlxsw_sp_rif_create(mlxsw_sp, &params);
4620 if (IS_ERR(rif))
4621 return PTR_ERR(rif);
4622 }
4623
4624 /* FID was already created, just take a reference */
4625 fid = rif->ops->fid_get(rif);
4626 err = mlxsw_sp_fid_port_vid_map(fid, mlxsw_sp_port, vid);
4627 if (err)
4628 goto err_fid_port_vid_map;
4629
4630 err = mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, false);
4631 if (err)
4632 goto err_port_vid_learning_set;
4633
4634 err = mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid,
4635 BR_STATE_FORWARDING);
4636 if (err)
4637 goto err_port_vid_stp_set;
4638
4639 mlxsw_sp_port_vlan->fid = fid;
4640
4641 return 0;
4642
4643 err_port_vid_stp_set:
4644 mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true);
4645 err_port_vid_learning_set:
4646 mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid);
4647 err_fid_port_vid_map:
4648 mlxsw_sp_fid_put(fid);
4649 return err;
4650 }
4651
4652 void
4653 mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
4654 {
4655 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
4656 struct mlxsw_sp_fid *fid = mlxsw_sp_port_vlan->fid;
4657 u16 vid = mlxsw_sp_port_vlan->vid;
4658
4659 if (WARN_ON(mlxsw_sp_fid_type(fid) != MLXSW_SP_FID_TYPE_RFID))
4660 return;
4661
4662 mlxsw_sp_port_vlan->fid = NULL;
4663 mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid, BR_STATE_BLOCKING);
4664 mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true);
4665 mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid);
4666 /* If router port holds the last reference on the rFID, then the
4667 * associated Sub-port RIF will be destroyed.
4668 */
4669 mlxsw_sp_fid_put(fid);
4670 }
4671
4672 static int mlxsw_sp_inetaddr_port_vlan_event(struct net_device *l3_dev,
4673 struct net_device *port_dev,
4674 unsigned long event, u16 vid)
4675 {
4676 struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(port_dev);
4677 struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan;
4678
4679 mlxsw_sp_port_vlan = mlxsw_sp_port_vlan_find_by_vid(mlxsw_sp_port, vid);
4680 if (WARN_ON(!mlxsw_sp_port_vlan))
4681 return -EINVAL;
4682
4683 switch (event) {
4684 case NETDEV_UP:
4685 return mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan,
4686 l3_dev);
4687 case NETDEV_DOWN:
4688 mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan);
4689 break;
4690 }
4691
4692 return 0;
4693 }
4694
4695 static int mlxsw_sp_inetaddr_port_event(struct net_device *port_dev,
4696 unsigned long event)
4697 {
4698 if (netif_is_bridge_port(port_dev) ||
4699 netif_is_lag_port(port_dev) ||
4700 netif_is_ovs_port(port_dev))
4701 return 0;
4702
4703 return mlxsw_sp_inetaddr_port_vlan_event(port_dev, port_dev, event, 1);
4704 }
4705
4706 static int __mlxsw_sp_inetaddr_lag_event(struct net_device *l3_dev,
4707 struct net_device *lag_dev,
4708 unsigned long event, u16 vid)
4709 {
4710 struct net_device *port_dev;
4711 struct list_head *iter;
4712 int err;
4713
4714 netdev_for_each_lower_dev(lag_dev, port_dev, iter) {
4715 if (mlxsw_sp_port_dev_check(port_dev)) {
4716 err = mlxsw_sp_inetaddr_port_vlan_event(l3_dev,
4717 port_dev,
4718 event, vid);
4719 if (err)
4720 return err;
4721 }
4722 }
4723
4724 return 0;
4725 }
4726
4727 static int mlxsw_sp_inetaddr_lag_event(struct net_device *lag_dev,
4728 unsigned long event)
4729 {
4730 if (netif_is_bridge_port(lag_dev))
4731 return 0;
4732
4733 return __mlxsw_sp_inetaddr_lag_event(lag_dev, lag_dev, event, 1);
4734 }
4735
4736 static int mlxsw_sp_inetaddr_bridge_event(struct net_device *l3_dev,
4737 unsigned long event)
4738 {
4739 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev);
4740 struct mlxsw_sp_rif_params params = {
4741 .dev = l3_dev,
4742 };
4743 struct mlxsw_sp_rif *rif;
4744
4745 switch (event) {
4746 case NETDEV_UP:
4747 rif = mlxsw_sp_rif_create(mlxsw_sp, &params);
4748 if (IS_ERR(rif))
4749 return PTR_ERR(rif);
4750 break;
4751 case NETDEV_DOWN:
4752 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
4753 mlxsw_sp_rif_destroy(rif);
4754 break;
4755 }
4756
4757 return 0;
4758 }
4759
4760 static int mlxsw_sp_inetaddr_vlan_event(struct net_device *vlan_dev,
4761 unsigned long event)
4762 {
4763 struct net_device *real_dev = vlan_dev_real_dev(vlan_dev);
4764 u16 vid = vlan_dev_vlan_id(vlan_dev);
4765
4766 if (netif_is_bridge_port(vlan_dev))
4767 return 0;
4768
4769 if (mlxsw_sp_port_dev_check(real_dev))
4770 return mlxsw_sp_inetaddr_port_vlan_event(vlan_dev, real_dev,
4771 event, vid);
4772 else if (netif_is_lag_master(real_dev))
4773 return __mlxsw_sp_inetaddr_lag_event(vlan_dev, real_dev, event,
4774 vid);
4775 else if (netif_is_bridge_master(real_dev) && br_vlan_enabled(real_dev))
4776 return mlxsw_sp_inetaddr_bridge_event(vlan_dev, event);
4777
4778 return 0;
4779 }
4780
4781 static int __mlxsw_sp_inetaddr_event(struct net_device *dev,
4782 unsigned long event)
4783 {
4784 if (mlxsw_sp_port_dev_check(dev))
4785 return mlxsw_sp_inetaddr_port_event(dev, event);
4786 else if (netif_is_lag_master(dev))
4787 return mlxsw_sp_inetaddr_lag_event(dev, event);
4788 else if (netif_is_bridge_master(dev))
4789 return mlxsw_sp_inetaddr_bridge_event(dev, event);
4790 else if (is_vlan_dev(dev))
4791 return mlxsw_sp_inetaddr_vlan_event(dev, event);
4792 else
4793 return 0;
4794 }
4795
4796 int mlxsw_sp_inetaddr_event(struct notifier_block *unused,
4797 unsigned long event, void *ptr)
4798 {
4799 struct in_ifaddr *ifa = (struct in_ifaddr *) ptr;
4800 struct net_device *dev = ifa->ifa_dev->dev;
4801 struct mlxsw_sp *mlxsw_sp;
4802 struct mlxsw_sp_rif *rif;
4803 int err = 0;
4804
4805 mlxsw_sp = mlxsw_sp_lower_get(dev);
4806 if (!mlxsw_sp)
4807 goto out;
4808
4809 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
4810 if (!mlxsw_sp_rif_should_config(rif, dev, event))
4811 goto out;
4812
4813 err = __mlxsw_sp_inetaddr_event(dev, event);
4814 out:
4815 return notifier_from_errno(err);
4816 }
4817
4818 struct mlxsw_sp_inet6addr_event_work {
4819 struct work_struct work;
4820 struct net_device *dev;
4821 unsigned long event;
4822 };
4823
4824 static void mlxsw_sp_inet6addr_event_work(struct work_struct *work)
4825 {
4826 struct mlxsw_sp_inet6addr_event_work *inet6addr_work =
4827 container_of(work, struct mlxsw_sp_inet6addr_event_work, work);
4828 struct net_device *dev = inet6addr_work->dev;
4829 unsigned long event = inet6addr_work->event;
4830 struct mlxsw_sp *mlxsw_sp;
4831 struct mlxsw_sp_rif *rif;
4832
4833 rtnl_lock();
4834 mlxsw_sp = mlxsw_sp_lower_get(dev);
4835 if (!mlxsw_sp)
4836 goto out;
4837
4838 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
4839 if (!mlxsw_sp_rif_should_config(rif, dev, event))
4840 goto out;
4841
4842 __mlxsw_sp_inetaddr_event(dev, event);
4843 out:
4844 rtnl_unlock();
4845 dev_put(dev);
4846 kfree(inet6addr_work);
4847 }
4848
4849 /* Called with rcu_read_lock() */
4850 int mlxsw_sp_inet6addr_event(struct notifier_block *unused,
4851 unsigned long event, void *ptr)
4852 {
4853 struct inet6_ifaddr *if6 = (struct inet6_ifaddr *) ptr;
4854 struct mlxsw_sp_inet6addr_event_work *inet6addr_work;
4855 struct net_device *dev = if6->idev->dev;
4856
4857 if (!mlxsw_sp_port_dev_lower_find_rcu(dev))
4858 return NOTIFY_DONE;
4859
4860 inet6addr_work = kzalloc(sizeof(*inet6addr_work), GFP_ATOMIC);
4861 if (!inet6addr_work)
4862 return NOTIFY_BAD;
4863
4864 INIT_WORK(&inet6addr_work->work, mlxsw_sp_inet6addr_event_work);
4865 inet6addr_work->dev = dev;
4866 inet6addr_work->event = event;
4867 dev_hold(dev);
4868 mlxsw_core_schedule_work(&inet6addr_work->work);
4869
4870 return NOTIFY_DONE;
4871 }
4872
4873 static int mlxsw_sp_rif_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
4874 const char *mac, int mtu)
4875 {
4876 char ritr_pl[MLXSW_REG_RITR_LEN];
4877 int err;
4878
4879 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
4880 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
4881 if (err)
4882 return err;
4883
4884 mlxsw_reg_ritr_mtu_set(ritr_pl, mtu);
4885 mlxsw_reg_ritr_if_mac_memcpy_to(ritr_pl, mac);
4886 mlxsw_reg_ritr_op_set(ritr_pl, MLXSW_REG_RITR_RIF_CREATE);
4887 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
4888 }
4889
4890 int mlxsw_sp_netdevice_router_port_event(struct net_device *dev)
4891 {
4892 struct mlxsw_sp *mlxsw_sp;
4893 struct mlxsw_sp_rif *rif;
4894 u16 fid_index;
4895 int err;
4896
4897 mlxsw_sp = mlxsw_sp_lower_get(dev);
4898 if (!mlxsw_sp)
4899 return 0;
4900
4901 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
4902 if (!rif)
4903 return 0;
4904 fid_index = mlxsw_sp_fid_index(rif->fid);
4905
4906 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, false);
4907 if (err)
4908 return err;
4909
4910 err = mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, dev->dev_addr,
4911 dev->mtu);
4912 if (err)
4913 goto err_rif_edit;
4914
4915 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, dev->dev_addr, fid_index, true);
4916 if (err)
4917 goto err_rif_fdb_op;
4918
4919 ether_addr_copy(rif->addr, dev->dev_addr);
4920 rif->mtu = dev->mtu;
4921
4922 netdev_dbg(dev, "Updated RIF=%d\n", rif->rif_index);
4923
4924 return 0;
4925
4926 err_rif_fdb_op:
4927 mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, rif->addr, rif->mtu);
4928 err_rif_edit:
4929 mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, true);
4930 return err;
4931 }
4932
4933 static int mlxsw_sp_port_vrf_join(struct mlxsw_sp *mlxsw_sp,
4934 struct net_device *l3_dev)
4935 {
4936 struct mlxsw_sp_rif *rif;
4937
4938 /* If netdev is already associated with a RIF, then we need to
4939 * destroy it and create a new one with the new virtual router ID.
4940 */
4941 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
4942 if (rif)
4943 __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_DOWN);
4944
4945 return __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_UP);
4946 }
4947
4948 static void mlxsw_sp_port_vrf_leave(struct mlxsw_sp *mlxsw_sp,
4949 struct net_device *l3_dev)
4950 {
4951 struct mlxsw_sp_rif *rif;
4952
4953 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
4954 if (!rif)
4955 return;
4956 __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_DOWN);
4957 }
4958
4959 int mlxsw_sp_netdevice_vrf_event(struct net_device *l3_dev, unsigned long event,
4960 struct netdev_notifier_changeupper_info *info)
4961 {
4962 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev);
4963 int err = 0;
4964
4965 if (!mlxsw_sp)
4966 return 0;
4967
4968 switch (event) {
4969 case NETDEV_PRECHANGEUPPER:
4970 return 0;
4971 case NETDEV_CHANGEUPPER:
4972 if (info->linking)
4973 err = mlxsw_sp_port_vrf_join(mlxsw_sp, l3_dev);
4974 else
4975 mlxsw_sp_port_vrf_leave(mlxsw_sp, l3_dev);
4976 break;
4977 }
4978
4979 return err;
4980 }
4981
4982 static struct mlxsw_sp_rif_subport *
4983 mlxsw_sp_rif_subport_rif(const struct mlxsw_sp_rif *rif)
4984 {
4985 return container_of(rif, struct mlxsw_sp_rif_subport, common);
4986 }
4987
4988 static void mlxsw_sp_rif_subport_setup(struct mlxsw_sp_rif *rif,
4989 const struct mlxsw_sp_rif_params *params)
4990 {
4991 struct mlxsw_sp_rif_subport *rif_subport;
4992
4993 rif_subport = mlxsw_sp_rif_subport_rif(rif);
4994 rif_subport->vid = params->vid;
4995 rif_subport->lag = params->lag;
4996 if (params->lag)
4997 rif_subport->lag_id = params->lag_id;
4998 else
4999 rif_subport->system_port = params->system_port;
5000 }
5001
5002 static int mlxsw_sp_rif_subport_op(struct mlxsw_sp_rif *rif, bool enable)
5003 {
5004 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5005 struct mlxsw_sp_rif_subport *rif_subport;
5006 char ritr_pl[MLXSW_REG_RITR_LEN];
5007
5008 rif_subport = mlxsw_sp_rif_subport_rif(rif);
5009 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_SP_IF,
5010 rif->rif_index, rif->vr_id, rif->dev->mtu);
5011 mlxsw_reg_ritr_mac_pack(ritr_pl, rif->dev->dev_addr);
5012 mlxsw_reg_ritr_sp_if_pack(ritr_pl, rif_subport->lag,
5013 rif_subport->lag ? rif_subport->lag_id :
5014 rif_subport->system_port,
5015 rif_subport->vid);
5016
5017 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
5018 }
5019
5020 static int mlxsw_sp_rif_subport_configure(struct mlxsw_sp_rif *rif)
5021 {
5022 int err;
5023
5024 err = mlxsw_sp_rif_subport_op(rif, true);
5025 if (err)
5026 return err;
5027
5028 err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
5029 mlxsw_sp_fid_index(rif->fid), true);
5030 if (err)
5031 goto err_rif_fdb_op;
5032
5033 mlxsw_sp_fid_rif_set(rif->fid, rif);
5034 return 0;
5035
5036 err_rif_fdb_op:
5037 mlxsw_sp_rif_subport_op(rif, false);
5038 return err;
5039 }
5040
5041 static void mlxsw_sp_rif_subport_deconfigure(struct mlxsw_sp_rif *rif)
5042 {
5043 struct mlxsw_sp_fid *fid = rif->fid;
5044
5045 mlxsw_sp_fid_rif_set(fid, NULL);
5046 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
5047 mlxsw_sp_fid_index(fid), false);
5048 mlxsw_sp_rif_subport_op(rif, false);
5049 }
5050
5051 static struct mlxsw_sp_fid *
5052 mlxsw_sp_rif_subport_fid_get(struct mlxsw_sp_rif *rif)
5053 {
5054 return mlxsw_sp_fid_rfid_get(rif->mlxsw_sp, rif->rif_index);
5055 }
5056
5057 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_subport_ops = {
5058 .type = MLXSW_SP_RIF_TYPE_SUBPORT,
5059 .rif_size = sizeof(struct mlxsw_sp_rif_subport),
5060 .setup = mlxsw_sp_rif_subport_setup,
5061 .configure = mlxsw_sp_rif_subport_configure,
5062 .deconfigure = mlxsw_sp_rif_subport_deconfigure,
5063 .fid_get = mlxsw_sp_rif_subport_fid_get,
5064 };
5065
5066 static int mlxsw_sp_rif_vlan_fid_op(struct mlxsw_sp_rif *rif,
5067 enum mlxsw_reg_ritr_if_type type,
5068 u16 vid_fid, bool enable)
5069 {
5070 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5071 char ritr_pl[MLXSW_REG_RITR_LEN];
5072
5073 mlxsw_reg_ritr_pack(ritr_pl, enable, type, rif->rif_index, rif->vr_id,
5074 rif->dev->mtu);
5075 mlxsw_reg_ritr_mac_pack(ritr_pl, rif->dev->dev_addr);
5076 mlxsw_reg_ritr_fid_set(ritr_pl, type, vid_fid);
5077
5078 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
5079 }
5080
5081 static u8 mlxsw_sp_router_port(const struct mlxsw_sp *mlxsw_sp)
5082 {
5083 return mlxsw_core_max_ports(mlxsw_sp->core) + 1;
5084 }
5085
5086 static int mlxsw_sp_rif_vlan_configure(struct mlxsw_sp_rif *rif)
5087 {
5088 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5089 u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid);
5090 int err;
5091
5092 err = mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, true);
5093 if (err)
5094 return err;
5095
5096 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
5097 mlxsw_sp_router_port(mlxsw_sp), true);
5098 if (err)
5099 goto err_fid_mc_flood_set;
5100
5101 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
5102 mlxsw_sp_router_port(mlxsw_sp), true);
5103 if (err)
5104 goto err_fid_bc_flood_set;
5105
5106 err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
5107 mlxsw_sp_fid_index(rif->fid), true);
5108 if (err)
5109 goto err_rif_fdb_op;
5110
5111 mlxsw_sp_fid_rif_set(rif->fid, rif);
5112 return 0;
5113
5114 err_rif_fdb_op:
5115 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
5116 mlxsw_sp_router_port(mlxsw_sp), false);
5117 err_fid_bc_flood_set:
5118 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
5119 mlxsw_sp_router_port(mlxsw_sp), false);
5120 err_fid_mc_flood_set:
5121 mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, false);
5122 return err;
5123 }
5124
5125 static void mlxsw_sp_rif_vlan_deconfigure(struct mlxsw_sp_rif *rif)
5126 {
5127 u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid);
5128 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5129 struct mlxsw_sp_fid *fid = rif->fid;
5130
5131 mlxsw_sp_fid_rif_set(fid, NULL);
5132 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
5133 mlxsw_sp_fid_index(fid), false);
5134 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
5135 mlxsw_sp_router_port(mlxsw_sp), false);
5136 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
5137 mlxsw_sp_router_port(mlxsw_sp), false);
5138 mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, false);
5139 }
5140
5141 static struct mlxsw_sp_fid *
5142 mlxsw_sp_rif_vlan_fid_get(struct mlxsw_sp_rif *rif)
5143 {
5144 u16 vid = is_vlan_dev(rif->dev) ? vlan_dev_vlan_id(rif->dev) : 1;
5145
5146 return mlxsw_sp_fid_8021q_get(rif->mlxsw_sp, vid);
5147 }
5148
5149 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_vlan_ops = {
5150 .type = MLXSW_SP_RIF_TYPE_VLAN,
5151 .rif_size = sizeof(struct mlxsw_sp_rif),
5152 .configure = mlxsw_sp_rif_vlan_configure,
5153 .deconfigure = mlxsw_sp_rif_vlan_deconfigure,
5154 .fid_get = mlxsw_sp_rif_vlan_fid_get,
5155 };
5156
5157 static int mlxsw_sp_rif_fid_configure(struct mlxsw_sp_rif *rif)
5158 {
5159 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5160 u16 fid_index = mlxsw_sp_fid_index(rif->fid);
5161 int err;
5162
5163 err = mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index,
5164 true);
5165 if (err)
5166 return err;
5167
5168 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
5169 mlxsw_sp_router_port(mlxsw_sp), true);
5170 if (err)
5171 goto err_fid_mc_flood_set;
5172
5173 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
5174 mlxsw_sp_router_port(mlxsw_sp), true);
5175 if (err)
5176 goto err_fid_bc_flood_set;
5177
5178 err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
5179 mlxsw_sp_fid_index(rif->fid), true);
5180 if (err)
5181 goto err_rif_fdb_op;
5182
5183 mlxsw_sp_fid_rif_set(rif->fid, rif);
5184 return 0;
5185
5186 err_rif_fdb_op:
5187 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
5188 mlxsw_sp_router_port(mlxsw_sp), false);
5189 err_fid_bc_flood_set:
5190 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
5191 mlxsw_sp_router_port(mlxsw_sp), false);
5192 err_fid_mc_flood_set:
5193 mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false);
5194 return err;
5195 }
5196
5197 static void mlxsw_sp_rif_fid_deconfigure(struct mlxsw_sp_rif *rif)
5198 {
5199 u16 fid_index = mlxsw_sp_fid_index(rif->fid);
5200 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5201 struct mlxsw_sp_fid *fid = rif->fid;
5202
5203 mlxsw_sp_fid_rif_set(fid, NULL);
5204 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
5205 mlxsw_sp_fid_index(fid), false);
5206 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
5207 mlxsw_sp_router_port(mlxsw_sp), false);
5208 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
5209 mlxsw_sp_router_port(mlxsw_sp), false);
5210 mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false);
5211 }
5212
5213 static struct mlxsw_sp_fid *
5214 mlxsw_sp_rif_fid_fid_get(struct mlxsw_sp_rif *rif)
5215 {
5216 return mlxsw_sp_fid_8021d_get(rif->mlxsw_sp, rif->dev->ifindex);
5217 }
5218
5219 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_fid_ops = {
5220 .type = MLXSW_SP_RIF_TYPE_FID,
5221 .rif_size = sizeof(struct mlxsw_sp_rif),
5222 .configure = mlxsw_sp_rif_fid_configure,
5223 .deconfigure = mlxsw_sp_rif_fid_deconfigure,
5224 .fid_get = mlxsw_sp_rif_fid_fid_get,
5225 };
5226
5227 static struct mlxsw_sp_rif_ipip_lb *
5228 mlxsw_sp_rif_ipip_lb_rif(struct mlxsw_sp_rif *rif)
5229 {
5230 return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common);
5231 }
5232
5233 static void
5234 mlxsw_sp_rif_ipip_lb_setup(struct mlxsw_sp_rif *rif,
5235 const struct mlxsw_sp_rif_params *params)
5236 {
5237 struct mlxsw_sp_rif_params_ipip_lb *params_lb;
5238 struct mlxsw_sp_rif_ipip_lb *rif_lb;
5239
5240 params_lb = container_of(params, struct mlxsw_sp_rif_params_ipip_lb,
5241 common);
5242 rif_lb = mlxsw_sp_rif_ipip_lb_rif(rif);
5243 rif_lb->lb_config = params_lb->lb_config;
5244 }
5245
5246 static int
5247 mlxsw_sp_rif_ipip_lb_op(struct mlxsw_sp_rif_ipip_lb *lb_rif,
5248 struct mlxsw_sp_vr *ul_vr, bool enable)
5249 {
5250 struct mlxsw_sp_rif_ipip_lb_config lb_cf = lb_rif->lb_config;
5251 struct mlxsw_sp_rif *rif = &lb_rif->common;
5252 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5253 char ritr_pl[MLXSW_REG_RITR_LEN];
5254 u32 saddr4;
5255
5256 switch (lb_cf.ul_protocol) {
5257 case MLXSW_SP_L3_PROTO_IPV4:
5258 saddr4 = be32_to_cpu(lb_cf.saddr.addr4);
5259 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF,
5260 rif->rif_index, rif->vr_id, rif->dev->mtu);
5261 mlxsw_reg_ritr_loopback_ipip4_pack(ritr_pl, lb_cf.lb_ipipt,
5262 MLXSW_REG_RITR_LOOPBACK_IPIP_OPTIONS_GRE_KEY_PRESET,
5263 ul_vr->id, saddr4, lb_cf.okey);
5264 break;
5265
5266 case MLXSW_SP_L3_PROTO_IPV6:
5267 return -EAFNOSUPPORT;
5268 }
5269
5270 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
5271 }
5272
5273 static int
5274 mlxsw_sp_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif)
5275 {
5276 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
5277 u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(rif->dev);
5278 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5279 struct mlxsw_sp_vr *ul_vr;
5280 int err;
5281
5282 ul_vr = mlxsw_sp_vr_get(mlxsw_sp, ul_tb_id);
5283 if (IS_ERR(ul_vr))
5284 return PTR_ERR(ul_vr);
5285
5286 err = mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr, true);
5287 if (err)
5288 goto err_loopback_op;
5289
5290 lb_rif->ul_vr_id = ul_vr->id;
5291 ++ul_vr->rif_count;
5292 return 0;
5293
5294 err_loopback_op:
5295 mlxsw_sp_vr_put(ul_vr);
5296 return err;
5297 }
5298
5299 static void mlxsw_sp_rif_ipip_lb_deconfigure(struct mlxsw_sp_rif *rif)
5300 {
5301 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
5302 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5303 struct mlxsw_sp_vr *ul_vr;
5304
5305 ul_vr = &mlxsw_sp->router->vrs[lb_rif->ul_vr_id];
5306 mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr, false);
5307
5308 --ul_vr->rif_count;
5309 mlxsw_sp_vr_put(ul_vr);
5310 }
5311
5312 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_ipip_lb_ops = {
5313 .type = MLXSW_SP_RIF_TYPE_IPIP_LB,
5314 .rif_size = sizeof(struct mlxsw_sp_rif_ipip_lb),
5315 .setup = mlxsw_sp_rif_ipip_lb_setup,
5316 .configure = mlxsw_sp_rif_ipip_lb_configure,
5317 .deconfigure = mlxsw_sp_rif_ipip_lb_deconfigure,
5318 };
5319
5320 static const struct mlxsw_sp_rif_ops *mlxsw_sp_rif_ops_arr[] = {
5321 [MLXSW_SP_RIF_TYPE_SUBPORT] = &mlxsw_sp_rif_subport_ops,
5322 [MLXSW_SP_RIF_TYPE_VLAN] = &mlxsw_sp_rif_vlan_ops,
5323 [MLXSW_SP_RIF_TYPE_FID] = &mlxsw_sp_rif_fid_ops,
5324 [MLXSW_SP_RIF_TYPE_IPIP_LB] = &mlxsw_sp_rif_ipip_lb_ops,
5325 };
5326
5327 static int mlxsw_sp_rifs_init(struct mlxsw_sp *mlxsw_sp)
5328 {
5329 u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
5330
5331 mlxsw_sp->router->rifs = kcalloc(max_rifs,
5332 sizeof(struct mlxsw_sp_rif *),
5333 GFP_KERNEL);
5334 if (!mlxsw_sp->router->rifs)
5335 return -ENOMEM;
5336
5337 mlxsw_sp->router->rif_ops_arr = mlxsw_sp_rif_ops_arr;
5338
5339 return 0;
5340 }
5341
5342 static void mlxsw_sp_rifs_fini(struct mlxsw_sp *mlxsw_sp)
5343 {
5344 int i;
5345
5346 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
5347 WARN_ON_ONCE(mlxsw_sp->router->rifs[i]);
5348
5349 kfree(mlxsw_sp->router->rifs);
5350 }
5351
5352 static int mlxsw_sp_ipips_init(struct mlxsw_sp *mlxsw_sp)
5353 {
5354 mlxsw_sp->router->ipip_ops_arr = mlxsw_sp_ipip_ops_arr;
5355 return 0;
5356 }
5357
5358 static void mlxsw_sp_ipips_fini(struct mlxsw_sp *mlxsw_sp)
5359 {
5360 }
5361
5362 static void mlxsw_sp_router_fib_dump_flush(struct notifier_block *nb)
5363 {
5364 struct mlxsw_sp_router *router;
5365
5366 /* Flush pending FIB notifications and then flush the device's
5367 * table before requesting another dump. The FIB notification
5368 * block is unregistered, so no need to take RTNL.
5369 */
5370 mlxsw_core_flush_owq();
5371 router = container_of(nb, struct mlxsw_sp_router, fib_nb);
5372 mlxsw_sp_router_fib_flush(router->mlxsw_sp);
5373 }
5374
5375 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
5376 {
5377 char rgcr_pl[MLXSW_REG_RGCR_LEN];
5378 u64 max_rifs;
5379 int err;
5380
5381 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_RIFS))
5382 return -EIO;
5383 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
5384
5385 mlxsw_reg_rgcr_pack(rgcr_pl, true, true);
5386 mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, max_rifs);
5387 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
5388 if (err)
5389 return err;
5390 return 0;
5391 }
5392
5393 static void __mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
5394 {
5395 char rgcr_pl[MLXSW_REG_RGCR_LEN];
5396
5397 mlxsw_reg_rgcr_pack(rgcr_pl, false, false);
5398 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
5399 }
5400
5401 int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
5402 {
5403 struct mlxsw_sp_router *router;
5404 int err;
5405
5406 router = kzalloc(sizeof(*mlxsw_sp->router), GFP_KERNEL);
5407 if (!router)
5408 return -ENOMEM;
5409 mlxsw_sp->router = router;
5410 router->mlxsw_sp = mlxsw_sp;
5411
5412 INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_neighs_list);
5413 err = __mlxsw_sp_router_init(mlxsw_sp);
5414 if (err)
5415 goto err_router_init;
5416
5417 err = mlxsw_sp_rifs_init(mlxsw_sp);
5418 if (err)
5419 goto err_rifs_init;
5420
5421 err = mlxsw_sp_ipips_init(mlxsw_sp);
5422 if (err)
5423 goto err_ipips_init;
5424
5425 err = rhashtable_init(&mlxsw_sp->router->nexthop_ht,
5426 &mlxsw_sp_nexthop_ht_params);
5427 if (err)
5428 goto err_nexthop_ht_init;
5429
5430 err = rhashtable_init(&mlxsw_sp->router->nexthop_group_ht,
5431 &mlxsw_sp_nexthop_group_ht_params);
5432 if (err)
5433 goto err_nexthop_group_ht_init;
5434
5435 err = mlxsw_sp_lpm_init(mlxsw_sp);
5436 if (err)
5437 goto err_lpm_init;
5438
5439 err = mlxsw_sp_vrs_init(mlxsw_sp);
5440 if (err)
5441 goto err_vrs_init;
5442
5443 err = mlxsw_sp_neigh_init(mlxsw_sp);
5444 if (err)
5445 goto err_neigh_init;
5446
5447 mlxsw_sp->router->fib_nb.notifier_call = mlxsw_sp_router_fib_event;
5448 err = register_fib_notifier(&mlxsw_sp->router->fib_nb,
5449 mlxsw_sp_router_fib_dump_flush);
5450 if (err)
5451 goto err_register_fib_notifier;
5452
5453 return 0;
5454
5455 err_register_fib_notifier:
5456 mlxsw_sp_neigh_fini(mlxsw_sp);
5457 err_neigh_init:
5458 mlxsw_sp_vrs_fini(mlxsw_sp);
5459 err_vrs_init:
5460 mlxsw_sp_lpm_fini(mlxsw_sp);
5461 err_lpm_init:
5462 rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht);
5463 err_nexthop_group_ht_init:
5464 rhashtable_destroy(&mlxsw_sp->router->nexthop_ht);
5465 err_nexthop_ht_init:
5466 mlxsw_sp_ipips_fini(mlxsw_sp);
5467 err_ipips_init:
5468 mlxsw_sp_rifs_fini(mlxsw_sp);
5469 err_rifs_init:
5470 __mlxsw_sp_router_fini(mlxsw_sp);
5471 err_router_init:
5472 kfree(mlxsw_sp->router);
5473 return err;
5474 }
5475
5476 void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
5477 {
5478 unregister_fib_notifier(&mlxsw_sp->router->fib_nb);
5479 mlxsw_sp_neigh_fini(mlxsw_sp);
5480 mlxsw_sp_vrs_fini(mlxsw_sp);
5481 mlxsw_sp_lpm_fini(mlxsw_sp);
5482 rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht);
5483 rhashtable_destroy(&mlxsw_sp->router->nexthop_ht);
5484 mlxsw_sp_ipips_fini(mlxsw_sp);
5485 mlxsw_sp_rifs_fini(mlxsw_sp);
5486 __mlxsw_sp_router_fini(mlxsw_sp);
5487 kfree(mlxsw_sp->router);
5488 }
(deprecated) Btrfs devel tree