drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c

   1 /*
   2  * drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
   3  * Copyright (c) 2016-2017 Mellanox Technologies. All rights reserved.
   4  * Copyright (c) 2016 Jiri Pirko <jiri@mellanox.com>
   5  * Copyright (c) 2016 Ido Schimmel <idosch@mellanox.com>
   6  * Copyright (c) 2016 Yotam Gigi <yotamg@mellanox.com>
   7  * Copyright (c) 2017 Petr Machata <petrm@mellanox.com>
   8  *
   9  * Redistribution and use in source and binary forms, with or without
  10  * modification, are permitted provided that the following conditions are met:
  11  *
  12  * 1. Redistributions of source code must retain the above copyright
  13  *    notice, this list of conditions and the following disclaimer.
  14  * 2. Redistributions in binary form must reproduce the above copyright
  15  *    notice, this list of conditions and the following disclaimer in the
  16  *    documentation and/or other materials provided with the distribution.
  17  * 3. Neither the names of the copyright holders nor the names of its
  18  *    contributors may be used to endorse or promote products derived from
  19  *    this software without specific prior written permission.
  20  *
  21  * Alternatively, this software may be distributed under the terms of the
  22  * GNU General Public License ("GPL") version 2 as published by the Free
  23  * Software Foundation.
  24  *
  25  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  26  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  27  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  28  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  29  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  30  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  31  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  32  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  33  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  34  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  35  * POSSIBILITY OF SUCH DAMAGE.
  36  */
  37
  38 #include <linux/kernel.h>
  39 #include <linux/types.h>
  40 #include <linux/rhashtable.h>
  41 #include <linux/bitops.h>
  42 #include <linux/in6.h>
  43 #include <linux/notifier.h>
  44 #include <linux/inetdevice.h>
  45 #include <linux/netdevice.h>
  46 #include <linux/if_bridge.h>
  47 #include <linux/socket.h>
  48 #include <linux/route.h>
  49 #include <net/netevent.h>
  50 #include <net/neighbour.h>
  51 #include <net/arp.h>
  52 #include <net/ip_fib.h>
  53 #include <net/ip6_fib.h>
  54 #include <net/fib_rules.h>
  55 #include <net/ip_tunnels.h>
  56 #include <net/l3mdev.h>
  57 #include <net/addrconf.h>
  58 #include <net/ndisc.h>
  59 #include <net/ipv6.h>
  60 #include <net/fib_notifier.h>
  61
  62 #include "spectrum.h"
  63 #include "core.h"
  64 #include "reg.h"
  65 #include "spectrum_cnt.h"
  66 #include "spectrum_dpipe.h"
  67 #include "spectrum_ipip.h"
  68 #include "spectrum_router.h"
  69
  70 struct mlxsw_sp_vr;
  71 struct mlxsw_sp_lpm_tree;
  72 struct mlxsw_sp_rif_ops;
  73
  74 struct mlxsw_sp_router {
  75         struct mlxsw_sp *mlxsw_sp;
  76         struct mlxsw_sp_rif **rifs;
  77         struct mlxsw_sp_vr *vrs;
  78         struct rhashtable neigh_ht;
  79         struct rhashtable nexthop_group_ht;
  80         struct rhashtable nexthop_ht;
  81         struct {
  82                 struct mlxsw_sp_lpm_tree *trees;
  83                 unsigned int tree_count;
  84         } lpm;
  85         struct {
  86                 struct delayed_work dw;
  87                 unsigned long interval; /* ms */
  88         } neighs_update;
  89         struct delayed_work nexthop_probe_dw;
  90 #define MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL 5000 /* ms */
  91         struct list_head nexthop_neighs_list;
  92         struct list_head ipip_list;
  93         bool aborted;
  94         struct notifier_block fib_nb;
  95         const struct mlxsw_sp_rif_ops **rif_ops_arr;
  96         const struct mlxsw_sp_ipip_ops **ipip_ops_arr;
  97 };
  98
  99 struct mlxsw_sp_rif {
 100         struct list_head nexthop_list;
 101         struct list_head neigh_list;
 102         struct net_device *dev;
 103         struct mlxsw_sp_fid *fid;
 104         unsigned char addr[ETH_ALEN];
 105         int mtu;
 106         u16 rif_index;
 107         u16 vr_id;
 108         const struct mlxsw_sp_rif_ops *ops;
 109         struct mlxsw_sp *mlxsw_sp;
 110
 111         unsigned int counter_ingress;
 112         bool counter_ingress_valid;
 113         unsigned int counter_egress;
 114         bool counter_egress_valid;
 115 };
 116
 117 struct mlxsw_sp_rif_params {
 118         struct net_device *dev;
 119         union {
 120                 u16 system_port;
 121                 u16 lag_id;
 122         };
 123         u16 vid;
 124         bool lag;
 125 };
 126
 127 struct mlxsw_sp_rif_subport {
 128         struct mlxsw_sp_rif common;
 129         union {
 130                 u16 system_port;
 131                 u16 lag_id;
 132         };
 133         u16 vid;
 134         bool lag;
 135 };
 136
 137 struct mlxsw_sp_rif_ipip_lb {
 138         struct mlxsw_sp_rif common;
 139         struct mlxsw_sp_rif_ipip_lb_config lb_config;
 140         u16 ul_vr_id; /* Reserved for Spectrum-2. */
 141 };
 142
 143 struct mlxsw_sp_rif_params_ipip_lb {
 144         struct mlxsw_sp_rif_params common;
 145         struct mlxsw_sp_rif_ipip_lb_config lb_config;
 146 };
 147
 148 struct mlxsw_sp_rif_ops {
 149         enum mlxsw_sp_rif_type type;
 150         size_t rif_size;
 151
 152         void (*setup)(struct mlxsw_sp_rif *rif,
 153                       const struct mlxsw_sp_rif_params *params);
 154         int (*configure)(struct mlxsw_sp_rif *rif);
 155         void (*deconfigure)(struct mlxsw_sp_rif *rif);
 156         struct mlxsw_sp_fid * (*fid_get)(struct mlxsw_sp_rif *rif);
 157 };
 158
 159 static unsigned int *
 160 mlxsw_sp_rif_p_counter_get(struct mlxsw_sp_rif *rif,
 161                            enum mlxsw_sp_rif_counter_dir dir)
 162 {
 163         switch (dir) {
 164         case MLXSW_SP_RIF_COUNTER_EGRESS:
 165                 return &rif->counter_egress;
 166         case MLXSW_SP_RIF_COUNTER_INGRESS:
 167                 return &rif->counter_ingress;
 168         }
 169         return NULL;
 170 }
 171
 172 static bool
 173 mlxsw_sp_rif_counter_valid_get(struct mlxsw_sp_rif *rif,
 174                                enum mlxsw_sp_rif_counter_dir dir)
 175 {
 176         switch (dir) {
 177         case MLXSW_SP_RIF_COUNTER_EGRESS:
 178                 return rif->counter_egress_valid;
 179         case MLXSW_SP_RIF_COUNTER_INGRESS:
 180                 return rif->counter_ingress_valid;
 181         }
 182         return false;
 183 }
 184
 185 static void
 186 mlxsw_sp_rif_counter_valid_set(struct mlxsw_sp_rif *rif,
 187                                enum mlxsw_sp_rif_counter_dir dir,
 188                                bool valid)
 189 {
 190         switch (dir) {
 191         case MLXSW_SP_RIF_COUNTER_EGRESS:
 192                 rif->counter_egress_valid = valid;
 193                 break;
 194         case MLXSW_SP_RIF_COUNTER_INGRESS:
 195                 rif->counter_ingress_valid = valid;
 196                 break;
 197         }
 198 }
 199
 200 static int mlxsw_sp_rif_counter_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
 201                                      unsigned int counter_index, bool enable,
 202                                      enum mlxsw_sp_rif_counter_dir dir)
 203 {
 204         char ritr_pl[MLXSW_REG_RITR_LEN];
 205         bool is_egress = false;
 206         int err;
 207
 208         if (dir == MLXSW_SP_RIF_COUNTER_EGRESS)
 209                 is_egress = true;
 210         mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
 211         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
 212         if (err)
 213                 return err;
 214
 215         mlxsw_reg_ritr_counter_pack(ritr_pl, counter_index, enable,
 216                                     is_egress);
 217         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
 218 }
 219
 220 int mlxsw_sp_rif_counter_value_get(struct mlxsw_sp *mlxsw_sp,
 221                                    struct mlxsw_sp_rif *rif,
 222                                    enum mlxsw_sp_rif_counter_dir dir, u64 *cnt)
 223 {
 224         char ricnt_pl[MLXSW_REG_RICNT_LEN];
 225         unsigned int *p_counter_index;
 226         bool valid;
 227         int err;
 228
 229         valid = mlxsw_sp_rif_counter_valid_get(rif, dir);
 230         if (!valid)
 231                 return -EINVAL;
 232
 233         p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
 234         if (!p_counter_index)
 235                 return -EINVAL;
 236         mlxsw_reg_ricnt_pack(ricnt_pl, *p_counter_index,
 237                              MLXSW_REG_RICNT_OPCODE_NOP);
 238         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl);
 239         if (err)
 240                 return err;
 241         *cnt = mlxsw_reg_ricnt_good_unicast_packets_get(ricnt_pl);
 242         return 0;
 243 }
 244
 245 static int mlxsw_sp_rif_counter_clear(struct mlxsw_sp *mlxsw_sp,
 246                                       unsigned int counter_index)
 247 {
 248         char ricnt_pl[MLXSW_REG_RICNT_LEN];
 249
 250         mlxsw_reg_ricnt_pack(ricnt_pl, counter_index,
 251                              MLXSW_REG_RICNT_OPCODE_CLEAR);
 252         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl);
 253 }
 254
 255 int mlxsw_sp_rif_counter_alloc(struct mlxsw_sp *mlxsw_sp,
 256                                struct mlxsw_sp_rif *rif,
 257                                enum mlxsw_sp_rif_counter_dir dir)
 258 {
 259         unsigned int *p_counter_index;
 260         int err;
 261
 262         p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
 263         if (!p_counter_index)
 264                 return -EINVAL;
 265         err = mlxsw_sp_counter_alloc(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
 266                                      p_counter_index);
 267         if (err)
 268                 return err;
 269
 270         err = mlxsw_sp_rif_counter_clear(mlxsw_sp, *p_counter_index);
 271         if (err)
 272                 goto err_counter_clear;
 273
 274         err = mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index,
 275                                         *p_counter_index, true, dir);
 276         if (err)
 277                 goto err_counter_edit;
 278         mlxsw_sp_rif_counter_valid_set(rif, dir, true);
 279         return 0;
 280
 281 err_counter_edit:
 282 err_counter_clear:
 283         mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
 284                               *p_counter_index);
 285         return err;
 286 }
 287
 288 void mlxsw_sp_rif_counter_free(struct mlxsw_sp *mlxsw_sp,
 289                                struct mlxsw_sp_rif *rif,
 290                                enum mlxsw_sp_rif_counter_dir dir)
 291 {
 292         unsigned int *p_counter_index;
 293
 294         if (!mlxsw_sp_rif_counter_valid_get(rif, dir))
 295                 return;
 296
 297         p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
 298         if (WARN_ON(!p_counter_index))
 299                 return;
 300         mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index,
 301                                   *p_counter_index, false, dir);
 302         mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
 303                               *p_counter_index);
 304         mlxsw_sp_rif_counter_valid_set(rif, dir, false);
 305 }
 306
 307 static void mlxsw_sp_rif_counters_alloc(struct mlxsw_sp_rif *rif)
 308 {
 309         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
 310         struct devlink *devlink;
 311
 312         devlink = priv_to_devlink(mlxsw_sp->core);
 313         if (!devlink_dpipe_table_counter_enabled(devlink,
 314                                                  MLXSW_SP_DPIPE_TABLE_NAME_ERIF))
 315                 return;
 316         mlxsw_sp_rif_counter_alloc(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS);
 317 }
 318
 319 static void mlxsw_sp_rif_counters_free(struct mlxsw_sp_rif *rif)
 320 {
 321         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
 322
 323         mlxsw_sp_rif_counter_free(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS);
 324 }
 325
 326 static struct mlxsw_sp_rif *
 327 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
 328                          const struct net_device *dev);
 329
 330 #define MLXSW_SP_PREFIX_COUNT (sizeof(struct in6_addr) * BITS_PER_BYTE + 1)
 331
 332 struct mlxsw_sp_prefix_usage {
 333         DECLARE_BITMAP(b, MLXSW_SP_PREFIX_COUNT);
 334 };
 335
 336 #define mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) \
 337         for_each_set_bit(prefix, (prefix_usage)->b, MLXSW_SP_PREFIX_COUNT)
 338
 339 static bool
 340 mlxsw_sp_prefix_usage_eq(struct mlxsw_sp_prefix_usage *prefix_usage1,
 341                          struct mlxsw_sp_prefix_usage *prefix_usage2)
 342 {
 343         return !memcmp(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
 344 }
 345
 346 static bool
 347 mlxsw_sp_prefix_usage_none(struct mlxsw_sp_prefix_usage *prefix_usage)
 348 {
 349         struct mlxsw_sp_prefix_usage prefix_usage_none = {{ 0 } };
 350
 351         return mlxsw_sp_prefix_usage_eq(prefix_usage, &prefix_usage_none);
 352 }
 353
 354 static void
 355 mlxsw_sp_prefix_usage_cpy(struct mlxsw_sp_prefix_usage *prefix_usage1,
 356                           struct mlxsw_sp_prefix_usage *prefix_usage2)
 357 {
 358         memcpy(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
 359 }
 360
 361 static void
 362 mlxsw_sp_prefix_usage_set(struct mlxsw_sp_prefix_usage *prefix_usage,
 363                           unsigned char prefix_len)
 364 {
 365         set_bit(prefix_len, prefix_usage->b);
 366 }
 367
 368 static void
 369 mlxsw_sp_prefix_usage_clear(struct mlxsw_sp_prefix_usage *prefix_usage,
 370                             unsigned char prefix_len)
 371 {
 372         clear_bit(prefix_len, prefix_usage->b);
 373 }
 374
 375 struct mlxsw_sp_fib_key {
 376         unsigned char addr[sizeof(struct in6_addr)];
 377         unsigned char prefix_len;
 378 };
 379
 380 enum mlxsw_sp_fib_entry_type {
 381         MLXSW_SP_FIB_ENTRY_TYPE_REMOTE,
 382         MLXSW_SP_FIB_ENTRY_TYPE_LOCAL,
 383         MLXSW_SP_FIB_ENTRY_TYPE_TRAP,
 384 };
 385
 386 struct mlxsw_sp_nexthop_group;
 387 struct mlxsw_sp_fib;
 388
 389 struct mlxsw_sp_fib_node {
 390         struct list_head entry_list;
 391         struct list_head list;
 392         struct rhash_head ht_node;
 393         struct mlxsw_sp_fib *fib;
 394         struct mlxsw_sp_fib_key key;
 395 };
 396
 397 struct mlxsw_sp_fib_entry {
 398         struct list_head list;
 399         struct mlxsw_sp_fib_node *fib_node;
 400         enum mlxsw_sp_fib_entry_type type;
 401         struct list_head nexthop_group_node;
 402         struct mlxsw_sp_nexthop_group *nh_group;
 403 };
 404
 405 struct mlxsw_sp_fib4_entry {
 406         struct mlxsw_sp_fib_entry common;
 407         u32 tb_id;
 408         u32 prio;
 409         u8 tos;
 410         u8 type;
 411 };
 412
 413 struct mlxsw_sp_fib6_entry {
 414         struct mlxsw_sp_fib_entry common;
 415         struct list_head rt6_list;
 416         unsigned int nrt6;
 417 };
 418
 419 struct mlxsw_sp_rt6 {
 420         struct list_head list;
 421         struct rt6_info *rt;
 422 };
 423
 424 struct mlxsw_sp_lpm_tree {
 425         u8 id; /* tree ID */
 426         unsigned int ref_count;
 427         enum mlxsw_sp_l3proto proto;
 428         struct mlxsw_sp_prefix_usage prefix_usage;
 429 };
 430
 431 struct mlxsw_sp_fib {
 432         struct rhashtable ht;
 433         struct list_head node_list;
 434         struct mlxsw_sp_vr *vr;
 435         struct mlxsw_sp_lpm_tree *lpm_tree;
 436         unsigned long prefix_ref_count[MLXSW_SP_PREFIX_COUNT];
 437         struct mlxsw_sp_prefix_usage prefix_usage;
 438         enum mlxsw_sp_l3proto proto;
 439 };
 440
 441 struct mlxsw_sp_vr {
 442         u16 id; /* virtual router ID */
 443         u32 tb_id; /* kernel fib table id */
 444         unsigned int rif_count;
 445         struct mlxsw_sp_fib *fib4;
 446         struct mlxsw_sp_fib *fib6;
 447 };
 448
 449 static const struct rhashtable_params mlxsw_sp_fib_ht_params;
 450
 451 static struct mlxsw_sp_fib *mlxsw_sp_fib_create(struct mlxsw_sp_vr *vr,
 452                                                 enum mlxsw_sp_l3proto proto)
 453 {
 454         struct mlxsw_sp_fib *fib;
 455         int err;
 456
 457         fib = kzalloc(sizeof(*fib), GFP_KERNEL);
 458         if (!fib)
 459                 return ERR_PTR(-ENOMEM);
 460         err = rhashtable_init(&fib->ht, &mlxsw_sp_fib_ht_params);
 461         if (err)
 462                 goto err_rhashtable_init;
 463         INIT_LIST_HEAD(&fib->node_list);
 464         fib->proto = proto;
 465         fib->vr = vr;
 466         return fib;
 467
 468 err_rhashtable_init:
 469         kfree(fib);
 470         return ERR_PTR(err);
 471 }
 472
 473 static void mlxsw_sp_fib_destroy(struct mlxsw_sp_fib *fib)
 474 {
 475         WARN_ON(!list_empty(&fib->node_list));
 476         WARN_ON(fib->lpm_tree);
 477         rhashtable_destroy(&fib->ht);
 478         kfree(fib);
 479 }
 480
 481 static struct mlxsw_sp_lpm_tree *
 482 mlxsw_sp_lpm_tree_find_unused(struct mlxsw_sp *mlxsw_sp)
 483 {
 484         static struct mlxsw_sp_lpm_tree *lpm_tree;
 485         int i;
 486
 487         for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
 488                 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
 489                 if (lpm_tree->ref_count == 0)
 490                         return lpm_tree;
 491         }
 492         return NULL;
 493 }
 494
 495 static int mlxsw_sp_lpm_tree_alloc(struct mlxsw_sp *mlxsw_sp,
 496                                    struct mlxsw_sp_lpm_tree *lpm_tree)
 497 {
 498         char ralta_pl[MLXSW_REG_RALTA_LEN];
 499
 500         mlxsw_reg_ralta_pack(ralta_pl, true,
 501                              (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
 502                              lpm_tree->id);
 503         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
 504 }
 505
 506 static void mlxsw_sp_lpm_tree_free(struct mlxsw_sp *mlxsw_sp,
 507                                    struct mlxsw_sp_lpm_tree *lpm_tree)
 508 {
 509         char ralta_pl[MLXSW_REG_RALTA_LEN];
 510
 511         mlxsw_reg_ralta_pack(ralta_pl, false,
 512                              (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
 513                              lpm_tree->id);
 514         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
 515 }
 516
 517 static int
 518 mlxsw_sp_lpm_tree_left_struct_set(struct mlxsw_sp *mlxsw_sp,
 519                                   struct mlxsw_sp_prefix_usage *prefix_usage,
 520                                   struct mlxsw_sp_lpm_tree *lpm_tree)
 521 {
 522         char ralst_pl[MLXSW_REG_RALST_LEN];
 523         u8 root_bin = 0;
 524         u8 prefix;
 525         u8 last_prefix = MLXSW_REG_RALST_BIN_NO_CHILD;
 526
 527         mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage)
 528                 root_bin = prefix;
 529
 530         mlxsw_reg_ralst_pack(ralst_pl, root_bin, lpm_tree->id);
 531         mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) {
 532                 if (prefix == 0)
 533                         continue;
 534                 mlxsw_reg_ralst_bin_pack(ralst_pl, prefix, last_prefix,
 535                                          MLXSW_REG_RALST_BIN_NO_CHILD);
 536                 last_prefix = prefix;
 537         }
 538         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
 539 }
 540
 541 static struct mlxsw_sp_lpm_tree *
 542 mlxsw_sp_lpm_tree_create(struct mlxsw_sp *mlxsw_sp,
 543                          struct mlxsw_sp_prefix_usage *prefix_usage,
 544                          enum mlxsw_sp_l3proto proto)
 545 {
 546         struct mlxsw_sp_lpm_tree *lpm_tree;
 547         int err;
 548
 549         lpm_tree = mlxsw_sp_lpm_tree_find_unused(mlxsw_sp);
 550         if (!lpm_tree)
 551                 return ERR_PTR(-EBUSY);
 552         lpm_tree->proto = proto;
 553         err = mlxsw_sp_lpm_tree_alloc(mlxsw_sp, lpm_tree);
 554         if (err)
 555                 return ERR_PTR(err);
 556
 557         err = mlxsw_sp_lpm_tree_left_struct_set(mlxsw_sp, prefix_usage,
 558                                                 lpm_tree);
 559         if (err)
 560                 goto err_left_struct_set;
 561         memcpy(&lpm_tree->prefix_usage, prefix_usage,
 562                sizeof(lpm_tree->prefix_usage));
 563         return lpm_tree;
 564
 565 err_left_struct_set:
 566         mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
 567         return ERR_PTR(err);
 568 }
 569
 570 static void mlxsw_sp_lpm_tree_destroy(struct mlxsw_sp *mlxsw_sp,
 571                                       struct mlxsw_sp_lpm_tree *lpm_tree)
 572 {
 573         mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
 574 }
 575
 576 static struct mlxsw_sp_lpm_tree *
 577 mlxsw_sp_lpm_tree_get(struct mlxsw_sp *mlxsw_sp,
 578                       struct mlxsw_sp_prefix_usage *prefix_usage,
 579                       enum mlxsw_sp_l3proto proto)
 580 {
 581         struct mlxsw_sp_lpm_tree *lpm_tree;
 582         int i;
 583
 584         for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
 585                 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
 586                 if (lpm_tree->ref_count != 0 &&
 587                     lpm_tree->proto == proto &&
 588                     mlxsw_sp_prefix_usage_eq(&lpm_tree->prefix_usage,
 589                                              prefix_usage))
 590                         return lpm_tree;
 591         }
 592         return mlxsw_sp_lpm_tree_create(mlxsw_sp, prefix_usage, proto);
 593 }
 594
 595 static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree)
 596 {
 597         lpm_tree->ref_count++;
 598 }
 599
 600 static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp,
 601                                   struct mlxsw_sp_lpm_tree *lpm_tree)
 602 {
 603         if (--lpm_tree->ref_count == 0)
 604                 mlxsw_sp_lpm_tree_destroy(mlxsw_sp, lpm_tree);
 605 }
 606
 607 #define MLXSW_SP_LPM_TREE_MIN 1 /* tree 0 is reserved */
 608
 609 static int mlxsw_sp_lpm_init(struct mlxsw_sp *mlxsw_sp)
 610 {
 611         struct mlxsw_sp_lpm_tree *lpm_tree;
 612         u64 max_trees;
 613         int i;
 614
 615         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_LPM_TREES))
 616                 return -EIO;
 617
 618         max_trees = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_LPM_TREES);
 619         mlxsw_sp->router->lpm.tree_count = max_trees - MLXSW_SP_LPM_TREE_MIN;
 620         mlxsw_sp->router->lpm.trees = kcalloc(mlxsw_sp->router->lpm.tree_count,
 621                                              sizeof(struct mlxsw_sp_lpm_tree),
 622                                              GFP_KERNEL);
 623         if (!mlxsw_sp->router->lpm.trees)
 624                 return -ENOMEM;
 625
 626         for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
 627                 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
 628                 lpm_tree->id = i + MLXSW_SP_LPM_TREE_MIN;
 629         }
 630
 631         return 0;
 632 }
 633
 634 static void mlxsw_sp_lpm_fini(struct mlxsw_sp *mlxsw_sp)
 635 {
 636         kfree(mlxsw_sp->router->lpm.trees);
 637 }
 638
 639 static bool mlxsw_sp_vr_is_used(const struct mlxsw_sp_vr *vr)
 640 {
 641         return !!vr->fib4 || !!vr->fib6;
 642 }
 643
 644 static struct mlxsw_sp_vr *mlxsw_sp_vr_find_unused(struct mlxsw_sp *mlxsw_sp)
 645 {
 646         struct mlxsw_sp_vr *vr;
 647         int i;
 648
 649         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
 650                 vr = &mlxsw_sp->router->vrs[i];
 651                 if (!mlxsw_sp_vr_is_used(vr))
 652                         return vr;
 653         }
 654         return NULL;
 655 }
 656
 657 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp,
 658                                      const struct mlxsw_sp_fib *fib, u8 tree_id)
 659 {
 660         char raltb_pl[MLXSW_REG_RALTB_LEN];
 661
 662         mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id,
 663                              (enum mlxsw_reg_ralxx_protocol) fib->proto,
 664                              tree_id);
 665         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
 666 }
 667
 668 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp,
 669                                        const struct mlxsw_sp_fib *fib)
 670 {
 671         char raltb_pl[MLXSW_REG_RALTB_LEN];
 672
 673         /* Bind to tree 0 which is default */
 674         mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id,
 675                              (enum mlxsw_reg_ralxx_protocol) fib->proto, 0);
 676         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
 677 }
 678
 679 static u32 mlxsw_sp_fix_tb_id(u32 tb_id)
 680 {
 681         /* For our purpose, squash main and local table into one */
 682         if (tb_id == RT_TABLE_LOCAL)
 683                 tb_id = RT_TABLE_MAIN;
 684         return tb_id;
 685 }
 686
 687 static struct mlxsw_sp_vr *mlxsw_sp_vr_find(struct mlxsw_sp *mlxsw_sp,
 688                                             u32 tb_id)
 689 {
 690         struct mlxsw_sp_vr *vr;
 691         int i;
 692
 693         tb_id = mlxsw_sp_fix_tb_id(tb_id);
 694
 695         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
 696                 vr = &mlxsw_sp->router->vrs[i];
 697                 if (mlxsw_sp_vr_is_used(vr) && vr->tb_id == tb_id)
 698                         return vr;
 699         }
 700         return NULL;
 701 }
 702
 703 static struct mlxsw_sp_fib *mlxsw_sp_vr_fib(const struct mlxsw_sp_vr *vr,
 704                                             enum mlxsw_sp_l3proto proto)
 705 {
 706         switch (proto) {
 707         case MLXSW_SP_L3_PROTO_IPV4:
 708                 return vr->fib4;
 709         case MLXSW_SP_L3_PROTO_IPV6:
 710                 return vr->fib6;
 711         }
 712         return NULL;
 713 }
 714
 715 static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp,
 716                                               u32 tb_id)
 717 {
 718         struct mlxsw_sp_vr *vr;
 719         int err;
 720
 721         vr = mlxsw_sp_vr_find_unused(mlxsw_sp);
 722         if (!vr)
 723                 return ERR_PTR(-EBUSY);
 724         vr->fib4 = mlxsw_sp_fib_create(vr, MLXSW_SP_L3_PROTO_IPV4);
 725         if (IS_ERR(vr->fib4))
 726                 return ERR_CAST(vr->fib4);
 727         vr->fib6 = mlxsw_sp_fib_create(vr, MLXSW_SP_L3_PROTO_IPV6);
 728         if (IS_ERR(vr->fib6)) {
 729                 err = PTR_ERR(vr->fib6);
 730                 goto err_fib6_create;
 731         }
 732         vr->tb_id = tb_id;
 733         return vr;
 734
 735 err_fib6_create:
 736         mlxsw_sp_fib_destroy(vr->fib4);
 737         vr->fib4 = NULL;
 738         return ERR_PTR(err);
 739 }
 740
 741 static void mlxsw_sp_vr_destroy(struct mlxsw_sp_vr *vr)
 742 {
 743         mlxsw_sp_fib_destroy(vr->fib6);
 744         vr->fib6 = NULL;
 745         mlxsw_sp_fib_destroy(vr->fib4);
 746         vr->fib4 = NULL;
 747 }
 748
 749 static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id)
 750 {
 751         struct mlxsw_sp_vr *vr;
 752
 753         tb_id = mlxsw_sp_fix_tb_id(tb_id);
 754         vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id);
 755         if (!vr)
 756                 vr = mlxsw_sp_vr_create(mlxsw_sp, tb_id);
 757         return vr;
 758 }
 759
 760 static void mlxsw_sp_vr_put(struct mlxsw_sp_vr *vr)
 761 {
 762         if (!vr->rif_count && list_empty(&vr->fib4->node_list) &&
 763             list_empty(&vr->fib6->node_list))
 764                 mlxsw_sp_vr_destroy(vr);
 765 }
 766
 767 static bool
 768 mlxsw_sp_vr_lpm_tree_should_replace(struct mlxsw_sp_vr *vr,
 769                                     enum mlxsw_sp_l3proto proto, u8 tree_id)
 770 {
 771         struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
 772
 773         if (!mlxsw_sp_vr_is_used(vr))
 774                 return false;
 775         if (fib->lpm_tree && fib->lpm_tree->id == tree_id)
 776                 return true;
 777         return false;
 778 }
 779
 780 static int mlxsw_sp_vr_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp,
 781                                         struct mlxsw_sp_fib *fib,
 782                                         struct mlxsw_sp_lpm_tree *new_tree)
 783 {
 784         struct mlxsw_sp_lpm_tree *old_tree = fib->lpm_tree;
 785         int err;
 786
 787         err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, new_tree->id);
 788         if (err)
 789                 return err;
 790         fib->lpm_tree = new_tree;
 791         mlxsw_sp_lpm_tree_hold(new_tree);
 792         mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree);
 793         return 0;
 794 }
 795
 796 static int mlxsw_sp_vrs_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp,
 797                                          struct mlxsw_sp_fib *fib,
 798                                          struct mlxsw_sp_lpm_tree *new_tree)
 799 {
 800         struct mlxsw_sp_lpm_tree *old_tree = fib->lpm_tree;
 801         enum mlxsw_sp_l3proto proto = fib->proto;
 802         u8 old_id, new_id = new_tree->id;
 803         struct mlxsw_sp_vr *vr;
 804         int i, err;
 805
 806         if (!old_tree)
 807                 goto no_replace;
 808         old_id = old_tree->id;
 809
 810         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
 811                 vr = &mlxsw_sp->router->vrs[i];
 812                 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, old_id))
 813                         continue;
 814                 err = mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp,
 815                                                    mlxsw_sp_vr_fib(vr, proto),
 816                                                    new_tree);
 817                 if (err)
 818                         goto err_tree_replace;
 819         }
 820
 821         return 0;
 822
 823 err_tree_replace:
 824         for (i--; i >= 0; i--) {
 825                 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, new_id))
 826                         continue;
 827                 mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp,
 828                                              mlxsw_sp_vr_fib(vr, proto),
 829                                              old_tree);
 830         }
 831         return err;
 832
 833 no_replace:
 834         err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, new_tree->id);
 835         if (err)
 836                 return err;
 837         fib->lpm_tree = new_tree;
 838         mlxsw_sp_lpm_tree_hold(new_tree);
 839         return 0;
 840 }
 841
 842 static void
 843 mlxsw_sp_vrs_prefixes(struct mlxsw_sp *mlxsw_sp,
 844                       enum mlxsw_sp_l3proto proto,
 845                       struct mlxsw_sp_prefix_usage *req_prefix_usage)
 846 {
 847         int i;
 848
 849         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
 850                 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
 851                 struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
 852                 unsigned char prefix;
 853
 854                 if (!mlxsw_sp_vr_is_used(vr))
 855                         continue;
 856                 mlxsw_sp_prefix_usage_for_each(prefix, &fib->prefix_usage)
 857                         mlxsw_sp_prefix_usage_set(req_prefix_usage, prefix);
 858         }
 859 }
 860
 861 static int mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp)
 862 {
 863         struct mlxsw_sp_vr *vr;
 864         u64 max_vrs;
 865         int i;
 866
 867         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_VRS))
 868                 return -EIO;
 869
 870         max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS);
 871         mlxsw_sp->router->vrs = kcalloc(max_vrs, sizeof(struct mlxsw_sp_vr),
 872                                         GFP_KERNEL);
 873         if (!mlxsw_sp->router->vrs)
 874                 return -ENOMEM;
 875
 876         for (i = 0; i < max_vrs; i++) {
 877                 vr = &mlxsw_sp->router->vrs[i];
 878                 vr->id = i;
 879         }
 880
 881         return 0;
 882 }
 883
 884 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp);
 885
 886 static void mlxsw_sp_vrs_fini(struct mlxsw_sp *mlxsw_sp)
 887 {
 888         /* At this stage we're guaranteed not to have new incoming
 889          * FIB notifications and the work queue is free from FIBs
 890          * sitting on top of mlxsw netdevs. However, we can still
 891          * have other FIBs queued. Flush the queue before flushing
 892          * the device's tables. No need for locks, as we're the only
 893          * writer.
 894          */
 895         mlxsw_core_flush_owq();
 896         mlxsw_sp_router_fib_flush(mlxsw_sp);
 897         kfree(mlxsw_sp->router->vrs);
 898 }
 899
 900 static struct net_device *
 901 __mlxsw_sp_ipip_netdev_ul_dev_get(const struct net_device *ol_dev)
 902 {
 903         struct ip_tunnel *tun = netdev_priv(ol_dev);
 904         struct net *net = dev_net(ol_dev);
 905
 906         return __dev_get_by_index(net, tun->parms.link);
 907 }
 908
 909 static u32 mlxsw_sp_ipip_dev_ul_tb_id(const struct net_device *ol_dev)
 910 {
 911         struct net_device *d = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev);
 912
 913         if (d)
 914                 return l3mdev_fib_table(d) ? : RT_TABLE_MAIN;
 915         else
 916                 return l3mdev_fib_table(ol_dev) ? : RT_TABLE_MAIN;
 917 }
 918
 919 static struct mlxsw_sp_rif *
 920 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp,
 921                     const struct mlxsw_sp_rif_params *params);
 922
 923 static struct mlxsw_sp_rif_ipip_lb *
 924 mlxsw_sp_ipip_ol_ipip_lb_create(struct mlxsw_sp *mlxsw_sp,
 925                                 enum mlxsw_sp_ipip_type ipipt,
 926                                 struct net_device *ol_dev)
 927 {
 928         struct mlxsw_sp_rif_params_ipip_lb lb_params;
 929         const struct mlxsw_sp_ipip_ops *ipip_ops;
 930         struct mlxsw_sp_rif *rif;
 931
 932         ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt];
 933         lb_params = (struct mlxsw_sp_rif_params_ipip_lb) {
 934                 .common.dev = ol_dev,
 935                 .common.lag = false,
 936                 .lb_config = ipip_ops->ol_loopback_config(mlxsw_sp, ol_dev),
 937         };
 938
 939         rif = mlxsw_sp_rif_create(mlxsw_sp, &lb_params.common);
 940         if (IS_ERR(rif))
 941                 return ERR_CAST(rif);
 942         return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common);
 943 }
 944
 945 static struct mlxsw_sp_ipip_entry *
 946 mlxsw_sp_ipip_entry_alloc(struct mlxsw_sp *mlxsw_sp,
 947                           enum mlxsw_sp_ipip_type ipipt,
 948                           struct net_device *ol_dev)
 949 {
 950         struct mlxsw_sp_ipip_entry *ipip_entry;
 951         struct mlxsw_sp_ipip_entry *ret = NULL;
 952
 953         ipip_entry = kzalloc(sizeof(*ipip_entry), GFP_KERNEL);
 954         if (!ipip_entry)
 955                 return ERR_PTR(-ENOMEM);
 956
 957         ipip_entry->ol_lb = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp, ipipt,
 958                                                             ol_dev);
 959         if (IS_ERR(ipip_entry->ol_lb)) {
 960                 ret = ERR_CAST(ipip_entry->ol_lb);
 961                 goto err_ol_ipip_lb_create;
 962         }
 963
 964         ipip_entry->ipipt = ipipt;
 965         ipip_entry->ol_dev = ol_dev;
 966
 967         return ipip_entry;
 968
 969 err_ol_ipip_lb_create:
 970         kfree(ipip_entry);
 971         return ret;
 972 }
 973
 974 static void
 975 mlxsw_sp_ipip_entry_destroy(struct mlxsw_sp_ipip_entry *ipip_entry)
 976 {
 977         WARN_ON(ipip_entry->ref_count > 0);
 978         mlxsw_sp_rif_destroy(&ipip_entry->ol_lb->common);
 979         kfree(ipip_entry);
 980 }
 981
 982 static __be32
 983 mlxsw_sp_ipip_netdev_saddr4(const struct net_device *ol_dev)
 984 {
 985         struct ip_tunnel *tun = netdev_priv(ol_dev);
 986
 987         return tun->parms.iph.saddr;
 988 }
 989
 990 union mlxsw_sp_l3addr
 991 mlxsw_sp_ipip_netdev_saddr(enum mlxsw_sp_l3proto proto,
 992                            const struct net_device *ol_dev)
 993 {
 994         switch (proto) {
 995         case MLXSW_SP_L3_PROTO_IPV4:
 996                 return (union mlxsw_sp_l3addr) {
 997                         .addr4 = mlxsw_sp_ipip_netdev_saddr4(ol_dev),
 998                 };
 999         case MLXSW_SP_L3_PROTO_IPV6:
1000                 break;
1001         };
1002
1003         WARN_ON(1);
1004         return (union mlxsw_sp_l3addr) {
1005                 .addr4 = 0,
1006         };
1007 }
1008
1009 static bool mlxsw_sp_l3addr_eq(const union mlxsw_sp_l3addr *addr1,
1010                                const union mlxsw_sp_l3addr *addr2)
1011 {
1012         return !memcmp(addr1, addr2, sizeof(*addr1));
1013 }
1014
1015 static bool
1016 mlxsw_sp_ipip_entry_saddr_matches(struct mlxsw_sp *mlxsw_sp,
1017                                   const enum mlxsw_sp_l3proto ul_proto,
1018                                   union mlxsw_sp_l3addr saddr,
1019                                   u32 ul_tb_id,
1020                                   struct mlxsw_sp_ipip_entry *ipip_entry)
1021 {
1022         u32 tun_ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev);
1023         enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt;
1024         union mlxsw_sp_l3addr tun_saddr;
1025
1026         if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto)
1027                 return false;
1028
1029         tun_saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ipip_entry->ol_dev);
1030         return tun_ul_tb_id == ul_tb_id &&
1031                mlxsw_sp_l3addr_eq(&tun_saddr, &saddr);
1032 }
1033
1034 static struct mlxsw_sp_ipip_entry *
1035 mlxsw_sp_ipip_entry_get(struct mlxsw_sp *mlxsw_sp,
1036                         enum mlxsw_sp_ipip_type ipipt,
1037                         struct net_device *ol_dev)
1038 {
1039         u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ol_dev);
1040         struct mlxsw_sp_router *router = mlxsw_sp->router;
1041         struct mlxsw_sp_ipip_entry *ipip_entry;
1042         enum mlxsw_sp_l3proto ul_proto;
1043         union mlxsw_sp_l3addr saddr;
1044
1045         list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list,
1046                             ipip_list_node) {
1047                 if (ipip_entry->ol_dev == ol_dev)
1048                         goto inc_ref_count;
1049
1050                 /* The configuration where several tunnels have the same local
1051                  * address in the same underlay table needs special treatment in
1052                  * the HW. That is currently not implemented in the driver.
1053                  */
1054                 ul_proto = router->ipip_ops_arr[ipip_entry->ipipt]->ul_proto;
1055                 saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ol_dev);
1056                 if (mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, saddr,
1057                                                       ul_tb_id, ipip_entry))
1058                         return ERR_PTR(-EEXIST);
1059         }
1060
1061         ipip_entry = mlxsw_sp_ipip_entry_alloc(mlxsw_sp, ipipt, ol_dev);
1062         if (IS_ERR(ipip_entry))
1063                 return ipip_entry;
1064
1065         list_add_tail(&ipip_entry->ipip_list_node,
1066                       &mlxsw_sp->router->ipip_list);
1067
1068 inc_ref_count:
1069         ++ipip_entry->ref_count;
1070         return ipip_entry;
1071 }
1072
1073 static void
1074 mlxsw_sp_ipip_entry_put(struct mlxsw_sp *mlxsw_sp,
1075                         struct mlxsw_sp_ipip_entry *ipip_entry)
1076 {
1077         if (--ipip_entry->ref_count == 0) {
1078                 list_del(&ipip_entry->ipip_list_node);
1079                 mlxsw_sp_ipip_entry_destroy(ipip_entry);
1080         }
1081 }
1082
1083 struct mlxsw_sp_neigh_key {
1084         struct neighbour *n;
1085 };
1086
1087 struct mlxsw_sp_neigh_entry {
1088         struct list_head rif_list_node;
1089         struct rhash_head ht_node;
1090         struct mlxsw_sp_neigh_key key;
1091         u16 rif;
1092         bool connected;
1093         unsigned char ha[ETH_ALEN];
1094         struct list_head nexthop_list; /* list of nexthops using
1095                                         * this neigh entry
1096                                         */
1097         struct list_head nexthop_neighs_list_node;
1098         unsigned int counter_index;
1099         bool counter_valid;
1100 };
1101
1102 static const struct rhashtable_params mlxsw_sp_neigh_ht_params = {
1103         .key_offset = offsetof(struct mlxsw_sp_neigh_entry, key),
1104         .head_offset = offsetof(struct mlxsw_sp_neigh_entry, ht_node),
1105         .key_len = sizeof(struct mlxsw_sp_neigh_key),
1106 };
1107
1108 struct mlxsw_sp_neigh_entry *
1109 mlxsw_sp_rif_neigh_next(struct mlxsw_sp_rif *rif,
1110                         struct mlxsw_sp_neigh_entry *neigh_entry)
1111 {
1112         if (!neigh_entry) {
1113                 if (list_empty(&rif->neigh_list))
1114                         return NULL;
1115                 else
1116                         return list_first_entry(&rif->neigh_list,
1117                                                 typeof(*neigh_entry),
1118                                                 rif_list_node);
1119         }
1120         if (neigh_entry->rif_list_node.next == &rif->neigh_list)
1121                 return NULL;
1122         return list_next_entry(neigh_entry, rif_list_node);
1123 }
1124
1125 int mlxsw_sp_neigh_entry_type(struct mlxsw_sp_neigh_entry *neigh_entry)
1126 {
1127         return neigh_entry->key.n->tbl->family;
1128 }
1129
1130 unsigned char *
1131 mlxsw_sp_neigh_entry_ha(struct mlxsw_sp_neigh_entry *neigh_entry)
1132 {
1133         return neigh_entry->ha;
1134 }
1135
1136 u32 mlxsw_sp_neigh4_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry)
1137 {
1138         struct neighbour *n;
1139
1140         n = neigh_entry->key.n;
1141         return ntohl(*((__be32 *) n->primary_key));
1142 }
1143
1144 struct in6_addr *
1145 mlxsw_sp_neigh6_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry)
1146 {
1147         struct neighbour *n;
1148
1149         n = neigh_entry->key.n;
1150         return (struct in6_addr *) &n->primary_key;
1151 }
1152
1153 int mlxsw_sp_neigh_counter_get(struct mlxsw_sp *mlxsw_sp,
1154                                struct mlxsw_sp_neigh_entry *neigh_entry,
1155                                u64 *p_counter)
1156 {
1157         if (!neigh_entry->counter_valid)
1158                 return -EINVAL;
1159
1160         return mlxsw_sp_flow_counter_get(mlxsw_sp, neigh_entry->counter_index,
1161                                          p_counter, NULL);
1162 }
1163
1164 static struct mlxsw_sp_neigh_entry *
1165 mlxsw_sp_neigh_entry_alloc(struct mlxsw_sp *mlxsw_sp, struct neighbour *n,
1166                            u16 rif)
1167 {
1168         struct mlxsw_sp_neigh_entry *neigh_entry;
1169
1170         neigh_entry = kzalloc(sizeof(*neigh_entry), GFP_KERNEL);
1171         if (!neigh_entry)
1172                 return NULL;
1173
1174         neigh_entry->key.n = n;
1175         neigh_entry->rif = rif;
1176         INIT_LIST_HEAD(&neigh_entry->nexthop_list);
1177
1178         return neigh_entry;
1179 }
1180
1181 static void mlxsw_sp_neigh_entry_free(struct mlxsw_sp_neigh_entry *neigh_entry)
1182 {
1183         kfree(neigh_entry);
1184 }
1185
1186 static int
1187 mlxsw_sp_neigh_entry_insert(struct mlxsw_sp *mlxsw_sp,
1188                             struct mlxsw_sp_neigh_entry *neigh_entry)
1189 {
1190         return rhashtable_insert_fast(&mlxsw_sp->router->neigh_ht,
1191                                       &neigh_entry->ht_node,
1192                                       mlxsw_sp_neigh_ht_params);
1193 }
1194
1195 static void
1196 mlxsw_sp_neigh_entry_remove(struct mlxsw_sp *mlxsw_sp,
1197                             struct mlxsw_sp_neigh_entry *neigh_entry)
1198 {
1199         rhashtable_remove_fast(&mlxsw_sp->router->neigh_ht,
1200                                &neigh_entry->ht_node,
1201                                mlxsw_sp_neigh_ht_params);
1202 }
1203
1204 static bool
1205 mlxsw_sp_neigh_counter_should_alloc(struct mlxsw_sp *mlxsw_sp,
1206                                     struct mlxsw_sp_neigh_entry *neigh_entry)
1207 {
1208         struct devlink *devlink;
1209         const char *table_name;
1210
1211         switch (mlxsw_sp_neigh_entry_type(neigh_entry)) {
1212         case AF_INET:
1213                 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST4;
1214                 break;
1215         case AF_INET6:
1216                 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST6;
1217                 break;
1218         default:
1219                 WARN_ON(1);
1220                 return false;
1221         }
1222
1223         devlink = priv_to_devlink(mlxsw_sp->core);
1224         return devlink_dpipe_table_counter_enabled(devlink, table_name);
1225 }
1226
1227 static void
1228 mlxsw_sp_neigh_counter_alloc(struct mlxsw_sp *mlxsw_sp,
1229                              struct mlxsw_sp_neigh_entry *neigh_entry)
1230 {
1231         if (!mlxsw_sp_neigh_counter_should_alloc(mlxsw_sp, neigh_entry))
1232                 return;
1233
1234         if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &neigh_entry->counter_index))
1235                 return;
1236
1237         neigh_entry->counter_valid = true;
1238 }
1239
1240 static void
1241 mlxsw_sp_neigh_counter_free(struct mlxsw_sp *mlxsw_sp,
1242                             struct mlxsw_sp_neigh_entry *neigh_entry)
1243 {
1244         if (!neigh_entry->counter_valid)
1245                 return;
1246         mlxsw_sp_flow_counter_free(mlxsw_sp,
1247                                    neigh_entry->counter_index);
1248         neigh_entry->counter_valid = false;
1249 }
1250
1251 static struct mlxsw_sp_neigh_entry *
1252 mlxsw_sp_neigh_entry_create(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
1253 {
1254         struct mlxsw_sp_neigh_entry *neigh_entry;
1255         struct mlxsw_sp_rif *rif;
1256         int err;
1257
1258         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, n->dev);
1259         if (!rif)
1260                 return ERR_PTR(-EINVAL);
1261
1262         neigh_entry = mlxsw_sp_neigh_entry_alloc(mlxsw_sp, n, rif->rif_index);
1263         if (!neigh_entry)
1264                 return ERR_PTR(-ENOMEM);
1265
1266         err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
1267         if (err)
1268                 goto err_neigh_entry_insert;
1269
1270         mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry);
1271         list_add(&neigh_entry->rif_list_node, &rif->neigh_list);
1272
1273         return neigh_entry;
1274
1275 err_neigh_entry_insert:
1276         mlxsw_sp_neigh_entry_free(neigh_entry);
1277         return ERR_PTR(err);
1278 }
1279
1280 static void
1281 mlxsw_sp_neigh_entry_destroy(struct mlxsw_sp *mlxsw_sp,
1282                              struct mlxsw_sp_neigh_entry *neigh_entry)
1283 {
1284         list_del(&neigh_entry->rif_list_node);
1285         mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry);
1286         mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry);
1287         mlxsw_sp_neigh_entry_free(neigh_entry);
1288 }
1289
1290 static struct mlxsw_sp_neigh_entry *
1291 mlxsw_sp_neigh_entry_lookup(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
1292 {
1293         struct mlxsw_sp_neigh_key key;
1294
1295         key.n = n;
1296         return rhashtable_lookup_fast(&mlxsw_sp->router->neigh_ht,
1297                                       &key, mlxsw_sp_neigh_ht_params);
1298 }
1299
1300 static void
1301 mlxsw_sp_router_neighs_update_interval_init(struct mlxsw_sp *mlxsw_sp)
1302 {
1303         unsigned long interval;
1304
1305 #if IS_ENABLED(CONFIG_IPV6)
1306         interval = min_t(unsigned long,
1307                          NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME),
1308                          NEIGH_VAR(&nd_tbl.parms, DELAY_PROBE_TIME));
1309 #else
1310         interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME);
1311 #endif
1312         mlxsw_sp->router->neighs_update.interval = jiffies_to_msecs(interval);
1313 }
1314
1315 static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp,
1316                                                    char *rauhtd_pl,
1317                                                    int ent_index)
1318 {
1319         struct net_device *dev;
1320         struct neighbour *n;
1321         __be32 dipn;
1322         u32 dip;
1323         u16 rif;
1324
1325         mlxsw_reg_rauhtd_ent_ipv4_unpack(rauhtd_pl, ent_index, &rif, &dip);
1326
1327         if (!mlxsw_sp->router->rifs[rif]) {
1328                 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
1329                 return;
1330         }
1331
1332         dipn = htonl(dip);
1333         dev = mlxsw_sp->router->rifs[rif]->dev;
1334         n = neigh_lookup(&arp_tbl, &dipn, dev);
1335         if (!n) {
1336                 netdev_err(dev, "Failed to find matching neighbour for IP=%pI4h\n",
1337                            &dip);
1338                 return;
1339         }
1340
1341         netdev_dbg(dev, "Updating neighbour with IP=%pI4h\n", &dip);
1342         neigh_event_send(n, NULL);
1343         neigh_release(n);
1344 }
1345
1346 #if IS_ENABLED(CONFIG_IPV6)
1347 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp,
1348                                                    char *rauhtd_pl,
1349                                                    int rec_index)
1350 {
1351         struct net_device *dev;
1352         struct neighbour *n;
1353         struct in6_addr dip;
1354         u16 rif;
1355
1356         mlxsw_reg_rauhtd_ent_ipv6_unpack(rauhtd_pl, rec_index, &rif,
1357                                          (char *) &dip);
1358
1359         if (!mlxsw_sp->router->rifs[rif]) {
1360                 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
1361                 return;
1362         }
1363
1364         dev = mlxsw_sp->router->rifs[rif]->dev;
1365         n = neigh_lookup(&nd_tbl, &dip, dev);
1366         if (!n) {
1367                 netdev_err(dev, "Failed to find matching neighbour for IP=%pI6c\n",
1368                            &dip);
1369                 return;
1370         }
1371
1372         netdev_dbg(dev, "Updating neighbour with IP=%pI6c\n", &dip);
1373         neigh_event_send(n, NULL);
1374         neigh_release(n);
1375 }
1376 #else
1377 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp,
1378                                                    char *rauhtd_pl,
1379                                                    int rec_index)
1380 {
1381 }
1382 #endif
1383
1384 static void mlxsw_sp_router_neigh_rec_ipv4_process(struct mlxsw_sp *mlxsw_sp,
1385                                                    char *rauhtd_pl,
1386                                                    int rec_index)
1387 {
1388         u8 num_entries;
1389         int i;
1390
1391         num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
1392                                                                 rec_index);
1393         /* Hardware starts counting at 0, so add 1. */
1394         num_entries++;
1395
1396         /* Each record consists of several neighbour entries. */
1397         for (i = 0; i < num_entries; i++) {
1398                 int ent_index;
1399
1400                 ent_index = rec_index * MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC + i;
1401                 mlxsw_sp_router_neigh_ent_ipv4_process(mlxsw_sp, rauhtd_pl,
1402                                                        ent_index);
1403         }
1404
1405 }
1406
1407 static void mlxsw_sp_router_neigh_rec_ipv6_process(struct mlxsw_sp *mlxsw_sp,
1408                                                    char *rauhtd_pl,
1409                                                    int rec_index)
1410 {
1411         /* One record contains one entry. */
1412         mlxsw_sp_router_neigh_ent_ipv6_process(mlxsw_sp, rauhtd_pl,
1413                                                rec_index);
1414 }
1415
1416 static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp,
1417                                               char *rauhtd_pl, int rec_index)
1418 {
1419         switch (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, rec_index)) {
1420         case MLXSW_REG_RAUHTD_TYPE_IPV4:
1421                 mlxsw_sp_router_neigh_rec_ipv4_process(mlxsw_sp, rauhtd_pl,
1422                                                        rec_index);
1423                 break;
1424         case MLXSW_REG_RAUHTD_TYPE_IPV6:
1425                 mlxsw_sp_router_neigh_rec_ipv6_process(mlxsw_sp, rauhtd_pl,
1426                                                        rec_index);
1427                 break;
1428         }
1429 }
1430
1431 static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl)
1432 {
1433         u8 num_rec, last_rec_index, num_entries;
1434
1435         num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
1436         last_rec_index = num_rec - 1;
1437
1438         if (num_rec < MLXSW_REG_RAUHTD_REC_MAX_NUM)
1439                 return false;
1440         if (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, last_rec_index) ==
1441             MLXSW_REG_RAUHTD_TYPE_IPV6)
1442                 return true;
1443
1444         num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
1445                                                                 last_rec_index);
1446         if (++num_entries == MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC)
1447                 return true;
1448         return false;
1449 }
1450
1451 static int
1452 __mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp,
1453                                        char *rauhtd_pl,
1454                                        enum mlxsw_reg_rauhtd_type type)
1455 {
1456         int i, num_rec;
1457         int err;
1458
1459         /* Make sure the neighbour's netdev isn't removed in the
1460          * process.
1461          */
1462         rtnl_lock();
1463         do {
1464                 mlxsw_reg_rauhtd_pack(rauhtd_pl, type);
1465                 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(rauhtd),
1466                                       rauhtd_pl);
1467                 if (err) {
1468                         dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to dump neighbour talbe\n");
1469                         break;
1470                 }
1471                 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
1472                 for (i = 0; i < num_rec; i++)
1473                         mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl,
1474                                                           i);
1475         } while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl));
1476         rtnl_unlock();
1477
1478         return err;
1479 }
1480
1481 static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp)
1482 {
1483         enum mlxsw_reg_rauhtd_type type;
1484         char *rauhtd_pl;
1485         int err;
1486
1487         rauhtd_pl = kmalloc(MLXSW_REG_RAUHTD_LEN, GFP_KERNEL);
1488         if (!rauhtd_pl)
1489                 return -ENOMEM;
1490
1491         type = MLXSW_REG_RAUHTD_TYPE_IPV4;
1492         err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type);
1493         if (err)
1494                 goto out;
1495
1496         type = MLXSW_REG_RAUHTD_TYPE_IPV6;
1497         err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type);
1498 out:
1499         kfree(rauhtd_pl);
1500         return err;
1501 }
1502
1503 static void mlxsw_sp_router_neighs_update_nh(struct mlxsw_sp *mlxsw_sp)
1504 {
1505         struct mlxsw_sp_neigh_entry *neigh_entry;
1506
1507         /* Take RTNL mutex here to prevent lists from changes */
1508         rtnl_lock();
1509         list_for_each_entry(neigh_entry, &mlxsw_sp->router->nexthop_neighs_list,
1510                             nexthop_neighs_list_node)
1511                 /* If this neigh have nexthops, make the kernel think this neigh
1512                  * is active regardless of the traffic.
1513                  */
1514                 neigh_event_send(neigh_entry->key.n, NULL);
1515         rtnl_unlock();
1516 }
1517
1518 static void
1519 mlxsw_sp_router_neighs_update_work_schedule(struct mlxsw_sp *mlxsw_sp)
1520 {
1521         unsigned long interval = mlxsw_sp->router->neighs_update.interval;
1522
1523         mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw,
1524                                msecs_to_jiffies(interval));
1525 }
1526
1527 static void mlxsw_sp_router_neighs_update_work(struct work_struct *work)
1528 {
1529         struct mlxsw_sp_router *router;
1530         int err;
1531
1532         router = container_of(work, struct mlxsw_sp_router,
1533                               neighs_update.dw.work);
1534         err = mlxsw_sp_router_neighs_update_rauhtd(router->mlxsw_sp);
1535         if (err)
1536                 dev_err(router->mlxsw_sp->bus_info->dev, "Could not update kernel for neigh activity");
1537
1538         mlxsw_sp_router_neighs_update_nh(router->mlxsw_sp);
1539
1540         mlxsw_sp_router_neighs_update_work_schedule(router->mlxsw_sp);
1541 }
1542
1543 static void mlxsw_sp_router_probe_unresolved_nexthops(struct work_struct *work)
1544 {
1545         struct mlxsw_sp_neigh_entry *neigh_entry;
1546         struct mlxsw_sp_router *router;
1547
1548         router = container_of(work, struct mlxsw_sp_router,
1549                               nexthop_probe_dw.work);
1550         /* Iterate over nexthop neighbours, find those who are unresolved and
1551          * send arp on them. This solves the chicken-egg problem when
1552          * the nexthop wouldn't get offloaded until the neighbor is resolved
1553          * but it wouldn't get resolved ever in case traffic is flowing in HW
1554          * using different nexthop.
1555          *
1556          * Take RTNL mutex here to prevent lists from changes.
1557          */
1558         rtnl_lock();
1559         list_for_each_entry(neigh_entry, &router->nexthop_neighs_list,
1560                             nexthop_neighs_list_node)
1561                 if (!neigh_entry->connected)
1562                         neigh_event_send(neigh_entry->key.n, NULL);
1563         rtnl_unlock();
1564
1565         mlxsw_core_schedule_dw(&router->nexthop_probe_dw,
1566                                MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL);
1567 }
1568
1569 static void
1570 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
1571                               struct mlxsw_sp_neigh_entry *neigh_entry,
1572                               bool removing);
1573
1574 static enum mlxsw_reg_rauht_op mlxsw_sp_rauht_op(bool adding)
1575 {
1576         return adding ? MLXSW_REG_RAUHT_OP_WRITE_ADD :
1577                         MLXSW_REG_RAUHT_OP_WRITE_DELETE;
1578 }
1579
1580 static void
1581 mlxsw_sp_router_neigh_entry_op4(struct mlxsw_sp *mlxsw_sp,
1582                                 struct mlxsw_sp_neigh_entry *neigh_entry,
1583                                 enum mlxsw_reg_rauht_op op)
1584 {
1585         struct neighbour *n = neigh_entry->key.n;
1586         u32 dip = ntohl(*((__be32 *) n->primary_key));
1587         char rauht_pl[MLXSW_REG_RAUHT_LEN];
1588
1589         mlxsw_reg_rauht_pack4(rauht_pl, op, neigh_entry->rif, neigh_entry->ha,
1590                               dip);
1591         if (neigh_entry->counter_valid)
1592                 mlxsw_reg_rauht_pack_counter(rauht_pl,
1593                                              neigh_entry->counter_index);
1594         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
1595 }
1596
1597 static void
1598 mlxsw_sp_router_neigh_entry_op6(struct mlxsw_sp *mlxsw_sp,
1599                                 struct mlxsw_sp_neigh_entry *neigh_entry,
1600                                 enum mlxsw_reg_rauht_op op)
1601 {
1602         struct neighbour *n = neigh_entry->key.n;
1603         char rauht_pl[MLXSW_REG_RAUHT_LEN];
1604         const char *dip = n->primary_key;
1605
1606         mlxsw_reg_rauht_pack6(rauht_pl, op, neigh_entry->rif, neigh_entry->ha,
1607                               dip);
1608         if (neigh_entry->counter_valid)
1609                 mlxsw_reg_rauht_pack_counter(rauht_pl,
1610                                              neigh_entry->counter_index);
1611         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
1612 }
1613
1614 bool mlxsw_sp_neigh_ipv6_ignore(struct mlxsw_sp_neigh_entry *neigh_entry)
1615 {
1616         struct neighbour *n = neigh_entry->key.n;
1617
1618         /* Packets with a link-local destination address are trapped
1619          * after LPM lookup and never reach the neighbour table, so
1620          * there is no need to program such neighbours to the device.
1621          */
1622         if (ipv6_addr_type((struct in6_addr *) &n->primary_key) &
1623             IPV6_ADDR_LINKLOCAL)
1624                 return true;
1625         return false;
1626 }
1627
1628 static void
1629 mlxsw_sp_neigh_entry_update(struct mlxsw_sp *mlxsw_sp,
1630                             struct mlxsw_sp_neigh_entry *neigh_entry,
1631                             bool adding)
1632 {
1633         if (!adding && !neigh_entry->connected)
1634                 return;
1635         neigh_entry->connected = adding;
1636         if (neigh_entry->key.n->tbl->family == AF_INET) {
1637                 mlxsw_sp_router_neigh_entry_op4(mlxsw_sp, neigh_entry,
1638                                                 mlxsw_sp_rauht_op(adding));
1639         } else if (neigh_entry->key.n->tbl->family == AF_INET6) {
1640                 if (mlxsw_sp_neigh_ipv6_ignore(neigh_entry))
1641                         return;
1642                 mlxsw_sp_router_neigh_entry_op6(mlxsw_sp, neigh_entry,
1643                                                 mlxsw_sp_rauht_op(adding));
1644         } else {
1645                 WARN_ON_ONCE(1);
1646         }
1647 }
1648
1649 void
1650 mlxsw_sp_neigh_entry_counter_update(struct mlxsw_sp *mlxsw_sp,
1651                                     struct mlxsw_sp_neigh_entry *neigh_entry,
1652                                     bool adding)
1653 {
1654         if (adding)
1655                 mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry);
1656         else
1657                 mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry);
1658         mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, true);
1659 }
1660
1661 struct mlxsw_sp_neigh_event_work {
1662         struct work_struct work;
1663         struct mlxsw_sp *mlxsw_sp;
1664         struct neighbour *n;
1665 };
1666
1667 static void mlxsw_sp_router_neigh_event_work(struct work_struct *work)
1668 {
1669         struct mlxsw_sp_neigh_event_work *neigh_work =
1670                 container_of(work, struct mlxsw_sp_neigh_event_work, work);
1671         struct mlxsw_sp *mlxsw_sp = neigh_work->mlxsw_sp;
1672         struct mlxsw_sp_neigh_entry *neigh_entry;
1673         struct neighbour *n = neigh_work->n;
1674         unsigned char ha[ETH_ALEN];
1675         bool entry_connected;
1676         u8 nud_state, dead;
1677
1678         /* If these parameters are changed after we release the lock,
1679          * then we are guaranteed to receive another event letting us
1680          * know about it.
1681          */
1682         read_lock_bh(&n->lock);
1683         memcpy(ha, n->ha, ETH_ALEN);
1684         nud_state = n->nud_state;
1685         dead = n->dead;
1686         read_unlock_bh(&n->lock);
1687
1688         rtnl_lock();
1689         entry_connected = nud_state & NUD_VALID && !dead;
1690         neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
1691         if (!entry_connected && !neigh_entry)
1692                 goto out;
1693         if (!neigh_entry) {
1694                 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
1695                 if (IS_ERR(neigh_entry))
1696                         goto out;
1697         }
1698
1699         memcpy(neigh_entry->ha, ha, ETH_ALEN);
1700         mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, entry_connected);
1701         mlxsw_sp_nexthop_neigh_update(mlxsw_sp, neigh_entry, !entry_connected);
1702
1703         if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
1704                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
1705
1706 out:
1707         rtnl_unlock();
1708         neigh_release(n);
1709         kfree(neigh_work);
1710 }
1711
1712 int mlxsw_sp_router_netevent_event(struct notifier_block *unused,
1713                                    unsigned long event, void *ptr)
1714 {
1715         struct mlxsw_sp_neigh_event_work *neigh_work;
1716         struct mlxsw_sp_port *mlxsw_sp_port;
1717         struct mlxsw_sp *mlxsw_sp;
1718         unsigned long interval;
1719         struct neigh_parms *p;
1720         struct neighbour *n;
1721
1722         switch (event) {
1723         case NETEVENT_DELAY_PROBE_TIME_UPDATE:
1724                 p = ptr;
1725
1726                 /* We don't care about changes in the default table. */
1727                 if (!p->dev || (p->tbl->family != AF_INET &&
1728                                 p->tbl->family != AF_INET6))
1729                         return NOTIFY_DONE;
1730
1731                 /* We are in atomic context and can't take RTNL mutex,
1732                  * so use RCU variant to walk the device chain.
1733                  */
1734                 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(p->dev);
1735                 if (!mlxsw_sp_port)
1736                         return NOTIFY_DONE;
1737
1738                 mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1739                 interval = jiffies_to_msecs(NEIGH_VAR(p, DELAY_PROBE_TIME));
1740                 mlxsw_sp->router->neighs_update.interval = interval;
1741
1742                 mlxsw_sp_port_dev_put(mlxsw_sp_port);
1743                 break;
1744         case NETEVENT_NEIGH_UPDATE:
1745                 n = ptr;
1746
1747                 if (n->tbl->family != AF_INET && n->tbl->family != AF_INET6)
1748                         return NOTIFY_DONE;
1749
1750                 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(n->dev);
1751                 if (!mlxsw_sp_port)
1752                         return NOTIFY_DONE;
1753
1754                 neigh_work = kzalloc(sizeof(*neigh_work), GFP_ATOMIC);
1755                 if (!neigh_work) {
1756                         mlxsw_sp_port_dev_put(mlxsw_sp_port);
1757                         return NOTIFY_BAD;
1758                 }
1759
1760                 INIT_WORK(&neigh_work->work, mlxsw_sp_router_neigh_event_work);
1761                 neigh_work->mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1762                 neigh_work->n = n;
1763
1764                 /* Take a reference to ensure the neighbour won't be
1765                  * destructed until we drop the reference in delayed
1766                  * work.
1767                  */
1768                 neigh_clone(n);
1769                 mlxsw_core_schedule_work(&neigh_work->work);
1770                 mlxsw_sp_port_dev_put(mlxsw_sp_port);
1771                 break;
1772         }
1773
1774         return NOTIFY_DONE;
1775 }
1776
1777 static int mlxsw_sp_neigh_init(struct mlxsw_sp *mlxsw_sp)
1778 {
1779         int err;
1780
1781         err = rhashtable_init(&mlxsw_sp->router->neigh_ht,
1782                               &mlxsw_sp_neigh_ht_params);
1783         if (err)
1784                 return err;
1785
1786         /* Initialize the polling interval according to the default
1787          * table.
1788          */
1789         mlxsw_sp_router_neighs_update_interval_init(mlxsw_sp);
1790
1791         /* Create the delayed works for the activity_update */
1792         INIT_DELAYED_WORK(&mlxsw_sp->router->neighs_update.dw,
1793                           mlxsw_sp_router_neighs_update_work);
1794         INIT_DELAYED_WORK(&mlxsw_sp->router->nexthop_probe_dw,
1795                           mlxsw_sp_router_probe_unresolved_nexthops);
1796         mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw, 0);
1797         mlxsw_core_schedule_dw(&mlxsw_sp->router->nexthop_probe_dw, 0);
1798         return 0;
1799 }
1800
1801 static void mlxsw_sp_neigh_fini(struct mlxsw_sp *mlxsw_sp)
1802 {
1803         cancel_delayed_work_sync(&mlxsw_sp->router->neighs_update.dw);
1804         cancel_delayed_work_sync(&mlxsw_sp->router->nexthop_probe_dw);
1805         rhashtable_destroy(&mlxsw_sp->router->neigh_ht);
1806 }
1807
1808 static void mlxsw_sp_neigh_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
1809                                          struct mlxsw_sp_rif *rif)
1810 {
1811         struct mlxsw_sp_neigh_entry *neigh_entry, *tmp;
1812
1813         list_for_each_entry_safe(neigh_entry, tmp, &rif->neigh_list,
1814                                  rif_list_node) {
1815                 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, false);
1816                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
1817         }
1818 }
1819
1820 enum mlxsw_sp_nexthop_type {
1821         MLXSW_SP_NEXTHOP_TYPE_ETH,
1822         MLXSW_SP_NEXTHOP_TYPE_IPIP,
1823 };
1824
1825 struct mlxsw_sp_nexthop_key {
1826         struct fib_nh *fib_nh;
1827 };
1828
1829 struct mlxsw_sp_nexthop {
1830         struct list_head neigh_list_node; /* member of neigh entry list */
1831         struct list_head rif_list_node;
1832         struct mlxsw_sp_nexthop_group *nh_grp; /* pointer back to the group
1833                                                 * this belongs to
1834                                                 */
1835         struct rhash_head ht_node;
1836         struct mlxsw_sp_nexthop_key key;
1837         unsigned char gw_addr[sizeof(struct in6_addr)];
1838         int ifindex;
1839         struct mlxsw_sp_rif *rif;
1840         u8 should_offload:1, /* set indicates this neigh is connected and
1841                               * should be put to KVD linear area of this group.
1842                               */
1843            offloaded:1, /* set in case the neigh is actually put into
1844                          * KVD linear area of this group.
1845                          */
1846            update:1; /* set indicates that MAC of this neigh should be
1847                       * updated in HW
1848                       */
1849         enum mlxsw_sp_nexthop_type type;
1850         union {
1851                 struct mlxsw_sp_neigh_entry *neigh_entry;
1852                 struct mlxsw_sp_ipip_entry *ipip_entry;
1853         };
1854 };
1855
1856 struct mlxsw_sp_nexthop_group {
1857         void *priv;
1858         struct rhash_head ht_node;
1859         struct list_head fib_list; /* list of fib entries that use this group */
1860         struct neigh_table *neigh_tbl;
1861         u8 adj_index_valid:1,
1862            gateway:1; /* routes using the group use a gateway */
1863         u32 adj_index;
1864         u16 ecmp_size;
1865         u16 count;
1866         struct mlxsw_sp_nexthop nexthops[0];
1867 #define nh_rif  nexthops[0].rif
1868 };
1869
1870 static struct fib_info *
1871 mlxsw_sp_nexthop4_group_fi(const struct mlxsw_sp_nexthop_group *nh_grp)
1872 {
1873         return nh_grp->priv;
1874 }
1875
1876 struct mlxsw_sp_nexthop_group_cmp_arg {
1877         enum mlxsw_sp_l3proto proto;
1878         union {
1879                 struct fib_info *fi;
1880                 struct mlxsw_sp_fib6_entry *fib6_entry;
1881         };
1882 };
1883
1884 static bool
1885 mlxsw_sp_nexthop6_group_has_nexthop(const struct mlxsw_sp_nexthop_group *nh_grp,
1886                                     const struct in6_addr *gw, int ifindex)
1887 {
1888         int i;
1889
1890         for (i = 0; i < nh_grp->count; i++) {
1891                 const struct mlxsw_sp_nexthop *nh;
1892
1893                 nh = &nh_grp->nexthops[i];
1894                 if (nh->ifindex == ifindex &&
1895                     ipv6_addr_equal(gw, (struct in6_addr *) nh->gw_addr))
1896                         return true;
1897         }
1898
1899         return false;
1900 }
1901
1902 static bool
1903 mlxsw_sp_nexthop6_group_cmp(const struct mlxsw_sp_nexthop_group *nh_grp,
1904                             const struct mlxsw_sp_fib6_entry *fib6_entry)
1905 {
1906         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
1907
1908         if (nh_grp->count != fib6_entry->nrt6)
1909                 return false;
1910
1911         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
1912                 struct in6_addr *gw;
1913                 int ifindex;
1914
1915                 ifindex = mlxsw_sp_rt6->rt->dst.dev->ifindex;
1916                 gw = &mlxsw_sp_rt6->rt->rt6i_gateway;
1917                 if (!mlxsw_sp_nexthop6_group_has_nexthop(nh_grp, gw, ifindex))
1918                         return false;
1919         }
1920
1921         return true;
1922 }
1923
1924 static int
1925 mlxsw_sp_nexthop_group_cmp(struct rhashtable_compare_arg *arg, const void *ptr)
1926 {
1927         const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = arg->key;
1928         const struct mlxsw_sp_nexthop_group *nh_grp = ptr;
1929
1930         switch (cmp_arg->proto) {
1931         case MLXSW_SP_L3_PROTO_IPV4:
1932                 return cmp_arg->fi != mlxsw_sp_nexthop4_group_fi(nh_grp);
1933         case MLXSW_SP_L3_PROTO_IPV6:
1934                 return !mlxsw_sp_nexthop6_group_cmp(nh_grp,
1935                                                     cmp_arg->fib6_entry);
1936         default:
1937                 WARN_ON(1);
1938                 return 1;
1939         }
1940 }
1941
1942 static int
1943 mlxsw_sp_nexthop_group_type(const struct mlxsw_sp_nexthop_group *nh_grp)
1944 {
1945         return nh_grp->neigh_tbl->family;
1946 }
1947
1948 static u32 mlxsw_sp_nexthop_group_hash_obj(const void *data, u32 len, u32 seed)
1949 {
1950         const struct mlxsw_sp_nexthop_group *nh_grp = data;
1951         const struct mlxsw_sp_nexthop *nh;
1952         struct fib_info *fi;
1953         unsigned int val;
1954         int i;
1955
1956         switch (mlxsw_sp_nexthop_group_type(nh_grp)) {
1957         case AF_INET:
1958                 fi = mlxsw_sp_nexthop4_group_fi(nh_grp);
1959                 return jhash(&fi, sizeof(fi), seed);
1960         case AF_INET6:
1961                 val = nh_grp->count;
1962                 for (i = 0; i < nh_grp->count; i++) {
1963                         nh = &nh_grp->nexthops[i];
1964                         val ^= nh->ifindex;
1965                 }
1966                 return jhash(&val, sizeof(val), seed);
1967         default:
1968                 WARN_ON(1);
1969                 return 0;
1970         }
1971 }
1972
1973 static u32
1974 mlxsw_sp_nexthop6_group_hash(struct mlxsw_sp_fib6_entry *fib6_entry, u32 seed)
1975 {
1976         unsigned int val = fib6_entry->nrt6;
1977         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
1978         struct net_device *dev;
1979
1980         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
1981                 dev = mlxsw_sp_rt6->rt->dst.dev;
1982                 val ^= dev->ifindex;
1983         }
1984
1985         return jhash(&val, sizeof(val), seed);
1986 }
1987
1988 static u32
1989 mlxsw_sp_nexthop_group_hash(const void *data, u32 len, u32 seed)
1990 {
1991         const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = data;
1992
1993         switch (cmp_arg->proto) {
1994         case MLXSW_SP_L3_PROTO_IPV4:
1995                 return jhash(&cmp_arg->fi, sizeof(cmp_arg->fi), seed);
1996         case MLXSW_SP_L3_PROTO_IPV6:
1997                 return mlxsw_sp_nexthop6_group_hash(cmp_arg->fib6_entry, seed);
1998         default:
1999                 WARN_ON(1);
2000                 return 0;
2001         }
2002 }
2003
2004 static const struct rhashtable_params mlxsw_sp_nexthop_group_ht_params = {
2005         .head_offset = offsetof(struct mlxsw_sp_nexthop_group, ht_node),
2006         .hashfn      = mlxsw_sp_nexthop_group_hash,
2007         .obj_hashfn  = mlxsw_sp_nexthop_group_hash_obj,
2008         .obj_cmpfn   = mlxsw_sp_nexthop_group_cmp,
2009 };
2010
2011 static int mlxsw_sp_nexthop_group_insert(struct mlxsw_sp *mlxsw_sp,
2012                                          struct mlxsw_sp_nexthop_group *nh_grp)
2013 {
2014         if (mlxsw_sp_nexthop_group_type(nh_grp) == AF_INET6 &&
2015             !nh_grp->gateway)
2016                 return 0;
2017
2018         return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_group_ht,
2019                                       &nh_grp->ht_node,
2020                                       mlxsw_sp_nexthop_group_ht_params);
2021 }
2022
2023 static void mlxsw_sp_nexthop_group_remove(struct mlxsw_sp *mlxsw_sp,
2024                                           struct mlxsw_sp_nexthop_group *nh_grp)
2025 {
2026         if (mlxsw_sp_nexthop_group_type(nh_grp) == AF_INET6 &&
2027             !nh_grp->gateway)
2028                 return;
2029
2030         rhashtable_remove_fast(&mlxsw_sp->router->nexthop_group_ht,
2031                                &nh_grp->ht_node,
2032                                mlxsw_sp_nexthop_group_ht_params);
2033 }
2034
2035 static struct mlxsw_sp_nexthop_group *
2036 mlxsw_sp_nexthop4_group_lookup(struct mlxsw_sp *mlxsw_sp,
2037                                struct fib_info *fi)
2038 {
2039         struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg;
2040
2041         cmp_arg.proto = MLXSW_SP_L3_PROTO_IPV4;
2042         cmp_arg.fi = fi;
2043         return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht,
2044                                       &cmp_arg,
2045                                       mlxsw_sp_nexthop_group_ht_params);
2046 }
2047
2048 static struct mlxsw_sp_nexthop_group *
2049 mlxsw_sp_nexthop6_group_lookup(struct mlxsw_sp *mlxsw_sp,
2050                                struct mlxsw_sp_fib6_entry *fib6_entry)
2051 {
2052         struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg;
2053
2054         cmp_arg.proto = MLXSW_SP_L3_PROTO_IPV6;
2055         cmp_arg.fib6_entry = fib6_entry;
2056         return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht,
2057                                       &cmp_arg,
2058                                       mlxsw_sp_nexthop_group_ht_params);
2059 }
2060
2061 static const struct rhashtable_params mlxsw_sp_nexthop_ht_params = {
2062         .key_offset = offsetof(struct mlxsw_sp_nexthop, key),
2063         .head_offset = offsetof(struct mlxsw_sp_nexthop, ht_node),
2064         .key_len = sizeof(struct mlxsw_sp_nexthop_key),
2065 };
2066
2067 static int mlxsw_sp_nexthop_insert(struct mlxsw_sp *mlxsw_sp,
2068                                    struct mlxsw_sp_nexthop *nh)
2069 {
2070         return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_ht,
2071                                       &nh->ht_node, mlxsw_sp_nexthop_ht_params);
2072 }
2073
2074 static void mlxsw_sp_nexthop_remove(struct mlxsw_sp *mlxsw_sp,
2075                                     struct mlxsw_sp_nexthop *nh)
2076 {
2077         rhashtable_remove_fast(&mlxsw_sp->router->nexthop_ht, &nh->ht_node,
2078                                mlxsw_sp_nexthop_ht_params);
2079 }
2080
2081 static struct mlxsw_sp_nexthop *
2082 mlxsw_sp_nexthop_lookup(struct mlxsw_sp *mlxsw_sp,
2083                         struct mlxsw_sp_nexthop_key key)
2084 {
2085         return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_ht, &key,
2086                                       mlxsw_sp_nexthop_ht_params);
2087 }
2088
2089 static int mlxsw_sp_adj_index_mass_update_vr(struct mlxsw_sp *mlxsw_sp,
2090                                              const struct mlxsw_sp_fib *fib,
2091                                              u32 adj_index, u16 ecmp_size,
2092                                              u32 new_adj_index,
2093                                              u16 new_ecmp_size)
2094 {
2095         char raleu_pl[MLXSW_REG_RALEU_LEN];
2096
2097         mlxsw_reg_raleu_pack(raleu_pl,
2098                              (enum mlxsw_reg_ralxx_protocol) fib->proto,
2099                              fib->vr->id, adj_index, ecmp_size, new_adj_index,
2100                              new_ecmp_size);
2101         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raleu), raleu_pl);
2102 }
2103
2104 static int mlxsw_sp_adj_index_mass_update(struct mlxsw_sp *mlxsw_sp,
2105                                           struct mlxsw_sp_nexthop_group *nh_grp,
2106                                           u32 old_adj_index, u16 old_ecmp_size)
2107 {
2108         struct mlxsw_sp_fib_entry *fib_entry;
2109         struct mlxsw_sp_fib *fib = NULL;
2110         int err;
2111
2112         list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
2113                 if (fib == fib_entry->fib_node->fib)
2114                         continue;
2115                 fib = fib_entry->fib_node->fib;
2116                 err = mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, fib,
2117                                                         old_adj_index,
2118                                                         old_ecmp_size,
2119                                                         nh_grp->adj_index,
2120                                                         nh_grp->ecmp_size);
2121                 if (err)
2122                         return err;
2123         }
2124         return 0;
2125 }
2126
2127 static int mlxsw_sp_nexthop_mac_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index,
2128                                        struct mlxsw_sp_nexthop *nh)
2129 {
2130         struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
2131         char ratr_pl[MLXSW_REG_RATR_LEN];
2132
2133         mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY,
2134                             true, MLXSW_REG_RATR_TYPE_ETHERNET,
2135                             adj_index, neigh_entry->rif);
2136         mlxsw_reg_ratr_eth_entry_pack(ratr_pl, neigh_entry->ha);
2137         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl);
2138 }
2139
2140 static int mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp,
2141                                         u32 adj_index,
2142                                         struct mlxsw_sp_nexthop *nh)
2143 {
2144         const struct mlxsw_sp_ipip_ops *ipip_ops;
2145
2146         ipip_ops = mlxsw_sp->router->ipip_ops_arr[nh->ipip_entry->ipipt];
2147         return ipip_ops->nexthop_update(mlxsw_sp, adj_index, nh->ipip_entry);
2148 }
2149
2150 static int
2151 mlxsw_sp_nexthop_group_update(struct mlxsw_sp *mlxsw_sp,
2152                               struct mlxsw_sp_nexthop_group *nh_grp,
2153                               bool reallocate)
2154 {
2155         u32 adj_index = nh_grp->adj_index; /* base */
2156         struct mlxsw_sp_nexthop *nh;
2157         int i;
2158         int err;
2159
2160         for (i = 0; i < nh_grp->count; i++) {
2161                 nh = &nh_grp->nexthops[i];
2162
2163                 if (!nh->should_offload) {
2164                         nh->offloaded = 0;
2165                         continue;
2166                 }
2167
2168                 if (nh->update || reallocate) {
2169                         switch (nh->type) {
2170                         case MLXSW_SP_NEXTHOP_TYPE_ETH:
2171                                 err = mlxsw_sp_nexthop_mac_update
2172                                             (mlxsw_sp, adj_index, nh);
2173                                 break;
2174                         case MLXSW_SP_NEXTHOP_TYPE_IPIP:
2175                                 err = mlxsw_sp_nexthop_ipip_update
2176                                             (mlxsw_sp, adj_index, nh);
2177                                 break;
2178                         }
2179                         if (err)
2180                                 return err;
2181                         nh->update = 0;
2182                         nh->offloaded = 1;
2183                 }
2184                 adj_index++;
2185         }
2186         return 0;
2187 }
2188
2189 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
2190                                      struct mlxsw_sp_fib_entry *fib_entry);
2191
2192 static bool
2193 mlxsw_sp_fib_node_entry_is_first(const struct mlxsw_sp_fib_node *fib_node,
2194                                  const struct mlxsw_sp_fib_entry *fib_entry);
2195
2196 static int
2197 mlxsw_sp_nexthop_fib_entries_update(struct mlxsw_sp *mlxsw_sp,
2198                                     struct mlxsw_sp_nexthop_group *nh_grp)
2199 {
2200         struct mlxsw_sp_fib_entry *fib_entry;
2201         int err;
2202
2203         list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
2204                 if (!mlxsw_sp_fib_node_entry_is_first(fib_entry->fib_node,
2205                                                       fib_entry))
2206                         continue;
2207                 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
2208                 if (err)
2209                         return err;
2210         }
2211         return 0;
2212 }
2213
2214 static void
2215 mlxsw_sp_fib_entry_offload_refresh(struct mlxsw_sp_fib_entry *fib_entry,
2216                                    enum mlxsw_reg_ralue_op op, int err);
2217
2218 static void
2219 mlxsw_sp_nexthop_fib_entries_refresh(struct mlxsw_sp_nexthop_group *nh_grp)
2220 {
2221         enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_WRITE;
2222         struct mlxsw_sp_fib_entry *fib_entry;
2223
2224         list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
2225                 if (!mlxsw_sp_fib_node_entry_is_first(fib_entry->fib_node,
2226                                                       fib_entry))
2227                         continue;
2228                 mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, 0);
2229         }
2230 }
2231
2232 static void
2233 mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp,
2234                                struct mlxsw_sp_nexthop_group *nh_grp)
2235 {
2236         struct mlxsw_sp_nexthop *nh;
2237         bool offload_change = false;
2238         u32 adj_index;
2239         u16 ecmp_size = 0;
2240         bool old_adj_index_valid;
2241         u32 old_adj_index;
2242         u16 old_ecmp_size;
2243         int i;
2244         int err;
2245
2246         if (!nh_grp->gateway) {
2247                 mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
2248                 return;
2249         }
2250
2251         for (i = 0; i < nh_grp->count; i++) {
2252                 nh = &nh_grp->nexthops[i];
2253
2254                 if (nh->should_offload != nh->offloaded) {
2255                         offload_change = true;
2256                         if (nh->should_offload)
2257                                 nh->update = 1;
2258                 }
2259                 if (nh->should_offload)
2260                         ecmp_size++;
2261         }
2262         if (!offload_change) {
2263                 /* Nothing was added or removed, so no need to reallocate. Just
2264                  * update MAC on existing adjacency indexes.
2265                  */
2266                 err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nh_grp, false);
2267                 if (err) {
2268                         dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
2269                         goto set_trap;
2270                 }
2271                 return;
2272         }
2273         if (!ecmp_size)
2274                 /* No neigh of this group is connected so we just set
2275                  * the trap and let everthing flow through kernel.
2276                  */
2277                 goto set_trap;
2278
2279         err = mlxsw_sp_kvdl_alloc(mlxsw_sp, ecmp_size, &adj_index);
2280         if (err) {
2281                 /* We ran out of KVD linear space, just set the
2282                  * trap and let everything flow through kernel.
2283                  */
2284                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to allocate KVD linear area for nexthop group.\n");
2285                 goto set_trap;
2286         }
2287         old_adj_index_valid = nh_grp->adj_index_valid;
2288         old_adj_index = nh_grp->adj_index;
2289         old_ecmp_size = nh_grp->ecmp_size;
2290         nh_grp->adj_index_valid = 1;
2291         nh_grp->adj_index = adj_index;
2292         nh_grp->ecmp_size = ecmp_size;
2293         err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nh_grp, true);
2294         if (err) {
2295                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
2296                 goto set_trap;
2297         }
2298
2299         if (!old_adj_index_valid) {
2300                 /* The trap was set for fib entries, so we have to call
2301                  * fib entry update to unset it and use adjacency index.
2302                  */
2303                 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
2304                 if (err) {
2305                         dev_warn(mlxsw_sp->bus_info->dev, "Failed to add adjacency index to fib entries.\n");
2306                         goto set_trap;
2307                 }
2308                 return;
2309         }
2310
2311         err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, nh_grp,
2312                                              old_adj_index, old_ecmp_size);
2313         mlxsw_sp_kvdl_free(mlxsw_sp, old_adj_index);
2314         if (err) {
2315                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to mass-update adjacency index for nexthop group.\n");
2316                 goto set_trap;
2317         }
2318
2319         /* Offload state within the group changed, so update the flags. */
2320         mlxsw_sp_nexthop_fib_entries_refresh(nh_grp);
2321
2322         return;
2323
2324 set_trap:
2325         old_adj_index_valid = nh_grp->adj_index_valid;
2326         nh_grp->adj_index_valid = 0;
2327         for (i = 0; i < nh_grp->count; i++) {
2328                 nh = &nh_grp->nexthops[i];
2329                 nh->offloaded = 0;
2330         }
2331         err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
2332         if (err)
2333                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set traps for fib entries.\n");
2334         if (old_adj_index_valid)
2335                 mlxsw_sp_kvdl_free(mlxsw_sp, nh_grp->adj_index);
2336 }
2337
2338 static void __mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp_nexthop *nh,
2339                                             bool removing)
2340 {
2341         if (!removing)
2342                 nh->should_offload = 1;
2343         else if (nh->offloaded)
2344                 nh->should_offload = 0;
2345         nh->update = 1;
2346 }
2347
2348 static void
2349 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
2350                               struct mlxsw_sp_neigh_entry *neigh_entry,
2351                               bool removing)
2352 {
2353         struct mlxsw_sp_nexthop *nh;
2354
2355         list_for_each_entry(nh, &neigh_entry->nexthop_list,
2356                             neigh_list_node) {
2357                 __mlxsw_sp_nexthop_neigh_update(nh, removing);
2358                 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
2359         }
2360 }
2361
2362 static void mlxsw_sp_nexthop_rif_init(struct mlxsw_sp_nexthop *nh,
2363                                       struct mlxsw_sp_rif *rif)
2364 {
2365         if (nh->rif)
2366                 return;
2367
2368         nh->rif = rif;
2369         list_add(&nh->rif_list_node, &rif->nexthop_list);
2370 }
2371
2372 static void mlxsw_sp_nexthop_rif_fini(struct mlxsw_sp_nexthop *nh)
2373 {
2374         if (!nh->rif)
2375                 return;
2376
2377         list_del(&nh->rif_list_node);
2378         nh->rif = NULL;
2379 }
2380
2381 static int mlxsw_sp_nexthop_neigh_init(struct mlxsw_sp *mlxsw_sp,
2382                                        struct mlxsw_sp_nexthop *nh)
2383 {
2384         struct mlxsw_sp_neigh_entry *neigh_entry;
2385         struct neighbour *n;
2386         u8 nud_state, dead;
2387         int err;
2388
2389         if (!nh->nh_grp->gateway || nh->neigh_entry)
2390                 return 0;
2391
2392         /* Take a reference of neigh here ensuring that neigh would
2393          * not be destructed before the nexthop entry is finished.
2394          * The reference is taken either in neigh_lookup() or
2395          * in neigh_create() in case n is not found.
2396          */
2397         n = neigh_lookup(nh->nh_grp->neigh_tbl, &nh->gw_addr, nh->rif->dev);
2398         if (!n) {
2399                 n = neigh_create(nh->nh_grp->neigh_tbl, &nh->gw_addr,
2400                                  nh->rif->dev);
2401                 if (IS_ERR(n))
2402                         return PTR_ERR(n);
2403                 neigh_event_send(n, NULL);
2404         }
2405         neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
2406         if (!neigh_entry) {
2407                 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
2408                 if (IS_ERR(neigh_entry)) {
2409                         err = -EINVAL;
2410                         goto err_neigh_entry_create;
2411                 }
2412         }
2413
2414         /* If that is the first nexthop connected to that neigh, add to
2415          * nexthop_neighs_list
2416          */
2417         if (list_empty(&neigh_entry->nexthop_list))
2418                 list_add_tail(&neigh_entry->nexthop_neighs_list_node,
2419                               &mlxsw_sp->router->nexthop_neighs_list);
2420
2421         nh->neigh_entry = neigh_entry;
2422         list_add_tail(&nh->neigh_list_node, &neigh_entry->nexthop_list);
2423         read_lock_bh(&n->lock);
2424         nud_state = n->nud_state;
2425         dead = n->dead;
2426         read_unlock_bh(&n->lock);
2427         __mlxsw_sp_nexthop_neigh_update(nh, !(nud_state & NUD_VALID && !dead));
2428
2429         return 0;
2430
2431 err_neigh_entry_create:
2432         neigh_release(n);
2433         return err;
2434 }
2435
2436 static void mlxsw_sp_nexthop_neigh_fini(struct mlxsw_sp *mlxsw_sp,
2437                                         struct mlxsw_sp_nexthop *nh)
2438 {
2439         struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
2440         struct neighbour *n;
2441
2442         if (!neigh_entry)
2443                 return;
2444         n = neigh_entry->key.n;
2445
2446         __mlxsw_sp_nexthop_neigh_update(nh, true);
2447         list_del(&nh->neigh_list_node);
2448         nh->neigh_entry = NULL;
2449
2450         /* If that is the last nexthop connected to that neigh, remove from
2451          * nexthop_neighs_list
2452          */
2453         if (list_empty(&neigh_entry->nexthop_list))
2454                 list_del(&neigh_entry->nexthop_neighs_list_node);
2455
2456         if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
2457                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
2458
2459         neigh_release(n);
2460 }
2461
2462 static bool mlxsw_sp_netdev_ipip_type(const struct mlxsw_sp *mlxsw_sp,
2463                                       const struct net_device *dev,
2464                                       enum mlxsw_sp_ipip_type *p_type)
2465 {
2466         struct mlxsw_sp_router *router = mlxsw_sp->router;
2467         const struct mlxsw_sp_ipip_ops *ipip_ops;
2468         enum mlxsw_sp_ipip_type ipipt;
2469
2470         for (ipipt = 0; ipipt < MLXSW_SP_IPIP_TYPE_MAX; ++ipipt) {
2471                 ipip_ops = router->ipip_ops_arr[ipipt];
2472                 if (dev->type == ipip_ops->dev_type) {
2473                         if (p_type)
2474                                 *p_type = ipipt;
2475                         return true;
2476                 }
2477         }
2478         return false;
2479 }
2480
2481 static int mlxsw_sp_nexthop_ipip_init(struct mlxsw_sp *mlxsw_sp,
2482                                       enum mlxsw_sp_ipip_type ipipt,
2483                                       struct mlxsw_sp_nexthop *nh,
2484                                       struct net_device *ol_dev)
2485 {
2486         if (!nh->nh_grp->gateway || nh->ipip_entry)
2487                 return 0;
2488
2489         nh->ipip_entry = mlxsw_sp_ipip_entry_get(mlxsw_sp, ipipt, ol_dev);
2490         if (IS_ERR(nh->ipip_entry))
2491                 return PTR_ERR(nh->ipip_entry);
2492
2493         __mlxsw_sp_nexthop_neigh_update(nh, false);
2494         return 0;
2495 }
2496
2497 static void mlxsw_sp_nexthop_ipip_fini(struct mlxsw_sp *mlxsw_sp,
2498                                        struct mlxsw_sp_nexthop *nh)
2499 {
2500         struct mlxsw_sp_ipip_entry *ipip_entry = nh->ipip_entry;
2501
2502         if (!ipip_entry)
2503                 return;
2504
2505         __mlxsw_sp_nexthop_neigh_update(nh, true);
2506         mlxsw_sp_ipip_entry_put(mlxsw_sp, ipip_entry);
2507         nh->ipip_entry = NULL;
2508 }
2509
2510 static bool mlxsw_sp_nexthop4_ipip_type(const struct mlxsw_sp *mlxsw_sp,
2511                                         const struct fib_nh *fib_nh,
2512                                         enum mlxsw_sp_ipip_type *p_ipipt)
2513 {
2514         struct net_device *dev = fib_nh->nh_dev;
2515
2516         return dev &&
2517                fib_nh->nh_parent->fib_type == RTN_UNICAST &&
2518                mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, p_ipipt);
2519 }
2520
2521 static void mlxsw_sp_nexthop_type_fini(struct mlxsw_sp *mlxsw_sp,
2522                                        struct mlxsw_sp_nexthop *nh)
2523 {
2524         switch (nh->type) {
2525         case MLXSW_SP_NEXTHOP_TYPE_ETH:
2526                 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
2527                 mlxsw_sp_nexthop_rif_fini(nh);
2528                 break;
2529         case MLXSW_SP_NEXTHOP_TYPE_IPIP:
2530                 mlxsw_sp_nexthop_ipip_fini(mlxsw_sp, nh);
2531                 break;
2532         }
2533 }
2534
2535 static int mlxsw_sp_nexthop4_type_init(struct mlxsw_sp *mlxsw_sp,
2536                                        struct mlxsw_sp_nexthop *nh,
2537                                        struct fib_nh *fib_nh)
2538 {
2539         struct mlxsw_sp_router *router = mlxsw_sp->router;
2540         struct net_device *dev = fib_nh->nh_dev;
2541         enum mlxsw_sp_ipip_type ipipt;
2542         struct mlxsw_sp_rif *rif;
2543         int err;
2544
2545         if (mlxsw_sp_nexthop4_ipip_type(mlxsw_sp, fib_nh, &ipipt) &&
2546             router->ipip_ops_arr[ipipt]->can_offload(mlxsw_sp, dev,
2547                                                      MLXSW_SP_L3_PROTO_IPV4)) {
2548                 nh->type = MLXSW_SP_NEXTHOP_TYPE_IPIP;
2549                 return mlxsw_sp_nexthop_ipip_init(mlxsw_sp, ipipt, nh, dev);
2550         }
2551
2552         nh->type = MLXSW_SP_NEXTHOP_TYPE_ETH;
2553         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
2554         if (!rif)
2555                 return 0;
2556
2557         mlxsw_sp_nexthop_rif_init(nh, rif);
2558         err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
2559         if (err)
2560                 goto err_neigh_init;
2561
2562         return 0;
2563
2564 err_neigh_init:
2565         mlxsw_sp_nexthop_rif_fini(nh);
2566         return err;
2567 }
2568
2569 static void mlxsw_sp_nexthop4_type_fini(struct mlxsw_sp *mlxsw_sp,
2570                                         struct mlxsw_sp_nexthop *nh)
2571 {
2572         mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
2573 }
2574
2575 static int mlxsw_sp_nexthop4_init(struct mlxsw_sp *mlxsw_sp,
2576                                   struct mlxsw_sp_nexthop_group *nh_grp,
2577                                   struct mlxsw_sp_nexthop *nh,
2578                                   struct fib_nh *fib_nh)
2579 {
2580         struct net_device *dev = fib_nh->nh_dev;
2581         struct in_device *in_dev;
2582         int err;
2583
2584         nh->nh_grp = nh_grp;
2585         nh->key.fib_nh = fib_nh;
2586         memcpy(&nh->gw_addr, &fib_nh->nh_gw, sizeof(fib_nh->nh_gw));
2587         err = mlxsw_sp_nexthop_insert(mlxsw_sp, nh);
2588         if (err)
2589                 return err;
2590
2591         if (!dev)
2592                 return 0;
2593
2594         in_dev = __in_dev_get_rtnl(dev);
2595         if (in_dev && IN_DEV_IGNORE_ROUTES_WITH_LINKDOWN(in_dev) &&
2596             fib_nh->nh_flags & RTNH_F_LINKDOWN)
2597                 return 0;
2598
2599         err = mlxsw_sp_nexthop4_type_init(mlxsw_sp, nh, fib_nh);
2600         if (err)
2601                 goto err_nexthop_neigh_init;
2602
2603         return 0;
2604
2605 err_nexthop_neigh_init:
2606         mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
2607         return err;
2608 }
2609
2610 static void mlxsw_sp_nexthop4_fini(struct mlxsw_sp *mlxsw_sp,
2611                                    struct mlxsw_sp_nexthop *nh)
2612 {
2613         mlxsw_sp_nexthop4_type_fini(mlxsw_sp, nh);
2614         mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
2615 }
2616
2617 static void mlxsw_sp_nexthop4_event(struct mlxsw_sp *mlxsw_sp,
2618                                     unsigned long event, struct fib_nh *fib_nh)
2619 {
2620         struct mlxsw_sp_nexthop_key key;
2621         struct mlxsw_sp_nexthop *nh;
2622
2623         if (mlxsw_sp->router->aborted)
2624                 return;
2625
2626         key.fib_nh = fib_nh;
2627         nh = mlxsw_sp_nexthop_lookup(mlxsw_sp, key);
2628         if (WARN_ON_ONCE(!nh))
2629                 return;
2630
2631         switch (event) {
2632         case FIB_EVENT_NH_ADD:
2633                 mlxsw_sp_nexthop4_type_init(mlxsw_sp, nh, fib_nh);
2634                 break;
2635         case FIB_EVENT_NH_DEL:
2636                 mlxsw_sp_nexthop4_type_fini(mlxsw_sp, nh);
2637                 break;
2638         }
2639
2640         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
2641 }
2642
2643 static void mlxsw_sp_nexthop_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
2644                                            struct mlxsw_sp_rif *rif)
2645 {
2646         struct mlxsw_sp_nexthop *nh, *tmp;
2647
2648         list_for_each_entry_safe(nh, tmp, &rif->nexthop_list, rif_list_node) {
2649                 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
2650                 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
2651         }
2652 }
2653
2654 static bool mlxsw_sp_fi_is_gateway(const struct mlxsw_sp *mlxsw_sp,
2655                                    const struct fib_info *fi)
2656 {
2657         return fi->fib_nh->nh_scope == RT_SCOPE_LINK ||
2658                mlxsw_sp_nexthop4_ipip_type(mlxsw_sp, fi->fib_nh, NULL);
2659 }
2660
2661 static struct mlxsw_sp_nexthop_group *
2662 mlxsw_sp_nexthop4_group_create(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi)
2663 {
2664         struct mlxsw_sp_nexthop_group *nh_grp;
2665         struct mlxsw_sp_nexthop *nh;
2666         struct fib_nh *fib_nh;
2667         size_t alloc_size;
2668         int i;
2669         int err;
2670
2671         alloc_size = sizeof(*nh_grp) +
2672                      fi->fib_nhs * sizeof(struct mlxsw_sp_nexthop);
2673         nh_grp = kzalloc(alloc_size, GFP_KERNEL);
2674         if (!nh_grp)
2675                 return ERR_PTR(-ENOMEM);
2676         nh_grp->priv = fi;
2677         INIT_LIST_HEAD(&nh_grp->fib_list);
2678         nh_grp->neigh_tbl = &arp_tbl;
2679
2680         nh_grp->gateway = mlxsw_sp_fi_is_gateway(mlxsw_sp, fi);
2681         nh_grp->count = fi->fib_nhs;
2682         fib_info_hold(fi);
2683         for (i = 0; i < nh_grp->count; i++) {
2684                 nh = &nh_grp->nexthops[i];
2685                 fib_nh = &fi->fib_nh[i];
2686                 err = mlxsw_sp_nexthop4_init(mlxsw_sp, nh_grp, nh, fib_nh);
2687                 if (err)
2688                         goto err_nexthop4_init;
2689         }
2690         err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
2691         if (err)
2692                 goto err_nexthop_group_insert;
2693         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
2694         return nh_grp;
2695
2696 err_nexthop_group_insert:
2697 err_nexthop4_init:
2698         for (i--; i >= 0; i--) {
2699                 nh = &nh_grp->nexthops[i];
2700                 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh);
2701         }
2702         fib_info_put(fi);
2703         kfree(nh_grp);
2704         return ERR_PTR(err);
2705 }
2706
2707 static void
2708 mlxsw_sp_nexthop4_group_destroy(struct mlxsw_sp *mlxsw_sp,
2709                                 struct mlxsw_sp_nexthop_group *nh_grp)
2710 {
2711         struct mlxsw_sp_nexthop *nh;
2712         int i;
2713
2714         mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
2715         for (i = 0; i < nh_grp->count; i++) {
2716                 nh = &nh_grp->nexthops[i];
2717                 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh);
2718         }
2719         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
2720         WARN_ON_ONCE(nh_grp->adj_index_valid);
2721         fib_info_put(mlxsw_sp_nexthop4_group_fi(nh_grp));
2722         kfree(nh_grp);
2723 }
2724
2725 static int mlxsw_sp_nexthop4_group_get(struct mlxsw_sp *mlxsw_sp,
2726                                        struct mlxsw_sp_fib_entry *fib_entry,
2727                                        struct fib_info *fi)
2728 {
2729         struct mlxsw_sp_nexthop_group *nh_grp;
2730
2731         nh_grp = mlxsw_sp_nexthop4_group_lookup(mlxsw_sp, fi);
2732         if (!nh_grp) {
2733                 nh_grp = mlxsw_sp_nexthop4_group_create(mlxsw_sp, fi);
2734                 if (IS_ERR(nh_grp))
2735                         return PTR_ERR(nh_grp);
2736         }
2737         list_add_tail(&fib_entry->nexthop_group_node, &nh_grp->fib_list);
2738         fib_entry->nh_group = nh_grp;
2739         return 0;
2740 }
2741
2742 static void mlxsw_sp_nexthop4_group_put(struct mlxsw_sp *mlxsw_sp,
2743                                         struct mlxsw_sp_fib_entry *fib_entry)
2744 {
2745         struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
2746
2747         list_del(&fib_entry->nexthop_group_node);
2748         if (!list_empty(&nh_grp->fib_list))
2749                 return;
2750         mlxsw_sp_nexthop4_group_destroy(mlxsw_sp, nh_grp);
2751 }
2752
2753 static bool
2754 mlxsw_sp_fib4_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry)
2755 {
2756         struct mlxsw_sp_fib4_entry *fib4_entry;
2757
2758         fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry,
2759                                   common);
2760         return !fib4_entry->tos;
2761 }
2762
2763 static bool
2764 mlxsw_sp_fib_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry)
2765 {
2766         struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group;
2767
2768         switch (fib_entry->fib_node->fib->proto) {
2769         case MLXSW_SP_L3_PROTO_IPV4:
2770                 if (!mlxsw_sp_fib4_entry_should_offload(fib_entry))
2771                         return false;
2772                 break;
2773         case MLXSW_SP_L3_PROTO_IPV6:
2774                 break;
2775         }
2776
2777         switch (fib_entry->type) {
2778         case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
2779                 return !!nh_group->adj_index_valid;
2780         case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
2781                 return !!nh_group->nh_rif;
2782         default:
2783                 return false;
2784         }
2785 }
2786
2787 static struct mlxsw_sp_nexthop *
2788 mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp,
2789                      const struct mlxsw_sp_rt6 *mlxsw_sp_rt6)
2790 {
2791         int i;
2792
2793         for (i = 0; i < nh_grp->count; i++) {
2794                 struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i];
2795                 struct rt6_info *rt = mlxsw_sp_rt6->rt;
2796
2797                 if (nh->rif && nh->rif->dev == rt->dst.dev &&
2798                     ipv6_addr_equal((const struct in6_addr *) &nh->gw_addr,
2799                                     &rt->rt6i_gateway))
2800                         return nh;
2801                 continue;
2802         }
2803
2804         return NULL;
2805 }
2806
2807 static void
2808 mlxsw_sp_fib4_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry)
2809 {
2810         struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
2811         int i;
2812
2813         if (fib_entry->type == MLXSW_SP_FIB_ENTRY_TYPE_LOCAL) {
2814                 nh_grp->nexthops->key.fib_nh->nh_flags |= RTNH_F_OFFLOAD;
2815                 return;
2816         }
2817
2818         for (i = 0; i < nh_grp->count; i++) {
2819                 struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i];
2820
2821                 if (nh->offloaded)
2822                         nh->key.fib_nh->nh_flags |= RTNH_F_OFFLOAD;
2823                 else
2824                         nh->key.fib_nh->nh_flags &= ~RTNH_F_OFFLOAD;
2825         }
2826 }
2827
2828 static void
2829 mlxsw_sp_fib4_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry)
2830 {
2831         struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
2832         int i;
2833
2834         for (i = 0; i < nh_grp->count; i++) {
2835                 struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i];
2836
2837                 nh->key.fib_nh->nh_flags &= ~RTNH_F_OFFLOAD;
2838         }
2839 }
2840
2841 static void
2842 mlxsw_sp_fib6_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry)
2843 {
2844         struct mlxsw_sp_fib6_entry *fib6_entry;
2845         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
2846
2847         fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry,
2848                                   common);
2849
2850         if (fib_entry->type == MLXSW_SP_FIB_ENTRY_TYPE_LOCAL) {
2851                 list_first_entry(&fib6_entry->rt6_list, struct mlxsw_sp_rt6,
2852                                  list)->rt->rt6i_nh_flags |= RTNH_F_OFFLOAD;
2853                 return;
2854         }
2855
2856         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
2857                 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
2858                 struct mlxsw_sp_nexthop *nh;
2859
2860                 nh = mlxsw_sp_rt6_nexthop(nh_grp, mlxsw_sp_rt6);
2861                 if (nh && nh->offloaded)
2862                         mlxsw_sp_rt6->rt->rt6i_nh_flags |= RTNH_F_OFFLOAD;
2863                 else
2864                         mlxsw_sp_rt6->rt->rt6i_nh_flags &= ~RTNH_F_OFFLOAD;
2865         }
2866 }
2867
2868 static void
2869 mlxsw_sp_fib6_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry)
2870 {
2871         struct mlxsw_sp_fib6_entry *fib6_entry;
2872         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
2873
2874         fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry,
2875                                   common);
2876         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
2877                 struct rt6_info *rt = mlxsw_sp_rt6->rt;
2878
2879                 rt->rt6i_nh_flags &= ~RTNH_F_OFFLOAD;
2880         }
2881 }
2882
2883 static void mlxsw_sp_fib_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry)
2884 {
2885         switch (fib_entry->fib_node->fib->proto) {
2886         case MLXSW_SP_L3_PROTO_IPV4:
2887                 mlxsw_sp_fib4_entry_offload_set(fib_entry);
2888                 break;
2889         case MLXSW_SP_L3_PROTO_IPV6:
2890                 mlxsw_sp_fib6_entry_offload_set(fib_entry);
2891                 break;
2892         }
2893 }
2894
2895 static void
2896 mlxsw_sp_fib_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry)
2897 {
2898         switch (fib_entry->fib_node->fib->proto) {
2899         case MLXSW_SP_L3_PROTO_IPV4:
2900                 mlxsw_sp_fib4_entry_offload_unset(fib_entry);
2901                 break;
2902         case MLXSW_SP_L3_PROTO_IPV6:
2903                 mlxsw_sp_fib6_entry_offload_unset(fib_entry);
2904                 break;
2905         }
2906 }
2907
2908 static void
2909 mlxsw_sp_fib_entry_offload_refresh(struct mlxsw_sp_fib_entry *fib_entry,
2910                                    enum mlxsw_reg_ralue_op op, int err)
2911 {
2912         switch (op) {
2913         case MLXSW_REG_RALUE_OP_WRITE_DELETE:
2914                 return mlxsw_sp_fib_entry_offload_unset(fib_entry);
2915         case MLXSW_REG_RALUE_OP_WRITE_WRITE:
2916                 if (err)
2917                         return;
2918                 if (mlxsw_sp_fib_entry_should_offload(fib_entry))
2919                         mlxsw_sp_fib_entry_offload_set(fib_entry);
2920                 else if (!mlxsw_sp_fib_entry_should_offload(fib_entry))
2921                         mlxsw_sp_fib_entry_offload_unset(fib_entry);
2922                 return;
2923         default:
2924                 return;
2925         }
2926 }
2927
2928 static void
2929 mlxsw_sp_fib_entry_ralue_pack(char *ralue_pl,
2930                               const struct mlxsw_sp_fib_entry *fib_entry,
2931                               enum mlxsw_reg_ralue_op op)
2932 {
2933         struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
2934         enum mlxsw_reg_ralxx_protocol proto;
2935         u32 *p_dip;
2936
2937         proto = (enum mlxsw_reg_ralxx_protocol) fib->proto;
2938
2939         switch (fib->proto) {
2940         case MLXSW_SP_L3_PROTO_IPV4:
2941                 p_dip = (u32 *) fib_entry->fib_node->key.addr;
2942                 mlxsw_reg_ralue_pack4(ralue_pl, proto, op, fib->vr->id,
2943                                       fib_entry->fib_node->key.prefix_len,
2944                                       *p_dip);
2945                 break;
2946         case MLXSW_SP_L3_PROTO_IPV6:
2947                 mlxsw_reg_ralue_pack6(ralue_pl, proto, op, fib->vr->id,
2948                                       fib_entry->fib_node->key.prefix_len,
2949                                       fib_entry->fib_node->key.addr);
2950                 break;
2951         }
2952 }
2953
2954 static int mlxsw_sp_fib_entry_op_remote(struct mlxsw_sp *mlxsw_sp,
2955                                         struct mlxsw_sp_fib_entry *fib_entry,
2956                                         enum mlxsw_reg_ralue_op op)
2957 {
2958         char ralue_pl[MLXSW_REG_RALUE_LEN];
2959         enum mlxsw_reg_ralue_trap_action trap_action;
2960         u16 trap_id = 0;
2961         u32 adjacency_index = 0;
2962         u16 ecmp_size = 0;
2963
2964         /* In case the nexthop group adjacency index is valid, use it
2965          * with provided ECMP size. Otherwise, setup trap and pass
2966          * traffic to kernel.
2967          */
2968         if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
2969                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
2970                 adjacency_index = fib_entry->nh_group->adj_index;
2971                 ecmp_size = fib_entry->nh_group->ecmp_size;
2972         } else {
2973                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
2974                 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
2975         }
2976
2977         mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
2978         mlxsw_reg_ralue_act_remote_pack(ralue_pl, trap_action, trap_id,
2979                                         adjacency_index, ecmp_size);
2980         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
2981 }
2982
2983 static int mlxsw_sp_fib_entry_op_local(struct mlxsw_sp *mlxsw_sp,
2984                                        struct mlxsw_sp_fib_entry *fib_entry,
2985                                        enum mlxsw_reg_ralue_op op)
2986 {
2987         struct mlxsw_sp_rif *rif = fib_entry->nh_group->nh_rif;
2988         enum mlxsw_reg_ralue_trap_action trap_action;
2989         char ralue_pl[MLXSW_REG_RALUE_LEN];
2990         u16 trap_id = 0;
2991         u16 rif_index = 0;
2992
2993         if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
2994                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
2995                 rif_index = rif->rif_index;
2996         } else {
2997                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
2998                 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
2999         }
3000
3001         mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
3002         mlxsw_reg_ralue_act_local_pack(ralue_pl, trap_action, trap_id,
3003                                        rif_index);
3004         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
3005 }
3006
3007 static int mlxsw_sp_fib_entry_op_trap(struct mlxsw_sp *mlxsw_sp,
3008                                       struct mlxsw_sp_fib_entry *fib_entry,
3009                                       enum mlxsw_reg_ralue_op op)
3010 {
3011         char ralue_pl[MLXSW_REG_RALUE_LEN];
3012
3013         mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
3014         mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
3015         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
3016 }
3017
3018 static int __mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
3019                                    struct mlxsw_sp_fib_entry *fib_entry,
3020                                    enum mlxsw_reg_ralue_op op)
3021 {
3022         switch (fib_entry->type) {
3023         case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
3024                 return mlxsw_sp_fib_entry_op_remote(mlxsw_sp, fib_entry, op);
3025         case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
3026                 return mlxsw_sp_fib_entry_op_local(mlxsw_sp, fib_entry, op);
3027         case MLXSW_SP_FIB_ENTRY_TYPE_TRAP:
3028                 return mlxsw_sp_fib_entry_op_trap(mlxsw_sp, fib_entry, op);
3029         }
3030         return -EINVAL;
3031 }
3032
3033 static int mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
3034                                  struct mlxsw_sp_fib_entry *fib_entry,
3035                                  enum mlxsw_reg_ralue_op op)
3036 {
3037         int err = __mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry, op);
3038
3039         mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, err);
3040
3041         return err;
3042 }
3043
3044 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
3045                                      struct mlxsw_sp_fib_entry *fib_entry)
3046 {
3047         return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
3048                                      MLXSW_REG_RALUE_OP_WRITE_WRITE);
3049 }
3050
3051 static int mlxsw_sp_fib_entry_del(struct mlxsw_sp *mlxsw_sp,
3052                                   struct mlxsw_sp_fib_entry *fib_entry)
3053 {
3054         return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
3055                                      MLXSW_REG_RALUE_OP_WRITE_DELETE);
3056 }
3057
3058 static int
3059 mlxsw_sp_fib4_entry_type_set(struct mlxsw_sp *mlxsw_sp,
3060                              const struct fib_entry_notifier_info *fen_info,
3061                              struct mlxsw_sp_fib_entry *fib_entry)
3062 {
3063         struct fib_info *fi = fen_info->fi;
3064
3065         switch (fen_info->type) {
3066         case RTN_BROADCAST: /* fall through */
3067         case RTN_LOCAL:
3068                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
3069                 return 0;
3070         case RTN_UNREACHABLE: /* fall through */
3071         case RTN_BLACKHOLE: /* fall through */
3072         case RTN_PROHIBIT:
3073                 /* Packets hitting these routes need to be trapped, but
3074                  * can do so with a lower priority than packets directed
3075                  * at the host, so use action type local instead of trap.
3076                  */
3077                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
3078                 return 0;
3079         case RTN_UNICAST:
3080                 if (mlxsw_sp_fi_is_gateway(mlxsw_sp, fi))
3081                         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
3082                 else
3083                         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
3084                 return 0;
3085         default:
3086                 return -EINVAL;
3087         }
3088 }
3089
3090 static struct mlxsw_sp_fib4_entry *
3091 mlxsw_sp_fib4_entry_create(struct mlxsw_sp *mlxsw_sp,
3092                            struct mlxsw_sp_fib_node *fib_node,
3093                            const struct fib_entry_notifier_info *fen_info)
3094 {
3095         struct mlxsw_sp_fib4_entry *fib4_entry;
3096         struct mlxsw_sp_fib_entry *fib_entry;
3097         int err;
3098
3099         fib4_entry = kzalloc(sizeof(*fib4_entry), GFP_KERNEL);
3100         if (!fib4_entry)
3101                 return ERR_PTR(-ENOMEM);
3102         fib_entry = &fib4_entry->common;
3103
3104         err = mlxsw_sp_fib4_entry_type_set(mlxsw_sp, fen_info, fib_entry);
3105         if (err)
3106                 goto err_fib4_entry_type_set;
3107
3108         err = mlxsw_sp_nexthop4_group_get(mlxsw_sp, fib_entry, fen_info->fi);
3109         if (err)
3110                 goto err_nexthop4_group_get;
3111
3112         fib4_entry->prio = fen_info->fi->fib_priority;
3113         fib4_entry->tb_id = fen_info->tb_id;
3114         fib4_entry->type = fen_info->type;
3115         fib4_entry->tos = fen_info->tos;
3116
3117         fib_entry->fib_node = fib_node;
3118
3119         return fib4_entry;
3120
3121 err_nexthop4_group_get:
3122 err_fib4_entry_type_set:
3123         kfree(fib4_entry);
3124         return ERR_PTR(err);
3125 }
3126
3127 static void mlxsw_sp_fib4_entry_destroy(struct mlxsw_sp *mlxsw_sp,
3128                                         struct mlxsw_sp_fib4_entry *fib4_entry)
3129 {
3130         mlxsw_sp_nexthop4_group_put(mlxsw_sp, &fib4_entry->common);
3131         kfree(fib4_entry);
3132 }
3133
3134 static struct mlxsw_sp_fib_node *
3135 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr,
3136                          size_t addr_len, unsigned char prefix_len);
3137
3138 static struct mlxsw_sp_fib4_entry *
3139 mlxsw_sp_fib4_entry_lookup(struct mlxsw_sp *mlxsw_sp,
3140                            const struct fib_entry_notifier_info *fen_info)
3141 {
3142         struct mlxsw_sp_fib4_entry *fib4_entry;
3143         struct mlxsw_sp_fib_node *fib_node;
3144         struct mlxsw_sp_fib *fib;
3145         struct mlxsw_sp_vr *vr;
3146
3147         vr = mlxsw_sp_vr_find(mlxsw_sp, fen_info->tb_id);
3148         if (!vr)
3149                 return NULL;
3150         fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV4);
3151
3152         fib_node = mlxsw_sp_fib_node_lookup(fib, &fen_info->dst,
3153                                             sizeof(fen_info->dst),
3154                                             fen_info->dst_len);
3155         if (!fib_node)
3156                 return NULL;
3157
3158         list_for_each_entry(fib4_entry, &fib_node->entry_list, common.list) {
3159                 if (fib4_entry->tb_id == fen_info->tb_id &&
3160                     fib4_entry->tos == fen_info->tos &&
3161                     fib4_entry->type == fen_info->type &&
3162                     mlxsw_sp_nexthop4_group_fi(fib4_entry->common.nh_group) ==
3163                     fen_info->fi) {
3164                         return fib4_entry;
3165                 }
3166         }
3167
3168         return NULL;
3169 }
3170
3171 static const struct rhashtable_params mlxsw_sp_fib_ht_params = {
3172         .key_offset = offsetof(struct mlxsw_sp_fib_node, key),
3173         .head_offset = offsetof(struct mlxsw_sp_fib_node, ht_node),
3174         .key_len = sizeof(struct mlxsw_sp_fib_key),
3175         .automatic_shrinking = true,
3176 };
3177
3178 static int mlxsw_sp_fib_node_insert(struct mlxsw_sp_fib *fib,
3179                                     struct mlxsw_sp_fib_node *fib_node)
3180 {
3181         return rhashtable_insert_fast(&fib->ht, &fib_node->ht_node,
3182                                       mlxsw_sp_fib_ht_params);
3183 }
3184
3185 static void mlxsw_sp_fib_node_remove(struct mlxsw_sp_fib *fib,
3186                                      struct mlxsw_sp_fib_node *fib_node)
3187 {
3188         rhashtable_remove_fast(&fib->ht, &fib_node->ht_node,
3189                                mlxsw_sp_fib_ht_params);
3190 }
3191
3192 static struct mlxsw_sp_fib_node *
3193 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr,
3194                          size_t addr_len, unsigned char prefix_len)
3195 {
3196         struct mlxsw_sp_fib_key key;
3197
3198         memset(&key, 0, sizeof(key));
3199         memcpy(key.addr, addr, addr_len);
3200         key.prefix_len = prefix_len;
3201         return rhashtable_lookup_fast(&fib->ht, &key, mlxsw_sp_fib_ht_params);
3202 }
3203
3204 static struct mlxsw_sp_fib_node *
3205 mlxsw_sp_fib_node_create(struct mlxsw_sp_fib *fib, const void *addr,
3206                          size_t addr_len, unsigned char prefix_len)
3207 {
3208         struct mlxsw_sp_fib_node *fib_node;
3209
3210         fib_node = kzalloc(sizeof(*fib_node), GFP_KERNEL);
3211         if (!fib_node)
3212                 return NULL;
3213
3214         INIT_LIST_HEAD(&fib_node->entry_list);
3215         list_add(&fib_node->list, &fib->node_list);
3216         memcpy(fib_node->key.addr, addr, addr_len);
3217         fib_node->key.prefix_len = prefix_len;
3218
3219         return fib_node;
3220 }
3221
3222 static void mlxsw_sp_fib_node_destroy(struct mlxsw_sp_fib_node *fib_node)
3223 {
3224         list_del(&fib_node->list);
3225         WARN_ON(!list_empty(&fib_node->entry_list));
3226         kfree(fib_node);
3227 }
3228
3229 static bool
3230 mlxsw_sp_fib_node_entry_is_first(const struct mlxsw_sp_fib_node *fib_node,
3231                                  const struct mlxsw_sp_fib_entry *fib_entry)
3232 {
3233         return list_first_entry(&fib_node->entry_list,
3234                                 struct mlxsw_sp_fib_entry, list) == fib_entry;
3235 }
3236
3237 static int mlxsw_sp_fib_lpm_tree_link(struct mlxsw_sp *mlxsw_sp,
3238                                       struct mlxsw_sp_fib *fib,
3239                                       struct mlxsw_sp_fib_node *fib_node)
3240 {
3241         struct mlxsw_sp_prefix_usage req_prefix_usage = {{ 0 } };
3242         struct mlxsw_sp_lpm_tree *lpm_tree;
3243         int err;
3244
3245         /* Since the tree is shared between all virtual routers we must
3246          * make sure it contains all the required prefix lengths. This
3247          * can be computed by either adding the new prefix length to the
3248          * existing prefix usage of a bound tree, or by aggregating the
3249          * prefix lengths across all virtual routers and adding the new
3250          * one as well.
3251          */
3252         if (fib->lpm_tree)
3253                 mlxsw_sp_prefix_usage_cpy(&req_prefix_usage,
3254                                           &fib->lpm_tree->prefix_usage);
3255         else
3256                 mlxsw_sp_vrs_prefixes(mlxsw_sp, fib->proto, &req_prefix_usage);
3257         mlxsw_sp_prefix_usage_set(&req_prefix_usage, fib_node->key.prefix_len);
3258
3259         lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
3260                                          fib->proto);
3261         if (IS_ERR(lpm_tree))
3262                 return PTR_ERR(lpm_tree);
3263
3264         if (fib->lpm_tree && fib->lpm_tree->id == lpm_tree->id)
3265                 return 0;
3266
3267         err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree);
3268         if (err)
3269                 return err;
3270
3271         return 0;
3272 }
3273
3274 static void mlxsw_sp_fib_lpm_tree_unlink(struct mlxsw_sp *mlxsw_sp,
3275                                          struct mlxsw_sp_fib *fib)
3276 {
3277         struct mlxsw_sp_prefix_usage req_prefix_usage = {{ 0 } };
3278         struct mlxsw_sp_lpm_tree *lpm_tree;
3279
3280         /* Aggregate prefix lengths across all virtual routers to make
3281          * sure we only have used prefix lengths in the LPM tree.
3282          */
3283         mlxsw_sp_vrs_prefixes(mlxsw_sp, fib->proto, &req_prefix_usage);
3284         lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
3285                                          fib->proto);
3286         if (IS_ERR(lpm_tree))
3287                 goto err_tree_get;
3288         mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree);
3289
3290 err_tree_get:
3291         if (!mlxsw_sp_prefix_usage_none(&fib->prefix_usage))
3292                 return;
3293         mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, fib);
3294         mlxsw_sp_lpm_tree_put(mlxsw_sp, fib->lpm_tree);
3295         fib->lpm_tree = NULL;
3296 }
3297
3298 static void mlxsw_sp_fib_node_prefix_inc(struct mlxsw_sp_fib_node *fib_node)
3299 {
3300         unsigned char prefix_len = fib_node->key.prefix_len;
3301         struct mlxsw_sp_fib *fib = fib_node->fib;
3302
3303         if (fib->prefix_ref_count[prefix_len]++ == 0)
3304                 mlxsw_sp_prefix_usage_set(&fib->prefix_usage, prefix_len);
3305 }
3306
3307 static void mlxsw_sp_fib_node_prefix_dec(struct mlxsw_sp_fib_node *fib_node)
3308 {
3309         unsigned char prefix_len = fib_node->key.prefix_len;
3310         struct mlxsw_sp_fib *fib = fib_node->fib;
3311
3312         if (--fib->prefix_ref_count[prefix_len] == 0)
3313                 mlxsw_sp_prefix_usage_clear(&fib->prefix_usage, prefix_len);
3314 }
3315
3316 static int mlxsw_sp_fib_node_init(struct mlxsw_sp *mlxsw_sp,
3317                                   struct mlxsw_sp_fib_node *fib_node,
3318                                   struct mlxsw_sp_fib *fib)
3319 {
3320         int err;
3321
3322         err = mlxsw_sp_fib_node_insert(fib, fib_node);
3323         if (err)
3324                 return err;
3325         fib_node->fib = fib;
3326
3327         err = mlxsw_sp_fib_lpm_tree_link(mlxsw_sp, fib, fib_node);
3328         if (err)
3329                 goto err_fib_lpm_tree_link;
3330
3331         mlxsw_sp_fib_node_prefix_inc(fib_node);
3332
3333         return 0;
3334
3335 err_fib_lpm_tree_link:
3336         fib_node->fib = NULL;
3337         mlxsw_sp_fib_node_remove(fib, fib_node);
3338         return err;
3339 }
3340
3341 static void mlxsw_sp_fib_node_fini(struct mlxsw_sp *mlxsw_sp,
3342                                    struct mlxsw_sp_fib_node *fib_node)
3343 {
3344         struct mlxsw_sp_fib *fib = fib_node->fib;
3345
3346         mlxsw_sp_fib_node_prefix_dec(fib_node);
3347         mlxsw_sp_fib_lpm_tree_unlink(mlxsw_sp, fib);
3348         fib_node->fib = NULL;
3349         mlxsw_sp_fib_node_remove(fib, fib_node);
3350 }
3351
3352 static struct mlxsw_sp_fib_node *
3353 mlxsw_sp_fib_node_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, const void *addr,
3354                       size_t addr_len, unsigned char prefix_len,
3355                       enum mlxsw_sp_l3proto proto)
3356 {
3357         struct mlxsw_sp_fib_node *fib_node;
3358         struct mlxsw_sp_fib *fib;
3359         struct mlxsw_sp_vr *vr;
3360         int err;
3361
3362         vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id);
3363         if (IS_ERR(vr))
3364                 return ERR_CAST(vr);
3365         fib = mlxsw_sp_vr_fib(vr, proto);
3366
3367         fib_node = mlxsw_sp_fib_node_lookup(fib, addr, addr_len, prefix_len);
3368         if (fib_node)
3369                 return fib_node;
3370
3371         fib_node = mlxsw_sp_fib_node_create(fib, addr, addr_len, prefix_len);
3372         if (!fib_node) {
3373                 err = -ENOMEM;
3374                 goto err_fib_node_create;
3375         }
3376
3377         err = mlxsw_sp_fib_node_init(mlxsw_sp, fib_node, fib);
3378         if (err)
3379                 goto err_fib_node_init;
3380
3381         return fib_node;
3382
3383 err_fib_node_init:
3384         mlxsw_sp_fib_node_destroy(fib_node);
3385 err_fib_node_create:
3386         mlxsw_sp_vr_put(vr);
3387         return ERR_PTR(err);
3388 }
3389
3390 static void mlxsw_sp_fib_node_put(struct mlxsw_sp *mlxsw_sp,
3391                                   struct mlxsw_sp_fib_node *fib_node)
3392 {
3393         struct mlxsw_sp_vr *vr = fib_node->fib->vr;
3394
3395         if (!list_empty(&fib_node->entry_list))
3396                 return;
3397         mlxsw_sp_fib_node_fini(mlxsw_sp, fib_node);
3398         mlxsw_sp_fib_node_destroy(fib_node);
3399         mlxsw_sp_vr_put(vr);
3400 }
3401
3402 static struct mlxsw_sp_fib4_entry *
3403 mlxsw_sp_fib4_node_entry_find(const struct mlxsw_sp_fib_node *fib_node,
3404                               const struct mlxsw_sp_fib4_entry *new4_entry)
3405 {
3406         struct mlxsw_sp_fib4_entry *fib4_entry;
3407
3408         list_for_each_entry(fib4_entry, &fib_node->entry_list, common.list) {
3409                 if (fib4_entry->tb_id > new4_entry->tb_id)
3410                         continue;
3411                 if (fib4_entry->tb_id != new4_entry->tb_id)
3412                         break;
3413                 if (fib4_entry->tos > new4_entry->tos)
3414                         continue;
3415                 if (fib4_entry->prio >= new4_entry->prio ||
3416                     fib4_entry->tos < new4_entry->tos)
3417                         return fib4_entry;
3418         }
3419
3420         return NULL;
3421 }
3422
3423 static int
3424 mlxsw_sp_fib4_node_list_append(struct mlxsw_sp_fib4_entry *fib4_entry,
3425                                struct mlxsw_sp_fib4_entry *new4_entry)
3426 {
3427         struct mlxsw_sp_fib_node *fib_node;
3428
3429         if (WARN_ON(!fib4_entry))
3430                 return -EINVAL;
3431
3432         fib_node = fib4_entry->common.fib_node;
3433         list_for_each_entry_from(fib4_entry, &fib_node->entry_list,
3434                                  common.list) {
3435                 if (fib4_entry->tb_id != new4_entry->tb_id ||
3436                     fib4_entry->tos != new4_entry->tos ||
3437                     fib4_entry->prio != new4_entry->prio)
3438                         break;
3439         }
3440
3441         list_add_tail(&new4_entry->common.list, &fib4_entry->common.list);
3442         return 0;
3443 }
3444
3445 static int
3446 mlxsw_sp_fib4_node_list_insert(struct mlxsw_sp_fib4_entry *new4_entry,
3447                                bool replace, bool append)
3448 {
3449         struct mlxsw_sp_fib_node *fib_node = new4_entry->common.fib_node;
3450         struct mlxsw_sp_fib4_entry *fib4_entry;
3451
3452         fib4_entry = mlxsw_sp_fib4_node_entry_find(fib_node, new4_entry);
3453
3454         if (append)
3455                 return mlxsw_sp_fib4_node_list_append(fib4_entry, new4_entry);
3456         if (replace && WARN_ON(!fib4_entry))
3457                 return -EINVAL;
3458
3459         /* Insert new entry before replaced one, so that we can later
3460          * remove the second.
3461          */
3462         if (fib4_entry) {
3463                 list_add_tail(&new4_entry->common.list,
3464                               &fib4_entry->common.list);
3465         } else {
3466                 struct mlxsw_sp_fib4_entry *last;
3467
3468                 list_for_each_entry(last, &fib_node->entry_list, common.list) {
3469                         if (new4_entry->tb_id > last->tb_id)
3470                                 break;
3471                         fib4_entry = last;
3472                 }
3473
3474                 if (fib4_entry)
3475                         list_add(&new4_entry->common.list,
3476                                  &fib4_entry->common.list);
3477                 else
3478                         list_add(&new4_entry->common.list,
3479                                  &fib_node->entry_list);
3480         }
3481
3482         return 0;
3483 }
3484
3485 static void
3486 mlxsw_sp_fib4_node_list_remove(struct mlxsw_sp_fib4_entry *fib4_entry)
3487 {
3488         list_del(&fib4_entry->common.list);
3489 }
3490
3491 static int mlxsw_sp_fib_node_entry_add(struct mlxsw_sp *mlxsw_sp,
3492                                        struct mlxsw_sp_fib_entry *fib_entry)
3493 {
3494         struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
3495
3496         if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry))
3497                 return 0;
3498
3499         /* To prevent packet loss, overwrite the previously offloaded
3500          * entry.
3501          */
3502         if (!list_is_singular(&fib_node->entry_list)) {
3503                 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_DELETE;
3504                 struct mlxsw_sp_fib_entry *n = list_next_entry(fib_entry, list);
3505
3506                 mlxsw_sp_fib_entry_offload_refresh(n, op, 0);
3507         }
3508
3509         return mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
3510 }
3511
3512 static void mlxsw_sp_fib_node_entry_del(struct mlxsw_sp *mlxsw_sp,
3513                                         struct mlxsw_sp_fib_entry *fib_entry)
3514 {
3515         struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
3516
3517         if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry))
3518                 return;
3519
3520         /* Promote the next entry by overwriting the deleted entry */
3521         if (!list_is_singular(&fib_node->entry_list)) {
3522                 struct mlxsw_sp_fib_entry *n = list_next_entry(fib_entry, list);
3523                 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_DELETE;
3524
3525                 mlxsw_sp_fib_entry_update(mlxsw_sp, n);
3526                 mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, 0);
3527                 return;
3528         }
3529
3530         mlxsw_sp_fib_entry_del(mlxsw_sp, fib_entry);
3531 }
3532
3533 static int mlxsw_sp_fib4_node_entry_link(struct mlxsw_sp *mlxsw_sp,
3534                                          struct mlxsw_sp_fib4_entry *fib4_entry,
3535                                          bool replace, bool append)
3536 {
3537         int err;
3538
3539         err = mlxsw_sp_fib4_node_list_insert(fib4_entry, replace, append);
3540         if (err)
3541                 return err;
3542
3543         err = mlxsw_sp_fib_node_entry_add(mlxsw_sp, &fib4_entry->common);
3544         if (err)
3545                 goto err_fib_node_entry_add;
3546
3547         return 0;
3548
3549 err_fib_node_entry_add:
3550         mlxsw_sp_fib4_node_list_remove(fib4_entry);
3551         return err;
3552 }
3553
3554 static void
3555 mlxsw_sp_fib4_node_entry_unlink(struct mlxsw_sp *mlxsw_sp,
3556                                 struct mlxsw_sp_fib4_entry *fib4_entry)
3557 {
3558         mlxsw_sp_fib_node_entry_del(mlxsw_sp, &fib4_entry->common);
3559         mlxsw_sp_fib4_node_list_remove(fib4_entry);
3560 }
3561
3562 static void mlxsw_sp_fib4_entry_replace(struct mlxsw_sp *mlxsw_sp,
3563                                         struct mlxsw_sp_fib4_entry *fib4_entry,
3564                                         bool replace)
3565 {
3566         struct mlxsw_sp_fib_node *fib_node = fib4_entry->common.fib_node;
3567         struct mlxsw_sp_fib4_entry *replaced;
3568
3569         if (!replace)
3570                 return;
3571
3572         /* We inserted the new entry before replaced one */
3573         replaced = list_next_entry(fib4_entry, common.list);
3574
3575         mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, replaced);
3576         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, replaced);
3577         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
3578 }
3579
3580 static int
3581 mlxsw_sp_router_fib4_add(struct mlxsw_sp *mlxsw_sp,
3582                          const struct fib_entry_notifier_info *fen_info,
3583                          bool replace, bool append)
3584 {
3585         struct mlxsw_sp_fib4_entry *fib4_entry;
3586         struct mlxsw_sp_fib_node *fib_node;
3587         int err;
3588
3589         if (mlxsw_sp->router->aborted)
3590                 return 0;
3591
3592         fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, fen_info->tb_id,
3593                                          &fen_info->dst, sizeof(fen_info->dst),
3594                                          fen_info->dst_len,
3595                                          MLXSW_SP_L3_PROTO_IPV4);
3596         if (IS_ERR(fib_node)) {
3597                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to get FIB node\n");
3598                 return PTR_ERR(fib_node);
3599         }
3600
3601         fib4_entry = mlxsw_sp_fib4_entry_create(mlxsw_sp, fib_node, fen_info);
3602         if (IS_ERR(fib4_entry)) {
3603                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to create FIB entry\n");
3604                 err = PTR_ERR(fib4_entry);
3605                 goto err_fib4_entry_create;
3606         }
3607
3608         err = mlxsw_sp_fib4_node_entry_link(mlxsw_sp, fib4_entry, replace,
3609                                             append);
3610         if (err) {
3611                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to link FIB entry to node\n");
3612                 goto err_fib4_node_entry_link;
3613         }
3614
3615         mlxsw_sp_fib4_entry_replace(mlxsw_sp, fib4_entry, replace);
3616
3617         return 0;
3618
3619 err_fib4_node_entry_link:
3620         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
3621 err_fib4_entry_create:
3622         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
3623         return err;
3624 }
3625
3626 static void mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp,
3627                                      struct fib_entry_notifier_info *fen_info)
3628 {
3629         struct mlxsw_sp_fib4_entry *fib4_entry;
3630         struct mlxsw_sp_fib_node *fib_node;
3631
3632         if (mlxsw_sp->router->aborted)
3633                 return;
3634
3635         fib4_entry = mlxsw_sp_fib4_entry_lookup(mlxsw_sp, fen_info);
3636         if (WARN_ON(!fib4_entry))
3637                 return;
3638         fib_node = fib4_entry->common.fib_node;
3639
3640         mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib4_entry);
3641         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
3642         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
3643 }
3644
3645 static bool mlxsw_sp_fib6_rt_should_ignore(const struct rt6_info *rt)
3646 {
3647         /* Packets with link-local destination IP arriving to the router
3648          * are trapped to the CPU, so no need to program specific routes
3649          * for them.
3650          */
3651         if (ipv6_addr_type(&rt->rt6i_dst.addr) & IPV6_ADDR_LINKLOCAL)
3652                 return true;
3653
3654         /* Multicast routes aren't supported, so ignore them. Neighbour
3655          * Discovery packets are specifically trapped.
3656          */
3657         if (ipv6_addr_type(&rt->rt6i_dst.addr) & IPV6_ADDR_MULTICAST)
3658                 return true;
3659
3660         /* Cloned routes are irrelevant in the forwarding path. */
3661         if (rt->rt6i_flags & RTF_CACHE)
3662                 return true;
3663
3664         return false;
3665 }
3666
3667 static struct mlxsw_sp_rt6 *mlxsw_sp_rt6_create(struct rt6_info *rt)
3668 {
3669         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3670
3671         mlxsw_sp_rt6 = kzalloc(sizeof(*mlxsw_sp_rt6), GFP_KERNEL);
3672         if (!mlxsw_sp_rt6)
3673                 return ERR_PTR(-ENOMEM);
3674
3675         /* In case of route replace, replaced route is deleted with
3676          * no notification. Take reference to prevent accessing freed
3677          * memory.
3678          */
3679         mlxsw_sp_rt6->rt = rt;
3680         rt6_hold(rt);
3681
3682         return mlxsw_sp_rt6;
3683 }
3684
3685 #if IS_ENABLED(CONFIG_IPV6)
3686 static void mlxsw_sp_rt6_release(struct rt6_info *rt)
3687 {
3688         rt6_release(rt);
3689 }
3690 #else
3691 static void mlxsw_sp_rt6_release(struct rt6_info *rt)
3692 {
3693 }
3694 #endif
3695
3696 static void mlxsw_sp_rt6_destroy(struct mlxsw_sp_rt6 *mlxsw_sp_rt6)
3697 {
3698         mlxsw_sp_rt6_release(mlxsw_sp_rt6->rt);
3699         kfree(mlxsw_sp_rt6);
3700 }
3701
3702 static bool mlxsw_sp_fib6_rt_can_mp(const struct rt6_info *rt)
3703 {
3704         /* RTF_CACHE routes are ignored */
3705         return (rt->rt6i_flags & (RTF_GATEWAY | RTF_ADDRCONF)) == RTF_GATEWAY;
3706 }
3707
3708 static struct rt6_info *
3709 mlxsw_sp_fib6_entry_rt(const struct mlxsw_sp_fib6_entry *fib6_entry)
3710 {
3711         return list_first_entry(&fib6_entry->rt6_list, struct mlxsw_sp_rt6,
3712                                 list)->rt;
3713 }
3714
3715 static struct mlxsw_sp_fib6_entry *
3716 mlxsw_sp_fib6_node_mp_entry_find(const struct mlxsw_sp_fib_node *fib_node,
3717                                  const struct rt6_info *nrt, bool replace)
3718 {
3719         struct mlxsw_sp_fib6_entry *fib6_entry;
3720
3721         if (!mlxsw_sp_fib6_rt_can_mp(nrt) || replace)
3722                 return NULL;
3723
3724         list_for_each_entry(fib6_entry, &fib_node->entry_list, common.list) {
3725                 struct rt6_info *rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
3726
3727                 /* RT6_TABLE_LOCAL and RT6_TABLE_MAIN share the same
3728                  * virtual router.
3729                  */
3730                 if (rt->rt6i_table->tb6_id > nrt->rt6i_table->tb6_id)
3731                         continue;
3732                 if (rt->rt6i_table->tb6_id != nrt->rt6i_table->tb6_id)
3733                         break;
3734                 if (rt->rt6i_metric < nrt->rt6i_metric)
3735                         continue;
3736                 if (rt->rt6i_metric == nrt->rt6i_metric &&
3737                     mlxsw_sp_fib6_rt_can_mp(rt))
3738                         return fib6_entry;
3739                 if (rt->rt6i_metric > nrt->rt6i_metric)
3740                         break;
3741         }
3742
3743         return NULL;
3744 }
3745
3746 static struct mlxsw_sp_rt6 *
3747 mlxsw_sp_fib6_entry_rt_find(const struct mlxsw_sp_fib6_entry *fib6_entry,
3748                             const struct rt6_info *rt)
3749 {
3750         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3751
3752         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
3753                 if (mlxsw_sp_rt6->rt == rt)
3754                         return mlxsw_sp_rt6;
3755         }
3756
3757         return NULL;
3758 }
3759
3760 static bool mlxsw_sp_nexthop6_ipip_type(const struct mlxsw_sp *mlxsw_sp,
3761                                         const struct rt6_info *rt,
3762                                         enum mlxsw_sp_ipip_type *ret)
3763 {
3764         return rt->dst.dev &&
3765                mlxsw_sp_netdev_ipip_type(mlxsw_sp, rt->dst.dev, ret);
3766 }
3767
3768 static int mlxsw_sp_nexthop6_type_init(struct mlxsw_sp *mlxsw_sp,
3769                                        struct mlxsw_sp_nexthop_group *nh_grp,
3770                                        struct mlxsw_sp_nexthop *nh,
3771                                        const struct rt6_info *rt)
3772 {
3773         struct mlxsw_sp_router *router = mlxsw_sp->router;
3774         struct net_device *dev = rt->dst.dev;
3775         enum mlxsw_sp_ipip_type ipipt;
3776         struct mlxsw_sp_rif *rif;
3777         int err;
3778
3779         if (mlxsw_sp_nexthop6_ipip_type(mlxsw_sp, rt, &ipipt) &&
3780             router->ipip_ops_arr[ipipt]->can_offload(mlxsw_sp, dev,
3781                                                      MLXSW_SP_L3_PROTO_IPV6)) {
3782                 nh->type = MLXSW_SP_NEXTHOP_TYPE_IPIP;
3783                 return mlxsw_sp_nexthop_ipip_init(mlxsw_sp, ipipt, nh, dev);
3784         }
3785
3786         nh->type = MLXSW_SP_NEXTHOP_TYPE_ETH;
3787         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
3788         if (!rif)
3789                 return 0;
3790         mlxsw_sp_nexthop_rif_init(nh, rif);
3791
3792         err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
3793         if (err)
3794                 goto err_nexthop_neigh_init;
3795
3796         return 0;
3797
3798 err_nexthop_neigh_init:
3799         mlxsw_sp_nexthop_rif_fini(nh);
3800         return err;
3801 }
3802
3803 static void mlxsw_sp_nexthop6_type_fini(struct mlxsw_sp *mlxsw_sp,
3804                                         struct mlxsw_sp_nexthop *nh)
3805 {
3806         mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
3807 }
3808
3809 static int mlxsw_sp_nexthop6_init(struct mlxsw_sp *mlxsw_sp,
3810                                   struct mlxsw_sp_nexthop_group *nh_grp,
3811                                   struct mlxsw_sp_nexthop *nh,
3812                                   const struct rt6_info *rt)
3813 {
3814         struct net_device *dev = rt->dst.dev;
3815
3816         nh->nh_grp = nh_grp;
3817         memcpy(&nh->gw_addr, &rt->rt6i_gateway, sizeof(nh->gw_addr));
3818
3819         if (!dev)
3820                 return 0;
3821         nh->ifindex = dev->ifindex;
3822
3823         return mlxsw_sp_nexthop6_type_init(mlxsw_sp, nh_grp, nh, rt);
3824 }
3825
3826 static void mlxsw_sp_nexthop6_fini(struct mlxsw_sp *mlxsw_sp,
3827                                    struct mlxsw_sp_nexthop *nh)
3828 {
3829         mlxsw_sp_nexthop6_type_fini(mlxsw_sp, nh);
3830 }
3831
3832 static bool mlxsw_sp_rt6_is_gateway(const struct mlxsw_sp *mlxsw_sp,
3833                                     const struct rt6_info *rt)
3834 {
3835         return rt->rt6i_flags & RTF_GATEWAY ||
3836                mlxsw_sp_nexthop6_ipip_type(mlxsw_sp, rt, NULL);
3837 }
3838
3839 static struct mlxsw_sp_nexthop_group *
3840 mlxsw_sp_nexthop6_group_create(struct mlxsw_sp *mlxsw_sp,
3841                                struct mlxsw_sp_fib6_entry *fib6_entry)
3842 {
3843         struct mlxsw_sp_nexthop_group *nh_grp;
3844         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3845         struct mlxsw_sp_nexthop *nh;
3846         size_t alloc_size;
3847         int i = 0;
3848         int err;
3849
3850         alloc_size = sizeof(*nh_grp) +
3851                      fib6_entry->nrt6 * sizeof(struct mlxsw_sp_nexthop);
3852         nh_grp = kzalloc(alloc_size, GFP_KERNEL);
3853         if (!nh_grp)
3854                 return ERR_PTR(-ENOMEM);
3855         INIT_LIST_HEAD(&nh_grp->fib_list);
3856 #if IS_ENABLED(CONFIG_IPV6)
3857         nh_grp->neigh_tbl = &nd_tbl;
3858 #endif
3859         mlxsw_sp_rt6 = list_first_entry(&fib6_entry->rt6_list,
3860                                         struct mlxsw_sp_rt6, list);
3861         nh_grp->gateway = mlxsw_sp_rt6_is_gateway(mlxsw_sp, mlxsw_sp_rt6->rt);
3862         nh_grp->count = fib6_entry->nrt6;
3863         for (i = 0; i < nh_grp->count; i++) {
3864                 struct rt6_info *rt = mlxsw_sp_rt6->rt;
3865
3866                 nh = &nh_grp->nexthops[i];
3867                 err = mlxsw_sp_nexthop6_init(mlxsw_sp, nh_grp, nh, rt);
3868                 if (err)
3869                         goto err_nexthop6_init;
3870                 mlxsw_sp_rt6 = list_next_entry(mlxsw_sp_rt6, list);
3871         }
3872
3873         err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
3874         if (err)
3875                 goto err_nexthop_group_insert;
3876
3877         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
3878         return nh_grp;
3879
3880 err_nexthop_group_insert:
3881 err_nexthop6_init:
3882         for (i--; i >= 0; i--) {
3883                 nh = &nh_grp->nexthops[i];
3884                 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh);
3885         }
3886         kfree(nh_grp);
3887         return ERR_PTR(err);
3888 }
3889
3890 static void
3891 mlxsw_sp_nexthop6_group_destroy(struct mlxsw_sp *mlxsw_sp,
3892                                 struct mlxsw_sp_nexthop_group *nh_grp)
3893 {
3894         struct mlxsw_sp_nexthop *nh;
3895         int i = nh_grp->count;
3896
3897         mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
3898         for (i--; i >= 0; i--) {
3899                 nh = &nh_grp->nexthops[i];
3900                 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh);
3901         }
3902         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
3903         WARN_ON(nh_grp->adj_index_valid);
3904         kfree(nh_grp);
3905 }
3906
3907 static int mlxsw_sp_nexthop6_group_get(struct mlxsw_sp *mlxsw_sp,
3908                                        struct mlxsw_sp_fib6_entry *fib6_entry)
3909 {
3910         struct mlxsw_sp_nexthop_group *nh_grp;
3911
3912         nh_grp = mlxsw_sp_nexthop6_group_lookup(mlxsw_sp, fib6_entry);
3913         if (!nh_grp) {
3914                 nh_grp = mlxsw_sp_nexthop6_group_create(mlxsw_sp, fib6_entry);
3915                 if (IS_ERR(nh_grp))
3916                         return PTR_ERR(nh_grp);
3917         }
3918
3919         list_add_tail(&fib6_entry->common.nexthop_group_node,
3920                       &nh_grp->fib_list);
3921         fib6_entry->common.nh_group = nh_grp;
3922
3923         return 0;
3924 }
3925
3926 static void mlxsw_sp_nexthop6_group_put(struct mlxsw_sp *mlxsw_sp,
3927                                         struct mlxsw_sp_fib_entry *fib_entry)
3928 {
3929         struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
3930
3931         list_del(&fib_entry->nexthop_group_node);
3932         if (!list_empty(&nh_grp->fib_list))
3933                 return;
3934         mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, nh_grp);
3935 }
3936
3937 static int
3938 mlxsw_sp_nexthop6_group_update(struct mlxsw_sp *mlxsw_sp,
3939                                struct mlxsw_sp_fib6_entry *fib6_entry)
3940 {
3941         struct mlxsw_sp_nexthop_group *old_nh_grp = fib6_entry->common.nh_group;
3942         int err;
3943
3944         fib6_entry->common.nh_group = NULL;
3945         list_del(&fib6_entry->common.nexthop_group_node);
3946
3947         err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry);
3948         if (err)
3949                 goto err_nexthop6_group_get;
3950
3951         /* In case this entry is offloaded, then the adjacency index
3952          * currently associated with it in the device's table is that
3953          * of the old group. Start using the new one instead.
3954          */
3955         err = mlxsw_sp_fib_node_entry_add(mlxsw_sp, &fib6_entry->common);
3956         if (err)
3957                 goto err_fib_node_entry_add;
3958
3959         if (list_empty(&old_nh_grp->fib_list))
3960                 mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, old_nh_grp);
3961
3962         return 0;
3963
3964 err_fib_node_entry_add:
3965         mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common);
3966 err_nexthop6_group_get:
3967         list_add_tail(&fib6_entry->common.nexthop_group_node,
3968                       &old_nh_grp->fib_list);
3969         fib6_entry->common.nh_group = old_nh_grp;
3970         return err;
3971 }
3972
3973 static int
3974 mlxsw_sp_fib6_entry_nexthop_add(struct mlxsw_sp *mlxsw_sp,
3975                                 struct mlxsw_sp_fib6_entry *fib6_entry,
3976                                 struct rt6_info *rt)
3977 {
3978         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3979         int err;
3980
3981         mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt);
3982         if (IS_ERR(mlxsw_sp_rt6))
3983                 return PTR_ERR(mlxsw_sp_rt6);
3984
3985         list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list);
3986         fib6_entry->nrt6++;
3987
3988         err = mlxsw_sp_nexthop6_group_update(mlxsw_sp, fib6_entry);
3989         if (err)
3990                 goto err_nexthop6_group_update;
3991
3992         return 0;
3993
3994 err_nexthop6_group_update:
3995         fib6_entry->nrt6--;
3996         list_del(&mlxsw_sp_rt6->list);
3997         mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
3998         return err;
3999 }
4000
4001 static void
4002 mlxsw_sp_fib6_entry_nexthop_del(struct mlxsw_sp *mlxsw_sp,
4003                                 struct mlxsw_sp_fib6_entry *fib6_entry,
4004                                 struct rt6_info *rt)
4005 {
4006         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
4007
4008         mlxsw_sp_rt6 = mlxsw_sp_fib6_entry_rt_find(fib6_entry, rt);
4009         if (WARN_ON(!mlxsw_sp_rt6))
4010                 return;
4011
4012         fib6_entry->nrt6--;
4013         list_del(&mlxsw_sp_rt6->list);
4014         mlxsw_sp_nexthop6_group_update(mlxsw_sp, fib6_entry);
4015         mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
4016 }
4017
4018 static void mlxsw_sp_fib6_entry_type_set(struct mlxsw_sp *mlxsw_sp,
4019                                          struct mlxsw_sp_fib_entry *fib_entry,
4020                                          const struct rt6_info *rt)
4021 {
4022         /* Packets hitting RTF_REJECT routes need to be discarded by the
4023          * stack. We can rely on their destination device not having a
4024          * RIF (it's the loopback device) and can thus use action type
4025          * local, which will cause them to be trapped with a lower
4026          * priority than packets that need to be locally received.
4027          */
4028         if (rt->rt6i_flags & (RTF_LOCAL | RTF_ANYCAST))
4029                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
4030         else if (rt->rt6i_flags & RTF_REJECT)
4031                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
4032         else if (mlxsw_sp_rt6_is_gateway(mlxsw_sp, rt))
4033                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
4034         else
4035                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
4036 }
4037
4038 static void
4039 mlxsw_sp_fib6_entry_rt_destroy_all(struct mlxsw_sp_fib6_entry *fib6_entry)
4040 {
4041         struct mlxsw_sp_rt6 *mlxsw_sp_rt6, *tmp;
4042
4043         list_for_each_entry_safe(mlxsw_sp_rt6, tmp, &fib6_entry->rt6_list,
4044                                  list) {
4045                 fib6_entry->nrt6--;
4046                 list_del(&mlxsw_sp_rt6->list);
4047                 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
4048         }
4049 }
4050
4051 static struct mlxsw_sp_fib6_entry *
4052 mlxsw_sp_fib6_entry_create(struct mlxsw_sp *mlxsw_sp,
4053                            struct mlxsw_sp_fib_node *fib_node,
4054                            struct rt6_info *rt)
4055 {
4056         struct mlxsw_sp_fib6_entry *fib6_entry;
4057         struct mlxsw_sp_fib_entry *fib_entry;
4058         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
4059         int err;
4060
4061         fib6_entry = kzalloc(sizeof(*fib6_entry), GFP_KERNEL);
4062         if (!fib6_entry)
4063                 return ERR_PTR(-ENOMEM);
4064         fib_entry = &fib6_entry->common;
4065
4066         mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt);
4067         if (IS_ERR(mlxsw_sp_rt6)) {
4068                 err = PTR_ERR(mlxsw_sp_rt6);
4069                 goto err_rt6_create;
4070         }
4071
4072         mlxsw_sp_fib6_entry_type_set(mlxsw_sp, fib_entry, mlxsw_sp_rt6->rt);
4073
4074         INIT_LIST_HEAD(&fib6_entry->rt6_list);
4075         list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list);
4076         fib6_entry->nrt6 = 1;
4077         err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry);
4078         if (err)
4079                 goto err_nexthop6_group_get;
4080
4081         fib_entry->fib_node = fib_node;
4082
4083         return fib6_entry;
4084
4085 err_nexthop6_group_get:
4086         list_del(&mlxsw_sp_rt6->list);
4087         mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
4088 err_rt6_create:
4089         kfree(fib6_entry);
4090         return ERR_PTR(err);
4091 }
4092
4093 static void mlxsw_sp_fib6_entry_destroy(struct mlxsw_sp *mlxsw_sp,
4094                                         struct mlxsw_sp_fib6_entry *fib6_entry)
4095 {
4096         mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common);
4097         mlxsw_sp_fib6_entry_rt_destroy_all(fib6_entry);
4098         WARN_ON(fib6_entry->nrt6);
4099         kfree(fib6_entry);
4100 }
4101
4102 static struct mlxsw_sp_fib6_entry *
4103 mlxsw_sp_fib6_node_entry_find(const struct mlxsw_sp_fib_node *fib_node,
4104                               const struct rt6_info *nrt, bool replace)
4105 {
4106         struct mlxsw_sp_fib6_entry *fib6_entry, *fallback = NULL;
4107
4108         list_for_each_entry(fib6_entry, &fib_node->entry_list, common.list) {
4109                 struct rt6_info *rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
4110
4111                 if (rt->rt6i_table->tb6_id > nrt->rt6i_table->tb6_id)
4112                         continue;
4113                 if (rt->rt6i_table->tb6_id != nrt->rt6i_table->tb6_id)
4114                         break;
4115                 if (replace && rt->rt6i_metric == nrt->rt6i_metric) {
4116                         if (mlxsw_sp_fib6_rt_can_mp(rt) ==
4117                             mlxsw_sp_fib6_rt_can_mp(nrt))
4118                                 return fib6_entry;
4119                         if (mlxsw_sp_fib6_rt_can_mp(nrt))
4120                                 fallback = fallback ?: fib6_entry;
4121                 }
4122                 if (rt->rt6i_metric > nrt->rt6i_metric)
4123                         return fallback ?: fib6_entry;
4124         }
4125
4126         return fallback;
4127 }
4128
4129 static int
4130 mlxsw_sp_fib6_node_list_insert(struct mlxsw_sp_fib6_entry *new6_entry,
4131                                bool replace)
4132 {
4133         struct mlxsw_sp_fib_node *fib_node = new6_entry->common.fib_node;
4134         struct rt6_info *nrt = mlxsw_sp_fib6_entry_rt(new6_entry);
4135         struct mlxsw_sp_fib6_entry *fib6_entry;
4136
4137         fib6_entry = mlxsw_sp_fib6_node_entry_find(fib_node, nrt, replace);
4138
4139         if (replace && WARN_ON(!fib6_entry))
4140                 return -EINVAL;
4141
4142         if (fib6_entry) {
4143                 list_add_tail(&new6_entry->common.list,
4144                               &fib6_entry->common.list);
4145         } else {
4146                 struct mlxsw_sp_fib6_entry *last;
4147
4148                 list_for_each_entry(last, &fib_node->entry_list, common.list) {
4149                         struct rt6_info *rt = mlxsw_sp_fib6_entry_rt(last);
4150
4151                         if (nrt->rt6i_table->tb6_id > rt->rt6i_table->tb6_id)
4152                                 break;
4153                         fib6_entry = last;
4154                 }
4155
4156                 if (fib6_entry)
4157                         list_add(&new6_entry->common.list,
4158                                  &fib6_entry->common.list);
4159                 else
4160                         list_add(&new6_entry->common.list,
4161                                  &fib_node->entry_list);
4162         }
4163
4164         return 0;
4165 }
4166
4167 static void
4168 mlxsw_sp_fib6_node_list_remove(struct mlxsw_sp_fib6_entry *fib6_entry)
4169 {
4170         list_del(&fib6_entry->common.list);
4171 }
4172
4173 static int mlxsw_sp_fib6_node_entry_link(struct mlxsw_sp *mlxsw_sp,
4174                                          struct mlxsw_sp_fib6_entry *fib6_entry,
4175                                          bool replace)
4176 {
4177         int err;
4178
4179         err = mlxsw_sp_fib6_node_list_insert(fib6_entry, replace);
4180         if (err)
4181                 return err;
4182
4183         err = mlxsw_sp_fib_node_entry_add(mlxsw_sp, &fib6_entry->common);
4184         if (err)
4185                 goto err_fib_node_entry_add;
4186
4187         return 0;
4188
4189 err_fib_node_entry_add:
4190         mlxsw_sp_fib6_node_list_remove(fib6_entry);
4191         return err;
4192 }
4193
4194 static void
4195 mlxsw_sp_fib6_node_entry_unlink(struct mlxsw_sp *mlxsw_sp,
4196                                 struct mlxsw_sp_fib6_entry *fib6_entry)
4197 {
4198         mlxsw_sp_fib_node_entry_del(mlxsw_sp, &fib6_entry->common);
4199         mlxsw_sp_fib6_node_list_remove(fib6_entry);
4200 }
4201
4202 static struct mlxsw_sp_fib6_entry *
4203 mlxsw_sp_fib6_entry_lookup(struct mlxsw_sp *mlxsw_sp,
4204                            const struct rt6_info *rt)
4205 {
4206         struct mlxsw_sp_fib6_entry *fib6_entry;
4207         struct mlxsw_sp_fib_node *fib_node;
4208         struct mlxsw_sp_fib *fib;
4209         struct mlxsw_sp_vr *vr;
4210
4211         vr = mlxsw_sp_vr_find(mlxsw_sp, rt->rt6i_table->tb6_id);
4212         if (!vr)
4213                 return NULL;
4214         fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV6);
4215
4216         fib_node = mlxsw_sp_fib_node_lookup(fib, &rt->rt6i_dst.addr,
4217                                             sizeof(rt->rt6i_dst.addr),
4218                                             rt->rt6i_dst.plen);
4219         if (!fib_node)
4220                 return NULL;
4221
4222         list_for_each_entry(fib6_entry, &fib_node->entry_list, common.list) {
4223                 struct rt6_info *iter_rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
4224
4225                 if (rt->rt6i_table->tb6_id == iter_rt->rt6i_table->tb6_id &&
4226                     rt->rt6i_metric == iter_rt->rt6i_metric &&
4227                     mlxsw_sp_fib6_entry_rt_find(fib6_entry, rt))
4228                         return fib6_entry;
4229         }
4230
4231         return NULL;
4232 }
4233
4234 static void mlxsw_sp_fib6_entry_replace(struct mlxsw_sp *mlxsw_sp,
4235                                         struct mlxsw_sp_fib6_entry *fib6_entry,
4236                                         bool replace)
4237 {
4238         struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node;
4239         struct mlxsw_sp_fib6_entry *replaced;
4240
4241         if (!replace)
4242                 return;
4243
4244         replaced = list_next_entry(fib6_entry, common.list);
4245
4246         mlxsw_sp_fib6_node_entry_unlink(mlxsw_sp, replaced);
4247         mlxsw_sp_fib6_entry_destroy(mlxsw_sp, replaced);
4248         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
4249 }
4250
4251 static int mlxsw_sp_router_fib6_add(struct mlxsw_sp *mlxsw_sp,
4252                                     struct rt6_info *rt, bool replace)
4253 {
4254         struct mlxsw_sp_fib6_entry *fib6_entry;
4255         struct mlxsw_sp_fib_node *fib_node;
4256         int err;
4257
4258         if (mlxsw_sp->router->aborted)
4259                 return 0;
4260
4261         if (rt->rt6i_src.plen)
4262                 return -EINVAL;
4263
4264         if (mlxsw_sp_fib6_rt_should_ignore(rt))
4265                 return 0;
4266
4267         fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, rt->rt6i_table->tb6_id,
4268                                          &rt->rt6i_dst.addr,
4269                                          sizeof(rt->rt6i_dst.addr),
4270                                          rt->rt6i_dst.plen,
4271                                          MLXSW_SP_L3_PROTO_IPV6);
4272         if (IS_ERR(fib_node))
4273                 return PTR_ERR(fib_node);
4274
4275         /* Before creating a new entry, try to append route to an existing
4276          * multipath entry.
4277          */
4278         fib6_entry = mlxsw_sp_fib6_node_mp_entry_find(fib_node, rt, replace);
4279         if (fib6_entry) {
4280                 err = mlxsw_sp_fib6_entry_nexthop_add(mlxsw_sp, fib6_entry, rt);
4281                 if (err)
4282                         goto err_fib6_entry_nexthop_add;
4283                 return 0;
4284         }
4285
4286         fib6_entry = mlxsw_sp_fib6_entry_create(mlxsw_sp, fib_node, rt);
4287         if (IS_ERR(fib6_entry)) {
4288                 err = PTR_ERR(fib6_entry);
4289                 goto err_fib6_entry_create;
4290         }
4291
4292         err = mlxsw_sp_fib6_node_entry_link(mlxsw_sp, fib6_entry, replace);
4293         if (err)
4294                 goto err_fib6_node_entry_link;
4295
4296         mlxsw_sp_fib6_entry_replace(mlxsw_sp, fib6_entry, replace);
4297
4298         return 0;
4299
4300 err_fib6_node_entry_link:
4301         mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
4302 err_fib6_entry_create:
4303 err_fib6_entry_nexthop_add:
4304         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
4305         return err;
4306 }
4307
4308 static void mlxsw_sp_router_fib6_del(struct mlxsw_sp *mlxsw_sp,
4309                                      struct rt6_info *rt)
4310 {
4311         struct mlxsw_sp_fib6_entry *fib6_entry;
4312         struct mlxsw_sp_fib_node *fib_node;
4313
4314         if (mlxsw_sp->router->aborted)
4315                 return;
4316
4317         if (mlxsw_sp_fib6_rt_should_ignore(rt))
4318                 return;
4319
4320         fib6_entry = mlxsw_sp_fib6_entry_lookup(mlxsw_sp, rt);
4321         if (WARN_ON(!fib6_entry))
4322                 return;
4323
4324         /* If route is part of a multipath entry, but not the last one
4325          * removed, then only reduce its nexthop group.
4326          */
4327         if (!list_is_singular(&fib6_entry->rt6_list)) {
4328                 mlxsw_sp_fib6_entry_nexthop_del(mlxsw_sp, fib6_entry, rt);
4329                 return;
4330         }
4331
4332         fib_node = fib6_entry->common.fib_node;
4333
4334         mlxsw_sp_fib6_node_entry_unlink(mlxsw_sp, fib6_entry);
4335         mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
4336         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
4337 }
4338
4339 static int __mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp,
4340                                             enum mlxsw_reg_ralxx_protocol proto,
4341                                             u8 tree_id)
4342 {
4343         char ralta_pl[MLXSW_REG_RALTA_LEN];
4344         char ralst_pl[MLXSW_REG_RALST_LEN];
4345         int i, err;
4346
4347         mlxsw_reg_ralta_pack(ralta_pl, true, proto, tree_id);
4348         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
4349         if (err)
4350                 return err;
4351
4352         mlxsw_reg_ralst_pack(ralst_pl, 0xff, tree_id);
4353         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
4354         if (err)
4355                 return err;
4356
4357         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
4358                 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
4359                 char raltb_pl[MLXSW_REG_RALTB_LEN];
4360                 char ralue_pl[MLXSW_REG_RALUE_LEN];
4361
4362                 mlxsw_reg_raltb_pack(raltb_pl, vr->id, proto, tree_id);
4363                 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb),
4364                                       raltb_pl);
4365                 if (err)
4366                         return err;
4367
4368                 mlxsw_reg_ralue_pack(ralue_pl, proto,
4369                                      MLXSW_REG_RALUE_OP_WRITE_WRITE, vr->id, 0);
4370                 mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
4371                 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue),
4372                                       ralue_pl);
4373                 if (err)
4374                         return err;
4375         }
4376
4377         return 0;
4378 }
4379
4380 static int mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp)
4381 {
4382         enum mlxsw_reg_ralxx_protocol proto = MLXSW_REG_RALXX_PROTOCOL_IPV4;
4383         int err;
4384
4385         err = __mlxsw_sp_router_set_abort_trap(mlxsw_sp, proto,
4386                                                MLXSW_SP_LPM_TREE_MIN);
4387         if (err)
4388                 return err;
4389
4390         proto = MLXSW_REG_RALXX_PROTOCOL_IPV6;
4391         return __mlxsw_sp_router_set_abort_trap(mlxsw_sp, proto,
4392                                                 MLXSW_SP_LPM_TREE_MIN + 1);
4393 }
4394
4395 static void mlxsw_sp_fib4_node_flush(struct mlxsw_sp *mlxsw_sp,
4396                                      struct mlxsw_sp_fib_node *fib_node)
4397 {
4398         struct mlxsw_sp_fib4_entry *fib4_entry, *tmp;
4399
4400         list_for_each_entry_safe(fib4_entry, tmp, &fib_node->entry_list,
4401                                  common.list) {
4402                 bool do_break = &tmp->common.list == &fib_node->entry_list;
4403
4404                 mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib4_entry);
4405                 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
4406                 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
4407                 /* Break when entry list is empty and node was freed.
4408                  * Otherwise, we'll access freed memory in the next
4409                  * iteration.
4410                  */
4411                 if (do_break)
4412                         break;
4413         }
4414 }
4415
4416 static void mlxsw_sp_fib6_node_flush(struct mlxsw_sp *mlxsw_sp,
4417                                      struct mlxsw_sp_fib_node *fib_node)
4418 {
4419         struct mlxsw_sp_fib6_entry *fib6_entry, *tmp;
4420
4421         list_for_each_entry_safe(fib6_entry, tmp, &fib_node->entry_list,
4422                                  common.list) {
4423                 bool do_break = &tmp->common.list == &fib_node->entry_list;
4424
4425                 mlxsw_sp_fib6_node_entry_unlink(mlxsw_sp, fib6_entry);
4426                 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
4427                 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
4428                 if (do_break)
4429                         break;
4430         }
4431 }
4432
4433 static void mlxsw_sp_fib_node_flush(struct mlxsw_sp *mlxsw_sp,
4434                                     struct mlxsw_sp_fib_node *fib_node)
4435 {
4436         switch (fib_node->fib->proto) {
4437         case MLXSW_SP_L3_PROTO_IPV4:
4438                 mlxsw_sp_fib4_node_flush(mlxsw_sp, fib_node);
4439                 break;
4440         case MLXSW_SP_L3_PROTO_IPV6:
4441                 mlxsw_sp_fib6_node_flush(mlxsw_sp, fib_node);
4442                 break;
4443         }
4444 }
4445
4446 static void mlxsw_sp_vr_fib_flush(struct mlxsw_sp *mlxsw_sp,
4447                                   struct mlxsw_sp_vr *vr,
4448                                   enum mlxsw_sp_l3proto proto)
4449 {
4450         struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
4451         struct mlxsw_sp_fib_node *fib_node, *tmp;
4452
4453         list_for_each_entry_safe(fib_node, tmp, &fib->node_list, list) {
4454                 bool do_break = &tmp->list == &fib->node_list;
4455
4456                 mlxsw_sp_fib_node_flush(mlxsw_sp, fib_node);
4457                 if (do_break)
4458                         break;
4459         }
4460 }
4461
4462 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp)
4463 {
4464         int i;
4465
4466         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
4467                 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
4468
4469                 if (!mlxsw_sp_vr_is_used(vr))
4470                         continue;
4471                 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4);
4472
4473                 /* If virtual router was only used for IPv4, then it's no
4474                  * longer used.
4475                  */
4476                 if (!mlxsw_sp_vr_is_used(vr))
4477                         continue;
4478                 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6);
4479         }
4480 }
4481
4482 static void mlxsw_sp_router_fib_abort(struct mlxsw_sp *mlxsw_sp)
4483 {
4484         int err;
4485
4486         if (mlxsw_sp->router->aborted)
4487                 return;
4488         dev_warn(mlxsw_sp->bus_info->dev, "FIB abort triggered. Note that FIB entries are no longer being offloaded to this device.\n");
4489         mlxsw_sp_router_fib_flush(mlxsw_sp);
4490         mlxsw_sp->router->aborted = true;
4491         err = mlxsw_sp_router_set_abort_trap(mlxsw_sp);
4492         if (err)
4493                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set abort trap.\n");
4494 }
4495
4496 struct mlxsw_sp_fib_event_work {
4497         struct work_struct work;
4498         union {
4499                 struct fib6_entry_notifier_info fen6_info;
4500                 struct fib_entry_notifier_info fen_info;
4501                 struct fib_rule_notifier_info fr_info;
4502                 struct fib_nh_notifier_info fnh_info;
4503         };
4504         struct mlxsw_sp *mlxsw_sp;
4505         unsigned long event;
4506 };
4507
4508 static void mlxsw_sp_router_fib4_event_work(struct work_struct *work)
4509 {
4510         struct mlxsw_sp_fib_event_work *fib_work =
4511                 container_of(work, struct mlxsw_sp_fib_event_work, work);
4512         struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp;
4513         struct fib_rule *rule;
4514         bool replace, append;
4515         int err;
4516
4517         /* Protect internal structures from changes */
4518         rtnl_lock();
4519         switch (fib_work->event) {
4520         case FIB_EVENT_ENTRY_REPLACE: /* fall through */
4521         case FIB_EVENT_ENTRY_APPEND: /* fall through */
4522         case FIB_EVENT_ENTRY_ADD:
4523                 replace = fib_work->event == FIB_EVENT_ENTRY_REPLACE;
4524                 append = fib_work->event == FIB_EVENT_ENTRY_APPEND;
4525                 err = mlxsw_sp_router_fib4_add(mlxsw_sp, &fib_work->fen_info,
4526                                                replace, append);
4527                 if (err)
4528                         mlxsw_sp_router_fib_abort(mlxsw_sp);
4529                 fib_info_put(fib_work->fen_info.fi);
4530                 break;
4531         case FIB_EVENT_ENTRY_DEL:
4532                 mlxsw_sp_router_fib4_del(mlxsw_sp, &fib_work->fen_info);
4533                 fib_info_put(fib_work->fen_info.fi);
4534                 break;
4535         case FIB_EVENT_RULE_ADD: /* fall through */
4536         case FIB_EVENT_RULE_DEL:
4537                 rule = fib_work->fr_info.rule;
4538                 if (!fib4_rule_default(rule) && !rule->l3mdev)
4539                         mlxsw_sp_router_fib_abort(mlxsw_sp);
4540                 fib_rule_put(rule);
4541                 break;
4542         case FIB_EVENT_NH_ADD: /* fall through */
4543         case FIB_EVENT_NH_DEL:
4544                 mlxsw_sp_nexthop4_event(mlxsw_sp, fib_work->event,
4545                                         fib_work->fnh_info.fib_nh);
4546                 fib_info_put(fib_work->fnh_info.fib_nh->nh_parent);
4547                 break;
4548         }
4549         rtnl_unlock();
4550         kfree(fib_work);
4551 }
4552
4553 static void mlxsw_sp_router_fib6_event_work(struct work_struct *work)
4554 {
4555         struct mlxsw_sp_fib_event_work *fib_work =
4556                 container_of(work, struct mlxsw_sp_fib_event_work, work);
4557         struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp;
4558         struct fib_rule *rule;
4559         bool replace;
4560         int err;
4561
4562         rtnl_lock();
4563         switch (fib_work->event) {
4564         case FIB_EVENT_ENTRY_REPLACE: /* fall through */
4565         case FIB_EVENT_ENTRY_ADD:
4566                 replace = fib_work->event == FIB_EVENT_ENTRY_REPLACE;
4567                 err = mlxsw_sp_router_fib6_add(mlxsw_sp,
4568                                                fib_work->fen6_info.rt, replace);
4569                 if (err)
4570                         mlxsw_sp_router_fib_abort(mlxsw_sp);
4571                 mlxsw_sp_rt6_release(fib_work->fen6_info.rt);
4572                 break;
4573         case FIB_EVENT_ENTRY_DEL:
4574                 mlxsw_sp_router_fib6_del(mlxsw_sp, fib_work->fen6_info.rt);
4575                 mlxsw_sp_rt6_release(fib_work->fen6_info.rt);
4576                 break;
4577         case FIB_EVENT_RULE_ADD: /* fall through */
4578         case FIB_EVENT_RULE_DEL:
4579                 rule = fib_work->fr_info.rule;
4580                 if (!fib6_rule_default(rule) && !rule->l3mdev)
4581                         mlxsw_sp_router_fib_abort(mlxsw_sp);
4582                 fib_rule_put(rule);
4583                 break;
4584         }
4585         rtnl_unlock();
4586         kfree(fib_work);
4587 }
4588
4589 static void mlxsw_sp_router_fib4_event(struct mlxsw_sp_fib_event_work *fib_work,
4590                                        struct fib_notifier_info *info)
4591 {
4592         switch (fib_work->event) {
4593         case FIB_EVENT_ENTRY_REPLACE: /* fall through */
4594         case FIB_EVENT_ENTRY_APPEND: /* fall through */
4595         case FIB_EVENT_ENTRY_ADD: /* fall through */
4596         case FIB_EVENT_ENTRY_DEL:
4597                 memcpy(&fib_work->fen_info, info, sizeof(fib_work->fen_info));
4598                 /* Take referece on fib_info to prevent it from being
4599                  * freed while work is queued. Release it afterwards.
4600                  */
4601                 fib_info_hold(fib_work->fen_info.fi);
4602                 break;
4603         case FIB_EVENT_RULE_ADD: /* fall through */
4604         case FIB_EVENT_RULE_DEL:
4605                 memcpy(&fib_work->fr_info, info, sizeof(fib_work->fr_info));
4606                 fib_rule_get(fib_work->fr_info.rule);
4607                 break;
4608         case FIB_EVENT_NH_ADD: /* fall through */
4609         case FIB_EVENT_NH_DEL:
4610                 memcpy(&fib_work->fnh_info, info, sizeof(fib_work->fnh_info));
4611                 fib_info_hold(fib_work->fnh_info.fib_nh->nh_parent);
4612                 break;
4613         }
4614 }
4615
4616 static void mlxsw_sp_router_fib6_event(struct mlxsw_sp_fib_event_work *fib_work,
4617                                        struct fib_notifier_info *info)
4618 {
4619         switch (fib_work->event) {
4620         case FIB_EVENT_ENTRY_REPLACE: /* fall through */
4621         case FIB_EVENT_ENTRY_ADD: /* fall through */
4622         case FIB_EVENT_ENTRY_DEL:
4623                 memcpy(&fib_work->fen6_info, info, sizeof(fib_work->fen6_info));
4624                 rt6_hold(fib_work->fen6_info.rt);
4625                 break;
4626         case FIB_EVENT_RULE_ADD: /* fall through */
4627         case FIB_EVENT_RULE_DEL:
4628                 memcpy(&fib_work->fr_info, info, sizeof(fib_work->fr_info));
4629                 fib_rule_get(fib_work->fr_info.rule);
4630                 break;
4631         }
4632 }
4633
4634 /* Called with rcu_read_lock() */
4635 static int mlxsw_sp_router_fib_event(struct notifier_block *nb,
4636                                      unsigned long event, void *ptr)
4637 {
4638         struct mlxsw_sp_fib_event_work *fib_work;
4639         struct fib_notifier_info *info = ptr;
4640         struct mlxsw_sp_router *router;
4641
4642         if (!net_eq(info->net, &init_net))
4643                 return NOTIFY_DONE;
4644
4645         fib_work = kzalloc(sizeof(*fib_work), GFP_ATOMIC);
4646         if (WARN_ON(!fib_work))
4647                 return NOTIFY_BAD;
4648
4649         router = container_of(nb, struct mlxsw_sp_router, fib_nb);
4650         fib_work->mlxsw_sp = router->mlxsw_sp;
4651         fib_work->event = event;
4652
4653         switch (info->family) {
4654         case AF_INET:
4655                 INIT_WORK(&fib_work->work, mlxsw_sp_router_fib4_event_work);
4656                 mlxsw_sp_router_fib4_event(fib_work, info);
4657                 break;
4658         case AF_INET6:
4659                 INIT_WORK(&fib_work->work, mlxsw_sp_router_fib6_event_work);
4660                 mlxsw_sp_router_fib6_event(fib_work, info);
4661                 break;
4662         }
4663
4664         mlxsw_core_schedule_work(&fib_work->work);
4665
4666         return NOTIFY_DONE;
4667 }
4668
4669 static struct mlxsw_sp_rif *
4670 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
4671                          const struct net_device *dev)
4672 {
4673         int i;
4674
4675         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
4676                 if (mlxsw_sp->router->rifs[i] &&
4677                     mlxsw_sp->router->rifs[i]->dev == dev)
4678                         return mlxsw_sp->router->rifs[i];
4679
4680         return NULL;
4681 }
4682
4683 static int mlxsw_sp_router_rif_disable(struct mlxsw_sp *mlxsw_sp, u16 rif)
4684 {
4685         char ritr_pl[MLXSW_REG_RITR_LEN];
4686         int err;
4687
4688         mlxsw_reg_ritr_rif_pack(ritr_pl, rif);
4689         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
4690         if (WARN_ON_ONCE(err))
4691                 return err;
4692
4693         mlxsw_reg_ritr_enable_set(ritr_pl, false);
4694         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
4695 }
4696
4697 static void mlxsw_sp_router_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
4698                                           struct mlxsw_sp_rif *rif)
4699 {
4700         mlxsw_sp_router_rif_disable(mlxsw_sp, rif->rif_index);
4701         mlxsw_sp_nexthop_rif_gone_sync(mlxsw_sp, rif);
4702         mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif);
4703 }
4704
4705 static bool
4706 mlxsw_sp_rif_should_config(struct mlxsw_sp_rif *rif, struct net_device *dev,
4707                            unsigned long event)
4708 {
4709         struct inet6_dev *inet6_dev;
4710         bool addr_list_empty = true;
4711         struct in_device *idev;
4712
4713         switch (event) {
4714         case NETDEV_UP:
4715                 return rif == NULL;
4716         case NETDEV_DOWN:
4717                 idev = __in_dev_get_rtnl(dev);
4718                 if (idev && idev->ifa_list)
4719                         addr_list_empty = false;
4720
4721                 inet6_dev = __in6_dev_get(dev);
4722                 if (addr_list_empty && inet6_dev &&
4723                     !list_empty(&inet6_dev->addr_list))
4724                         addr_list_empty = false;
4725
4726                 if (rif && addr_list_empty &&
4727                     !netif_is_l3_slave(rif->dev))
4728                         return true;
4729                 /* It is possible we already removed the RIF ourselves
4730                  * if it was assigned to a netdev that is now a bridge
4731                  * or LAG slave.
4732                  */
4733                 return false;
4734         }
4735
4736         return false;
4737 }
4738
4739 static enum mlxsw_sp_rif_type
4740 mlxsw_sp_dev_rif_type(const struct mlxsw_sp *mlxsw_sp,
4741                       const struct net_device *dev)
4742 {
4743         enum mlxsw_sp_fid_type type;
4744
4745         if (mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL))
4746                 return MLXSW_SP_RIF_TYPE_IPIP_LB;
4747
4748         /* Otherwise RIF type is derived from the type of the underlying FID. */
4749         if (is_vlan_dev(dev) && netif_is_bridge_master(vlan_dev_real_dev(dev)))
4750                 type = MLXSW_SP_FID_TYPE_8021Q;
4751         else if (netif_is_bridge_master(dev) && br_vlan_enabled(dev))
4752                 type = MLXSW_SP_FID_TYPE_8021Q;
4753         else if (netif_is_bridge_master(dev))
4754                 type = MLXSW_SP_FID_TYPE_8021D;
4755         else
4756                 type = MLXSW_SP_FID_TYPE_RFID;
4757
4758         return mlxsw_sp_fid_type_rif_type(mlxsw_sp, type);
4759 }
4760
4761 static int mlxsw_sp_rif_index_alloc(struct mlxsw_sp *mlxsw_sp, u16 *p_rif_index)
4762 {
4763         int i;
4764
4765         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) {
4766                 if (!mlxsw_sp->router->rifs[i]) {
4767                         *p_rif_index = i;
4768                         return 0;
4769                 }
4770         }
4771
4772         return -ENOBUFS;
4773 }
4774
4775 static struct mlxsw_sp_rif *mlxsw_sp_rif_alloc(size_t rif_size, u16 rif_index,
4776                                                u16 vr_id,
4777                                                struct net_device *l3_dev)
4778 {
4779         struct mlxsw_sp_rif *rif;
4780
4781         rif = kzalloc(rif_size, GFP_KERNEL);
4782         if (!rif)
4783                 return NULL;
4784
4785         INIT_LIST_HEAD(&rif->nexthop_list);
4786         INIT_LIST_HEAD(&rif->neigh_list);
4787         ether_addr_copy(rif->addr, l3_dev->dev_addr);
4788         rif->mtu = l3_dev->mtu;
4789         rif->vr_id = vr_id;
4790         rif->dev = l3_dev;
4791         rif->rif_index = rif_index;
4792
4793         return rif;
4794 }
4795
4796 struct mlxsw_sp_rif *mlxsw_sp_rif_by_index(const struct mlxsw_sp *mlxsw_sp,
4797                                            u16 rif_index)
4798 {
4799         return mlxsw_sp->router->rifs[rif_index];
4800 }
4801
4802 u16 mlxsw_sp_rif_index(const struct mlxsw_sp_rif *rif)
4803 {
4804         return rif->rif_index;
4805 }
4806
4807 int mlxsw_sp_rif_dev_ifindex(const struct mlxsw_sp_rif *rif)
4808 {
4809         return rif->dev->ifindex;
4810 }
4811
4812 static struct mlxsw_sp_rif *
4813 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp,
4814                     const struct mlxsw_sp_rif_params *params)
4815 {
4816         u32 tb_id = l3mdev_fib_table(params->dev);
4817         const struct mlxsw_sp_rif_ops *ops;
4818         struct mlxsw_sp_fid *fid = NULL;
4819         enum mlxsw_sp_rif_type type;
4820         struct mlxsw_sp_rif *rif;
4821         struct mlxsw_sp_vr *vr;
4822         u16 rif_index;
4823         int err;
4824
4825         type = mlxsw_sp_dev_rif_type(mlxsw_sp, params->dev);
4826         ops = mlxsw_sp->router->rif_ops_arr[type];
4827
4828         vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id ? : RT_TABLE_MAIN);
4829         if (IS_ERR(vr))
4830                 return ERR_CAST(vr);
4831
4832         err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index);
4833         if (err)
4834                 goto err_rif_index_alloc;
4835
4836         rif = mlxsw_sp_rif_alloc(ops->rif_size, rif_index, vr->id, params->dev);
4837         if (!rif) {
4838                 err = -ENOMEM;
4839                 goto err_rif_alloc;
4840         }
4841         rif->mlxsw_sp = mlxsw_sp;
4842         rif->ops = ops;
4843
4844         if (ops->fid_get) {
4845                 fid = ops->fid_get(rif);
4846                 if (IS_ERR(fid)) {
4847                         err = PTR_ERR(fid);
4848                         goto err_fid_get;
4849                 }
4850                 rif->fid = fid;
4851         }
4852
4853         if (ops->setup)
4854                 ops->setup(rif, params);
4855
4856         err = ops->configure(rif);
4857         if (err)
4858                 goto err_configure;
4859
4860         mlxsw_sp_rif_counters_alloc(rif);
4861         mlxsw_sp->router->rifs[rif_index] = rif;
4862         vr->rif_count++;
4863
4864         return rif;
4865
4866 err_configure:
4867         if (fid)
4868                 mlxsw_sp_fid_put(fid);
4869 err_fid_get:
4870         kfree(rif);
4871 err_rif_alloc:
4872 err_rif_index_alloc:
4873         mlxsw_sp_vr_put(vr);
4874         return ERR_PTR(err);
4875 }
4876
4877 void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif)
4878 {
4879         const struct mlxsw_sp_rif_ops *ops = rif->ops;
4880         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
4881         struct mlxsw_sp_fid *fid = rif->fid;
4882         struct mlxsw_sp_vr *vr;
4883
4884         mlxsw_sp_router_rif_gone_sync(mlxsw_sp, rif);
4885         vr = &mlxsw_sp->router->vrs[rif->vr_id];
4886
4887         vr->rif_count--;
4888         mlxsw_sp->router->rifs[rif->rif_index] = NULL;
4889         mlxsw_sp_rif_counters_free(rif);
4890         ops->deconfigure(rif);
4891         if (fid)
4892                 /* Loopback RIFs are not associated with a FID. */
4893                 mlxsw_sp_fid_put(fid);
4894         kfree(rif);
4895         mlxsw_sp_vr_put(vr);
4896 }
4897
4898 static void
4899 mlxsw_sp_rif_subport_params_init(struct mlxsw_sp_rif_params *params,
4900                                  struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
4901 {
4902         struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
4903
4904         params->vid = mlxsw_sp_port_vlan->vid;
4905         params->lag = mlxsw_sp_port->lagged;
4906         if (params->lag)
4907                 params->lag_id = mlxsw_sp_port->lag_id;
4908         else
4909                 params->system_port = mlxsw_sp_port->local_port;
4910 }
4911
4912 static int
4913 mlxsw_sp_port_vlan_router_join(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan,
4914                                struct net_device *l3_dev)
4915 {
4916         struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
4917         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
4918         u16 vid = mlxsw_sp_port_vlan->vid;
4919         struct mlxsw_sp_rif *rif;
4920         struct mlxsw_sp_fid *fid;
4921         int err;
4922
4923         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
4924         if (!rif) {
4925                 struct mlxsw_sp_rif_params params = {
4926                         .dev = l3_dev,
4927                 };
4928
4929                 mlxsw_sp_rif_subport_params_init(&params, mlxsw_sp_port_vlan);
4930                 rif = mlxsw_sp_rif_create(mlxsw_sp, &params);
4931                 if (IS_ERR(rif))
4932                         return PTR_ERR(rif);
4933         }
4934
4935         /* FID was already created, just take a reference */
4936         fid = rif->ops->fid_get(rif);
4937         err = mlxsw_sp_fid_port_vid_map(fid, mlxsw_sp_port, vid);
4938         if (err)
4939                 goto err_fid_port_vid_map;
4940
4941         err = mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, false);
4942         if (err)
4943                 goto err_port_vid_learning_set;
4944
4945         err = mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid,
4946                                         BR_STATE_FORWARDING);
4947         if (err)
4948                 goto err_port_vid_stp_set;
4949
4950         mlxsw_sp_port_vlan->fid = fid;
4951
4952         return 0;
4953
4954 err_port_vid_stp_set:
4955         mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true);
4956 err_port_vid_learning_set:
4957         mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid);
4958 err_fid_port_vid_map:
4959         mlxsw_sp_fid_put(fid);
4960         return err;
4961 }
4962
4963 void
4964 mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
4965 {
4966         struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
4967         struct mlxsw_sp_fid *fid = mlxsw_sp_port_vlan->fid;
4968         u16 vid = mlxsw_sp_port_vlan->vid;
4969
4970         if (WARN_ON(mlxsw_sp_fid_type(fid) != MLXSW_SP_FID_TYPE_RFID))
4971                 return;
4972
4973         mlxsw_sp_port_vlan->fid = NULL;
4974         mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid, BR_STATE_BLOCKING);
4975         mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true);
4976         mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid);
4977         /* If router port holds the last reference on the rFID, then the
4978          * associated Sub-port RIF will be destroyed.
4979          */
4980         mlxsw_sp_fid_put(fid);
4981 }
4982
4983 static int mlxsw_sp_inetaddr_port_vlan_event(struct net_device *l3_dev,
4984                                              struct net_device *port_dev,
4985                                              unsigned long event, u16 vid)
4986 {
4987         struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(port_dev);
4988         struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan;
4989
4990         mlxsw_sp_port_vlan = mlxsw_sp_port_vlan_find_by_vid(mlxsw_sp_port, vid);
4991         if (WARN_ON(!mlxsw_sp_port_vlan))
4992                 return -EINVAL;
4993
4994         switch (event) {
4995         case NETDEV_UP:
4996                 return mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan,
4997                                                       l3_dev);
4998         case NETDEV_DOWN:
4999                 mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan);
5000                 break;
5001         }
5002
5003         return 0;
5004 }
5005
5006 static int mlxsw_sp_inetaddr_port_event(struct net_device *port_dev,
5007                                         unsigned long event)
5008 {
5009         if (netif_is_bridge_port(port_dev) ||
5010             netif_is_lag_port(port_dev) ||
5011             netif_is_ovs_port(port_dev))
5012                 return 0;
5013
5014         return mlxsw_sp_inetaddr_port_vlan_event(port_dev, port_dev, event, 1);
5015 }
5016
5017 static int __mlxsw_sp_inetaddr_lag_event(struct net_device *l3_dev,
5018                                          struct net_device *lag_dev,
5019                                          unsigned long event, u16 vid)
5020 {
5021         struct net_device *port_dev;
5022         struct list_head *iter;
5023         int err;
5024
5025         netdev_for_each_lower_dev(lag_dev, port_dev, iter) {
5026                 if (mlxsw_sp_port_dev_check(port_dev)) {
5027                         err = mlxsw_sp_inetaddr_port_vlan_event(l3_dev,
5028                                                                 port_dev,
5029                                                                 event, vid);
5030                         if (err)
5031                                 return err;
5032                 }
5033         }
5034
5035         return 0;
5036 }
5037
5038 static int mlxsw_sp_inetaddr_lag_event(struct net_device *lag_dev,
5039                                        unsigned long event)
5040 {
5041         if (netif_is_bridge_port(lag_dev))
5042                 return 0;
5043
5044         return __mlxsw_sp_inetaddr_lag_event(lag_dev, lag_dev, event, 1);
5045 }
5046
5047 static int mlxsw_sp_inetaddr_bridge_event(struct net_device *l3_dev,
5048                                           unsigned long event)
5049 {
5050         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev);
5051         struct mlxsw_sp_rif_params params = {
5052                 .dev = l3_dev,
5053         };
5054         struct mlxsw_sp_rif *rif;
5055
5056         switch (event) {
5057         case NETDEV_UP:
5058                 rif = mlxsw_sp_rif_create(mlxsw_sp, &params);
5059                 if (IS_ERR(rif))
5060                         return PTR_ERR(rif);
5061                 break;
5062         case NETDEV_DOWN:
5063                 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
5064                 mlxsw_sp_rif_destroy(rif);
5065                 break;
5066         }
5067
5068         return 0;
5069 }
5070
5071 static int mlxsw_sp_inetaddr_vlan_event(struct net_device *vlan_dev,
5072                                         unsigned long event)
5073 {
5074         struct net_device *real_dev = vlan_dev_real_dev(vlan_dev);
5075         u16 vid = vlan_dev_vlan_id(vlan_dev);
5076
5077         if (netif_is_bridge_port(vlan_dev))
5078                 return 0;
5079
5080         if (mlxsw_sp_port_dev_check(real_dev))
5081                 return mlxsw_sp_inetaddr_port_vlan_event(vlan_dev, real_dev,
5082                                                          event, vid);
5083         else if (netif_is_lag_master(real_dev))
5084                 return __mlxsw_sp_inetaddr_lag_event(vlan_dev, real_dev, event,
5085                                                      vid);
5086         else if (netif_is_bridge_master(real_dev) && br_vlan_enabled(real_dev))
5087                 return mlxsw_sp_inetaddr_bridge_event(vlan_dev, event);
5088
5089         return 0;
5090 }
5091
5092 static int __mlxsw_sp_inetaddr_event(struct net_device *dev,
5093                                      unsigned long event)
5094 {
5095         if (mlxsw_sp_port_dev_check(dev))
5096                 return mlxsw_sp_inetaddr_port_event(dev, event);
5097         else if (netif_is_lag_master(dev))
5098                 return mlxsw_sp_inetaddr_lag_event(dev, event);
5099         else if (netif_is_bridge_master(dev))
5100                 return mlxsw_sp_inetaddr_bridge_event(dev, event);
5101         else if (is_vlan_dev(dev))
5102                 return mlxsw_sp_inetaddr_vlan_event(dev, event);
5103         else
5104                 return 0;
5105 }
5106
5107 int mlxsw_sp_inetaddr_event(struct notifier_block *unused,
5108                             unsigned long event, void *ptr)
5109 {
5110         struct in_ifaddr *ifa = (struct in_ifaddr *) ptr;
5111         struct net_device *dev = ifa->ifa_dev->dev;
5112         struct mlxsw_sp *mlxsw_sp;
5113         struct mlxsw_sp_rif *rif;
5114         int err = 0;
5115
5116         mlxsw_sp = mlxsw_sp_lower_get(dev);
5117         if (!mlxsw_sp)
5118                 goto out;
5119
5120         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
5121         if (!mlxsw_sp_rif_should_config(rif, dev, event))
5122                 goto out;
5123
5124         err = __mlxsw_sp_inetaddr_event(dev, event);
5125 out:
5126         return notifier_from_errno(err);
5127 }
5128
5129 struct mlxsw_sp_inet6addr_event_work {
5130         struct work_struct work;
5131         struct net_device *dev;
5132         unsigned long event;
5133 };
5134
5135 static void mlxsw_sp_inet6addr_event_work(struct work_struct *work)
5136 {
5137         struct mlxsw_sp_inet6addr_event_work *inet6addr_work =
5138                 container_of(work, struct mlxsw_sp_inet6addr_event_work, work);
5139         struct net_device *dev = inet6addr_work->dev;
5140         unsigned long event = inet6addr_work->event;
5141         struct mlxsw_sp *mlxsw_sp;
5142         struct mlxsw_sp_rif *rif;
5143
5144         rtnl_lock();
5145         mlxsw_sp = mlxsw_sp_lower_get(dev);
5146         if (!mlxsw_sp)
5147                 goto out;
5148
5149         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
5150         if (!mlxsw_sp_rif_should_config(rif, dev, event))
5151                 goto out;
5152
5153         __mlxsw_sp_inetaddr_event(dev, event);
5154 out:
5155         rtnl_unlock();
5156         dev_put(dev);
5157         kfree(inet6addr_work);
5158 }
5159
5160 /* Called with rcu_read_lock() */
5161 int mlxsw_sp_inet6addr_event(struct notifier_block *unused,
5162                              unsigned long event, void *ptr)
5163 {
5164         struct inet6_ifaddr *if6 = (struct inet6_ifaddr *) ptr;
5165         struct mlxsw_sp_inet6addr_event_work *inet6addr_work;
5166         struct net_device *dev = if6->idev->dev;
5167
5168         if (!mlxsw_sp_port_dev_lower_find_rcu(dev))
5169                 return NOTIFY_DONE;
5170
5171         inet6addr_work = kzalloc(sizeof(*inet6addr_work), GFP_ATOMIC);
5172         if (!inet6addr_work)
5173                 return NOTIFY_BAD;
5174
5175         INIT_WORK(&inet6addr_work->work, mlxsw_sp_inet6addr_event_work);
5176         inet6addr_work->dev = dev;
5177         inet6addr_work->event = event;
5178         dev_hold(dev);
5179         mlxsw_core_schedule_work(&inet6addr_work->work);
5180
5181         return NOTIFY_DONE;
5182 }
5183
5184 static int mlxsw_sp_rif_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
5185                              const char *mac, int mtu)
5186 {
5187         char ritr_pl[MLXSW_REG_RITR_LEN];
5188         int err;
5189
5190         mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
5191         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
5192         if (err)
5193                 return err;
5194
5195         mlxsw_reg_ritr_mtu_set(ritr_pl, mtu);
5196         mlxsw_reg_ritr_if_mac_memcpy_to(ritr_pl, mac);
5197         mlxsw_reg_ritr_op_set(ritr_pl, MLXSW_REG_RITR_RIF_CREATE);
5198         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
5199 }
5200
5201 int mlxsw_sp_netdevice_router_port_event(struct net_device *dev)
5202 {
5203         struct mlxsw_sp *mlxsw_sp;
5204         struct mlxsw_sp_rif *rif;
5205         u16 fid_index;
5206         int err;
5207
5208         mlxsw_sp = mlxsw_sp_lower_get(dev);
5209         if (!mlxsw_sp)
5210                 return 0;
5211
5212         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
5213         if (!rif)
5214                 return 0;
5215         fid_index = mlxsw_sp_fid_index(rif->fid);
5216
5217         err = mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, false);
5218         if (err)
5219                 return err;
5220
5221         err = mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, dev->dev_addr,
5222                                 dev->mtu);
5223         if (err)
5224                 goto err_rif_edit;
5225
5226         err = mlxsw_sp_rif_fdb_op(mlxsw_sp, dev->dev_addr, fid_index, true);
5227         if (err)
5228                 goto err_rif_fdb_op;
5229
5230         ether_addr_copy(rif->addr, dev->dev_addr);
5231         rif->mtu = dev->mtu;
5232
5233         netdev_dbg(dev, "Updated RIF=%d\n", rif->rif_index);
5234
5235         return 0;
5236
5237 err_rif_fdb_op:
5238         mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, rif->addr, rif->mtu);
5239 err_rif_edit:
5240         mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, true);
5241         return err;
5242 }
5243
5244 static int mlxsw_sp_port_vrf_join(struct mlxsw_sp *mlxsw_sp,
5245                                   struct net_device *l3_dev)
5246 {
5247         struct mlxsw_sp_rif *rif;
5248
5249         /* If netdev is already associated with a RIF, then we need to
5250          * destroy it and create a new one with the new virtual router ID.
5251          */
5252         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
5253         if (rif)
5254                 __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_DOWN);
5255
5256         return __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_UP);
5257 }
5258
5259 static void mlxsw_sp_port_vrf_leave(struct mlxsw_sp *mlxsw_sp,
5260                                     struct net_device *l3_dev)
5261 {
5262         struct mlxsw_sp_rif *rif;
5263
5264         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
5265         if (!rif)
5266                 return;
5267         __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_DOWN);
5268 }
5269
5270 int mlxsw_sp_netdevice_vrf_event(struct net_device *l3_dev, unsigned long event,
5271                                  struct netdev_notifier_changeupper_info *info)
5272 {
5273         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev);
5274         int err = 0;
5275
5276         if (!mlxsw_sp)
5277                 return 0;
5278
5279         switch (event) {
5280         case NETDEV_PRECHANGEUPPER:
5281                 return 0;
5282         case NETDEV_CHANGEUPPER:
5283                 if (info->linking)
5284                         err = mlxsw_sp_port_vrf_join(mlxsw_sp, l3_dev);
5285                 else
5286                         mlxsw_sp_port_vrf_leave(mlxsw_sp, l3_dev);
5287                 break;
5288         }
5289
5290         return err;
5291 }
5292
5293 static struct mlxsw_sp_rif_subport *
5294 mlxsw_sp_rif_subport_rif(const struct mlxsw_sp_rif *rif)
5295 {
5296         return container_of(rif, struct mlxsw_sp_rif_subport, common);
5297 }
5298
5299 static void mlxsw_sp_rif_subport_setup(struct mlxsw_sp_rif *rif,
5300                                        const struct mlxsw_sp_rif_params *params)
5301 {
5302         struct mlxsw_sp_rif_subport *rif_subport;
5303
5304         rif_subport = mlxsw_sp_rif_subport_rif(rif);
5305         rif_subport->vid = params->vid;
5306         rif_subport->lag = params->lag;
5307         if (params->lag)
5308                 rif_subport->lag_id = params->lag_id;
5309         else
5310                 rif_subport->system_port = params->system_port;
5311 }
5312
5313 static int mlxsw_sp_rif_subport_op(struct mlxsw_sp_rif *rif, bool enable)
5314 {
5315         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5316         struct mlxsw_sp_rif_subport *rif_subport;
5317         char ritr_pl[MLXSW_REG_RITR_LEN];
5318
5319         rif_subport = mlxsw_sp_rif_subport_rif(rif);
5320         mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_SP_IF,
5321                             rif->rif_index, rif->vr_id, rif->dev->mtu);
5322         mlxsw_reg_ritr_mac_pack(ritr_pl, rif->dev->dev_addr);
5323         mlxsw_reg_ritr_sp_if_pack(ritr_pl, rif_subport->lag,
5324                                   rif_subport->lag ? rif_subport->lag_id :
5325                                                      rif_subport->system_port,
5326                                   rif_subport->vid);
5327
5328         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
5329 }
5330
5331 static int mlxsw_sp_rif_subport_configure(struct mlxsw_sp_rif *rif)
5332 {
5333         int err;
5334
5335         err = mlxsw_sp_rif_subport_op(rif, true);
5336         if (err)
5337                 return err;
5338
5339         err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
5340                                   mlxsw_sp_fid_index(rif->fid), true);
5341         if (err)
5342                 goto err_rif_fdb_op;
5343
5344         mlxsw_sp_fid_rif_set(rif->fid, rif);
5345         return 0;
5346
5347 err_rif_fdb_op:
5348         mlxsw_sp_rif_subport_op(rif, false);
5349         return err;
5350 }
5351
5352 static void mlxsw_sp_rif_subport_deconfigure(struct mlxsw_sp_rif *rif)
5353 {
5354         struct mlxsw_sp_fid *fid = rif->fid;
5355
5356         mlxsw_sp_fid_rif_set(fid, NULL);
5357         mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
5358                             mlxsw_sp_fid_index(fid), false);
5359         mlxsw_sp_rif_subport_op(rif, false);
5360 }
5361
5362 static struct mlxsw_sp_fid *
5363 mlxsw_sp_rif_subport_fid_get(struct mlxsw_sp_rif *rif)
5364 {
5365         return mlxsw_sp_fid_rfid_get(rif->mlxsw_sp, rif->rif_index);
5366 }
5367
5368 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_subport_ops = {
5369         .type                   = MLXSW_SP_RIF_TYPE_SUBPORT,
5370         .rif_size               = sizeof(struct mlxsw_sp_rif_subport),
5371         .setup                  = mlxsw_sp_rif_subport_setup,
5372         .configure              = mlxsw_sp_rif_subport_configure,
5373         .deconfigure            = mlxsw_sp_rif_subport_deconfigure,
5374         .fid_get                = mlxsw_sp_rif_subport_fid_get,
5375 };
5376
5377 static int mlxsw_sp_rif_vlan_fid_op(struct mlxsw_sp_rif *rif,
5378                                     enum mlxsw_reg_ritr_if_type type,
5379                                     u16 vid_fid, bool enable)
5380 {
5381         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5382         char ritr_pl[MLXSW_REG_RITR_LEN];
5383
5384         mlxsw_reg_ritr_pack(ritr_pl, enable, type, rif->rif_index, rif->vr_id,
5385                             rif->dev->mtu);
5386         mlxsw_reg_ritr_mac_pack(ritr_pl, rif->dev->dev_addr);
5387         mlxsw_reg_ritr_fid_set(ritr_pl, type, vid_fid);
5388
5389         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
5390 }
5391
5392 static u8 mlxsw_sp_router_port(const struct mlxsw_sp *mlxsw_sp)
5393 {
5394         return mlxsw_core_max_ports(mlxsw_sp->core) + 1;
5395 }
5396
5397 static int mlxsw_sp_rif_vlan_configure(struct mlxsw_sp_rif *rif)
5398 {
5399         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5400         u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid);
5401         int err;
5402
5403         err = mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, true);
5404         if (err)
5405                 return err;
5406
5407         err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
5408                                      mlxsw_sp_router_port(mlxsw_sp), true);
5409         if (err)
5410                 goto err_fid_mc_flood_set;
5411
5412         err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
5413                                      mlxsw_sp_router_port(mlxsw_sp), true);
5414         if (err)
5415                 goto err_fid_bc_flood_set;
5416
5417         err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
5418                                   mlxsw_sp_fid_index(rif->fid), true);
5419         if (err)
5420                 goto err_rif_fdb_op;
5421
5422         mlxsw_sp_fid_rif_set(rif->fid, rif);
5423         return 0;
5424
5425 err_rif_fdb_op:
5426         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
5427                                mlxsw_sp_router_port(mlxsw_sp), false);
5428 err_fid_bc_flood_set:
5429         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
5430                                mlxsw_sp_router_port(mlxsw_sp), false);
5431 err_fid_mc_flood_set:
5432         mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, false);
5433         return err;
5434 }
5435
5436 static void mlxsw_sp_rif_vlan_deconfigure(struct mlxsw_sp_rif *rif)
5437 {
5438         u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid);
5439         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5440         struct mlxsw_sp_fid *fid = rif->fid;
5441
5442         mlxsw_sp_fid_rif_set(fid, NULL);
5443         mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
5444                             mlxsw_sp_fid_index(fid), false);
5445         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
5446                                mlxsw_sp_router_port(mlxsw_sp), false);
5447         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
5448                                mlxsw_sp_router_port(mlxsw_sp), false);
5449         mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, false);
5450 }
5451
5452 static struct mlxsw_sp_fid *
5453 mlxsw_sp_rif_vlan_fid_get(struct mlxsw_sp_rif *rif)
5454 {
5455         u16 vid = is_vlan_dev(rif->dev) ? vlan_dev_vlan_id(rif->dev) : 1;
5456
5457         return mlxsw_sp_fid_8021q_get(rif->mlxsw_sp, vid);
5458 }
5459
5460 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_vlan_ops = {
5461         .type                   = MLXSW_SP_RIF_TYPE_VLAN,
5462         .rif_size               = sizeof(struct mlxsw_sp_rif),
5463         .configure              = mlxsw_sp_rif_vlan_configure,
5464         .deconfigure            = mlxsw_sp_rif_vlan_deconfigure,
5465         .fid_get                = mlxsw_sp_rif_vlan_fid_get,
5466 };
5467
5468 static int mlxsw_sp_rif_fid_configure(struct mlxsw_sp_rif *rif)
5469 {
5470         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5471         u16 fid_index = mlxsw_sp_fid_index(rif->fid);
5472         int err;
5473
5474         err = mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index,
5475                                        true);
5476         if (err)
5477                 return err;
5478
5479         err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
5480                                      mlxsw_sp_router_port(mlxsw_sp), true);
5481         if (err)
5482                 goto err_fid_mc_flood_set;
5483
5484         err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
5485                                      mlxsw_sp_router_port(mlxsw_sp), true);
5486         if (err)
5487                 goto err_fid_bc_flood_set;
5488
5489         err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
5490                                   mlxsw_sp_fid_index(rif->fid), true);
5491         if (err)
5492                 goto err_rif_fdb_op;
5493
5494         mlxsw_sp_fid_rif_set(rif->fid, rif);
5495         return 0;
5496
5497 err_rif_fdb_op:
5498         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
5499                                mlxsw_sp_router_port(mlxsw_sp), false);
5500 err_fid_bc_flood_set:
5501         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
5502                                mlxsw_sp_router_port(mlxsw_sp), false);
5503 err_fid_mc_flood_set:
5504         mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false);
5505         return err;
5506 }
5507
5508 static void mlxsw_sp_rif_fid_deconfigure(struct mlxsw_sp_rif *rif)
5509 {
5510         u16 fid_index = mlxsw_sp_fid_index(rif->fid);
5511         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5512         struct mlxsw_sp_fid *fid = rif->fid;
5513
5514         mlxsw_sp_fid_rif_set(fid, NULL);
5515         mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
5516                             mlxsw_sp_fid_index(fid), false);
5517         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
5518                                mlxsw_sp_router_port(mlxsw_sp), false);
5519         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
5520                                mlxsw_sp_router_port(mlxsw_sp), false);
5521         mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false);
5522 }
5523
5524 static struct mlxsw_sp_fid *
5525 mlxsw_sp_rif_fid_fid_get(struct mlxsw_sp_rif *rif)
5526 {
5527         return mlxsw_sp_fid_8021d_get(rif->mlxsw_sp, rif->dev->ifindex);
5528 }
5529
5530 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_fid_ops = {
5531         .type                   = MLXSW_SP_RIF_TYPE_FID,
5532         .rif_size               = sizeof(struct mlxsw_sp_rif),
5533         .configure              = mlxsw_sp_rif_fid_configure,
5534         .deconfigure            = mlxsw_sp_rif_fid_deconfigure,
5535         .fid_get                = mlxsw_sp_rif_fid_fid_get,
5536 };
5537
5538 static struct mlxsw_sp_rif_ipip_lb *
5539 mlxsw_sp_rif_ipip_lb_rif(struct mlxsw_sp_rif *rif)
5540 {
5541         return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common);
5542 }
5543
5544 static void
5545 mlxsw_sp_rif_ipip_lb_setup(struct mlxsw_sp_rif *rif,
5546                            const struct mlxsw_sp_rif_params *params)
5547 {
5548         struct mlxsw_sp_rif_params_ipip_lb *params_lb;
5549         struct mlxsw_sp_rif_ipip_lb *rif_lb;
5550
5551         params_lb = container_of(params, struct mlxsw_sp_rif_params_ipip_lb,
5552                                  common);
5553         rif_lb = mlxsw_sp_rif_ipip_lb_rif(rif);
5554         rif_lb->lb_config = params_lb->lb_config;
5555 }
5556
5557 static int
5558 mlxsw_sp_rif_ipip_lb_op(struct mlxsw_sp_rif_ipip_lb *lb_rif,
5559                         struct mlxsw_sp_vr *ul_vr, bool enable)
5560 {
5561         struct mlxsw_sp_rif_ipip_lb_config lb_cf = lb_rif->lb_config;
5562         struct mlxsw_sp_rif *rif = &lb_rif->common;
5563         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5564         char ritr_pl[MLXSW_REG_RITR_LEN];
5565         u32 saddr4;
5566
5567         switch (lb_cf.ul_protocol) {
5568         case MLXSW_SP_L3_PROTO_IPV4:
5569                 saddr4 = be32_to_cpu(lb_cf.saddr.addr4);
5570                 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF,
5571                                     rif->rif_index, rif->vr_id, rif->dev->mtu);
5572                 mlxsw_reg_ritr_loopback_ipip4_pack(ritr_pl, lb_cf.lb_ipipt,
5573                             MLXSW_REG_RITR_LOOPBACK_IPIP_OPTIONS_GRE_KEY_PRESET,
5574                             ul_vr->id, saddr4, lb_cf.okey);
5575                 break;
5576
5577         case MLXSW_SP_L3_PROTO_IPV6:
5578                 return -EAFNOSUPPORT;
5579         }
5580
5581         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
5582 }
5583
5584 static int
5585 mlxsw_sp_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif)
5586 {
5587         struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
5588         u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(rif->dev);
5589         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5590         struct mlxsw_sp_vr *ul_vr;
5591         int err;
5592
5593         ul_vr = mlxsw_sp_vr_get(mlxsw_sp, ul_tb_id);
5594         if (IS_ERR(ul_vr))
5595                 return PTR_ERR(ul_vr);
5596
5597         err = mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr, true);
5598         if (err)
5599                 goto err_loopback_op;
5600
5601         lb_rif->ul_vr_id = ul_vr->id;
5602         ++ul_vr->rif_count;
5603         return 0;
5604
5605 err_loopback_op:
5606         mlxsw_sp_vr_put(ul_vr);
5607         return err;
5608 }
5609
5610 static void mlxsw_sp_rif_ipip_lb_deconfigure(struct mlxsw_sp_rif *rif)
5611 {
5612         struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
5613         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
5614         struct mlxsw_sp_vr *ul_vr;
5615
5616         ul_vr = &mlxsw_sp->router->vrs[lb_rif->ul_vr_id];
5617         mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr, false);
5618
5619         --ul_vr->rif_count;
5620         mlxsw_sp_vr_put(ul_vr);
5621 }
5622
5623 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_ipip_lb_ops = {
5624         .type                   = MLXSW_SP_RIF_TYPE_IPIP_LB,
5625         .rif_size               = sizeof(struct mlxsw_sp_rif_ipip_lb),
5626         .setup                  = mlxsw_sp_rif_ipip_lb_setup,
5627         .configure              = mlxsw_sp_rif_ipip_lb_configure,
5628         .deconfigure            = mlxsw_sp_rif_ipip_lb_deconfigure,
5629 };
5630
5631 static const struct mlxsw_sp_rif_ops *mlxsw_sp_rif_ops_arr[] = {
5632         [MLXSW_SP_RIF_TYPE_SUBPORT]     = &mlxsw_sp_rif_subport_ops,
5633         [MLXSW_SP_RIF_TYPE_VLAN]        = &mlxsw_sp_rif_vlan_ops,
5634         [MLXSW_SP_RIF_TYPE_FID]         = &mlxsw_sp_rif_fid_ops,
5635         [MLXSW_SP_RIF_TYPE_IPIP_LB]     = &mlxsw_sp_rif_ipip_lb_ops,
5636 };
5637
5638 static int mlxsw_sp_rifs_init(struct mlxsw_sp *mlxsw_sp)
5639 {
5640         u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
5641
5642         mlxsw_sp->router->rifs = kcalloc(max_rifs,
5643                                          sizeof(struct mlxsw_sp_rif *),
5644                                          GFP_KERNEL);
5645         if (!mlxsw_sp->router->rifs)
5646                 return -ENOMEM;
5647
5648         mlxsw_sp->router->rif_ops_arr = mlxsw_sp_rif_ops_arr;
5649
5650         return 0;
5651 }
5652
5653 static void mlxsw_sp_rifs_fini(struct mlxsw_sp *mlxsw_sp)
5654 {
5655         int i;
5656
5657         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
5658                 WARN_ON_ONCE(mlxsw_sp->router->rifs[i]);
5659
5660         kfree(mlxsw_sp->router->rifs);
5661 }
5662
5663 static int mlxsw_sp_ipips_init(struct mlxsw_sp *mlxsw_sp)
5664 {
5665         mlxsw_sp->router->ipip_ops_arr = mlxsw_sp_ipip_ops_arr;
5666         INIT_LIST_HEAD(&mlxsw_sp->router->ipip_list);
5667         return 0;
5668 }
5669
5670 static void mlxsw_sp_ipips_fini(struct mlxsw_sp *mlxsw_sp)
5671 {
5672         WARN_ON(!list_empty(&mlxsw_sp->router->ipip_list));
5673 }
5674
5675 static void mlxsw_sp_router_fib_dump_flush(struct notifier_block *nb)
5676 {
5677         struct mlxsw_sp_router *router;
5678
5679         /* Flush pending FIB notifications and then flush the device's
5680          * table before requesting another dump. The FIB notification
5681          * block is unregistered, so no need to take RTNL.
5682          */
5683         mlxsw_core_flush_owq();
5684         router = container_of(nb, struct mlxsw_sp_router, fib_nb);
5685         mlxsw_sp_router_fib_flush(router->mlxsw_sp);
5686 }
5687
5688 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
5689 {
5690         char rgcr_pl[MLXSW_REG_RGCR_LEN];
5691         u64 max_rifs;
5692         int err;
5693
5694         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_RIFS))
5695                 return -EIO;
5696         max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
5697
5698         mlxsw_reg_rgcr_pack(rgcr_pl, true, true);
5699         mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, max_rifs);
5700         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
5701         if (err)
5702                 return err;
5703         return 0;
5704 }
5705
5706 static void __mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
5707 {
5708         char rgcr_pl[MLXSW_REG_RGCR_LEN];
5709
5710         mlxsw_reg_rgcr_pack(rgcr_pl, false, false);
5711         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
5712 }
5713
5714 int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
5715 {
5716         struct mlxsw_sp_router *router;
5717         int err;
5718
5719         router = kzalloc(sizeof(*mlxsw_sp->router), GFP_KERNEL);
5720         if (!router)
5721                 return -ENOMEM;
5722         mlxsw_sp->router = router;
5723         router->mlxsw_sp = mlxsw_sp;
5724
5725         INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_neighs_list);
5726         err = __mlxsw_sp_router_init(mlxsw_sp);
5727         if (err)
5728                 goto err_router_init;
5729
5730         err = mlxsw_sp_rifs_init(mlxsw_sp);
5731         if (err)
5732                 goto err_rifs_init;
5733
5734         err = mlxsw_sp_ipips_init(mlxsw_sp);
5735         if (err)
5736                 goto err_ipips_init;
5737
5738         err = rhashtable_init(&mlxsw_sp->router->nexthop_ht,
5739                               &mlxsw_sp_nexthop_ht_params);
5740         if (err)
5741                 goto err_nexthop_ht_init;
5742
5743         err = rhashtable_init(&mlxsw_sp->router->nexthop_group_ht,
5744                               &mlxsw_sp_nexthop_group_ht_params);
5745         if (err)
5746                 goto err_nexthop_group_ht_init;
5747
5748         err = mlxsw_sp_lpm_init(mlxsw_sp);
5749         if (err)
5750                 goto err_lpm_init;
5751
5752         err = mlxsw_sp_vrs_init(mlxsw_sp);
5753         if (err)
5754                 goto err_vrs_init;
5755
5756         err = mlxsw_sp_neigh_init(mlxsw_sp);
5757         if (err)
5758                 goto err_neigh_init;
5759
5760         mlxsw_sp->router->fib_nb.notifier_call = mlxsw_sp_router_fib_event;
5761         err = register_fib_notifier(&mlxsw_sp->router->fib_nb,
5762                                     mlxsw_sp_router_fib_dump_flush);
5763         if (err)
5764                 goto err_register_fib_notifier;
5765
5766         return 0;
5767
5768 err_register_fib_notifier:
5769         mlxsw_sp_neigh_fini(mlxsw_sp);
5770 err_neigh_init:
5771         mlxsw_sp_vrs_fini(mlxsw_sp);
5772 err_vrs_init:
5773         mlxsw_sp_lpm_fini(mlxsw_sp);
5774 err_lpm_init:
5775         rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht);
5776 err_nexthop_group_ht_init:
5777         rhashtable_destroy(&mlxsw_sp->router->nexthop_ht);
5778 err_nexthop_ht_init:
5779         mlxsw_sp_ipips_fini(mlxsw_sp);
5780 err_ipips_init:
5781         mlxsw_sp_rifs_fini(mlxsw_sp);
5782 err_rifs_init:
5783         __mlxsw_sp_router_fini(mlxsw_sp);
5784 err_router_init:
5785         kfree(mlxsw_sp->router);
5786         return err;
5787 }
5788
5789 void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
5790 {
5791         unregister_fib_notifier(&mlxsw_sp->router->fib_nb);
5792         mlxsw_sp_neigh_fini(mlxsw_sp);
5793         mlxsw_sp_vrs_fini(mlxsw_sp);
5794         mlxsw_sp_lpm_fini(mlxsw_sp);
5795         rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht);
5796         rhashtable_destroy(&mlxsw_sp->router->nexthop_ht);
5797         mlxsw_sp_ipips_fini(mlxsw_sp);
5798         mlxsw_sp_rifs_fini(mlxsw_sp);
5799         __mlxsw_sp_router_fini(mlxsw_sp);
5800         kfree(mlxsw_sp->router);
5801 }