drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c

   1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
   2 /* Copyright (c) 2016-2018 Mellanox Technologies. All rights reserved */
   3
   4 #include <linux/kernel.h>
   5 #include <linux/types.h>
   6 #include <linux/rhashtable.h>
   7 #include <linux/bitops.h>
   8 #include <linux/in6.h>
   9 #include <linux/notifier.h>
  10 #include <linux/inetdevice.h>
  11 #include <linux/netdevice.h>
  12 #include <linux/if_bridge.h>
  13 #include <linux/socket.h>
  14 #include <linux/route.h>
  15 #include <linux/gcd.h>
  16 #include <linux/random.h>
  17 #include <linux/if_macvlan.h>
  18 #include <linux/refcount.h>
  19 #include <net/netevent.h>
  20 #include <net/neighbour.h>
  21 #include <net/arp.h>
  22 #include <net/ip_fib.h>
  23 #include <net/ip6_fib.h>
  24 #include <net/fib_rules.h>
  25 #include <net/ip_tunnels.h>
  26 #include <net/l3mdev.h>
  27 #include <net/addrconf.h>
  28 #include <net/ndisc.h>
  29 #include <net/ipv6.h>
  30 #include <net/fib_notifier.h>
  31 #include <net/switchdev.h>
  32
  33 #include "spectrum.h"
  34 #include "core.h"
  35 #include "reg.h"
  36 #include "spectrum_cnt.h"
  37 #include "spectrum_dpipe.h"
  38 #include "spectrum_ipip.h"
  39 #include "spectrum_mr.h"
  40 #include "spectrum_mr_tcam.h"
  41 #include "spectrum_router.h"
  42 #include "spectrum_span.h"
  43
  44 struct mlxsw_sp_fib;
  45 struct mlxsw_sp_vr;
  46 struct mlxsw_sp_lpm_tree;
  47 struct mlxsw_sp_rif_ops;
  48
  49 struct mlxsw_sp_router {
  50         struct mlxsw_sp *mlxsw_sp;
  51         struct mlxsw_sp_rif **rifs;
  52         struct mlxsw_sp_vr *vrs;
  53         struct rhashtable neigh_ht;
  54         struct rhashtable nexthop_group_ht;
  55         struct rhashtable nexthop_ht;
  56         struct list_head nexthop_list;
  57         struct {
  58                 /* One tree for each protocol: IPv4 and IPv6 */
  59                 struct mlxsw_sp_lpm_tree *proto_trees[2];
  60                 struct mlxsw_sp_lpm_tree *trees;
  61                 unsigned int tree_count;
  62         } lpm;
  63         struct {
  64                 struct delayed_work dw;
  65                 unsigned long interval; /* ms */
  66         } neighs_update;
  67         struct delayed_work nexthop_probe_dw;
  68 #define MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL 5000 /* ms */
  69         struct list_head nexthop_neighs_list;
  70         struct list_head ipip_list;
  71         bool aborted;
  72         struct notifier_block fib_nb;
  73         struct notifier_block netevent_nb;
  74         struct notifier_block inetaddr_nb;
  75         struct notifier_block inet6addr_nb;
  76         const struct mlxsw_sp_rif_ops **rif_ops_arr;
  77         const struct mlxsw_sp_ipip_ops **ipip_ops_arr;
  78 };
  79
  80 struct mlxsw_sp_rif {
  81         struct list_head nexthop_list;
  82         struct list_head neigh_list;
  83         struct net_device *dev; /* NULL for underlay RIF */
  84         struct mlxsw_sp_fid *fid;
  85         unsigned char addr[ETH_ALEN];
  86         int mtu;
  87         u16 rif_index;
  88         u16 vr_id;
  89         const struct mlxsw_sp_rif_ops *ops;
  90         struct mlxsw_sp *mlxsw_sp;
  91
  92         unsigned int counter_ingress;
  93         bool counter_ingress_valid;
  94         unsigned int counter_egress;
  95         bool counter_egress_valid;
  96 };
  97
  98 struct mlxsw_sp_rif_params {
  99         struct net_device *dev;
 100         union {
 101                 u16 system_port;
 102                 u16 lag_id;
 103         };
 104         u16 vid;
 105         bool lag;
 106 };
 107
 108 struct mlxsw_sp_rif_subport {
 109         struct mlxsw_sp_rif common;
 110         refcount_t ref_count;
 111         union {
 112                 u16 system_port;
 113                 u16 lag_id;
 114         };
 115         u16 vid;
 116         bool lag;
 117 };
 118
 119 struct mlxsw_sp_rif_ipip_lb {
 120         struct mlxsw_sp_rif common;
 121         struct mlxsw_sp_rif_ipip_lb_config lb_config;
 122         u16 ul_vr_id; /* Reserved for Spectrum-2. */
 123         u16 ul_rif_id; /* Reserved for Spectrum. */
 124 };
 125
 126 struct mlxsw_sp_rif_params_ipip_lb {
 127         struct mlxsw_sp_rif_params common;
 128         struct mlxsw_sp_rif_ipip_lb_config lb_config;
 129 };
 130
 131 struct mlxsw_sp_rif_ops {
 132         enum mlxsw_sp_rif_type type;
 133         size_t rif_size;
 134
 135         void (*setup)(struct mlxsw_sp_rif *rif,
 136                       const struct mlxsw_sp_rif_params *params);
 137         int (*configure)(struct mlxsw_sp_rif *rif);
 138         void (*deconfigure)(struct mlxsw_sp_rif *rif);
 139         struct mlxsw_sp_fid * (*fid_get)(struct mlxsw_sp_rif *rif,
 140                                          struct netlink_ext_ack *extack);
 141         void (*fdb_del)(struct mlxsw_sp_rif *rif, const char *mac);
 142 };
 143
 144 static void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif);
 145 static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree);
 146 static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp,
 147                                   struct mlxsw_sp_lpm_tree *lpm_tree);
 148 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp,
 149                                      const struct mlxsw_sp_fib *fib,
 150                                      u8 tree_id);
 151 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp,
 152                                        const struct mlxsw_sp_fib *fib);
 153
 154 static unsigned int *
 155 mlxsw_sp_rif_p_counter_get(struct mlxsw_sp_rif *rif,
 156                            enum mlxsw_sp_rif_counter_dir dir)
 157 {
 158         switch (dir) {
 159         case MLXSW_SP_RIF_COUNTER_EGRESS:
 160                 return &rif->counter_egress;
 161         case MLXSW_SP_RIF_COUNTER_INGRESS:
 162                 return &rif->counter_ingress;
 163         }
 164         return NULL;
 165 }
 166
 167 static bool
 168 mlxsw_sp_rif_counter_valid_get(struct mlxsw_sp_rif *rif,
 169                                enum mlxsw_sp_rif_counter_dir dir)
 170 {
 171         switch (dir) {
 172         case MLXSW_SP_RIF_COUNTER_EGRESS:
 173                 return rif->counter_egress_valid;
 174         case MLXSW_SP_RIF_COUNTER_INGRESS:
 175                 return rif->counter_ingress_valid;
 176         }
 177         return false;
 178 }
 179
 180 static void
 181 mlxsw_sp_rif_counter_valid_set(struct mlxsw_sp_rif *rif,
 182                                enum mlxsw_sp_rif_counter_dir dir,
 183                                bool valid)
 184 {
 185         switch (dir) {
 186         case MLXSW_SP_RIF_COUNTER_EGRESS:
 187                 rif->counter_egress_valid = valid;
 188                 break;
 189         case MLXSW_SP_RIF_COUNTER_INGRESS:
 190                 rif->counter_ingress_valid = valid;
 191                 break;
 192         }
 193 }
 194
 195 static int mlxsw_sp_rif_counter_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
 196                                      unsigned int counter_index, bool enable,
 197                                      enum mlxsw_sp_rif_counter_dir dir)
 198 {
 199         char ritr_pl[MLXSW_REG_RITR_LEN];
 200         bool is_egress = false;
 201         int err;
 202
 203         if (dir == MLXSW_SP_RIF_COUNTER_EGRESS)
 204                 is_egress = true;
 205         mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
 206         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
 207         if (err)
 208                 return err;
 209
 210         mlxsw_reg_ritr_counter_pack(ritr_pl, counter_index, enable,
 211                                     is_egress);
 212         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
 213 }
 214
 215 int mlxsw_sp_rif_counter_value_get(struct mlxsw_sp *mlxsw_sp,
 216                                    struct mlxsw_sp_rif *rif,
 217                                    enum mlxsw_sp_rif_counter_dir dir, u64 *cnt)
 218 {
 219         char ricnt_pl[MLXSW_REG_RICNT_LEN];
 220         unsigned int *p_counter_index;
 221         bool valid;
 222         int err;
 223
 224         valid = mlxsw_sp_rif_counter_valid_get(rif, dir);
 225         if (!valid)
 226                 return -EINVAL;
 227
 228         p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
 229         if (!p_counter_index)
 230                 return -EINVAL;
 231         mlxsw_reg_ricnt_pack(ricnt_pl, *p_counter_index,
 232                              MLXSW_REG_RICNT_OPCODE_NOP);
 233         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl);
 234         if (err)
 235                 return err;
 236         *cnt = mlxsw_reg_ricnt_good_unicast_packets_get(ricnt_pl);
 237         return 0;
 238 }
 239
 240 static int mlxsw_sp_rif_counter_clear(struct mlxsw_sp *mlxsw_sp,
 241                                       unsigned int counter_index)
 242 {
 243         char ricnt_pl[MLXSW_REG_RICNT_LEN];
 244
 245         mlxsw_reg_ricnt_pack(ricnt_pl, counter_index,
 246                              MLXSW_REG_RICNT_OPCODE_CLEAR);
 247         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl);
 248 }
 249
 250 int mlxsw_sp_rif_counter_alloc(struct mlxsw_sp *mlxsw_sp,
 251                                struct mlxsw_sp_rif *rif,
 252                                enum mlxsw_sp_rif_counter_dir dir)
 253 {
 254         unsigned int *p_counter_index;
 255         int err;
 256
 257         p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
 258         if (!p_counter_index)
 259                 return -EINVAL;
 260         err = mlxsw_sp_counter_alloc(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
 261                                      p_counter_index);
 262         if (err)
 263                 return err;
 264
 265         err = mlxsw_sp_rif_counter_clear(mlxsw_sp, *p_counter_index);
 266         if (err)
 267                 goto err_counter_clear;
 268
 269         err = mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index,
 270                                         *p_counter_index, true, dir);
 271         if (err)
 272                 goto err_counter_edit;
 273         mlxsw_sp_rif_counter_valid_set(rif, dir, true);
 274         return 0;
 275
 276 err_counter_edit:
 277 err_counter_clear:
 278         mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
 279                               *p_counter_index);
 280         return err;
 281 }
 282
 283 void mlxsw_sp_rif_counter_free(struct mlxsw_sp *mlxsw_sp,
 284                                struct mlxsw_sp_rif *rif,
 285                                enum mlxsw_sp_rif_counter_dir dir)
 286 {
 287         unsigned int *p_counter_index;
 288
 289         if (!mlxsw_sp_rif_counter_valid_get(rif, dir))
 290                 return;
 291
 292         p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
 293         if (WARN_ON(!p_counter_index))
 294                 return;
 295         mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index,
 296                                   *p_counter_index, false, dir);
 297         mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
 298                               *p_counter_index);
 299         mlxsw_sp_rif_counter_valid_set(rif, dir, false);
 300 }
 301
 302 static void mlxsw_sp_rif_counters_alloc(struct mlxsw_sp_rif *rif)
 303 {
 304         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
 305         struct devlink *devlink;
 306
 307         devlink = priv_to_devlink(mlxsw_sp->core);
 308         if (!devlink_dpipe_table_counter_enabled(devlink,
 309                                                  MLXSW_SP_DPIPE_TABLE_NAME_ERIF))
 310                 return;
 311         mlxsw_sp_rif_counter_alloc(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS);
 312 }
 313
 314 static void mlxsw_sp_rif_counters_free(struct mlxsw_sp_rif *rif)
 315 {
 316         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
 317
 318         mlxsw_sp_rif_counter_free(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS);
 319 }
 320
 321 #define MLXSW_SP_PREFIX_COUNT (sizeof(struct in6_addr) * BITS_PER_BYTE + 1)
 322
 323 struct mlxsw_sp_prefix_usage {
 324         DECLARE_BITMAP(b, MLXSW_SP_PREFIX_COUNT);
 325 };
 326
 327 #define mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) \
 328         for_each_set_bit(prefix, (prefix_usage)->b, MLXSW_SP_PREFIX_COUNT)
 329
 330 static bool
 331 mlxsw_sp_prefix_usage_eq(struct mlxsw_sp_prefix_usage *prefix_usage1,
 332                          struct mlxsw_sp_prefix_usage *prefix_usage2)
 333 {
 334         return !memcmp(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
 335 }
 336
 337 static void
 338 mlxsw_sp_prefix_usage_cpy(struct mlxsw_sp_prefix_usage *prefix_usage1,
 339                           struct mlxsw_sp_prefix_usage *prefix_usage2)
 340 {
 341         memcpy(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
 342 }
 343
 344 static void
 345 mlxsw_sp_prefix_usage_set(struct mlxsw_sp_prefix_usage *prefix_usage,
 346                           unsigned char prefix_len)
 347 {
 348         set_bit(prefix_len, prefix_usage->b);
 349 }
 350
 351 static void
 352 mlxsw_sp_prefix_usage_clear(struct mlxsw_sp_prefix_usage *prefix_usage,
 353                             unsigned char prefix_len)
 354 {
 355         clear_bit(prefix_len, prefix_usage->b);
 356 }
 357
 358 struct mlxsw_sp_fib_key {
 359         unsigned char addr[sizeof(struct in6_addr)];
 360         unsigned char prefix_len;
 361 };
 362
 363 enum mlxsw_sp_fib_entry_type {
 364         MLXSW_SP_FIB_ENTRY_TYPE_REMOTE,
 365         MLXSW_SP_FIB_ENTRY_TYPE_LOCAL,
 366         MLXSW_SP_FIB_ENTRY_TYPE_TRAP,
 367         MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE,
 368
 369         /* This is a special case of local delivery, where a packet should be
 370          * decapsulated on reception. Note that there is no corresponding ENCAP,
 371          * because that's a type of next hop, not of FIB entry. (There can be
 372          * several next hops in a REMOTE entry, and some of them may be
 373          * encapsulating entries.)
 374          */
 375         MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP,
 376         MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP,
 377 };
 378
 379 struct mlxsw_sp_nexthop_group;
 380
 381 struct mlxsw_sp_fib_node {
 382         struct list_head entry_list;
 383         struct list_head list;
 384         struct rhash_head ht_node;
 385         struct mlxsw_sp_fib *fib;
 386         struct mlxsw_sp_fib_key key;
 387 };
 388
 389 struct mlxsw_sp_fib_entry_decap {
 390         struct mlxsw_sp_ipip_entry *ipip_entry;
 391         u32 tunnel_index;
 392 };
 393
 394 struct mlxsw_sp_fib_entry {
 395         struct list_head list;
 396         struct mlxsw_sp_fib_node *fib_node;
 397         enum mlxsw_sp_fib_entry_type type;
 398         struct list_head nexthop_group_node;
 399         struct mlxsw_sp_nexthop_group *nh_group;
 400         struct mlxsw_sp_fib_entry_decap decap; /* Valid for decap entries. */
 401 };
 402
 403 struct mlxsw_sp_fib4_entry {
 404         struct mlxsw_sp_fib_entry common;
 405         u32 tb_id;
 406         u32 prio;
 407         u8 tos;
 408         u8 type;
 409 };
 410
 411 struct mlxsw_sp_fib6_entry {
 412         struct mlxsw_sp_fib_entry common;
 413         struct list_head rt6_list;
 414         unsigned int nrt6;
 415 };
 416
 417 struct mlxsw_sp_rt6 {
 418         struct list_head list;
 419         struct fib6_info *rt;
 420 };
 421
 422 struct mlxsw_sp_lpm_tree {
 423         u8 id; /* tree ID */
 424         unsigned int ref_count;
 425         enum mlxsw_sp_l3proto proto;
 426         unsigned long prefix_ref_count[MLXSW_SP_PREFIX_COUNT];
 427         struct mlxsw_sp_prefix_usage prefix_usage;
 428 };
 429
 430 struct mlxsw_sp_fib {
 431         struct rhashtable ht;
 432         struct list_head node_list;
 433         struct mlxsw_sp_vr *vr;
 434         struct mlxsw_sp_lpm_tree *lpm_tree;
 435         enum mlxsw_sp_l3proto proto;
 436 };
 437
 438 struct mlxsw_sp_vr {
 439         u16 id; /* virtual router ID */
 440         u32 tb_id; /* kernel fib table id */
 441         unsigned int rif_count;
 442         struct mlxsw_sp_fib *fib4;
 443         struct mlxsw_sp_fib *fib6;
 444         struct mlxsw_sp_mr_table *mr_table[MLXSW_SP_L3_PROTO_MAX];
 445         struct mlxsw_sp_rif *ul_rif;
 446         refcount_t ul_rif_refcnt;
 447 };
 448
 449 static const struct rhashtable_params mlxsw_sp_fib_ht_params;
 450
 451 static struct mlxsw_sp_fib *mlxsw_sp_fib_create(struct mlxsw_sp *mlxsw_sp,
 452                                                 struct mlxsw_sp_vr *vr,
 453                                                 enum mlxsw_sp_l3proto proto)
 454 {
 455         struct mlxsw_sp_lpm_tree *lpm_tree;
 456         struct mlxsw_sp_fib *fib;
 457         int err;
 458
 459         lpm_tree = mlxsw_sp->router->lpm.proto_trees[proto];
 460         fib = kzalloc(sizeof(*fib), GFP_KERNEL);
 461         if (!fib)
 462                 return ERR_PTR(-ENOMEM);
 463         err = rhashtable_init(&fib->ht, &mlxsw_sp_fib_ht_params);
 464         if (err)
 465                 goto err_rhashtable_init;
 466         INIT_LIST_HEAD(&fib->node_list);
 467         fib->proto = proto;
 468         fib->vr = vr;
 469         fib->lpm_tree = lpm_tree;
 470         mlxsw_sp_lpm_tree_hold(lpm_tree);
 471         err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, lpm_tree->id);
 472         if (err)
 473                 goto err_lpm_tree_bind;
 474         return fib;
 475
 476 err_lpm_tree_bind:
 477         mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
 478 err_rhashtable_init:
 479         kfree(fib);
 480         return ERR_PTR(err);
 481 }
 482
 483 static void mlxsw_sp_fib_destroy(struct mlxsw_sp *mlxsw_sp,
 484                                  struct mlxsw_sp_fib *fib)
 485 {
 486         mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, fib);
 487         mlxsw_sp_lpm_tree_put(mlxsw_sp, fib->lpm_tree);
 488         WARN_ON(!list_empty(&fib->node_list));
 489         rhashtable_destroy(&fib->ht);
 490         kfree(fib);
 491 }
 492
 493 static struct mlxsw_sp_lpm_tree *
 494 mlxsw_sp_lpm_tree_find_unused(struct mlxsw_sp *mlxsw_sp)
 495 {
 496         static struct mlxsw_sp_lpm_tree *lpm_tree;
 497         int i;
 498
 499         for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
 500                 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
 501                 if (lpm_tree->ref_count == 0)
 502                         return lpm_tree;
 503         }
 504         return NULL;
 505 }
 506
 507 static int mlxsw_sp_lpm_tree_alloc(struct mlxsw_sp *mlxsw_sp,
 508                                    struct mlxsw_sp_lpm_tree *lpm_tree)
 509 {
 510         char ralta_pl[MLXSW_REG_RALTA_LEN];
 511
 512         mlxsw_reg_ralta_pack(ralta_pl, true,
 513                              (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
 514                              lpm_tree->id);
 515         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
 516 }
 517
 518 static void mlxsw_sp_lpm_tree_free(struct mlxsw_sp *mlxsw_sp,
 519                                    struct mlxsw_sp_lpm_tree *lpm_tree)
 520 {
 521         char ralta_pl[MLXSW_REG_RALTA_LEN];
 522
 523         mlxsw_reg_ralta_pack(ralta_pl, false,
 524                              (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
 525                              lpm_tree->id);
 526         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
 527 }
 528
 529 static int
 530 mlxsw_sp_lpm_tree_left_struct_set(struct mlxsw_sp *mlxsw_sp,
 531                                   struct mlxsw_sp_prefix_usage *prefix_usage,
 532                                   struct mlxsw_sp_lpm_tree *lpm_tree)
 533 {
 534         char ralst_pl[MLXSW_REG_RALST_LEN];
 535         u8 root_bin = 0;
 536         u8 prefix;
 537         u8 last_prefix = MLXSW_REG_RALST_BIN_NO_CHILD;
 538
 539         mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage)
 540                 root_bin = prefix;
 541
 542         mlxsw_reg_ralst_pack(ralst_pl, root_bin, lpm_tree->id);
 543         mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) {
 544                 if (prefix == 0)
 545                         continue;
 546                 mlxsw_reg_ralst_bin_pack(ralst_pl, prefix, last_prefix,
 547                                          MLXSW_REG_RALST_BIN_NO_CHILD);
 548                 last_prefix = prefix;
 549         }
 550         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
 551 }
 552
 553 static struct mlxsw_sp_lpm_tree *
 554 mlxsw_sp_lpm_tree_create(struct mlxsw_sp *mlxsw_sp,
 555                          struct mlxsw_sp_prefix_usage *prefix_usage,
 556                          enum mlxsw_sp_l3proto proto)
 557 {
 558         struct mlxsw_sp_lpm_tree *lpm_tree;
 559         int err;
 560
 561         lpm_tree = mlxsw_sp_lpm_tree_find_unused(mlxsw_sp);
 562         if (!lpm_tree)
 563                 return ERR_PTR(-EBUSY);
 564         lpm_tree->proto = proto;
 565         err = mlxsw_sp_lpm_tree_alloc(mlxsw_sp, lpm_tree);
 566         if (err)
 567                 return ERR_PTR(err);
 568
 569         err = mlxsw_sp_lpm_tree_left_struct_set(mlxsw_sp, prefix_usage,
 570                                                 lpm_tree);
 571         if (err)
 572                 goto err_left_struct_set;
 573         memcpy(&lpm_tree->prefix_usage, prefix_usage,
 574                sizeof(lpm_tree->prefix_usage));
 575         memset(&lpm_tree->prefix_ref_count, 0,
 576                sizeof(lpm_tree->prefix_ref_count));
 577         lpm_tree->ref_count = 1;
 578         return lpm_tree;
 579
 580 err_left_struct_set:
 581         mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
 582         return ERR_PTR(err);
 583 }
 584
 585 static void mlxsw_sp_lpm_tree_destroy(struct mlxsw_sp *mlxsw_sp,
 586                                       struct mlxsw_sp_lpm_tree *lpm_tree)
 587 {
 588         mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
 589 }
 590
 591 static struct mlxsw_sp_lpm_tree *
 592 mlxsw_sp_lpm_tree_get(struct mlxsw_sp *mlxsw_sp,
 593                       struct mlxsw_sp_prefix_usage *prefix_usage,
 594                       enum mlxsw_sp_l3proto proto)
 595 {
 596         struct mlxsw_sp_lpm_tree *lpm_tree;
 597         int i;
 598
 599         for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
 600                 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
 601                 if (lpm_tree->ref_count != 0 &&
 602                     lpm_tree->proto == proto &&
 603                     mlxsw_sp_prefix_usage_eq(&lpm_tree->prefix_usage,
 604                                              prefix_usage)) {
 605                         mlxsw_sp_lpm_tree_hold(lpm_tree);
 606                         return lpm_tree;
 607                 }
 608         }
 609         return mlxsw_sp_lpm_tree_create(mlxsw_sp, prefix_usage, proto);
 610 }
 611
 612 static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree)
 613 {
 614         lpm_tree->ref_count++;
 615 }
 616
 617 static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp,
 618                                   struct mlxsw_sp_lpm_tree *lpm_tree)
 619 {
 620         if (--lpm_tree->ref_count == 0)
 621                 mlxsw_sp_lpm_tree_destroy(mlxsw_sp, lpm_tree);
 622 }
 623
 624 #define MLXSW_SP_LPM_TREE_MIN 1 /* tree 0 is reserved */
 625
 626 static int mlxsw_sp_lpm_init(struct mlxsw_sp *mlxsw_sp)
 627 {
 628         struct mlxsw_sp_prefix_usage req_prefix_usage = {{ 0 } };
 629         struct mlxsw_sp_lpm_tree *lpm_tree;
 630         u64 max_trees;
 631         int err, i;
 632
 633         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_LPM_TREES))
 634                 return -EIO;
 635
 636         max_trees = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_LPM_TREES);
 637         mlxsw_sp->router->lpm.tree_count = max_trees - MLXSW_SP_LPM_TREE_MIN;
 638         mlxsw_sp->router->lpm.trees = kcalloc(mlxsw_sp->router->lpm.tree_count,
 639                                              sizeof(struct mlxsw_sp_lpm_tree),
 640                                              GFP_KERNEL);
 641         if (!mlxsw_sp->router->lpm.trees)
 642                 return -ENOMEM;
 643
 644         for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
 645                 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
 646                 lpm_tree->id = i + MLXSW_SP_LPM_TREE_MIN;
 647         }
 648
 649         lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
 650                                          MLXSW_SP_L3_PROTO_IPV4);
 651         if (IS_ERR(lpm_tree)) {
 652                 err = PTR_ERR(lpm_tree);
 653                 goto err_ipv4_tree_get;
 654         }
 655         mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4] = lpm_tree;
 656
 657         lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
 658                                          MLXSW_SP_L3_PROTO_IPV6);
 659         if (IS_ERR(lpm_tree)) {
 660                 err = PTR_ERR(lpm_tree);
 661                 goto err_ipv6_tree_get;
 662         }
 663         mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV6] = lpm_tree;
 664
 665         return 0;
 666
 667 err_ipv6_tree_get:
 668         lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4];
 669         mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
 670 err_ipv4_tree_get:
 671         kfree(mlxsw_sp->router->lpm.trees);
 672         return err;
 673 }
 674
 675 static void mlxsw_sp_lpm_fini(struct mlxsw_sp *mlxsw_sp)
 676 {
 677         struct mlxsw_sp_lpm_tree *lpm_tree;
 678
 679         lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV6];
 680         mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
 681
 682         lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4];
 683         mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
 684
 685         kfree(mlxsw_sp->router->lpm.trees);
 686 }
 687
 688 static bool mlxsw_sp_vr_is_used(const struct mlxsw_sp_vr *vr)
 689 {
 690         return !!vr->fib4 || !!vr->fib6 ||
 691                !!vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] ||
 692                !!vr->mr_table[MLXSW_SP_L3_PROTO_IPV6];
 693 }
 694
 695 static struct mlxsw_sp_vr *mlxsw_sp_vr_find_unused(struct mlxsw_sp *mlxsw_sp)
 696 {
 697         struct mlxsw_sp_vr *vr;
 698         int i;
 699
 700         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
 701                 vr = &mlxsw_sp->router->vrs[i];
 702                 if (!mlxsw_sp_vr_is_used(vr))
 703                         return vr;
 704         }
 705         return NULL;
 706 }
 707
 708 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp,
 709                                      const struct mlxsw_sp_fib *fib, u8 tree_id)
 710 {
 711         char raltb_pl[MLXSW_REG_RALTB_LEN];
 712
 713         mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id,
 714                              (enum mlxsw_reg_ralxx_protocol) fib->proto,
 715                              tree_id);
 716         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
 717 }
 718
 719 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp,
 720                                        const struct mlxsw_sp_fib *fib)
 721 {
 722         char raltb_pl[MLXSW_REG_RALTB_LEN];
 723
 724         /* Bind to tree 0 which is default */
 725         mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id,
 726                              (enum mlxsw_reg_ralxx_protocol) fib->proto, 0);
 727         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
 728 }
 729
 730 static u32 mlxsw_sp_fix_tb_id(u32 tb_id)
 731 {
 732         /* For our purpose, squash main, default and local tables into one */
 733         if (tb_id == RT_TABLE_LOCAL || tb_id == RT_TABLE_DEFAULT)
 734                 tb_id = RT_TABLE_MAIN;
 735         return tb_id;
 736 }
 737
 738 static struct mlxsw_sp_vr *mlxsw_sp_vr_find(struct mlxsw_sp *mlxsw_sp,
 739                                             u32 tb_id)
 740 {
 741         struct mlxsw_sp_vr *vr;
 742         int i;
 743
 744         tb_id = mlxsw_sp_fix_tb_id(tb_id);
 745
 746         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
 747                 vr = &mlxsw_sp->router->vrs[i];
 748                 if (mlxsw_sp_vr_is_used(vr) && vr->tb_id == tb_id)
 749                         return vr;
 750         }
 751         return NULL;
 752 }
 753
 754 int mlxsw_sp_router_tb_id_vr_id(struct mlxsw_sp *mlxsw_sp, u32 tb_id,
 755                                 u16 *vr_id)
 756 {
 757         struct mlxsw_sp_vr *vr;
 758
 759         vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id);
 760         if (!vr)
 761                 return -ESRCH;
 762         *vr_id = vr->id;
 763
 764         return 0;
 765 }
 766
 767 static struct mlxsw_sp_fib *mlxsw_sp_vr_fib(const struct mlxsw_sp_vr *vr,
 768                                             enum mlxsw_sp_l3proto proto)
 769 {
 770         switch (proto) {
 771         case MLXSW_SP_L3_PROTO_IPV4:
 772                 return vr->fib4;
 773         case MLXSW_SP_L3_PROTO_IPV6:
 774                 return vr->fib6;
 775         }
 776         return NULL;
 777 }
 778
 779 static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp,
 780                                               u32 tb_id,
 781                                               struct netlink_ext_ack *extack)
 782 {
 783         struct mlxsw_sp_mr_table *mr4_table, *mr6_table;
 784         struct mlxsw_sp_fib *fib4;
 785         struct mlxsw_sp_fib *fib6;
 786         struct mlxsw_sp_vr *vr;
 787         int err;
 788
 789         vr = mlxsw_sp_vr_find_unused(mlxsw_sp);
 790         if (!vr) {
 791                 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported virtual routers");
 792                 return ERR_PTR(-EBUSY);
 793         }
 794         fib4 = mlxsw_sp_fib_create(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4);
 795         if (IS_ERR(fib4))
 796                 return ERR_CAST(fib4);
 797         fib6 = mlxsw_sp_fib_create(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6);
 798         if (IS_ERR(fib6)) {
 799                 err = PTR_ERR(fib6);
 800                 goto err_fib6_create;
 801         }
 802         mr4_table = mlxsw_sp_mr_table_create(mlxsw_sp, vr->id,
 803                                              MLXSW_SP_L3_PROTO_IPV4);
 804         if (IS_ERR(mr4_table)) {
 805                 err = PTR_ERR(mr4_table);
 806                 goto err_mr4_table_create;
 807         }
 808         mr6_table = mlxsw_sp_mr_table_create(mlxsw_sp, vr->id,
 809                                              MLXSW_SP_L3_PROTO_IPV6);
 810         if (IS_ERR(mr6_table)) {
 811                 err = PTR_ERR(mr6_table);
 812                 goto err_mr6_table_create;
 813         }
 814
 815         vr->fib4 = fib4;
 816         vr->fib6 = fib6;
 817         vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] = mr4_table;
 818         vr->mr_table[MLXSW_SP_L3_PROTO_IPV6] = mr6_table;
 819         vr->tb_id = tb_id;
 820         return vr;
 821
 822 err_mr6_table_create:
 823         mlxsw_sp_mr_table_destroy(mr4_table);
 824 err_mr4_table_create:
 825         mlxsw_sp_fib_destroy(mlxsw_sp, fib6);
 826 err_fib6_create:
 827         mlxsw_sp_fib_destroy(mlxsw_sp, fib4);
 828         return ERR_PTR(err);
 829 }
 830
 831 static void mlxsw_sp_vr_destroy(struct mlxsw_sp *mlxsw_sp,
 832                                 struct mlxsw_sp_vr *vr)
 833 {
 834         mlxsw_sp_mr_table_destroy(vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]);
 835         vr->mr_table[MLXSW_SP_L3_PROTO_IPV6] = NULL;
 836         mlxsw_sp_mr_table_destroy(vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]);
 837         vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] = NULL;
 838         mlxsw_sp_fib_destroy(mlxsw_sp, vr->fib6);
 839         vr->fib6 = NULL;
 840         mlxsw_sp_fib_destroy(mlxsw_sp, vr->fib4);
 841         vr->fib4 = NULL;
 842 }
 843
 844 static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id,
 845                                            struct netlink_ext_ack *extack)
 846 {
 847         struct mlxsw_sp_vr *vr;
 848
 849         tb_id = mlxsw_sp_fix_tb_id(tb_id);
 850         vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id);
 851         if (!vr)
 852                 vr = mlxsw_sp_vr_create(mlxsw_sp, tb_id, extack);
 853         return vr;
 854 }
 855
 856 static void mlxsw_sp_vr_put(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr)
 857 {
 858         if (!vr->rif_count && list_empty(&vr->fib4->node_list) &&
 859             list_empty(&vr->fib6->node_list) &&
 860             mlxsw_sp_mr_table_empty(vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]) &&
 861             mlxsw_sp_mr_table_empty(vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]))
 862                 mlxsw_sp_vr_destroy(mlxsw_sp, vr);
 863 }
 864
 865 static bool
 866 mlxsw_sp_vr_lpm_tree_should_replace(struct mlxsw_sp_vr *vr,
 867                                     enum mlxsw_sp_l3proto proto, u8 tree_id)
 868 {
 869         struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
 870
 871         if (!mlxsw_sp_vr_is_used(vr))
 872                 return false;
 873         if (fib->lpm_tree->id == tree_id)
 874                 return true;
 875         return false;
 876 }
 877
 878 static int mlxsw_sp_vr_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp,
 879                                         struct mlxsw_sp_fib *fib,
 880                                         struct mlxsw_sp_lpm_tree *new_tree)
 881 {
 882         struct mlxsw_sp_lpm_tree *old_tree = fib->lpm_tree;
 883         int err;
 884
 885         fib->lpm_tree = new_tree;
 886         mlxsw_sp_lpm_tree_hold(new_tree);
 887         err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, new_tree->id);
 888         if (err)
 889                 goto err_tree_bind;
 890         mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree);
 891         return 0;
 892
 893 err_tree_bind:
 894         mlxsw_sp_lpm_tree_put(mlxsw_sp, new_tree);
 895         fib->lpm_tree = old_tree;
 896         return err;
 897 }
 898
 899 static int mlxsw_sp_vrs_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp,
 900                                          struct mlxsw_sp_fib *fib,
 901                                          struct mlxsw_sp_lpm_tree *new_tree)
 902 {
 903         enum mlxsw_sp_l3proto proto = fib->proto;
 904         struct mlxsw_sp_lpm_tree *old_tree;
 905         u8 old_id, new_id = new_tree->id;
 906         struct mlxsw_sp_vr *vr;
 907         int i, err;
 908
 909         old_tree = mlxsw_sp->router->lpm.proto_trees[proto];
 910         old_id = old_tree->id;
 911
 912         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
 913                 vr = &mlxsw_sp->router->vrs[i];
 914                 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, old_id))
 915                         continue;
 916                 err = mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp,
 917                                                    mlxsw_sp_vr_fib(vr, proto),
 918                                                    new_tree);
 919                 if (err)
 920                         goto err_tree_replace;
 921         }
 922
 923         memcpy(new_tree->prefix_ref_count, old_tree->prefix_ref_count,
 924                sizeof(new_tree->prefix_ref_count));
 925         mlxsw_sp->router->lpm.proto_trees[proto] = new_tree;
 926         mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree);
 927
 928         return 0;
 929
 930 err_tree_replace:
 931         for (i--; i >= 0; i--) {
 932                 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, new_id))
 933                         continue;
 934                 mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp,
 935                                              mlxsw_sp_vr_fib(vr, proto),
 936                                              old_tree);
 937         }
 938         return err;
 939 }
 940
 941 static int mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp)
 942 {
 943         struct mlxsw_sp_vr *vr;
 944         u64 max_vrs;
 945         int i;
 946
 947         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_VRS))
 948                 return -EIO;
 949
 950         max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS);
 951         mlxsw_sp->router->vrs = kcalloc(max_vrs, sizeof(struct mlxsw_sp_vr),
 952                                         GFP_KERNEL);
 953         if (!mlxsw_sp->router->vrs)
 954                 return -ENOMEM;
 955
 956         for (i = 0; i < max_vrs; i++) {
 957                 vr = &mlxsw_sp->router->vrs[i];
 958                 vr->id = i;
 959         }
 960
 961         return 0;
 962 }
 963
 964 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp);
 965
 966 static void mlxsw_sp_vrs_fini(struct mlxsw_sp *mlxsw_sp)
 967 {
 968         /* At this stage we're guaranteed not to have new incoming
 969          * FIB notifications and the work queue is free from FIBs
 970          * sitting on top of mlxsw netdevs. However, we can still
 971          * have other FIBs queued. Flush the queue before flushing
 972          * the device's tables. No need for locks, as we're the only
 973          * writer.
 974          */
 975         mlxsw_core_flush_owq();
 976         mlxsw_sp_router_fib_flush(mlxsw_sp);
 977         kfree(mlxsw_sp->router->vrs);
 978 }
 979
 980 static struct net_device *
 981 __mlxsw_sp_ipip_netdev_ul_dev_get(const struct net_device *ol_dev)
 982 {
 983         struct ip_tunnel *tun = netdev_priv(ol_dev);
 984         struct net *net = dev_net(ol_dev);
 985
 986         return __dev_get_by_index(net, tun->parms.link);
 987 }
 988
 989 u32 mlxsw_sp_ipip_dev_ul_tb_id(const struct net_device *ol_dev)
 990 {
 991         struct net_device *d = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev);
 992
 993         if (d)
 994                 return l3mdev_fib_table(d) ? : RT_TABLE_MAIN;
 995         else
 996                 return l3mdev_fib_table(ol_dev) ? : RT_TABLE_MAIN;
 997 }
 998
 999 static struct mlxsw_sp_rif *
1000 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp,
1001                     const struct mlxsw_sp_rif_params *params,
1002                     struct netlink_ext_ack *extack);
1003
1004 static struct mlxsw_sp_rif_ipip_lb *
1005 mlxsw_sp_ipip_ol_ipip_lb_create(struct mlxsw_sp *mlxsw_sp,
1006                                 enum mlxsw_sp_ipip_type ipipt,
1007                                 struct net_device *ol_dev,
1008                                 struct netlink_ext_ack *extack)
1009 {
1010         struct mlxsw_sp_rif_params_ipip_lb lb_params;
1011         const struct mlxsw_sp_ipip_ops *ipip_ops;
1012         struct mlxsw_sp_rif *rif;
1013
1014         ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt];
1015         lb_params = (struct mlxsw_sp_rif_params_ipip_lb) {
1016                 .common.dev = ol_dev,
1017                 .common.lag = false,
1018                 .lb_config = ipip_ops->ol_loopback_config(mlxsw_sp, ol_dev),
1019         };
1020
1021         rif = mlxsw_sp_rif_create(mlxsw_sp, &lb_params.common, extack);
1022         if (IS_ERR(rif))
1023                 return ERR_CAST(rif);
1024         return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common);
1025 }
1026
1027 static struct mlxsw_sp_ipip_entry *
1028 mlxsw_sp_ipip_entry_alloc(struct mlxsw_sp *mlxsw_sp,
1029                           enum mlxsw_sp_ipip_type ipipt,
1030                           struct net_device *ol_dev)
1031 {
1032         const struct mlxsw_sp_ipip_ops *ipip_ops;
1033         struct mlxsw_sp_ipip_entry *ipip_entry;
1034         struct mlxsw_sp_ipip_entry *ret = NULL;
1035
1036         ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt];
1037         ipip_entry = kzalloc(sizeof(*ipip_entry), GFP_KERNEL);
1038         if (!ipip_entry)
1039                 return ERR_PTR(-ENOMEM);
1040
1041         ipip_entry->ol_lb = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp, ipipt,
1042                                                             ol_dev, NULL);
1043         if (IS_ERR(ipip_entry->ol_lb)) {
1044                 ret = ERR_CAST(ipip_entry->ol_lb);
1045                 goto err_ol_ipip_lb_create;
1046         }
1047
1048         ipip_entry->ipipt = ipipt;
1049         ipip_entry->ol_dev = ol_dev;
1050
1051         switch (ipip_ops->ul_proto) {
1052         case MLXSW_SP_L3_PROTO_IPV4:
1053                 ipip_entry->parms4 = mlxsw_sp_ipip_netdev_parms4(ol_dev);
1054                 break;
1055         case MLXSW_SP_L3_PROTO_IPV6:
1056                 WARN_ON(1);
1057                 break;
1058         }
1059
1060         return ipip_entry;
1061
1062 err_ol_ipip_lb_create:
1063         kfree(ipip_entry);
1064         return ret;
1065 }
1066
1067 static void
1068 mlxsw_sp_ipip_entry_dealloc(struct mlxsw_sp_ipip_entry *ipip_entry)
1069 {
1070         mlxsw_sp_rif_destroy(&ipip_entry->ol_lb->common);
1071         kfree(ipip_entry);
1072 }
1073
1074 static bool
1075 mlxsw_sp_ipip_entry_saddr_matches(struct mlxsw_sp *mlxsw_sp,
1076                                   const enum mlxsw_sp_l3proto ul_proto,
1077                                   union mlxsw_sp_l3addr saddr,
1078                                   u32 ul_tb_id,
1079                                   struct mlxsw_sp_ipip_entry *ipip_entry)
1080 {
1081         u32 tun_ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev);
1082         enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt;
1083         union mlxsw_sp_l3addr tun_saddr;
1084
1085         if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto)
1086                 return false;
1087
1088         tun_saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ipip_entry->ol_dev);
1089         return tun_ul_tb_id == ul_tb_id &&
1090                mlxsw_sp_l3addr_eq(&tun_saddr, &saddr);
1091 }
1092
1093 static int
1094 mlxsw_sp_fib_entry_decap_init(struct mlxsw_sp *mlxsw_sp,
1095                               struct mlxsw_sp_fib_entry *fib_entry,
1096                               struct mlxsw_sp_ipip_entry *ipip_entry)
1097 {
1098         u32 tunnel_index;
1099         int err;
1100
1101         err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
1102                                   1, &tunnel_index);
1103         if (err)
1104                 return err;
1105
1106         ipip_entry->decap_fib_entry = fib_entry;
1107         fib_entry->decap.ipip_entry = ipip_entry;
1108         fib_entry->decap.tunnel_index = tunnel_index;
1109         return 0;
1110 }
1111
1112 static void mlxsw_sp_fib_entry_decap_fini(struct mlxsw_sp *mlxsw_sp,
1113                                           struct mlxsw_sp_fib_entry *fib_entry)
1114 {
1115         /* Unlink this node from the IPIP entry that it's the decap entry of. */
1116         fib_entry->decap.ipip_entry->decap_fib_entry = NULL;
1117         fib_entry->decap.ipip_entry = NULL;
1118         mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
1119                            1, fib_entry->decap.tunnel_index);
1120 }
1121
1122 static struct mlxsw_sp_fib_node *
1123 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr,
1124                          size_t addr_len, unsigned char prefix_len);
1125 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
1126                                      struct mlxsw_sp_fib_entry *fib_entry);
1127
1128 static void
1129 mlxsw_sp_ipip_entry_demote_decap(struct mlxsw_sp *mlxsw_sp,
1130                                  struct mlxsw_sp_ipip_entry *ipip_entry)
1131 {
1132         struct mlxsw_sp_fib_entry *fib_entry = ipip_entry->decap_fib_entry;
1133
1134         mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, fib_entry);
1135         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
1136
1137         mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
1138 }
1139
1140 static void
1141 mlxsw_sp_ipip_entry_promote_decap(struct mlxsw_sp *mlxsw_sp,
1142                                   struct mlxsw_sp_ipip_entry *ipip_entry,
1143                                   struct mlxsw_sp_fib_entry *decap_fib_entry)
1144 {
1145         if (mlxsw_sp_fib_entry_decap_init(mlxsw_sp, decap_fib_entry,
1146                                           ipip_entry))
1147                 return;
1148         decap_fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP;
1149
1150         if (mlxsw_sp_fib_entry_update(mlxsw_sp, decap_fib_entry))
1151                 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry);
1152 }
1153
1154 static struct mlxsw_sp_fib_entry *
1155 mlxsw_sp_router_ip2me_fib_entry_find(struct mlxsw_sp *mlxsw_sp, u32 tb_id,
1156                                      enum mlxsw_sp_l3proto proto,
1157                                      const union mlxsw_sp_l3addr *addr,
1158                                      enum mlxsw_sp_fib_entry_type type)
1159 {
1160         struct mlxsw_sp_fib_entry *fib_entry;
1161         struct mlxsw_sp_fib_node *fib_node;
1162         unsigned char addr_prefix_len;
1163         struct mlxsw_sp_fib *fib;
1164         struct mlxsw_sp_vr *vr;
1165         const void *addrp;
1166         size_t addr_len;
1167         u32 addr4;
1168
1169         vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id);
1170         if (!vr)
1171                 return NULL;
1172         fib = mlxsw_sp_vr_fib(vr, proto);
1173
1174         switch (proto) {
1175         case MLXSW_SP_L3_PROTO_IPV4:
1176                 addr4 = be32_to_cpu(addr->addr4);
1177                 addrp = &addr4;
1178                 addr_len = 4;
1179                 addr_prefix_len = 32;
1180                 break;
1181         case MLXSW_SP_L3_PROTO_IPV6: /* fall through */
1182         default:
1183                 WARN_ON(1);
1184                 return NULL;
1185         }
1186
1187         fib_node = mlxsw_sp_fib_node_lookup(fib, addrp, addr_len,
1188                                             addr_prefix_len);
1189         if (!fib_node || list_empty(&fib_node->entry_list))
1190                 return NULL;
1191
1192         fib_entry = list_first_entry(&fib_node->entry_list,
1193                                      struct mlxsw_sp_fib_entry, list);
1194         if (fib_entry->type != type)
1195                 return NULL;
1196
1197         return fib_entry;
1198 }
1199
1200 /* Given an IPIP entry, find the corresponding decap route. */
1201 static struct mlxsw_sp_fib_entry *
1202 mlxsw_sp_ipip_entry_find_decap(struct mlxsw_sp *mlxsw_sp,
1203                                struct mlxsw_sp_ipip_entry *ipip_entry)
1204 {
1205         static struct mlxsw_sp_fib_node *fib_node;
1206         const struct mlxsw_sp_ipip_ops *ipip_ops;
1207         struct mlxsw_sp_fib_entry *fib_entry;
1208         unsigned char saddr_prefix_len;
1209         union mlxsw_sp_l3addr saddr;
1210         struct mlxsw_sp_fib *ul_fib;
1211         struct mlxsw_sp_vr *ul_vr;
1212         const void *saddrp;
1213         size_t saddr_len;
1214         u32 ul_tb_id;
1215         u32 saddr4;
1216
1217         ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
1218
1219         ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev);
1220         ul_vr = mlxsw_sp_vr_find(mlxsw_sp, ul_tb_id);
1221         if (!ul_vr)
1222                 return NULL;
1223
1224         ul_fib = mlxsw_sp_vr_fib(ul_vr, ipip_ops->ul_proto);
1225         saddr = mlxsw_sp_ipip_netdev_saddr(ipip_ops->ul_proto,
1226                                            ipip_entry->ol_dev);
1227
1228         switch (ipip_ops->ul_proto) {
1229         case MLXSW_SP_L3_PROTO_IPV4:
1230                 saddr4 = be32_to_cpu(saddr.addr4);
1231                 saddrp = &saddr4;
1232                 saddr_len = 4;
1233                 saddr_prefix_len = 32;
1234                 break;
1235         case MLXSW_SP_L3_PROTO_IPV6:
1236                 WARN_ON(1);
1237                 return NULL;
1238         }
1239
1240         fib_node = mlxsw_sp_fib_node_lookup(ul_fib, saddrp, saddr_len,
1241                                             saddr_prefix_len);
1242         if (!fib_node || list_empty(&fib_node->entry_list))
1243                 return NULL;
1244
1245         fib_entry = list_first_entry(&fib_node->entry_list,
1246                                      struct mlxsw_sp_fib_entry, list);
1247         if (fib_entry->type != MLXSW_SP_FIB_ENTRY_TYPE_TRAP)
1248                 return NULL;
1249
1250         return fib_entry;
1251 }
1252
1253 static struct mlxsw_sp_ipip_entry *
1254 mlxsw_sp_ipip_entry_create(struct mlxsw_sp *mlxsw_sp,
1255                            enum mlxsw_sp_ipip_type ipipt,
1256                            struct net_device *ol_dev)
1257 {
1258         struct mlxsw_sp_ipip_entry *ipip_entry;
1259
1260         ipip_entry = mlxsw_sp_ipip_entry_alloc(mlxsw_sp, ipipt, ol_dev);
1261         if (IS_ERR(ipip_entry))
1262                 return ipip_entry;
1263
1264         list_add_tail(&ipip_entry->ipip_list_node,
1265                       &mlxsw_sp->router->ipip_list);
1266
1267         return ipip_entry;
1268 }
1269
1270 static void
1271 mlxsw_sp_ipip_entry_destroy(struct mlxsw_sp *mlxsw_sp,
1272                             struct mlxsw_sp_ipip_entry *ipip_entry)
1273 {
1274         list_del(&ipip_entry->ipip_list_node);
1275         mlxsw_sp_ipip_entry_dealloc(ipip_entry);
1276 }
1277
1278 static bool
1279 mlxsw_sp_ipip_entry_matches_decap(struct mlxsw_sp *mlxsw_sp,
1280                                   const struct net_device *ul_dev,
1281                                   enum mlxsw_sp_l3proto ul_proto,
1282                                   union mlxsw_sp_l3addr ul_dip,
1283                                   struct mlxsw_sp_ipip_entry *ipip_entry)
1284 {
1285         u32 ul_tb_id = l3mdev_fib_table(ul_dev) ? : RT_TABLE_MAIN;
1286         enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt;
1287
1288         if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto)
1289                 return false;
1290
1291         return mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, ul_dip,
1292                                                  ul_tb_id, ipip_entry);
1293 }
1294
1295 /* Given decap parameters, find the corresponding IPIP entry. */
1296 static struct mlxsw_sp_ipip_entry *
1297 mlxsw_sp_ipip_entry_find_by_decap(struct mlxsw_sp *mlxsw_sp,
1298                                   const struct net_device *ul_dev,
1299                                   enum mlxsw_sp_l3proto ul_proto,
1300                                   union mlxsw_sp_l3addr ul_dip)
1301 {
1302         struct mlxsw_sp_ipip_entry *ipip_entry;
1303
1304         list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list,
1305                             ipip_list_node)
1306                 if (mlxsw_sp_ipip_entry_matches_decap(mlxsw_sp, ul_dev,
1307                                                       ul_proto, ul_dip,
1308                                                       ipip_entry))
1309                         return ipip_entry;
1310
1311         return NULL;
1312 }
1313
1314 static bool mlxsw_sp_netdev_ipip_type(const struct mlxsw_sp *mlxsw_sp,
1315                                       const struct net_device *dev,
1316                                       enum mlxsw_sp_ipip_type *p_type)
1317 {
1318         struct mlxsw_sp_router *router = mlxsw_sp->router;
1319         const struct mlxsw_sp_ipip_ops *ipip_ops;
1320         enum mlxsw_sp_ipip_type ipipt;
1321
1322         for (ipipt = 0; ipipt < MLXSW_SP_IPIP_TYPE_MAX; ++ipipt) {
1323                 ipip_ops = router->ipip_ops_arr[ipipt];
1324                 if (dev->type == ipip_ops->dev_type) {
1325                         if (p_type)
1326                                 *p_type = ipipt;
1327                         return true;
1328                 }
1329         }
1330         return false;
1331 }
1332
1333 bool mlxsw_sp_netdev_is_ipip_ol(const struct mlxsw_sp *mlxsw_sp,
1334                                 const struct net_device *dev)
1335 {
1336         return mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL);
1337 }
1338
1339 static struct mlxsw_sp_ipip_entry *
1340 mlxsw_sp_ipip_entry_find_by_ol_dev(struct mlxsw_sp *mlxsw_sp,
1341                                    const struct net_device *ol_dev)
1342 {
1343         struct mlxsw_sp_ipip_entry *ipip_entry;
1344
1345         list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list,
1346                             ipip_list_node)
1347                 if (ipip_entry->ol_dev == ol_dev)
1348                         return ipip_entry;
1349
1350         return NULL;
1351 }
1352
1353 static struct mlxsw_sp_ipip_entry *
1354 mlxsw_sp_ipip_entry_find_by_ul_dev(const struct mlxsw_sp *mlxsw_sp,
1355                                    const struct net_device *ul_dev,
1356                                    struct mlxsw_sp_ipip_entry *start)
1357 {
1358         struct mlxsw_sp_ipip_entry *ipip_entry;
1359
1360         ipip_entry = list_prepare_entry(start, &mlxsw_sp->router->ipip_list,
1361                                         ipip_list_node);
1362         list_for_each_entry_continue(ipip_entry, &mlxsw_sp->router->ipip_list,
1363                                      ipip_list_node) {
1364                 struct net_device *ipip_ul_dev =
1365                         __mlxsw_sp_ipip_netdev_ul_dev_get(ipip_entry->ol_dev);
1366
1367                 if (ipip_ul_dev == ul_dev)
1368                         return ipip_entry;
1369         }
1370
1371         return NULL;
1372 }
1373
1374 bool mlxsw_sp_netdev_is_ipip_ul(const struct mlxsw_sp *mlxsw_sp,
1375                                 const struct net_device *dev)
1376 {
1377         return mlxsw_sp_ipip_entry_find_by_ul_dev(mlxsw_sp, dev, NULL);
1378 }
1379
1380 static bool mlxsw_sp_netdevice_ipip_can_offload(struct mlxsw_sp *mlxsw_sp,
1381                                                 const struct net_device *ol_dev,
1382                                                 enum mlxsw_sp_ipip_type ipipt)
1383 {
1384         const struct mlxsw_sp_ipip_ops *ops
1385                 = mlxsw_sp->router->ipip_ops_arr[ipipt];
1386
1387         /* For deciding whether decap should be offloaded, we don't care about
1388          * overlay protocol, so ask whether either one is supported.
1389          */
1390         return ops->can_offload(mlxsw_sp, ol_dev, MLXSW_SP_L3_PROTO_IPV4) ||
1391                ops->can_offload(mlxsw_sp, ol_dev, MLXSW_SP_L3_PROTO_IPV6);
1392 }
1393
1394 static int mlxsw_sp_netdevice_ipip_ol_reg_event(struct mlxsw_sp *mlxsw_sp,
1395                                                 struct net_device *ol_dev)
1396 {
1397         struct mlxsw_sp_ipip_entry *ipip_entry;
1398         enum mlxsw_sp_l3proto ul_proto;
1399         enum mlxsw_sp_ipip_type ipipt;
1400         union mlxsw_sp_l3addr saddr;
1401         u32 ul_tb_id;
1402
1403         mlxsw_sp_netdev_ipip_type(mlxsw_sp, ol_dev, &ipipt);
1404         if (mlxsw_sp_netdevice_ipip_can_offload(mlxsw_sp, ol_dev, ipipt)) {
1405                 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ol_dev);
1406                 ul_proto = mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto;
1407                 saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ol_dev);
1408                 if (!mlxsw_sp_ipip_demote_tunnel_by_saddr(mlxsw_sp, ul_proto,
1409                                                           saddr, ul_tb_id,
1410                                                           NULL)) {
1411                         ipip_entry = mlxsw_sp_ipip_entry_create(mlxsw_sp, ipipt,
1412                                                                 ol_dev);
1413                         if (IS_ERR(ipip_entry))
1414                                 return PTR_ERR(ipip_entry);
1415                 }
1416         }
1417
1418         return 0;
1419 }
1420
1421 static void mlxsw_sp_netdevice_ipip_ol_unreg_event(struct mlxsw_sp *mlxsw_sp,
1422                                                    struct net_device *ol_dev)
1423 {
1424         struct mlxsw_sp_ipip_entry *ipip_entry;
1425
1426         ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1427         if (ipip_entry)
1428                 mlxsw_sp_ipip_entry_destroy(mlxsw_sp, ipip_entry);
1429 }
1430
1431 static void
1432 mlxsw_sp_ipip_entry_ol_up_event(struct mlxsw_sp *mlxsw_sp,
1433                                 struct mlxsw_sp_ipip_entry *ipip_entry)
1434 {
1435         struct mlxsw_sp_fib_entry *decap_fib_entry;
1436
1437         decap_fib_entry = mlxsw_sp_ipip_entry_find_decap(mlxsw_sp, ipip_entry);
1438         if (decap_fib_entry)
1439                 mlxsw_sp_ipip_entry_promote_decap(mlxsw_sp, ipip_entry,
1440                                                   decap_fib_entry);
1441 }
1442
1443 static int
1444 mlxsw_sp_rif_ipip_lb_op(struct mlxsw_sp_rif_ipip_lb *lb_rif, u16 ul_vr_id,
1445                         u16 ul_rif_id, bool enable)
1446 {
1447         struct mlxsw_sp_rif_ipip_lb_config lb_cf = lb_rif->lb_config;
1448         struct mlxsw_sp_rif *rif = &lb_rif->common;
1449         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
1450         char ritr_pl[MLXSW_REG_RITR_LEN];
1451         u32 saddr4;
1452
1453         switch (lb_cf.ul_protocol) {
1454         case MLXSW_SP_L3_PROTO_IPV4:
1455                 saddr4 = be32_to_cpu(lb_cf.saddr.addr4);
1456                 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF,
1457                                     rif->rif_index, rif->vr_id, rif->dev->mtu);
1458                 mlxsw_reg_ritr_loopback_ipip4_pack(ritr_pl, lb_cf.lb_ipipt,
1459                             MLXSW_REG_RITR_LOOPBACK_IPIP_OPTIONS_GRE_KEY_PRESET,
1460                             ul_vr_id, ul_rif_id, saddr4, lb_cf.okey);
1461                 break;
1462
1463         case MLXSW_SP_L3_PROTO_IPV6:
1464                 return -EAFNOSUPPORT;
1465         }
1466
1467         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
1468 }
1469
1470 static int mlxsw_sp_netdevice_ipip_ol_update_mtu(struct mlxsw_sp *mlxsw_sp,
1471                                                  struct net_device *ol_dev)
1472 {
1473         struct mlxsw_sp_ipip_entry *ipip_entry;
1474         struct mlxsw_sp_rif_ipip_lb *lb_rif;
1475         int err = 0;
1476
1477         ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1478         if (ipip_entry) {
1479                 lb_rif = ipip_entry->ol_lb;
1480                 err = mlxsw_sp_rif_ipip_lb_op(lb_rif, lb_rif->ul_vr_id,
1481                                               lb_rif->ul_rif_id, true);
1482                 if (err)
1483                         goto out;
1484                 lb_rif->common.mtu = ol_dev->mtu;
1485         }
1486
1487 out:
1488         return err;
1489 }
1490
1491 static void mlxsw_sp_netdevice_ipip_ol_up_event(struct mlxsw_sp *mlxsw_sp,
1492                                                 struct net_device *ol_dev)
1493 {
1494         struct mlxsw_sp_ipip_entry *ipip_entry;
1495
1496         ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1497         if (ipip_entry)
1498                 mlxsw_sp_ipip_entry_ol_up_event(mlxsw_sp, ipip_entry);
1499 }
1500
1501 static void
1502 mlxsw_sp_ipip_entry_ol_down_event(struct mlxsw_sp *mlxsw_sp,
1503                                   struct mlxsw_sp_ipip_entry *ipip_entry)
1504 {
1505         if (ipip_entry->decap_fib_entry)
1506                 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry);
1507 }
1508
1509 static void mlxsw_sp_netdevice_ipip_ol_down_event(struct mlxsw_sp *mlxsw_sp,
1510                                                   struct net_device *ol_dev)
1511 {
1512         struct mlxsw_sp_ipip_entry *ipip_entry;
1513
1514         ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1515         if (ipip_entry)
1516                 mlxsw_sp_ipip_entry_ol_down_event(mlxsw_sp, ipip_entry);
1517 }
1518
1519 static void mlxsw_sp_nexthop_rif_migrate(struct mlxsw_sp *mlxsw_sp,
1520                                          struct mlxsw_sp_rif *old_rif,
1521                                          struct mlxsw_sp_rif *new_rif);
1522 static int
1523 mlxsw_sp_ipip_entry_ol_lb_update(struct mlxsw_sp *mlxsw_sp,
1524                                  struct mlxsw_sp_ipip_entry *ipip_entry,
1525                                  bool keep_encap,
1526                                  struct netlink_ext_ack *extack)
1527 {
1528         struct mlxsw_sp_rif_ipip_lb *old_lb_rif = ipip_entry->ol_lb;
1529         struct mlxsw_sp_rif_ipip_lb *new_lb_rif;
1530
1531         new_lb_rif = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp,
1532                                                      ipip_entry->ipipt,
1533                                                      ipip_entry->ol_dev,
1534                                                      extack);
1535         if (IS_ERR(new_lb_rif))
1536                 return PTR_ERR(new_lb_rif);
1537         ipip_entry->ol_lb = new_lb_rif;
1538
1539         if (keep_encap)
1540                 mlxsw_sp_nexthop_rif_migrate(mlxsw_sp, &old_lb_rif->common,
1541                                              &new_lb_rif->common);
1542
1543         mlxsw_sp_rif_destroy(&old_lb_rif->common);
1544
1545         return 0;
1546 }
1547
1548 static void mlxsw_sp_nexthop_rif_update(struct mlxsw_sp *mlxsw_sp,
1549                                         struct mlxsw_sp_rif *rif);
1550
1551 /**
1552  * Update the offload related to an IPIP entry. This always updates decap, and
1553  * in addition to that it also:
1554  * @recreate_loopback: recreates the associated loopback RIF
1555  * @keep_encap: updates next hops that use the tunnel netdevice. This is only
1556  *              relevant when recreate_loopback is true.
1557  * @update_nexthops: updates next hops, keeping the current loopback RIF. This
1558  *                   is only relevant when recreate_loopback is false.
1559  */
1560 int __mlxsw_sp_ipip_entry_update_tunnel(struct mlxsw_sp *mlxsw_sp,
1561                                         struct mlxsw_sp_ipip_entry *ipip_entry,
1562                                         bool recreate_loopback,
1563                                         bool keep_encap,
1564                                         bool update_nexthops,
1565                                         struct netlink_ext_ack *extack)
1566 {
1567         int err;
1568
1569         /* RIFs can't be edited, so to update loopback, we need to destroy and
1570          * recreate it. That creates a window of opportunity where RALUE and
1571          * RATR registers end up referencing a RIF that's already gone. RATRs
1572          * are handled in mlxsw_sp_ipip_entry_ol_lb_update(), and to take care
1573          * of RALUE, demote the decap route back.
1574          */
1575         if (ipip_entry->decap_fib_entry)
1576                 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry);
1577
1578         if (recreate_loopback) {
1579                 err = mlxsw_sp_ipip_entry_ol_lb_update(mlxsw_sp, ipip_entry,
1580                                                        keep_encap, extack);
1581                 if (err)
1582                         return err;
1583         } else if (update_nexthops) {
1584                 mlxsw_sp_nexthop_rif_update(mlxsw_sp,
1585                                             &ipip_entry->ol_lb->common);
1586         }
1587
1588         if (ipip_entry->ol_dev->flags & IFF_UP)
1589                 mlxsw_sp_ipip_entry_ol_up_event(mlxsw_sp, ipip_entry);
1590
1591         return 0;
1592 }
1593
1594 static int mlxsw_sp_netdevice_ipip_ol_vrf_event(struct mlxsw_sp *mlxsw_sp,
1595                                                 struct net_device *ol_dev,
1596                                                 struct netlink_ext_ack *extack)
1597 {
1598         struct mlxsw_sp_ipip_entry *ipip_entry =
1599                 mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1600         enum mlxsw_sp_l3proto ul_proto;
1601         union mlxsw_sp_l3addr saddr;
1602         u32 ul_tb_id;
1603
1604         if (!ipip_entry)
1605                 return 0;
1606
1607         /* For flat configuration cases, moving overlay to a different VRF might
1608          * cause local address conflict, and the conflicting tunnels need to be
1609          * demoted.
1610          */
1611         ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ol_dev);
1612         ul_proto = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]->ul_proto;
1613         saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ol_dev);
1614         if (mlxsw_sp_ipip_demote_tunnel_by_saddr(mlxsw_sp, ul_proto,
1615                                                  saddr, ul_tb_id,
1616                                                  ipip_entry)) {
1617                 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry);
1618                 return 0;
1619         }
1620
1621         return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry,
1622                                                    true, false, false, extack);
1623 }
1624
1625 static int
1626 mlxsw_sp_netdevice_ipip_ul_vrf_event(struct mlxsw_sp *mlxsw_sp,
1627                                      struct mlxsw_sp_ipip_entry *ipip_entry,
1628                                      struct net_device *ul_dev,
1629                                      struct netlink_ext_ack *extack)
1630 {
1631         return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry,
1632                                                    true, true, false, extack);
1633 }
1634
1635 static int
1636 mlxsw_sp_netdevice_ipip_ul_up_event(struct mlxsw_sp *mlxsw_sp,
1637                                     struct mlxsw_sp_ipip_entry *ipip_entry,
1638                                     struct net_device *ul_dev)
1639 {
1640         return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry,
1641                                                    false, false, true, NULL);
1642 }
1643
1644 static int
1645 mlxsw_sp_netdevice_ipip_ul_down_event(struct mlxsw_sp *mlxsw_sp,
1646                                       struct mlxsw_sp_ipip_entry *ipip_entry,
1647                                       struct net_device *ul_dev)
1648 {
1649         /* A down underlay device causes encapsulated packets to not be
1650          * forwarded, but decap still works. So refresh next hops without
1651          * touching anything else.
1652          */
1653         return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry,
1654                                                    false, false, true, NULL);
1655 }
1656
1657 static int
1658 mlxsw_sp_netdevice_ipip_ol_change_event(struct mlxsw_sp *mlxsw_sp,
1659                                         struct net_device *ol_dev,
1660                                         struct netlink_ext_ack *extack)
1661 {
1662         const struct mlxsw_sp_ipip_ops *ipip_ops;
1663         struct mlxsw_sp_ipip_entry *ipip_entry;
1664         int err;
1665
1666         ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1667         if (!ipip_entry)
1668                 /* A change might make a tunnel eligible for offloading, but
1669                  * that is currently not implemented. What falls to slow path
1670                  * stays there.
1671                  */
1672                 return 0;
1673
1674         /* A change might make a tunnel not eligible for offloading. */
1675         if (!mlxsw_sp_netdevice_ipip_can_offload(mlxsw_sp, ol_dev,
1676                                                  ipip_entry->ipipt)) {
1677                 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry);
1678                 return 0;
1679         }
1680
1681         ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
1682         err = ipip_ops->ol_netdev_change(mlxsw_sp, ipip_entry, extack);
1683         return err;
1684 }
1685
1686 void mlxsw_sp_ipip_entry_demote_tunnel(struct mlxsw_sp *mlxsw_sp,
1687                                        struct mlxsw_sp_ipip_entry *ipip_entry)
1688 {
1689         struct net_device *ol_dev = ipip_entry->ol_dev;
1690
1691         if (ol_dev->flags & IFF_UP)
1692                 mlxsw_sp_ipip_entry_ol_down_event(mlxsw_sp, ipip_entry);
1693         mlxsw_sp_ipip_entry_destroy(mlxsw_sp, ipip_entry);
1694 }
1695
1696 /* The configuration where several tunnels have the same local address in the
1697  * same underlay table needs special treatment in the HW. That is currently not
1698  * implemented in the driver. This function finds and demotes the first tunnel
1699  * with a given source address, except the one passed in in the argument
1700  * `except'.
1701  */
1702 bool
1703 mlxsw_sp_ipip_demote_tunnel_by_saddr(struct mlxsw_sp *mlxsw_sp,
1704                                      enum mlxsw_sp_l3proto ul_proto,
1705                                      union mlxsw_sp_l3addr saddr,
1706                                      u32 ul_tb_id,
1707                                      const struct mlxsw_sp_ipip_entry *except)
1708 {
1709         struct mlxsw_sp_ipip_entry *ipip_entry, *tmp;
1710
1711         list_for_each_entry_safe(ipip_entry, tmp, &mlxsw_sp->router->ipip_list,
1712                                  ipip_list_node) {
1713                 if (ipip_entry != except &&
1714                     mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, saddr,
1715                                                       ul_tb_id, ipip_entry)) {
1716                         mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry);
1717                         return true;
1718                 }
1719         }
1720
1721         return false;
1722 }
1723
1724 static void mlxsw_sp_ipip_demote_tunnel_by_ul_netdev(struct mlxsw_sp *mlxsw_sp,
1725                                                      struct net_device *ul_dev)
1726 {
1727         struct mlxsw_sp_ipip_entry *ipip_entry, *tmp;
1728
1729         list_for_each_entry_safe(ipip_entry, tmp, &mlxsw_sp->router->ipip_list,
1730                                  ipip_list_node) {
1731                 struct net_device *ipip_ul_dev =
1732                         __mlxsw_sp_ipip_netdev_ul_dev_get(ipip_entry->ol_dev);
1733
1734                 if (ipip_ul_dev == ul_dev)
1735                         mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry);
1736         }
1737 }
1738
1739 int mlxsw_sp_netdevice_ipip_ol_event(struct mlxsw_sp *mlxsw_sp,
1740                                      struct net_device *ol_dev,
1741                                      unsigned long event,
1742                                      struct netdev_notifier_info *info)
1743 {
1744         struct netdev_notifier_changeupper_info *chup;
1745         struct netlink_ext_ack *extack;
1746
1747         switch (event) {
1748         case NETDEV_REGISTER:
1749                 return mlxsw_sp_netdevice_ipip_ol_reg_event(mlxsw_sp, ol_dev);
1750         case NETDEV_UNREGISTER:
1751                 mlxsw_sp_netdevice_ipip_ol_unreg_event(mlxsw_sp, ol_dev);
1752                 return 0;
1753         case NETDEV_UP:
1754                 mlxsw_sp_netdevice_ipip_ol_up_event(mlxsw_sp, ol_dev);
1755                 return 0;
1756         case NETDEV_DOWN:
1757                 mlxsw_sp_netdevice_ipip_ol_down_event(mlxsw_sp, ol_dev);
1758                 return 0;
1759         case NETDEV_CHANGEUPPER:
1760                 chup = container_of(info, typeof(*chup), info);
1761                 extack = info->extack;
1762                 if (netif_is_l3_master(chup->upper_dev))
1763                         return mlxsw_sp_netdevice_ipip_ol_vrf_event(mlxsw_sp,
1764                                                                     ol_dev,
1765                                                                     extack);
1766                 return 0;
1767         case NETDEV_CHANGE:
1768                 extack = info->extack;
1769                 return mlxsw_sp_netdevice_ipip_ol_change_event(mlxsw_sp,
1770                                                                ol_dev, extack);
1771         case NETDEV_CHANGEMTU:
1772                 return mlxsw_sp_netdevice_ipip_ol_update_mtu(mlxsw_sp, ol_dev);
1773         }
1774         return 0;
1775 }
1776
1777 static int
1778 __mlxsw_sp_netdevice_ipip_ul_event(struct mlxsw_sp *mlxsw_sp,
1779                                    struct mlxsw_sp_ipip_entry *ipip_entry,
1780                                    struct net_device *ul_dev,
1781                                    unsigned long event,
1782                                    struct netdev_notifier_info *info)
1783 {
1784         struct netdev_notifier_changeupper_info *chup;
1785         struct netlink_ext_ack *extack;
1786
1787         switch (event) {
1788         case NETDEV_CHANGEUPPER:
1789                 chup = container_of(info, typeof(*chup), info);
1790                 extack = info->extack;
1791                 if (netif_is_l3_master(chup->upper_dev))
1792                         return mlxsw_sp_netdevice_ipip_ul_vrf_event(mlxsw_sp,
1793                                                                     ipip_entry,
1794                                                                     ul_dev,
1795                                                                     extack);
1796                 break;
1797
1798         case NETDEV_UP:
1799                 return mlxsw_sp_netdevice_ipip_ul_up_event(mlxsw_sp, ipip_entry,
1800                                                            ul_dev);
1801         case NETDEV_DOWN:
1802                 return mlxsw_sp_netdevice_ipip_ul_down_event(mlxsw_sp,
1803                                                              ipip_entry,
1804                                                              ul_dev);
1805         }
1806         return 0;
1807 }
1808
1809 int
1810 mlxsw_sp_netdevice_ipip_ul_event(struct mlxsw_sp *mlxsw_sp,
1811                                  struct net_device *ul_dev,
1812                                  unsigned long event,
1813                                  struct netdev_notifier_info *info)
1814 {
1815         struct mlxsw_sp_ipip_entry *ipip_entry = NULL;
1816         int err;
1817
1818         while ((ipip_entry = mlxsw_sp_ipip_entry_find_by_ul_dev(mlxsw_sp,
1819                                                                 ul_dev,
1820                                                                 ipip_entry))) {
1821                 err = __mlxsw_sp_netdevice_ipip_ul_event(mlxsw_sp, ipip_entry,
1822                                                          ul_dev, event, info);
1823                 if (err) {
1824                         mlxsw_sp_ipip_demote_tunnel_by_ul_netdev(mlxsw_sp,
1825                                                                  ul_dev);
1826                         return err;
1827                 }
1828         }
1829
1830         return 0;
1831 }
1832
1833 int mlxsw_sp_router_nve_promote_decap(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id,
1834                                       enum mlxsw_sp_l3proto ul_proto,
1835                                       const union mlxsw_sp_l3addr *ul_sip,
1836                                       u32 tunnel_index)
1837 {
1838         enum mlxsw_sp_fib_entry_type type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
1839         struct mlxsw_sp_fib_entry *fib_entry;
1840         int err;
1841
1842         /* It is valid to create a tunnel with a local IP and only later
1843          * assign this IP address to a local interface
1844          */
1845         fib_entry = mlxsw_sp_router_ip2me_fib_entry_find(mlxsw_sp, ul_tb_id,
1846                                                          ul_proto, ul_sip,
1847                                                          type);
1848         if (!fib_entry)
1849                 return 0;
1850
1851         fib_entry->decap.tunnel_index = tunnel_index;
1852         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP;
1853
1854         err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
1855         if (err)
1856                 goto err_fib_entry_update;
1857
1858         return 0;
1859
1860 err_fib_entry_update:
1861         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
1862         mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
1863         return err;
1864 }
1865
1866 void mlxsw_sp_router_nve_demote_decap(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id,
1867                                       enum mlxsw_sp_l3proto ul_proto,
1868                                       const union mlxsw_sp_l3addr *ul_sip)
1869 {
1870         enum mlxsw_sp_fib_entry_type type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP;
1871         struct mlxsw_sp_fib_entry *fib_entry;
1872
1873         fib_entry = mlxsw_sp_router_ip2me_fib_entry_find(mlxsw_sp, ul_tb_id,
1874                                                          ul_proto, ul_sip,
1875                                                          type);
1876         if (!fib_entry)
1877                 return;
1878
1879         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
1880         mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
1881 }
1882
1883 struct mlxsw_sp_neigh_key {
1884         struct neighbour *n;
1885 };
1886
1887 struct mlxsw_sp_neigh_entry {
1888         struct list_head rif_list_node;
1889         struct rhash_head ht_node;
1890         struct mlxsw_sp_neigh_key key;
1891         u16 rif;
1892         bool connected;
1893         unsigned char ha[ETH_ALEN];
1894         struct list_head nexthop_list; /* list of nexthops using
1895                                         * this neigh entry
1896                                         */
1897         struct list_head nexthop_neighs_list_node;
1898         unsigned int counter_index;
1899         bool counter_valid;
1900 };
1901
1902 static const struct rhashtable_params mlxsw_sp_neigh_ht_params = {
1903         .key_offset = offsetof(struct mlxsw_sp_neigh_entry, key),
1904         .head_offset = offsetof(struct mlxsw_sp_neigh_entry, ht_node),
1905         .key_len = sizeof(struct mlxsw_sp_neigh_key),
1906 };
1907
1908 struct mlxsw_sp_neigh_entry *
1909 mlxsw_sp_rif_neigh_next(struct mlxsw_sp_rif *rif,
1910                         struct mlxsw_sp_neigh_entry *neigh_entry)
1911 {
1912         if (!neigh_entry) {
1913                 if (list_empty(&rif->neigh_list))
1914                         return NULL;
1915                 else
1916                         return list_first_entry(&rif->neigh_list,
1917                                                 typeof(*neigh_entry),
1918                                                 rif_list_node);
1919         }
1920         if (list_is_last(&neigh_entry->rif_list_node, &rif->neigh_list))
1921                 return NULL;
1922         return list_next_entry(neigh_entry, rif_list_node);
1923 }
1924
1925 int mlxsw_sp_neigh_entry_type(struct mlxsw_sp_neigh_entry *neigh_entry)
1926 {
1927         return neigh_entry->key.n->tbl->family;
1928 }
1929
1930 unsigned char *
1931 mlxsw_sp_neigh_entry_ha(struct mlxsw_sp_neigh_entry *neigh_entry)
1932 {
1933         return neigh_entry->ha;
1934 }
1935
1936 u32 mlxsw_sp_neigh4_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry)
1937 {
1938         struct neighbour *n;
1939
1940         n = neigh_entry->key.n;
1941         return ntohl(*((__be32 *) n->primary_key));
1942 }
1943
1944 struct in6_addr *
1945 mlxsw_sp_neigh6_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry)
1946 {
1947         struct neighbour *n;
1948
1949         n = neigh_entry->key.n;
1950         return (struct in6_addr *) &n->primary_key;
1951 }
1952
1953 int mlxsw_sp_neigh_counter_get(struct mlxsw_sp *mlxsw_sp,
1954                                struct mlxsw_sp_neigh_entry *neigh_entry,
1955                                u64 *p_counter)
1956 {
1957         if (!neigh_entry->counter_valid)
1958                 return -EINVAL;
1959
1960         return mlxsw_sp_flow_counter_get(mlxsw_sp, neigh_entry->counter_index,
1961                                          p_counter, NULL);
1962 }
1963
1964 static struct mlxsw_sp_neigh_entry *
1965 mlxsw_sp_neigh_entry_alloc(struct mlxsw_sp *mlxsw_sp, struct neighbour *n,
1966                            u16 rif)
1967 {
1968         struct mlxsw_sp_neigh_entry *neigh_entry;
1969
1970         neigh_entry = kzalloc(sizeof(*neigh_entry), GFP_KERNEL);
1971         if (!neigh_entry)
1972                 return NULL;
1973
1974         neigh_entry->key.n = n;
1975         neigh_entry->rif = rif;
1976         INIT_LIST_HEAD(&neigh_entry->nexthop_list);
1977
1978         return neigh_entry;
1979 }
1980
1981 static void mlxsw_sp_neigh_entry_free(struct mlxsw_sp_neigh_entry *neigh_entry)
1982 {
1983         kfree(neigh_entry);
1984 }
1985
1986 static int
1987 mlxsw_sp_neigh_entry_insert(struct mlxsw_sp *mlxsw_sp,
1988                             struct mlxsw_sp_neigh_entry *neigh_entry)
1989 {
1990         return rhashtable_insert_fast(&mlxsw_sp->router->neigh_ht,
1991                                       &neigh_entry->ht_node,
1992                                       mlxsw_sp_neigh_ht_params);
1993 }
1994
1995 static void
1996 mlxsw_sp_neigh_entry_remove(struct mlxsw_sp *mlxsw_sp,
1997                             struct mlxsw_sp_neigh_entry *neigh_entry)
1998 {
1999         rhashtable_remove_fast(&mlxsw_sp->router->neigh_ht,
2000                                &neigh_entry->ht_node,
2001                                mlxsw_sp_neigh_ht_params);
2002 }
2003
2004 static bool
2005 mlxsw_sp_neigh_counter_should_alloc(struct mlxsw_sp *mlxsw_sp,
2006                                     struct mlxsw_sp_neigh_entry *neigh_entry)
2007 {
2008         struct devlink *devlink;
2009         const char *table_name;
2010
2011         switch (mlxsw_sp_neigh_entry_type(neigh_entry)) {
2012         case AF_INET:
2013                 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST4;
2014                 break;
2015         case AF_INET6:
2016                 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST6;
2017                 break;
2018         default:
2019                 WARN_ON(1);
2020                 return false;
2021         }
2022
2023         devlink = priv_to_devlink(mlxsw_sp->core);
2024         return devlink_dpipe_table_counter_enabled(devlink, table_name);
2025 }
2026
2027 static void
2028 mlxsw_sp_neigh_counter_alloc(struct mlxsw_sp *mlxsw_sp,
2029                              struct mlxsw_sp_neigh_entry *neigh_entry)
2030 {
2031         if (!mlxsw_sp_neigh_counter_should_alloc(mlxsw_sp, neigh_entry))
2032                 return;
2033
2034         if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &neigh_entry->counter_index))
2035                 return;
2036
2037         neigh_entry->counter_valid = true;
2038 }
2039
2040 static void
2041 mlxsw_sp_neigh_counter_free(struct mlxsw_sp *mlxsw_sp,
2042                             struct mlxsw_sp_neigh_entry *neigh_entry)
2043 {
2044         if (!neigh_entry->counter_valid)
2045                 return;
2046         mlxsw_sp_flow_counter_free(mlxsw_sp,
2047                                    neigh_entry->counter_index);
2048         neigh_entry->counter_valid = false;
2049 }
2050
2051 static struct mlxsw_sp_neigh_entry *
2052 mlxsw_sp_neigh_entry_create(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
2053 {
2054         struct mlxsw_sp_neigh_entry *neigh_entry;
2055         struct mlxsw_sp_rif *rif;
2056         int err;
2057
2058         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, n->dev);
2059         if (!rif)
2060                 return ERR_PTR(-EINVAL);
2061
2062         neigh_entry = mlxsw_sp_neigh_entry_alloc(mlxsw_sp, n, rif->rif_index);
2063         if (!neigh_entry)
2064                 return ERR_PTR(-ENOMEM);
2065
2066         err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
2067         if (err)
2068                 goto err_neigh_entry_insert;
2069
2070         mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry);
2071         list_add(&neigh_entry->rif_list_node, &rif->neigh_list);
2072
2073         return neigh_entry;
2074
2075 err_neigh_entry_insert:
2076         mlxsw_sp_neigh_entry_free(neigh_entry);
2077         return ERR_PTR(err);
2078 }
2079
2080 static void
2081 mlxsw_sp_neigh_entry_destroy(struct mlxsw_sp *mlxsw_sp,
2082                              struct mlxsw_sp_neigh_entry *neigh_entry)
2083 {
2084         list_del(&neigh_entry->rif_list_node);
2085         mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry);
2086         mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry);
2087         mlxsw_sp_neigh_entry_free(neigh_entry);
2088 }
2089
2090 static struct mlxsw_sp_neigh_entry *
2091 mlxsw_sp_neigh_entry_lookup(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
2092 {
2093         struct mlxsw_sp_neigh_key key;
2094
2095         key.n = n;
2096         return rhashtable_lookup_fast(&mlxsw_sp->router->neigh_ht,
2097                                       &key, mlxsw_sp_neigh_ht_params);
2098 }
2099
2100 static void
2101 mlxsw_sp_router_neighs_update_interval_init(struct mlxsw_sp *mlxsw_sp)
2102 {
2103         unsigned long interval;
2104
2105 #if IS_ENABLED(CONFIG_IPV6)
2106         interval = min_t(unsigned long,
2107                          NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME),
2108                          NEIGH_VAR(&nd_tbl.parms, DELAY_PROBE_TIME));
2109 #else
2110         interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME);
2111 #endif
2112         mlxsw_sp->router->neighs_update.interval = jiffies_to_msecs(interval);
2113 }
2114
2115 static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp,
2116                                                    char *rauhtd_pl,
2117                                                    int ent_index)
2118 {
2119         struct net_device *dev;
2120         struct neighbour *n;
2121         __be32 dipn;
2122         u32 dip;
2123         u16 rif;
2124
2125         mlxsw_reg_rauhtd_ent_ipv4_unpack(rauhtd_pl, ent_index, &rif, &dip);
2126
2127         if (!mlxsw_sp->router->rifs[rif]) {
2128                 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
2129                 return;
2130         }
2131
2132         dipn = htonl(dip);
2133         dev = mlxsw_sp->router->rifs[rif]->dev;
2134         n = neigh_lookup(&arp_tbl, &dipn, dev);
2135         if (!n)
2136                 return;
2137
2138         netdev_dbg(dev, "Updating neighbour with IP=%pI4h\n", &dip);
2139         neigh_event_send(n, NULL);
2140         neigh_release(n);
2141 }
2142
2143 #if IS_ENABLED(CONFIG_IPV6)
2144 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp,
2145                                                    char *rauhtd_pl,
2146                                                    int rec_index)
2147 {
2148         struct net_device *dev;
2149         struct neighbour *n;
2150         struct in6_addr dip;
2151         u16 rif;
2152
2153         mlxsw_reg_rauhtd_ent_ipv6_unpack(rauhtd_pl, rec_index, &rif,
2154                                          (char *) &dip);
2155
2156         if (!mlxsw_sp->router->rifs[rif]) {
2157                 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
2158                 return;
2159         }
2160
2161         dev = mlxsw_sp->router->rifs[rif]->dev;
2162         n = neigh_lookup(&nd_tbl, &dip, dev);
2163         if (!n)
2164                 return;
2165
2166         netdev_dbg(dev, "Updating neighbour with IP=%pI6c\n", &dip);
2167         neigh_event_send(n, NULL);
2168         neigh_release(n);
2169 }
2170 #else
2171 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp,
2172                                                    char *rauhtd_pl,
2173                                                    int rec_index)
2174 {
2175 }
2176 #endif
2177
2178 static void mlxsw_sp_router_neigh_rec_ipv4_process(struct mlxsw_sp *mlxsw_sp,
2179                                                    char *rauhtd_pl,
2180                                                    int rec_index)
2181 {
2182         u8 num_entries;
2183         int i;
2184
2185         num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
2186                                                                 rec_index);
2187         /* Hardware starts counting at 0, so add 1. */
2188         num_entries++;
2189
2190         /* Each record consists of several neighbour entries. */
2191         for (i = 0; i < num_entries; i++) {
2192                 int ent_index;
2193
2194                 ent_index = rec_index * MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC + i;
2195                 mlxsw_sp_router_neigh_ent_ipv4_process(mlxsw_sp, rauhtd_pl,
2196                                                        ent_index);
2197         }
2198
2199 }
2200
2201 static void mlxsw_sp_router_neigh_rec_ipv6_process(struct mlxsw_sp *mlxsw_sp,
2202                                                    char *rauhtd_pl,
2203                                                    int rec_index)
2204 {
2205         /* One record contains one entry. */
2206         mlxsw_sp_router_neigh_ent_ipv6_process(mlxsw_sp, rauhtd_pl,
2207                                                rec_index);
2208 }
2209
2210 static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp,
2211                                               char *rauhtd_pl, int rec_index)
2212 {
2213         switch (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, rec_index)) {
2214         case MLXSW_REG_RAUHTD_TYPE_IPV4:
2215                 mlxsw_sp_router_neigh_rec_ipv4_process(mlxsw_sp, rauhtd_pl,
2216                                                        rec_index);
2217                 break;
2218         case MLXSW_REG_RAUHTD_TYPE_IPV6:
2219                 mlxsw_sp_router_neigh_rec_ipv6_process(mlxsw_sp, rauhtd_pl,
2220                                                        rec_index);
2221                 break;
2222         }
2223 }
2224
2225 static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl)
2226 {
2227         u8 num_rec, last_rec_index, num_entries;
2228
2229         num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
2230         last_rec_index = num_rec - 1;
2231
2232         if (num_rec < MLXSW_REG_RAUHTD_REC_MAX_NUM)
2233                 return false;
2234         if (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, last_rec_index) ==
2235             MLXSW_REG_RAUHTD_TYPE_IPV6)
2236                 return true;
2237
2238         num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
2239                                                                 last_rec_index);
2240         if (++num_entries == MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC)
2241                 return true;
2242         return false;
2243 }
2244
2245 static int
2246 __mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp,
2247                                        char *rauhtd_pl,
2248                                        enum mlxsw_reg_rauhtd_type type)
2249 {
2250         int i, num_rec;
2251         int err;
2252
2253         /* Make sure the neighbour's netdev isn't removed in the
2254          * process.
2255          */
2256         rtnl_lock();
2257         do {
2258                 mlxsw_reg_rauhtd_pack(rauhtd_pl, type);
2259                 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(rauhtd),
2260                                       rauhtd_pl);
2261                 if (err) {
2262                         dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to dump neighbour table\n");
2263                         break;
2264                 }
2265                 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
2266                 for (i = 0; i < num_rec; i++)
2267                         mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl,
2268                                                           i);
2269         } while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl));
2270         rtnl_unlock();
2271
2272         return err;
2273 }
2274
2275 static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp)
2276 {
2277         enum mlxsw_reg_rauhtd_type type;
2278         char *rauhtd_pl;
2279         int err;
2280
2281         rauhtd_pl = kmalloc(MLXSW_REG_RAUHTD_LEN, GFP_KERNEL);
2282         if (!rauhtd_pl)
2283                 return -ENOMEM;
2284
2285         type = MLXSW_REG_RAUHTD_TYPE_IPV4;
2286         err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type);
2287         if (err)
2288                 goto out;
2289
2290         type = MLXSW_REG_RAUHTD_TYPE_IPV6;
2291         err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type);
2292 out:
2293         kfree(rauhtd_pl);
2294         return err;
2295 }
2296
2297 static void mlxsw_sp_router_neighs_update_nh(struct mlxsw_sp *mlxsw_sp)
2298 {
2299         struct mlxsw_sp_neigh_entry *neigh_entry;
2300
2301         /* Take RTNL mutex here to prevent lists from changes */
2302         rtnl_lock();
2303         list_for_each_entry(neigh_entry, &mlxsw_sp->router->nexthop_neighs_list,
2304                             nexthop_neighs_list_node)
2305                 /* If this neigh have nexthops, make the kernel think this neigh
2306                  * is active regardless of the traffic.
2307                  */
2308                 neigh_event_send(neigh_entry->key.n, NULL);
2309         rtnl_unlock();
2310 }
2311
2312 static void
2313 mlxsw_sp_router_neighs_update_work_schedule(struct mlxsw_sp *mlxsw_sp)
2314 {
2315         unsigned long interval = mlxsw_sp->router->neighs_update.interval;
2316
2317         mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw,
2318                                msecs_to_jiffies(interval));
2319 }
2320
2321 static void mlxsw_sp_router_neighs_update_work(struct work_struct *work)
2322 {
2323         struct mlxsw_sp_router *router;
2324         int err;
2325
2326         router = container_of(work, struct mlxsw_sp_router,
2327                               neighs_update.dw.work);
2328         err = mlxsw_sp_router_neighs_update_rauhtd(router->mlxsw_sp);
2329         if (err)
2330                 dev_err(router->mlxsw_sp->bus_info->dev, "Could not update kernel for neigh activity");
2331
2332         mlxsw_sp_router_neighs_update_nh(router->mlxsw_sp);
2333
2334         mlxsw_sp_router_neighs_update_work_schedule(router->mlxsw_sp);
2335 }
2336
2337 static void mlxsw_sp_router_probe_unresolved_nexthops(struct work_struct *work)
2338 {
2339         struct mlxsw_sp_neigh_entry *neigh_entry;
2340         struct mlxsw_sp_router *router;
2341
2342         router = container_of(work, struct mlxsw_sp_router,
2343                               nexthop_probe_dw.work);
2344         /* Iterate over nexthop neighbours, find those who are unresolved and
2345          * send arp on them. This solves the chicken-egg problem when
2346          * the nexthop wouldn't get offloaded until the neighbor is resolved
2347          * but it wouldn't get resolved ever in case traffic is flowing in HW
2348          * using different nexthop.
2349          *
2350          * Take RTNL mutex here to prevent lists from changes.
2351          */
2352         rtnl_lock();
2353         list_for_each_entry(neigh_entry, &router->nexthop_neighs_list,
2354                             nexthop_neighs_list_node)
2355                 if (!neigh_entry->connected)
2356                         neigh_event_send(neigh_entry->key.n, NULL);
2357         rtnl_unlock();
2358
2359         mlxsw_core_schedule_dw(&router->nexthop_probe_dw,
2360                                MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL);
2361 }
2362
2363 static void
2364 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
2365                               struct mlxsw_sp_neigh_entry *neigh_entry,
2366                               bool removing);
2367
2368 static enum mlxsw_reg_rauht_op mlxsw_sp_rauht_op(bool adding)
2369 {
2370         return adding ? MLXSW_REG_RAUHT_OP_WRITE_ADD :
2371                         MLXSW_REG_RAUHT_OP_WRITE_DELETE;
2372 }
2373
2374 static void
2375 mlxsw_sp_router_neigh_entry_op4(struct mlxsw_sp *mlxsw_sp,
2376                                 struct mlxsw_sp_neigh_entry *neigh_entry,
2377                                 enum mlxsw_reg_rauht_op op)
2378 {
2379         struct neighbour *n = neigh_entry->key.n;
2380         u32 dip = ntohl(*((__be32 *) n->primary_key));
2381         char rauht_pl[MLXSW_REG_RAUHT_LEN];
2382
2383         mlxsw_reg_rauht_pack4(rauht_pl, op, neigh_entry->rif, neigh_entry->ha,
2384                               dip);
2385         if (neigh_entry->counter_valid)
2386                 mlxsw_reg_rauht_pack_counter(rauht_pl,
2387                                              neigh_entry->counter_index);
2388         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
2389 }
2390
2391 static void
2392 mlxsw_sp_router_neigh_entry_op6(struct mlxsw_sp *mlxsw_sp,
2393                                 struct mlxsw_sp_neigh_entry *neigh_entry,
2394                                 enum mlxsw_reg_rauht_op op)
2395 {
2396         struct neighbour *n = neigh_entry->key.n;
2397         char rauht_pl[MLXSW_REG_RAUHT_LEN];
2398         const char *dip = n->primary_key;
2399
2400         mlxsw_reg_rauht_pack6(rauht_pl, op, neigh_entry->rif, neigh_entry->ha,
2401                               dip);
2402         if (neigh_entry->counter_valid)
2403                 mlxsw_reg_rauht_pack_counter(rauht_pl,
2404                                              neigh_entry->counter_index);
2405         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
2406 }
2407
2408 bool mlxsw_sp_neigh_ipv6_ignore(struct mlxsw_sp_neigh_entry *neigh_entry)
2409 {
2410         struct neighbour *n = neigh_entry->key.n;
2411
2412         /* Packets with a link-local destination address are trapped
2413          * after LPM lookup and never reach the neighbour table, so
2414          * there is no need to program such neighbours to the device.
2415          */
2416         if (ipv6_addr_type((struct in6_addr *) &n->primary_key) &
2417             IPV6_ADDR_LINKLOCAL)
2418                 return true;
2419         return false;
2420 }
2421
2422 static void
2423 mlxsw_sp_neigh_entry_update(struct mlxsw_sp *mlxsw_sp,
2424                             struct mlxsw_sp_neigh_entry *neigh_entry,
2425                             bool adding)
2426 {
2427         if (!adding && !neigh_entry->connected)
2428                 return;
2429         neigh_entry->connected = adding;
2430         if (neigh_entry->key.n->tbl->family == AF_INET) {
2431                 mlxsw_sp_router_neigh_entry_op4(mlxsw_sp, neigh_entry,
2432                                                 mlxsw_sp_rauht_op(adding));
2433         } else if (neigh_entry->key.n->tbl->family == AF_INET6) {
2434                 if (mlxsw_sp_neigh_ipv6_ignore(neigh_entry))
2435                         return;
2436                 mlxsw_sp_router_neigh_entry_op6(mlxsw_sp, neigh_entry,
2437                                                 mlxsw_sp_rauht_op(adding));
2438         } else {
2439                 WARN_ON_ONCE(1);
2440         }
2441 }
2442
2443 void
2444 mlxsw_sp_neigh_entry_counter_update(struct mlxsw_sp *mlxsw_sp,
2445                                     struct mlxsw_sp_neigh_entry *neigh_entry,
2446                                     bool adding)
2447 {
2448         if (adding)
2449                 mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry);
2450         else
2451                 mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry);
2452         mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, true);
2453 }
2454
2455 struct mlxsw_sp_netevent_work {
2456         struct work_struct work;
2457         struct mlxsw_sp *mlxsw_sp;
2458         struct neighbour *n;
2459 };
2460
2461 static void mlxsw_sp_router_neigh_event_work(struct work_struct *work)
2462 {
2463         struct mlxsw_sp_netevent_work *net_work =
2464                 container_of(work, struct mlxsw_sp_netevent_work, work);
2465         struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp;
2466         struct mlxsw_sp_neigh_entry *neigh_entry;
2467         struct neighbour *n = net_work->n;
2468         unsigned char ha[ETH_ALEN];
2469         bool entry_connected;
2470         u8 nud_state, dead;
2471
2472         /* If these parameters are changed after we release the lock,
2473          * then we are guaranteed to receive another event letting us
2474          * know about it.
2475          */
2476         read_lock_bh(&n->lock);
2477         memcpy(ha, n->ha, ETH_ALEN);
2478         nud_state = n->nud_state;
2479         dead = n->dead;
2480         read_unlock_bh(&n->lock);
2481
2482         rtnl_lock();
2483         mlxsw_sp_span_respin(mlxsw_sp);
2484
2485         entry_connected = nud_state & NUD_VALID && !dead;
2486         neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
2487         if (!entry_connected && !neigh_entry)
2488                 goto out;
2489         if (!neigh_entry) {
2490                 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
2491                 if (IS_ERR(neigh_entry))
2492                         goto out;
2493         }
2494
2495         memcpy(neigh_entry->ha, ha, ETH_ALEN);
2496         mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, entry_connected);
2497         mlxsw_sp_nexthop_neigh_update(mlxsw_sp, neigh_entry, !entry_connected);
2498
2499         if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
2500                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
2501
2502 out:
2503         rtnl_unlock();
2504         neigh_release(n);
2505         kfree(net_work);
2506 }
2507
2508 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp);
2509
2510 static void mlxsw_sp_router_mp_hash_event_work(struct work_struct *work)
2511 {
2512         struct mlxsw_sp_netevent_work *net_work =
2513                 container_of(work, struct mlxsw_sp_netevent_work, work);
2514         struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp;
2515
2516         mlxsw_sp_mp_hash_init(mlxsw_sp);
2517         kfree(net_work);
2518 }
2519
2520 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp);
2521
2522 static void mlxsw_sp_router_update_priority_work(struct work_struct *work)
2523 {
2524         struct mlxsw_sp_netevent_work *net_work =
2525                 container_of(work, struct mlxsw_sp_netevent_work, work);
2526         struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp;
2527
2528         __mlxsw_sp_router_init(mlxsw_sp);
2529         kfree(net_work);
2530 }
2531
2532 static int mlxsw_sp_router_schedule_work(struct net *net,
2533                                          struct notifier_block *nb,
2534                                          void (*cb)(struct work_struct *))
2535 {
2536         struct mlxsw_sp_netevent_work *net_work;
2537         struct mlxsw_sp_router *router;
2538
2539         if (!net_eq(net, &init_net))
2540                 return NOTIFY_DONE;
2541
2542         net_work = kzalloc(sizeof(*net_work), GFP_ATOMIC);
2543         if (!net_work)
2544                 return NOTIFY_BAD;
2545
2546         router = container_of(nb, struct mlxsw_sp_router, netevent_nb);
2547         INIT_WORK(&net_work->work, cb);
2548         net_work->mlxsw_sp = router->mlxsw_sp;
2549         mlxsw_core_schedule_work(&net_work->work);
2550         return NOTIFY_DONE;
2551 }
2552
2553 static int mlxsw_sp_router_netevent_event(struct notifier_block *nb,
2554                                           unsigned long event, void *ptr)
2555 {
2556         struct mlxsw_sp_netevent_work *net_work;
2557         struct mlxsw_sp_port *mlxsw_sp_port;
2558         struct mlxsw_sp *mlxsw_sp;
2559         unsigned long interval;
2560         struct neigh_parms *p;
2561         struct neighbour *n;
2562
2563         switch (event) {
2564         case NETEVENT_DELAY_PROBE_TIME_UPDATE:
2565                 p = ptr;
2566
2567                 /* We don't care about changes in the default table. */
2568                 if (!p->dev || (p->tbl->family != AF_INET &&
2569                                 p->tbl->family != AF_INET6))
2570                         return NOTIFY_DONE;
2571
2572                 /* We are in atomic context and can't take RTNL mutex,
2573                  * so use RCU variant to walk the device chain.
2574                  */
2575                 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(p->dev);
2576                 if (!mlxsw_sp_port)
2577                         return NOTIFY_DONE;
2578
2579                 mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
2580                 interval = jiffies_to_msecs(NEIGH_VAR(p, DELAY_PROBE_TIME));
2581                 mlxsw_sp->router->neighs_update.interval = interval;
2582
2583                 mlxsw_sp_port_dev_put(mlxsw_sp_port);
2584                 break;
2585         case NETEVENT_NEIGH_UPDATE:
2586                 n = ptr;
2587
2588                 if (n->tbl->family != AF_INET && n->tbl->family != AF_INET6)
2589                         return NOTIFY_DONE;
2590
2591                 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(n->dev);
2592                 if (!mlxsw_sp_port)
2593                         return NOTIFY_DONE;
2594
2595                 net_work = kzalloc(sizeof(*net_work), GFP_ATOMIC);
2596                 if (!net_work) {
2597                         mlxsw_sp_port_dev_put(mlxsw_sp_port);
2598                         return NOTIFY_BAD;
2599                 }
2600
2601                 INIT_WORK(&net_work->work, mlxsw_sp_router_neigh_event_work);
2602                 net_work->mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
2603                 net_work->n = n;
2604
2605                 /* Take a reference to ensure the neighbour won't be
2606                  * destructed until we drop the reference in delayed
2607                  * work.
2608                  */
2609                 neigh_clone(n);
2610                 mlxsw_core_schedule_work(&net_work->work);
2611                 mlxsw_sp_port_dev_put(mlxsw_sp_port);
2612                 break;
2613         case NETEVENT_IPV4_MPATH_HASH_UPDATE:
2614         case NETEVENT_IPV6_MPATH_HASH_UPDATE:
2615                 return mlxsw_sp_router_schedule_work(ptr, nb,
2616                                 mlxsw_sp_router_mp_hash_event_work);
2617
2618         case NETEVENT_IPV4_FWD_UPDATE_PRIORITY_UPDATE:
2619                 return mlxsw_sp_router_schedule_work(ptr, nb,
2620                                 mlxsw_sp_router_update_priority_work);
2621         }
2622
2623         return NOTIFY_DONE;
2624 }
2625
2626 static int mlxsw_sp_neigh_init(struct mlxsw_sp *mlxsw_sp)
2627 {
2628         int err;
2629
2630         err = rhashtable_init(&mlxsw_sp->router->neigh_ht,
2631                               &mlxsw_sp_neigh_ht_params);
2632         if (err)
2633                 return err;
2634
2635         /* Initialize the polling interval according to the default
2636          * table.
2637          */
2638         mlxsw_sp_router_neighs_update_interval_init(mlxsw_sp);
2639
2640         /* Create the delayed works for the activity_update */
2641         INIT_DELAYED_WORK(&mlxsw_sp->router->neighs_update.dw,
2642                           mlxsw_sp_router_neighs_update_work);
2643         INIT_DELAYED_WORK(&mlxsw_sp->router->nexthop_probe_dw,
2644                           mlxsw_sp_router_probe_unresolved_nexthops);
2645         mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw, 0);
2646         mlxsw_core_schedule_dw(&mlxsw_sp->router->nexthop_probe_dw, 0);
2647         return 0;
2648 }
2649
2650 static void mlxsw_sp_neigh_fini(struct mlxsw_sp *mlxsw_sp)
2651 {
2652         cancel_delayed_work_sync(&mlxsw_sp->router->neighs_update.dw);
2653         cancel_delayed_work_sync(&mlxsw_sp->router->nexthop_probe_dw);
2654         rhashtable_destroy(&mlxsw_sp->router->neigh_ht);
2655 }
2656
2657 static void mlxsw_sp_neigh_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
2658                                          struct mlxsw_sp_rif *rif)
2659 {
2660         struct mlxsw_sp_neigh_entry *neigh_entry, *tmp;
2661
2662         list_for_each_entry_safe(neigh_entry, tmp, &rif->neigh_list,
2663                                  rif_list_node) {
2664                 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, false);
2665                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
2666         }
2667 }
2668
2669 enum mlxsw_sp_nexthop_type {
2670         MLXSW_SP_NEXTHOP_TYPE_ETH,
2671         MLXSW_SP_NEXTHOP_TYPE_IPIP,
2672 };
2673
2674 struct mlxsw_sp_nexthop_key {
2675         struct fib_nh *fib_nh;
2676 };
2677
2678 struct mlxsw_sp_nexthop {
2679         struct list_head neigh_list_node; /* member of neigh entry list */
2680         struct list_head rif_list_node;
2681         struct list_head router_list_node;
2682         struct mlxsw_sp_nexthop_group *nh_grp; /* pointer back to the group
2683                                                 * this belongs to
2684                                                 */
2685         struct rhash_head ht_node;
2686         struct mlxsw_sp_nexthop_key key;
2687         unsigned char gw_addr[sizeof(struct in6_addr)];
2688         int ifindex;
2689         int nh_weight;
2690         int norm_nh_weight;
2691         int num_adj_entries;
2692         struct mlxsw_sp_rif *rif;
2693         u8 should_offload:1, /* set indicates this neigh is connected and
2694                               * should be put to KVD linear area of this group.
2695                               */
2696            offloaded:1, /* set in case the neigh is actually put into
2697                          * KVD linear area of this group.
2698                          */
2699            update:1; /* set indicates that MAC of this neigh should be
2700                       * updated in HW
2701                       */
2702         enum mlxsw_sp_nexthop_type type;
2703         union {
2704                 struct mlxsw_sp_neigh_entry *neigh_entry;
2705                 struct mlxsw_sp_ipip_entry *ipip_entry;
2706         };
2707         unsigned int counter_index;
2708         bool counter_valid;
2709 };
2710
2711 struct mlxsw_sp_nexthop_group {
2712         void *priv;
2713         struct rhash_head ht_node;
2714         struct list_head fib_list; /* list of fib entries that use this group */
2715         struct neigh_table *neigh_tbl;
2716         u8 adj_index_valid:1,
2717            gateway:1; /* routes using the group use a gateway */
2718         u32 adj_index;
2719         u16 ecmp_size;
2720         u16 count;
2721         int sum_norm_weight;
2722         struct mlxsw_sp_nexthop nexthops[0];
2723 #define nh_rif  nexthops[0].rif
2724 };
2725
2726 void mlxsw_sp_nexthop_counter_alloc(struct mlxsw_sp *mlxsw_sp,
2727                                     struct mlxsw_sp_nexthop *nh)
2728 {
2729         struct devlink *devlink;
2730
2731         devlink = priv_to_devlink(mlxsw_sp->core);
2732         if (!devlink_dpipe_table_counter_enabled(devlink,
2733                                                  MLXSW_SP_DPIPE_TABLE_NAME_ADJ))
2734                 return;
2735
2736         if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &nh->counter_index))
2737                 return;
2738
2739         nh->counter_valid = true;
2740 }
2741
2742 void mlxsw_sp_nexthop_counter_free(struct mlxsw_sp *mlxsw_sp,
2743                                    struct mlxsw_sp_nexthop *nh)
2744 {
2745         if (!nh->counter_valid)
2746                 return;
2747         mlxsw_sp_flow_counter_free(mlxsw_sp, nh->counter_index);
2748         nh->counter_valid = false;
2749 }
2750
2751 int mlxsw_sp_nexthop_counter_get(struct mlxsw_sp *mlxsw_sp,
2752                                  struct mlxsw_sp_nexthop *nh, u64 *p_counter)
2753 {
2754         if (!nh->counter_valid)
2755                 return -EINVAL;
2756
2757         return mlxsw_sp_flow_counter_get(mlxsw_sp, nh->counter_index,
2758                                          p_counter, NULL);
2759 }
2760
2761 struct mlxsw_sp_nexthop *mlxsw_sp_nexthop_next(struct mlxsw_sp_router *router,
2762                                                struct mlxsw_sp_nexthop *nh)
2763 {
2764         if (!nh) {
2765                 if (list_empty(&router->nexthop_list))
2766                         return NULL;
2767                 else
2768                         return list_first_entry(&router->nexthop_list,
2769                                                 typeof(*nh), router_list_node);
2770         }
2771         if (list_is_last(&nh->router_list_node, &router->nexthop_list))
2772                 return NULL;
2773         return list_next_entry(nh, router_list_node);
2774 }
2775
2776 bool mlxsw_sp_nexthop_offload(struct mlxsw_sp_nexthop *nh)
2777 {
2778         return nh->offloaded;
2779 }
2780
2781 unsigned char *mlxsw_sp_nexthop_ha(struct mlxsw_sp_nexthop *nh)
2782 {
2783         if (!nh->offloaded)
2784                 return NULL;
2785         return nh->neigh_entry->ha;
2786 }
2787
2788 int mlxsw_sp_nexthop_indexes(struct mlxsw_sp_nexthop *nh, u32 *p_adj_index,
2789                              u32 *p_adj_size, u32 *p_adj_hash_index)
2790 {
2791         struct mlxsw_sp_nexthop_group *nh_grp = nh->nh_grp;
2792         u32 adj_hash_index = 0;
2793         int i;
2794
2795         if (!nh->offloaded || !nh_grp->adj_index_valid)
2796                 return -EINVAL;
2797
2798         *p_adj_index = nh_grp->adj_index;
2799         *p_adj_size = nh_grp->ecmp_size;
2800
2801         for (i = 0; i < nh_grp->count; i++) {
2802                 struct mlxsw_sp_nexthop *nh_iter = &nh_grp->nexthops[i];
2803
2804                 if (nh_iter == nh)
2805                         break;
2806                 if (nh_iter->offloaded)
2807                         adj_hash_index += nh_iter->num_adj_entries;
2808         }
2809
2810         *p_adj_hash_index = adj_hash_index;
2811         return 0;
2812 }
2813
2814 struct mlxsw_sp_rif *mlxsw_sp_nexthop_rif(struct mlxsw_sp_nexthop *nh)
2815 {
2816         return nh->rif;
2817 }
2818
2819 bool mlxsw_sp_nexthop_group_has_ipip(struct mlxsw_sp_nexthop *nh)
2820 {
2821         struct mlxsw_sp_nexthop_group *nh_grp = nh->nh_grp;
2822         int i;
2823
2824         for (i = 0; i < nh_grp->count; i++) {
2825                 struct mlxsw_sp_nexthop *nh_iter = &nh_grp->nexthops[i];
2826
2827                 if (nh_iter->type == MLXSW_SP_NEXTHOP_TYPE_IPIP)
2828                         return true;
2829         }
2830         return false;
2831 }
2832
2833 static struct fib_info *
2834 mlxsw_sp_nexthop4_group_fi(const struct mlxsw_sp_nexthop_group *nh_grp)
2835 {
2836         return nh_grp->priv;
2837 }
2838
2839 struct mlxsw_sp_nexthop_group_cmp_arg {
2840         enum mlxsw_sp_l3proto proto;
2841         union {
2842                 struct fib_info *fi;
2843                 struct mlxsw_sp_fib6_entry *fib6_entry;
2844         };
2845 };
2846
2847 static bool
2848 mlxsw_sp_nexthop6_group_has_nexthop(const struct mlxsw_sp_nexthop_group *nh_grp,
2849                                     const struct in6_addr *gw, int ifindex,
2850                                     int weight)
2851 {
2852         int i;
2853
2854         for (i = 0; i < nh_grp->count; i++) {
2855                 const struct mlxsw_sp_nexthop *nh;
2856
2857                 nh = &nh_grp->nexthops[i];
2858                 if (nh->ifindex == ifindex && nh->nh_weight == weight &&
2859                     ipv6_addr_equal(gw, (struct in6_addr *) nh->gw_addr))
2860                         return true;
2861         }
2862
2863         return false;
2864 }
2865
2866 static bool
2867 mlxsw_sp_nexthop6_group_cmp(const struct mlxsw_sp_nexthop_group *nh_grp,
2868                             const struct mlxsw_sp_fib6_entry *fib6_entry)
2869 {
2870         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
2871
2872         if (nh_grp->count != fib6_entry->nrt6)
2873                 return false;
2874
2875         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
2876                 struct in6_addr *gw;
2877                 int ifindex, weight;
2878
2879                 ifindex = mlxsw_sp_rt6->rt->fib6_nh.nh_dev->ifindex;
2880                 weight = mlxsw_sp_rt6->rt->fib6_nh.nh_weight;
2881                 gw = &mlxsw_sp_rt6->rt->fib6_nh.nh_gw;
2882                 if (!mlxsw_sp_nexthop6_group_has_nexthop(nh_grp, gw, ifindex,
2883                                                          weight))
2884                         return false;
2885         }
2886
2887         return true;
2888 }
2889
2890 static int
2891 mlxsw_sp_nexthop_group_cmp(struct rhashtable_compare_arg *arg, const void *ptr)
2892 {
2893         const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = arg->key;
2894         const struct mlxsw_sp_nexthop_group *nh_grp = ptr;
2895
2896         switch (cmp_arg->proto) {
2897         case MLXSW_SP_L3_PROTO_IPV4:
2898                 return cmp_arg->fi != mlxsw_sp_nexthop4_group_fi(nh_grp);
2899         case MLXSW_SP_L3_PROTO_IPV6:
2900                 return !mlxsw_sp_nexthop6_group_cmp(nh_grp,
2901                                                     cmp_arg->fib6_entry);
2902         default:
2903                 WARN_ON(1);
2904                 return 1;
2905         }
2906 }
2907
2908 static int
2909 mlxsw_sp_nexthop_group_type(const struct mlxsw_sp_nexthop_group *nh_grp)
2910 {
2911         return nh_grp->neigh_tbl->family;
2912 }
2913
2914 static u32 mlxsw_sp_nexthop_group_hash_obj(const void *data, u32 len, u32 seed)
2915 {
2916         const struct mlxsw_sp_nexthop_group *nh_grp = data;
2917         const struct mlxsw_sp_nexthop *nh;
2918         struct fib_info *fi;
2919         unsigned int val;
2920         int i;
2921
2922         switch (mlxsw_sp_nexthop_group_type(nh_grp)) {
2923         case AF_INET:
2924                 fi = mlxsw_sp_nexthop4_group_fi(nh_grp);
2925                 return jhash(&fi, sizeof(fi), seed);
2926         case AF_INET6:
2927                 val = nh_grp->count;
2928                 for (i = 0; i < nh_grp->count; i++) {
2929                         nh = &nh_grp->nexthops[i];
2930                         val ^= nh->ifindex;
2931                 }
2932                 return jhash(&val, sizeof(val), seed);
2933         default:
2934                 WARN_ON(1);
2935                 return 0;
2936         }
2937 }
2938
2939 static u32
2940 mlxsw_sp_nexthop6_group_hash(struct mlxsw_sp_fib6_entry *fib6_entry, u32 seed)
2941 {
2942         unsigned int val = fib6_entry->nrt6;
2943         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
2944         struct net_device *dev;
2945
2946         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
2947                 dev = mlxsw_sp_rt6->rt->fib6_nh.nh_dev;
2948                 val ^= dev->ifindex;
2949         }
2950
2951         return jhash(&val, sizeof(val), seed);
2952 }
2953
2954 static u32
2955 mlxsw_sp_nexthop_group_hash(const void *data, u32 len, u32 seed)
2956 {
2957         const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = data;
2958
2959         switch (cmp_arg->proto) {
2960         case MLXSW_SP_L3_PROTO_IPV4:
2961                 return jhash(&cmp_arg->fi, sizeof(cmp_arg->fi), seed);
2962         case MLXSW_SP_L3_PROTO_IPV6:
2963                 return mlxsw_sp_nexthop6_group_hash(cmp_arg->fib6_entry, seed);
2964         default:
2965                 WARN_ON(1);
2966                 return 0;
2967         }
2968 }
2969
2970 static const struct rhashtable_params mlxsw_sp_nexthop_group_ht_params = {
2971         .head_offset = offsetof(struct mlxsw_sp_nexthop_group, ht_node),
2972         .hashfn      = mlxsw_sp_nexthop_group_hash,
2973         .obj_hashfn  = mlxsw_sp_nexthop_group_hash_obj,
2974         .obj_cmpfn   = mlxsw_sp_nexthop_group_cmp,
2975 };
2976
2977 static int mlxsw_sp_nexthop_group_insert(struct mlxsw_sp *mlxsw_sp,
2978                                          struct mlxsw_sp_nexthop_group *nh_grp)
2979 {
2980         if (mlxsw_sp_nexthop_group_type(nh_grp) == AF_INET6 &&
2981             !nh_grp->gateway)
2982                 return 0;
2983
2984         return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_group_ht,
2985                                       &nh_grp->ht_node,
2986                                       mlxsw_sp_nexthop_group_ht_params);
2987 }
2988
2989 static void mlxsw_sp_nexthop_group_remove(struct mlxsw_sp *mlxsw_sp,
2990                                           struct mlxsw_sp_nexthop_group *nh_grp)
2991 {
2992         if (mlxsw_sp_nexthop_group_type(nh_grp) == AF_INET6 &&
2993             !nh_grp->gateway)
2994                 return;
2995
2996         rhashtable_remove_fast(&mlxsw_sp->router->nexthop_group_ht,
2997                                &nh_grp->ht_node,
2998                                mlxsw_sp_nexthop_group_ht_params);
2999 }
3000
3001 static struct mlxsw_sp_nexthop_group *
3002 mlxsw_sp_nexthop4_group_lookup(struct mlxsw_sp *mlxsw_sp,
3003                                struct fib_info *fi)
3004 {
3005         struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg;
3006
3007         cmp_arg.proto = MLXSW_SP_L3_PROTO_IPV4;
3008         cmp_arg.fi = fi;
3009         return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht,
3010                                       &cmp_arg,
3011                                       mlxsw_sp_nexthop_group_ht_params);
3012 }
3013
3014 static struct mlxsw_sp_nexthop_group *
3015 mlxsw_sp_nexthop6_group_lookup(struct mlxsw_sp *mlxsw_sp,
3016                                struct mlxsw_sp_fib6_entry *fib6_entry)
3017 {
3018         struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg;
3019
3020         cmp_arg.proto = MLXSW_SP_L3_PROTO_IPV6;
3021         cmp_arg.fib6_entry = fib6_entry;
3022         return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht,
3023                                       &cmp_arg,
3024                                       mlxsw_sp_nexthop_group_ht_params);
3025 }
3026
3027 static const struct rhashtable_params mlxsw_sp_nexthop_ht_params = {
3028         .key_offset = offsetof(struct mlxsw_sp_nexthop, key),
3029         .head_offset = offsetof(struct mlxsw_sp_nexthop, ht_node),
3030         .key_len = sizeof(struct mlxsw_sp_nexthop_key),
3031 };
3032
3033 static int mlxsw_sp_nexthop_insert(struct mlxsw_sp *mlxsw_sp,
3034                                    struct mlxsw_sp_nexthop *nh)
3035 {
3036         return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_ht,
3037                                       &nh->ht_node, mlxsw_sp_nexthop_ht_params);
3038 }
3039
3040 static void mlxsw_sp_nexthop_remove(struct mlxsw_sp *mlxsw_sp,
3041                                     struct mlxsw_sp_nexthop *nh)
3042 {
3043         rhashtable_remove_fast(&mlxsw_sp->router->nexthop_ht, &nh->ht_node,
3044                                mlxsw_sp_nexthop_ht_params);
3045 }
3046
3047 static struct mlxsw_sp_nexthop *
3048 mlxsw_sp_nexthop_lookup(struct mlxsw_sp *mlxsw_sp,
3049                         struct mlxsw_sp_nexthop_key key)
3050 {
3051         return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_ht, &key,
3052                                       mlxsw_sp_nexthop_ht_params);
3053 }
3054
3055 static int mlxsw_sp_adj_index_mass_update_vr(struct mlxsw_sp *mlxsw_sp,
3056                                              const struct mlxsw_sp_fib *fib,
3057                                              u32 adj_index, u16 ecmp_size,
3058                                              u32 new_adj_index,
3059                                              u16 new_ecmp_size)
3060 {
3061         char raleu_pl[MLXSW_REG_RALEU_LEN];
3062
3063         mlxsw_reg_raleu_pack(raleu_pl,
3064                              (enum mlxsw_reg_ralxx_protocol) fib->proto,
3065                              fib->vr->id, adj_index, ecmp_size, new_adj_index,
3066                              new_ecmp_size);
3067         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raleu), raleu_pl);
3068 }
3069
3070 static int mlxsw_sp_adj_index_mass_update(struct mlxsw_sp *mlxsw_sp,
3071                                           struct mlxsw_sp_nexthop_group *nh_grp,
3072                                           u32 old_adj_index, u16 old_ecmp_size)
3073 {
3074         struct mlxsw_sp_fib_entry *fib_entry;
3075         struct mlxsw_sp_fib *fib = NULL;
3076         int err;
3077
3078         list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
3079                 if (fib == fib_entry->fib_node->fib)
3080                         continue;
3081                 fib = fib_entry->fib_node->fib;
3082                 err = mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, fib,
3083                                                         old_adj_index,
3084                                                         old_ecmp_size,
3085                                                         nh_grp->adj_index,
3086                                                         nh_grp->ecmp_size);
3087                 if (err)
3088                         return err;
3089         }
3090         return 0;
3091 }
3092
3093 static int __mlxsw_sp_nexthop_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index,
3094                                      struct mlxsw_sp_nexthop *nh)
3095 {
3096         struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
3097         char ratr_pl[MLXSW_REG_RATR_LEN];
3098
3099         mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY,
3100                             true, MLXSW_REG_RATR_TYPE_ETHERNET,
3101                             adj_index, neigh_entry->rif);
3102         mlxsw_reg_ratr_eth_entry_pack(ratr_pl, neigh_entry->ha);
3103         if (nh->counter_valid)
3104                 mlxsw_reg_ratr_counter_pack(ratr_pl, nh->counter_index, true);
3105         else
3106                 mlxsw_reg_ratr_counter_pack(ratr_pl, 0, false);
3107
3108         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl);
3109 }
3110
3111 int mlxsw_sp_nexthop_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index,
3112                             struct mlxsw_sp_nexthop *nh)
3113 {
3114         int i;
3115
3116         for (i = 0; i < nh->num_adj_entries; i++) {
3117                 int err;
3118
3119                 err = __mlxsw_sp_nexthop_update(mlxsw_sp, adj_index + i, nh);
3120                 if (err)
3121                         return err;
3122         }
3123
3124         return 0;
3125 }
3126
3127 static int __mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp,
3128                                           u32 adj_index,
3129                                           struct mlxsw_sp_nexthop *nh)
3130 {
3131         const struct mlxsw_sp_ipip_ops *ipip_ops;
3132
3133         ipip_ops = mlxsw_sp->router->ipip_ops_arr[nh->ipip_entry->ipipt];
3134         return ipip_ops->nexthop_update(mlxsw_sp, adj_index, nh->ipip_entry);
3135 }
3136
3137 static int mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp,
3138                                         u32 adj_index,
3139                                         struct mlxsw_sp_nexthop *nh)
3140 {
3141         int i;
3142
3143         for (i = 0; i < nh->num_adj_entries; i++) {
3144                 int err;
3145
3146                 err = __mlxsw_sp_nexthop_ipip_update(mlxsw_sp, adj_index + i,
3147                                                      nh);
3148                 if (err)
3149                         return err;
3150         }
3151
3152         return 0;
3153 }
3154
3155 static int
3156 mlxsw_sp_nexthop_group_update(struct mlxsw_sp *mlxsw_sp,
3157                               struct mlxsw_sp_nexthop_group *nh_grp,
3158                               bool reallocate)
3159 {
3160         u32 adj_index = nh_grp->adj_index; /* base */
3161         struct mlxsw_sp_nexthop *nh;
3162         int i;
3163         int err;
3164
3165         for (i = 0; i < nh_grp->count; i++) {
3166                 nh = &nh_grp->nexthops[i];
3167
3168                 if (!nh->should_offload) {
3169                         nh->offloaded = 0;
3170                         continue;
3171                 }
3172
3173                 if (nh->update || reallocate) {
3174                         switch (nh->type) {
3175                         case MLXSW_SP_NEXTHOP_TYPE_ETH:
3176                                 err = mlxsw_sp_nexthop_update
3177                                             (mlxsw_sp, adj_index, nh);
3178                                 break;
3179                         case MLXSW_SP_NEXTHOP_TYPE_IPIP:
3180                                 err = mlxsw_sp_nexthop_ipip_update
3181                                             (mlxsw_sp, adj_index, nh);
3182                                 break;
3183                         }
3184                         if (err)
3185                                 return err;
3186                         nh->update = 0;
3187                         nh->offloaded = 1;
3188                 }
3189                 adj_index += nh->num_adj_entries;
3190         }
3191         return 0;
3192 }
3193
3194 static bool
3195 mlxsw_sp_fib_node_entry_is_first(const struct mlxsw_sp_fib_node *fib_node,
3196                                  const struct mlxsw_sp_fib_entry *fib_entry);
3197
3198 static int
3199 mlxsw_sp_nexthop_fib_entries_update(struct mlxsw_sp *mlxsw_sp,
3200                                     struct mlxsw_sp_nexthop_group *nh_grp)
3201 {
3202         struct mlxsw_sp_fib_entry *fib_entry;
3203         int err;
3204
3205         list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
3206                 if (!mlxsw_sp_fib_node_entry_is_first(fib_entry->fib_node,
3207                                                       fib_entry))
3208                         continue;
3209                 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
3210                 if (err)
3211                         return err;
3212         }
3213         return 0;
3214 }
3215
3216 static void
3217 mlxsw_sp_fib_entry_offload_refresh(struct mlxsw_sp_fib_entry *fib_entry,
3218                                    enum mlxsw_reg_ralue_op op, int err);
3219
3220 static void
3221 mlxsw_sp_nexthop_fib_entries_refresh(struct mlxsw_sp_nexthop_group *nh_grp)
3222 {
3223         enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_WRITE;
3224         struct mlxsw_sp_fib_entry *fib_entry;
3225
3226         list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
3227                 if (!mlxsw_sp_fib_node_entry_is_first(fib_entry->fib_node,
3228                                                       fib_entry))
3229                         continue;
3230                 mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, 0);
3231         }
3232 }
3233
3234 static void mlxsw_sp_adj_grp_size_round_up(u16 *p_adj_grp_size)
3235 {
3236         /* Valid sizes for an adjacency group are:
3237          * 1-64, 512, 1024, 2048 and 4096.
3238          */
3239         if (*p_adj_grp_size <= 64)
3240                 return;
3241         else if (*p_adj_grp_size <= 512)
3242                 *p_adj_grp_size = 512;
3243         else if (*p_adj_grp_size <= 1024)
3244                 *p_adj_grp_size = 1024;
3245         else if (*p_adj_grp_size <= 2048)
3246                 *p_adj_grp_size = 2048;
3247         else
3248                 *p_adj_grp_size = 4096;
3249 }
3250
3251 static void mlxsw_sp_adj_grp_size_round_down(u16 *p_adj_grp_size,
3252                                              unsigned int alloc_size)
3253 {
3254         if (alloc_size >= 4096)
3255                 *p_adj_grp_size = 4096;
3256         else if (alloc_size >= 2048)
3257                 *p_adj_grp_size = 2048;
3258         else if (alloc_size >= 1024)
3259                 *p_adj_grp_size = 1024;
3260         else if (alloc_size >= 512)
3261                 *p_adj_grp_size = 512;
3262 }
3263
3264 static int mlxsw_sp_fix_adj_grp_size(struct mlxsw_sp *mlxsw_sp,
3265                                      u16 *p_adj_grp_size)
3266 {
3267         unsigned int alloc_size;
3268         int err;
3269
3270         /* Round up the requested group size to the next size supported
3271          * by the device and make sure the request can be satisfied.
3272          */
3273         mlxsw_sp_adj_grp_size_round_up(p_adj_grp_size);
3274         err = mlxsw_sp_kvdl_alloc_count_query(mlxsw_sp,
3275                                               MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
3276                                               *p_adj_grp_size, &alloc_size);
3277         if (err)
3278                 return err;
3279         /* It is possible the allocation results in more allocated
3280          * entries than requested. Try to use as much of them as
3281          * possible.
3282          */
3283         mlxsw_sp_adj_grp_size_round_down(p_adj_grp_size, alloc_size);
3284
3285         return 0;
3286 }
3287
3288 static void
3289 mlxsw_sp_nexthop_group_normalize(struct mlxsw_sp_nexthop_group *nh_grp)
3290 {
3291         int i, g = 0, sum_norm_weight = 0;
3292         struct mlxsw_sp_nexthop *nh;
3293
3294         for (i = 0; i < nh_grp->count; i++) {
3295                 nh = &nh_grp->nexthops[i];
3296
3297                 if (!nh->should_offload)
3298                         continue;
3299                 if (g > 0)
3300                         g = gcd(nh->nh_weight, g);
3301                 else
3302                         g = nh->nh_weight;
3303         }
3304
3305         for (i = 0; i < nh_grp->count; i++) {
3306                 nh = &nh_grp->nexthops[i];
3307
3308                 if (!nh->should_offload)
3309                         continue;
3310                 nh->norm_nh_weight = nh->nh_weight / g;
3311                 sum_norm_weight += nh->norm_nh_weight;
3312         }
3313
3314         nh_grp->sum_norm_weight = sum_norm_weight;
3315 }
3316
3317 static void
3318 mlxsw_sp_nexthop_group_rebalance(struct mlxsw_sp_nexthop_group *nh_grp)
3319 {
3320         int total = nh_grp->sum_norm_weight;
3321         u16 ecmp_size = nh_grp->ecmp_size;
3322         int i, weight = 0, lower_bound = 0;
3323
3324         for (i = 0; i < nh_grp->count; i++) {
3325                 struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i];
3326                 int upper_bound;
3327
3328                 if (!nh->should_offload)
3329                         continue;
3330                 weight += nh->norm_nh_weight;
3331                 upper_bound = DIV_ROUND_CLOSEST(ecmp_size * weight, total);
3332                 nh->num_adj_entries = upper_bound - lower_bound;
3333                 lower_bound = upper_bound;
3334         }
3335 }
3336
3337 static void
3338 mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp,
3339                                struct mlxsw_sp_nexthop_group *nh_grp)
3340 {
3341         u16 ecmp_size, old_ecmp_size;
3342         struct mlxsw_sp_nexthop *nh;
3343         bool offload_change = false;
3344         u32 adj_index;
3345         bool old_adj_index_valid;
3346         u32 old_adj_index;
3347         int i;
3348         int err;
3349
3350         if (!nh_grp->gateway) {
3351                 mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
3352                 return;
3353         }
3354
3355         for (i = 0; i < nh_grp->count; i++) {
3356                 nh = &nh_grp->nexthops[i];
3357
3358                 if (nh->should_offload != nh->offloaded) {
3359                         offload_change = true;
3360                         if (nh->should_offload)
3361                                 nh->update = 1;
3362                 }
3363         }
3364         if (!offload_change) {
3365                 /* Nothing was added or removed, so no need to reallocate. Just
3366                  * update MAC on existing adjacency indexes.
3367                  */
3368                 err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nh_grp, false);
3369                 if (err) {
3370                         dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
3371                         goto set_trap;
3372                 }
3373                 return;
3374         }
3375         mlxsw_sp_nexthop_group_normalize(nh_grp);
3376         if (!nh_grp->sum_norm_weight)
3377                 /* No neigh of this group is connected so we just set
3378                  * the trap and let everthing flow through kernel.
3379                  */
3380                 goto set_trap;
3381
3382         ecmp_size = nh_grp->sum_norm_weight;
3383         err = mlxsw_sp_fix_adj_grp_size(mlxsw_sp, &ecmp_size);
3384         if (err)
3385                 /* No valid allocation size available. */
3386                 goto set_trap;
3387
3388         err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
3389                                   ecmp_size, &adj_index);
3390         if (err) {
3391                 /* We ran out of KVD linear space, just set the
3392                  * trap and let everything flow through kernel.
3393                  */
3394                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to allocate KVD linear area for nexthop group.\n");
3395                 goto set_trap;
3396         }
3397         old_adj_index_valid = nh_grp->adj_index_valid;
3398         old_adj_index = nh_grp->adj_index;
3399         old_ecmp_size = nh_grp->ecmp_size;
3400         nh_grp->adj_index_valid = 1;
3401         nh_grp->adj_index = adj_index;
3402         nh_grp->ecmp_size = ecmp_size;
3403         mlxsw_sp_nexthop_group_rebalance(nh_grp);
3404         err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nh_grp, true);
3405         if (err) {
3406                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
3407                 goto set_trap;
3408         }
3409
3410         if (!old_adj_index_valid) {
3411                 /* The trap was set for fib entries, so we have to call
3412                  * fib entry update to unset it and use adjacency index.
3413                  */
3414                 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
3415                 if (err) {
3416                         dev_warn(mlxsw_sp->bus_info->dev, "Failed to add adjacency index to fib entries.\n");
3417                         goto set_trap;
3418                 }
3419                 return;
3420         }
3421
3422         err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, nh_grp,
3423                                              old_adj_index, old_ecmp_size);
3424         mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
3425                            old_ecmp_size, old_adj_index);
3426         if (err) {
3427                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to mass-update adjacency index for nexthop group.\n");
3428                 goto set_trap;
3429         }
3430
3431         /* Offload state within the group changed, so update the flags. */
3432         mlxsw_sp_nexthop_fib_entries_refresh(nh_grp);
3433
3434         return;
3435
3436 set_trap:
3437         old_adj_index_valid = nh_grp->adj_index_valid;
3438         nh_grp->adj_index_valid = 0;
3439         for (i = 0; i < nh_grp->count; i++) {
3440                 nh = &nh_grp->nexthops[i];
3441                 nh->offloaded = 0;
3442         }
3443         err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
3444         if (err)
3445                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set traps for fib entries.\n");
3446         if (old_adj_index_valid)
3447                 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ,
3448                                    nh_grp->ecmp_size, nh_grp->adj_index);
3449 }
3450
3451 static void __mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp_nexthop *nh,
3452                                             bool removing)
3453 {
3454         if (!removing)
3455                 nh->should_offload = 1;
3456         else
3457                 nh->should_offload = 0;
3458         nh->update = 1;
3459 }
3460
3461 static void
3462 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
3463                               struct mlxsw_sp_neigh_entry *neigh_entry,
3464                               bool removing)
3465 {
3466         struct mlxsw_sp_nexthop *nh;
3467
3468         list_for_each_entry(nh, &neigh_entry->nexthop_list,
3469                             neigh_list_node) {
3470                 __mlxsw_sp_nexthop_neigh_update(nh, removing);
3471                 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
3472         }
3473 }
3474
3475 static void mlxsw_sp_nexthop_rif_init(struct mlxsw_sp_nexthop *nh,
3476                                       struct mlxsw_sp_rif *rif)
3477 {
3478         if (nh->rif)
3479                 return;
3480
3481         nh->rif = rif;
3482         list_add(&nh->rif_list_node, &rif->nexthop_list);
3483 }
3484
3485 static void mlxsw_sp_nexthop_rif_fini(struct mlxsw_sp_nexthop *nh)
3486 {
3487         if (!nh->rif)
3488                 return;
3489
3490         list_del(&nh->rif_list_node);
3491         nh->rif = NULL;
3492 }
3493
3494 static int mlxsw_sp_nexthop_neigh_init(struct mlxsw_sp *mlxsw_sp,
3495                                        struct mlxsw_sp_nexthop *nh)
3496 {
3497         struct mlxsw_sp_neigh_entry *neigh_entry;
3498         struct neighbour *n;
3499         u8 nud_state, dead;
3500         int err;
3501
3502         if (!nh->nh_grp->gateway || nh->neigh_entry)
3503                 return 0;
3504
3505         /* Take a reference of neigh here ensuring that neigh would
3506          * not be destructed before the nexthop entry is finished.
3507          * The reference is taken either in neigh_lookup() or
3508          * in neigh_create() in case n is not found.
3509          */
3510         n = neigh_lookup(nh->nh_grp->neigh_tbl, &nh->gw_addr, nh->rif->dev);
3511         if (!n) {
3512                 n = neigh_create(nh->nh_grp->neigh_tbl, &nh->gw_addr,
3513                                  nh->rif->dev);
3514                 if (IS_ERR(n))
3515                         return PTR_ERR(n);
3516                 neigh_event_send(n, NULL);
3517         }
3518         neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
3519         if (!neigh_entry) {
3520                 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
3521                 if (IS_ERR(neigh_entry)) {
3522                         err = -EINVAL;
3523                         goto err_neigh_entry_create;
3524                 }
3525         }
3526
3527         /* If that is the first nexthop connected to that neigh, add to
3528          * nexthop_neighs_list
3529          */
3530         if (list_empty(&neigh_entry->nexthop_list))
3531                 list_add_tail(&neigh_entry->nexthop_neighs_list_node,
3532                               &mlxsw_sp->router->nexthop_neighs_list);
3533
3534         nh->neigh_entry = neigh_entry;
3535         list_add_tail(&nh->neigh_list_node, &neigh_entry->nexthop_list);
3536         read_lock_bh(&n->lock);
3537         nud_state = n->nud_state;
3538         dead = n->dead;
3539         read_unlock_bh(&n->lock);
3540         __mlxsw_sp_nexthop_neigh_update(nh, !(nud_state & NUD_VALID && !dead));
3541
3542         return 0;
3543
3544 err_neigh_entry_create:
3545         neigh_release(n);
3546         return err;
3547 }
3548
3549 static void mlxsw_sp_nexthop_neigh_fini(struct mlxsw_sp *mlxsw_sp,
3550                                         struct mlxsw_sp_nexthop *nh)
3551 {
3552         struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
3553         struct neighbour *n;
3554
3555         if (!neigh_entry)
3556                 return;
3557         n = neigh_entry->key.n;
3558
3559         __mlxsw_sp_nexthop_neigh_update(nh, true);
3560         list_del(&nh->neigh_list_node);
3561         nh->neigh_entry = NULL;
3562
3563         /* If that is the last nexthop connected to that neigh, remove from
3564          * nexthop_neighs_list
3565          */
3566         if (list_empty(&neigh_entry->nexthop_list))
3567                 list_del(&neigh_entry->nexthop_neighs_list_node);
3568
3569         if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
3570                 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
3571
3572         neigh_release(n);
3573 }
3574
3575 static bool mlxsw_sp_ipip_netdev_ul_up(struct net_device *ol_dev)
3576 {
3577         struct net_device *ul_dev = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev);
3578
3579         return ul_dev ? (ul_dev->flags & IFF_UP) : true;
3580 }
3581
3582 static void mlxsw_sp_nexthop_ipip_init(struct mlxsw_sp *mlxsw_sp,
3583                                        struct mlxsw_sp_nexthop *nh,
3584                                        struct mlxsw_sp_ipip_entry *ipip_entry)
3585 {
3586         bool removing;
3587
3588         if (!nh->nh_grp->gateway || nh->ipip_entry)
3589                 return;
3590
3591         nh->ipip_entry = ipip_entry;
3592         removing = !mlxsw_sp_ipip_netdev_ul_up(ipip_entry->ol_dev);
3593         __mlxsw_sp_nexthop_neigh_update(nh, removing);
3594         mlxsw_sp_nexthop_rif_init(nh, &ipip_entry->ol_lb->common);
3595 }
3596
3597 static void mlxsw_sp_nexthop_ipip_fini(struct mlxsw_sp *mlxsw_sp,
3598                                        struct mlxsw_sp_nexthop *nh)
3599 {
3600         struct mlxsw_sp_ipip_entry *ipip_entry = nh->ipip_entry;
3601
3602         if (!ipip_entry)
3603                 return;
3604
3605         __mlxsw_sp_nexthop_neigh_update(nh, true);
3606         nh->ipip_entry = NULL;
3607 }
3608
3609 static bool mlxsw_sp_nexthop4_ipip_type(const struct mlxsw_sp *mlxsw_sp,
3610                                         const struct fib_nh *fib_nh,
3611                                         enum mlxsw_sp_ipip_type *p_ipipt)
3612 {
3613         struct net_device *dev = fib_nh->nh_dev;
3614
3615         return dev &&
3616                fib_nh->nh_parent->fib_type == RTN_UNICAST &&
3617                mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, p_ipipt);
3618 }
3619
3620 static void mlxsw_sp_nexthop_type_fini(struct mlxsw_sp *mlxsw_sp,
3621                                        struct mlxsw_sp_nexthop *nh)
3622 {
3623         switch (nh->type) {
3624         case MLXSW_SP_NEXTHOP_TYPE_ETH:
3625                 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
3626                 mlxsw_sp_nexthop_rif_fini(nh);
3627                 break;
3628         case MLXSW_SP_NEXTHOP_TYPE_IPIP:
3629                 mlxsw_sp_nexthop_rif_fini(nh);
3630                 mlxsw_sp_nexthop_ipip_fini(mlxsw_sp, nh);
3631                 break;
3632         }
3633 }
3634
3635 static int mlxsw_sp_nexthop4_type_init(struct mlxsw_sp *mlxsw_sp,
3636                                        struct mlxsw_sp_nexthop *nh,
3637                                        struct fib_nh *fib_nh)
3638 {
3639         const struct mlxsw_sp_ipip_ops *ipip_ops;
3640         struct net_device *dev = fib_nh->nh_dev;
3641         struct mlxsw_sp_ipip_entry *ipip_entry;
3642         struct mlxsw_sp_rif *rif;
3643         int err;
3644
3645         ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, dev);
3646         if (ipip_entry) {
3647                 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
3648                 if (ipip_ops->can_offload(mlxsw_sp, dev,
3649                                           MLXSW_SP_L3_PROTO_IPV4)) {
3650                         nh->type = MLXSW_SP_NEXTHOP_TYPE_IPIP;
3651                         mlxsw_sp_nexthop_ipip_init(mlxsw_sp, nh, ipip_entry);
3652                         return 0;
3653                 }
3654         }
3655
3656         nh->type = MLXSW_SP_NEXTHOP_TYPE_ETH;
3657         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
3658         if (!rif)
3659                 return 0;
3660
3661         mlxsw_sp_nexthop_rif_init(nh, rif);
3662         err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
3663         if (err)
3664                 goto err_neigh_init;
3665
3666         return 0;
3667
3668 err_neigh_init:
3669         mlxsw_sp_nexthop_rif_fini(nh);
3670         return err;
3671 }
3672
3673 static void mlxsw_sp_nexthop4_type_fini(struct mlxsw_sp *mlxsw_sp,
3674                                         struct mlxsw_sp_nexthop *nh)
3675 {
3676         mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
3677 }
3678
3679 static int mlxsw_sp_nexthop4_init(struct mlxsw_sp *mlxsw_sp,
3680                                   struct mlxsw_sp_nexthop_group *nh_grp,
3681                                   struct mlxsw_sp_nexthop *nh,
3682                                   struct fib_nh *fib_nh)
3683 {
3684         struct net_device *dev = fib_nh->nh_dev;
3685         struct in_device *in_dev;
3686         int err;
3687
3688         nh->nh_grp = nh_grp;
3689         nh->key.fib_nh = fib_nh;
3690 #ifdef CONFIG_IP_ROUTE_MULTIPATH
3691         nh->nh_weight = fib_nh->nh_weight;
3692 #else
3693         nh->nh_weight = 1;
3694 #endif
3695         memcpy(&nh->gw_addr, &fib_nh->nh_gw, sizeof(fib_nh->nh_gw));
3696         err = mlxsw_sp_nexthop_insert(mlxsw_sp, nh);
3697         if (err)
3698                 return err;
3699
3700         mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh);
3701         list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list);
3702
3703         if (!dev)
3704                 return 0;
3705
3706         in_dev = __in_dev_get_rtnl(dev);
3707         if (in_dev && IN_DEV_IGNORE_ROUTES_WITH_LINKDOWN(in_dev) &&
3708             fib_nh->nh_flags & RTNH_F_LINKDOWN)
3709                 return 0;
3710
3711         err = mlxsw_sp_nexthop4_type_init(mlxsw_sp, nh, fib_nh);
3712         if (err)
3713                 goto err_nexthop_neigh_init;
3714
3715         return 0;
3716
3717 err_nexthop_neigh_init:
3718         mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
3719         return err;
3720 }
3721
3722 static void mlxsw_sp_nexthop4_fini(struct mlxsw_sp *mlxsw_sp,
3723                                    struct mlxsw_sp_nexthop *nh)
3724 {
3725         mlxsw_sp_nexthop4_type_fini(mlxsw_sp, nh);
3726         list_del(&nh->router_list_node);
3727         mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh);
3728         mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
3729 }
3730
3731 static void mlxsw_sp_nexthop4_event(struct mlxsw_sp *mlxsw_sp,
3732                                     unsigned long event, struct fib_nh *fib_nh)
3733 {
3734         struct mlxsw_sp_nexthop_key key;
3735         struct mlxsw_sp_nexthop *nh;
3736
3737         if (mlxsw_sp->router->aborted)
3738                 return;
3739
3740         key.fib_nh = fib_nh;
3741         nh = mlxsw_sp_nexthop_lookup(mlxsw_sp, key);
3742         if (WARN_ON_ONCE(!nh))
3743                 return;
3744
3745         switch (event) {
3746         case FIB_EVENT_NH_ADD:
3747                 mlxsw_sp_nexthop4_type_init(mlxsw_sp, nh, fib_nh);
3748                 break;
3749         case FIB_EVENT_NH_DEL:
3750                 mlxsw_sp_nexthop4_type_fini(mlxsw_sp, nh);
3751                 break;
3752         }
3753
3754         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
3755 }
3756
3757 static void mlxsw_sp_nexthop_rif_update(struct mlxsw_sp *mlxsw_sp,
3758                                         struct mlxsw_sp_rif *rif)
3759 {
3760         struct mlxsw_sp_nexthop *nh;
3761         bool removing;
3762
3763         list_for_each_entry(nh, &rif->nexthop_list, rif_list_node) {
3764                 switch (nh->type) {
3765                 case MLXSW_SP_NEXTHOP_TYPE_ETH:
3766                         removing = false;
3767                         break;
3768                 case MLXSW_SP_NEXTHOP_TYPE_IPIP:
3769                         removing = !mlxsw_sp_ipip_netdev_ul_up(rif->dev);
3770                         break;
3771                 default:
3772                         WARN_ON(1);
3773                         continue;
3774                 }
3775
3776                 __mlxsw_sp_nexthop_neigh_update(nh, removing);
3777                 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
3778         }
3779 }
3780
3781 static void mlxsw_sp_nexthop_rif_migrate(struct mlxsw_sp *mlxsw_sp,
3782                                          struct mlxsw_sp_rif *old_rif,
3783                                          struct mlxsw_sp_rif *new_rif)
3784 {
3785         struct mlxsw_sp_nexthop *nh;
3786
3787         list_splice_init(&old_rif->nexthop_list, &new_rif->nexthop_list);
3788         list_for_each_entry(nh, &new_rif->nexthop_list, rif_list_node)
3789                 nh->rif = new_rif;
3790         mlxsw_sp_nexthop_rif_update(mlxsw_sp, new_rif);
3791 }
3792
3793 static void mlxsw_sp_nexthop_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
3794                                            struct mlxsw_sp_rif *rif)
3795 {
3796         struct mlxsw_sp_nexthop *nh, *tmp;
3797
3798         list_for_each_entry_safe(nh, tmp, &rif->nexthop_list, rif_list_node) {
3799                 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
3800                 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
3801         }
3802 }
3803
3804 static bool mlxsw_sp_fi_is_gateway(const struct mlxsw_sp *mlxsw_sp,
3805                                    const struct fib_info *fi)
3806 {
3807         return fi->fib_nh->nh_scope == RT_SCOPE_LINK ||
3808                mlxsw_sp_nexthop4_ipip_type(mlxsw_sp, fi->fib_nh, NULL);
3809 }
3810
3811 static struct mlxsw_sp_nexthop_group *
3812 mlxsw_sp_nexthop4_group_create(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi)
3813 {
3814         struct mlxsw_sp_nexthop_group *nh_grp;
3815         struct mlxsw_sp_nexthop *nh;
3816         struct fib_nh *fib_nh;
3817         int i;
3818         int err;
3819
3820         nh_grp = kzalloc(struct_size(nh_grp, nexthops, fi->fib_nhs),
3821                          GFP_KERNEL);
3822         if (!nh_grp)
3823                 return ERR_PTR(-ENOMEM);
3824         nh_grp->priv = fi;
3825         INIT_LIST_HEAD(&nh_grp->fib_list);
3826         nh_grp->neigh_tbl = &arp_tbl;
3827
3828         nh_grp->gateway = mlxsw_sp_fi_is_gateway(mlxsw_sp, fi);
3829         nh_grp->count = fi->fib_nhs;
3830         fib_info_hold(fi);
3831         for (i = 0; i < nh_grp->count; i++) {
3832                 nh = &nh_grp->nexthops[i];
3833                 fib_nh = &fi->fib_nh[i];
3834                 err = mlxsw_sp_nexthop4_init(mlxsw_sp, nh_grp, nh, fib_nh);
3835                 if (err)
3836                         goto err_nexthop4_init;
3837         }
3838         err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
3839         if (err)
3840                 goto err_nexthop_group_insert;
3841         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
3842         return nh_grp;
3843
3844 err_nexthop_group_insert:
3845 err_nexthop4_init:
3846         for (i--; i >= 0; i--) {
3847                 nh = &nh_grp->nexthops[i];
3848                 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh);
3849         }
3850         fib_info_put(fi);
3851         kfree(nh_grp);
3852         return ERR_PTR(err);
3853 }
3854
3855 static void
3856 mlxsw_sp_nexthop4_group_destroy(struct mlxsw_sp *mlxsw_sp,
3857                                 struct mlxsw_sp_nexthop_group *nh_grp)
3858 {
3859         struct mlxsw_sp_nexthop *nh;
3860         int i;
3861
3862         mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
3863         for (i = 0; i < nh_grp->count; i++) {
3864                 nh = &nh_grp->nexthops[i];
3865                 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh);
3866         }
3867         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
3868         WARN_ON_ONCE(nh_grp->adj_index_valid);
3869         fib_info_put(mlxsw_sp_nexthop4_group_fi(nh_grp));
3870         kfree(nh_grp);
3871 }
3872
3873 static int mlxsw_sp_nexthop4_group_get(struct mlxsw_sp *mlxsw_sp,
3874                                        struct mlxsw_sp_fib_entry *fib_entry,
3875                                        struct fib_info *fi)
3876 {
3877         struct mlxsw_sp_nexthop_group *nh_grp;
3878
3879         nh_grp = mlxsw_sp_nexthop4_group_lookup(mlxsw_sp, fi);
3880         if (!nh_grp) {
3881                 nh_grp = mlxsw_sp_nexthop4_group_create(mlxsw_sp, fi);
3882                 if (IS_ERR(nh_grp))
3883                         return PTR_ERR(nh_grp);
3884         }
3885         list_add_tail(&fib_entry->nexthop_group_node, &nh_grp->fib_list);
3886         fib_entry->nh_group = nh_grp;
3887         return 0;
3888 }
3889
3890 static void mlxsw_sp_nexthop4_group_put(struct mlxsw_sp *mlxsw_sp,
3891                                         struct mlxsw_sp_fib_entry *fib_entry)
3892 {
3893         struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
3894
3895         list_del(&fib_entry->nexthop_group_node);
3896         if (!list_empty(&nh_grp->fib_list))
3897                 return;
3898         mlxsw_sp_nexthop4_group_destroy(mlxsw_sp, nh_grp);
3899 }
3900
3901 static bool
3902 mlxsw_sp_fib4_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry)
3903 {
3904         struct mlxsw_sp_fib4_entry *fib4_entry;
3905
3906         fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry,
3907                                   common);
3908         return !fib4_entry->tos;
3909 }
3910
3911 static bool
3912 mlxsw_sp_fib_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry)
3913 {
3914         struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group;
3915
3916         switch (fib_entry->fib_node->fib->proto) {
3917         case MLXSW_SP_L3_PROTO_IPV4:
3918                 if (!mlxsw_sp_fib4_entry_should_offload(fib_entry))
3919                         return false;
3920                 break;
3921         case MLXSW_SP_L3_PROTO_IPV6:
3922                 break;
3923         }
3924
3925         switch (fib_entry->type) {
3926         case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
3927                 return !!nh_group->adj_index_valid;
3928         case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
3929                 return !!nh_group->nh_rif;
3930         case MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE:
3931         case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP:
3932         case MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP:
3933                 return true;
3934         default:
3935                 return false;
3936         }
3937 }
3938
3939 static struct mlxsw_sp_nexthop *
3940 mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp,
3941                      const struct mlxsw_sp_rt6 *mlxsw_sp_rt6)
3942 {
3943         int i;
3944
3945         for (i = 0; i < nh_grp->count; i++) {
3946                 struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i];
3947                 struct fib6_info *rt = mlxsw_sp_rt6->rt;
3948
3949                 if (nh->rif && nh->rif->dev == rt->fib6_nh.nh_dev &&
3950                     ipv6_addr_equal((const struct in6_addr *) &nh->gw_addr,
3951                                     &rt->fib6_nh.nh_gw))
3952                         return nh;
3953                 continue;
3954         }
3955
3956         return NULL;
3957 }
3958
3959 static void
3960 mlxsw_sp_fib4_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry)
3961 {
3962         struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
3963         int i;
3964
3965         if (fib_entry->type == MLXSW_SP_FIB_ENTRY_TYPE_LOCAL ||
3966             fib_entry->type == MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE ||
3967             fib_entry->type == MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP ||
3968             fib_entry->type == MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP) {
3969                 nh_grp->nexthops->key.fib_nh->nh_flags |= RTNH_F_OFFLOAD;
3970                 return;
3971         }
3972
3973         for (i = 0; i < nh_grp->count; i++) {
3974                 struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i];
3975
3976                 if (nh->offloaded)
3977                         nh->key.fib_nh->nh_flags |= RTNH_F_OFFLOAD;
3978                 else
3979                         nh->key.fib_nh->nh_flags &= ~RTNH_F_OFFLOAD;
3980         }
3981 }
3982
3983 static void
3984 mlxsw_sp_fib4_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry)
3985 {
3986         struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
3987         int i;
3988
3989         if (!list_is_singular(&nh_grp->fib_list))
3990                 return;
3991
3992         for (i = 0; i < nh_grp->count; i++) {
3993                 struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i];
3994
3995                 nh->key.fib_nh->nh_flags &= ~RTNH_F_OFFLOAD;
3996         }
3997 }
3998
3999 static void
4000 mlxsw_sp_fib6_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry)
4001 {
4002         struct mlxsw_sp_fib6_entry *fib6_entry;
4003         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
4004
4005         fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry,
4006                                   common);
4007
4008         if (fib_entry->type == MLXSW_SP_FIB_ENTRY_TYPE_LOCAL ||
4009             fib_entry->type == MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE) {
4010                 list_first_entry(&fib6_entry->rt6_list, struct mlxsw_sp_rt6,
4011                                  list)->rt->fib6_nh.nh_flags |= RTNH_F_OFFLOAD;
4012                 return;
4013         }
4014
4015         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
4016                 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
4017                 struct mlxsw_sp_nexthop *nh;
4018
4019                 nh = mlxsw_sp_rt6_nexthop(nh_grp, mlxsw_sp_rt6);
4020                 if (nh && nh->offloaded)
4021                         mlxsw_sp_rt6->rt->fib6_nh.nh_flags |= RTNH_F_OFFLOAD;
4022                 else
4023                         mlxsw_sp_rt6->rt->fib6_nh.nh_flags &= ~RTNH_F_OFFLOAD;
4024         }
4025 }
4026
4027 static void
4028 mlxsw_sp_fib6_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry)
4029 {
4030         struct mlxsw_sp_fib6_entry *fib6_entry;
4031         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
4032
4033         fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry,
4034                                   common);
4035         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
4036                 struct fib6_info *rt = mlxsw_sp_rt6->rt;
4037
4038                 rt->fib6_nh.nh_flags &= ~RTNH_F_OFFLOAD;
4039         }
4040 }
4041
4042 static void mlxsw_sp_fib_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry)
4043 {
4044         switch (fib_entry->fib_node->fib->proto) {
4045         case MLXSW_SP_L3_PROTO_IPV4:
4046                 mlxsw_sp_fib4_entry_offload_set(fib_entry);
4047                 break;
4048         case MLXSW_SP_L3_PROTO_IPV6:
4049                 mlxsw_sp_fib6_entry_offload_set(fib_entry);
4050                 break;
4051         }
4052 }
4053
4054 static void
4055 mlxsw_sp_fib_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry)
4056 {
4057         switch (fib_entry->fib_node->fib->proto) {
4058         case MLXSW_SP_L3_PROTO_IPV4:
4059                 mlxsw_sp_fib4_entry_offload_unset(fib_entry);
4060                 break;
4061         case MLXSW_SP_L3_PROTO_IPV6:
4062                 mlxsw_sp_fib6_entry_offload_unset(fib_entry);
4063                 break;
4064         }
4065 }
4066
4067 static void
4068 mlxsw_sp_fib_entry_offload_refresh(struct mlxsw_sp_fib_entry *fib_entry,
4069                                    enum mlxsw_reg_ralue_op op, int err)
4070 {
4071         switch (op) {
4072         case MLXSW_REG_RALUE_OP_WRITE_DELETE:
4073                 return mlxsw_sp_fib_entry_offload_unset(fib_entry);
4074         case MLXSW_REG_RALUE_OP_WRITE_WRITE:
4075                 if (err)
4076                         return;
4077                 if (mlxsw_sp_fib_entry_should_offload(fib_entry))
4078                         mlxsw_sp_fib_entry_offload_set(fib_entry);
4079                 else
4080                         mlxsw_sp_fib_entry_offload_unset(fib_entry);
4081                 return;
4082         default:
4083                 return;
4084         }
4085 }
4086
4087 static void
4088 mlxsw_sp_fib_entry_ralue_pack(char *ralue_pl,
4089                               const struct mlxsw_sp_fib_entry *fib_entry,
4090                               enum mlxsw_reg_ralue_op op)
4091 {
4092         struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
4093         enum mlxsw_reg_ralxx_protocol proto;
4094         u32 *p_dip;
4095
4096         proto = (enum mlxsw_reg_ralxx_protocol) fib->proto;
4097
4098         switch (fib->proto) {
4099         case MLXSW_SP_L3_PROTO_IPV4:
4100                 p_dip = (u32 *) fib_entry->fib_node->key.addr;
4101                 mlxsw_reg_ralue_pack4(ralue_pl, proto, op, fib->vr->id,
4102                                       fib_entry->fib_node->key.prefix_len,
4103                                       *p_dip);
4104                 break;
4105         case MLXSW_SP_L3_PROTO_IPV6:
4106                 mlxsw_reg_ralue_pack6(ralue_pl, proto, op, fib->vr->id,
4107                                       fib_entry->fib_node->key.prefix_len,
4108                                       fib_entry->fib_node->key.addr);
4109                 break;
4110         }
4111 }
4112
4113 static int mlxsw_sp_fib_entry_op_remote(struct mlxsw_sp *mlxsw_sp,
4114                                         struct mlxsw_sp_fib_entry *fib_entry,
4115                                         enum mlxsw_reg_ralue_op op)
4116 {
4117         char ralue_pl[MLXSW_REG_RALUE_LEN];
4118         enum mlxsw_reg_ralue_trap_action trap_action;
4119         u16 trap_id = 0;
4120         u32 adjacency_index = 0;
4121         u16 ecmp_size = 0;
4122
4123         /* In case the nexthop group adjacency index is valid, use it
4124          * with provided ECMP size. Otherwise, setup trap and pass
4125          * traffic to kernel.
4126          */
4127         if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
4128                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
4129                 adjacency_index = fib_entry->nh_group->adj_index;
4130                 ecmp_size = fib_entry->nh_group->ecmp_size;
4131         } else {
4132                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
4133                 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
4134         }
4135
4136         mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
4137         mlxsw_reg_ralue_act_remote_pack(ralue_pl, trap_action, trap_id,
4138                                         adjacency_index, ecmp_size);
4139         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
4140 }
4141
4142 static int mlxsw_sp_fib_entry_op_local(struct mlxsw_sp *mlxsw_sp,
4143                                        struct mlxsw_sp_fib_entry *fib_entry,
4144                                        enum mlxsw_reg_ralue_op op)
4145 {
4146         struct mlxsw_sp_rif *rif = fib_entry->nh_group->nh_rif;
4147         enum mlxsw_reg_ralue_trap_action trap_action;
4148         char ralue_pl[MLXSW_REG_RALUE_LEN];
4149         u16 trap_id = 0;
4150         u16 rif_index = 0;
4151
4152         if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
4153                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
4154                 rif_index = rif->rif_index;
4155         } else {
4156                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
4157                 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
4158         }
4159
4160         mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
4161         mlxsw_reg_ralue_act_local_pack(ralue_pl, trap_action, trap_id,
4162                                        rif_index);
4163         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
4164 }
4165
4166 static int mlxsw_sp_fib_entry_op_trap(struct mlxsw_sp *mlxsw_sp,
4167                                       struct mlxsw_sp_fib_entry *fib_entry,
4168                                       enum mlxsw_reg_ralue_op op)
4169 {
4170         char ralue_pl[MLXSW_REG_RALUE_LEN];
4171
4172         mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
4173         mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
4174         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
4175 }
4176
4177 static int mlxsw_sp_fib_entry_op_blackhole(struct mlxsw_sp *mlxsw_sp,
4178                                            struct mlxsw_sp_fib_entry *fib_entry,
4179                                            enum mlxsw_reg_ralue_op op)
4180 {
4181         enum mlxsw_reg_ralue_trap_action trap_action;
4182         char ralue_pl[MLXSW_REG_RALUE_LEN];
4183
4184         trap_action = MLXSW_REG_RALUE_TRAP_ACTION_DISCARD_ERROR;
4185         mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
4186         mlxsw_reg_ralue_act_local_pack(ralue_pl, trap_action, 0, 0);
4187         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
4188 }
4189
4190 static int
4191 mlxsw_sp_fib_entry_op_ipip_decap(struct mlxsw_sp *mlxsw_sp,
4192                                  struct mlxsw_sp_fib_entry *fib_entry,
4193                                  enum mlxsw_reg_ralue_op op)
4194 {
4195         struct mlxsw_sp_ipip_entry *ipip_entry = fib_entry->decap.ipip_entry;
4196         const struct mlxsw_sp_ipip_ops *ipip_ops;
4197
4198         if (WARN_ON(!ipip_entry))
4199                 return -EINVAL;
4200
4201         ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
4202         return ipip_ops->fib_entry_op(mlxsw_sp, ipip_entry, op,
4203                                       fib_entry->decap.tunnel_index);
4204 }
4205
4206 static int mlxsw_sp_fib_entry_op_nve_decap(struct mlxsw_sp *mlxsw_sp,
4207                                            struct mlxsw_sp_fib_entry *fib_entry,
4208                                            enum mlxsw_reg_ralue_op op)
4209 {
4210         char ralue_pl[MLXSW_REG_RALUE_LEN];
4211
4212         mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
4213         mlxsw_reg_ralue_act_ip2me_tun_pack(ralue_pl,
4214                                            fib_entry->decap.tunnel_index);
4215         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
4216 }
4217
4218 static int __mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
4219                                    struct mlxsw_sp_fib_entry *fib_entry,
4220                                    enum mlxsw_reg_ralue_op op)
4221 {
4222         switch (fib_entry->type) {
4223         case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
4224                 return mlxsw_sp_fib_entry_op_remote(mlxsw_sp, fib_entry, op);
4225         case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
4226                 return mlxsw_sp_fib_entry_op_local(mlxsw_sp, fib_entry, op);
4227         case MLXSW_SP_FIB_ENTRY_TYPE_TRAP:
4228                 return mlxsw_sp_fib_entry_op_trap(mlxsw_sp, fib_entry, op);
4229         case MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE:
4230                 return mlxsw_sp_fib_entry_op_blackhole(mlxsw_sp, fib_entry, op);
4231         case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP:
4232                 return mlxsw_sp_fib_entry_op_ipip_decap(mlxsw_sp,
4233                                                         fib_entry, op);
4234         case MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP:
4235                 return mlxsw_sp_fib_entry_op_nve_decap(mlxsw_sp, fib_entry, op);
4236         }
4237         return -EINVAL;
4238 }
4239
4240 static int mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
4241                                  struct mlxsw_sp_fib_entry *fib_entry,
4242                                  enum mlxsw_reg_ralue_op op)
4243 {
4244         int err = __mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry, op);
4245
4246         mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, err);
4247
4248         return err;
4249 }
4250
4251 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
4252                                      struct mlxsw_sp_fib_entry *fib_entry)
4253 {
4254         return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
4255                                      MLXSW_REG_RALUE_OP_WRITE_WRITE);
4256 }
4257
4258 static int mlxsw_sp_fib_entry_del(struct mlxsw_sp *mlxsw_sp,
4259                                   struct mlxsw_sp_fib_entry *fib_entry)
4260 {
4261         return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
4262                                      MLXSW_REG_RALUE_OP_WRITE_DELETE);
4263 }
4264
4265 static int
4266 mlxsw_sp_fib4_entry_type_set(struct mlxsw_sp *mlxsw_sp,
4267                              const struct fib_entry_notifier_info *fen_info,
4268                              struct mlxsw_sp_fib_entry *fib_entry)
4269 {
4270         union mlxsw_sp_l3addr dip = { .addr4 = htonl(fen_info->dst) };
4271         u32 tb_id = mlxsw_sp_fix_tb_id(fen_info->tb_id);
4272         struct net_device *dev = fen_info->fi->fib_dev;
4273         struct mlxsw_sp_ipip_entry *ipip_entry;
4274         struct fib_info *fi = fen_info->fi;
4275
4276         switch (fen_info->type) {
4277         case RTN_LOCAL:
4278                 ipip_entry = mlxsw_sp_ipip_entry_find_by_decap(mlxsw_sp, dev,
4279                                                  MLXSW_SP_L3_PROTO_IPV4, dip);
4280                 if (ipip_entry && ipip_entry->ol_dev->flags & IFF_UP) {
4281                         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP;
4282                         return mlxsw_sp_fib_entry_decap_init(mlxsw_sp,
4283                                                              fib_entry,
4284                                                              ipip_entry);
4285                 }
4286                 if (mlxsw_sp_nve_ipv4_route_is_decap(mlxsw_sp, tb_id,
4287                                                      dip.addr4)) {
4288                         u32 t_index;
4289
4290                         t_index = mlxsw_sp_nve_decap_tunnel_index_get(mlxsw_sp);
4291                         fib_entry->decap.tunnel_index = t_index;
4292                         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_NVE_DECAP;
4293                         return 0;
4294                 }
4295                 /* fall through */
4296         case RTN_BROADCAST:
4297                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
4298                 return 0;
4299         case RTN_BLACKHOLE:
4300                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE;
4301                 return 0;
4302         case RTN_UNREACHABLE: /* fall through */
4303         case RTN_PROHIBIT:
4304                 /* Packets hitting these routes need to be trapped, but
4305                  * can do so with a lower priority than packets directed
4306                  * at the host, so use action type local instead of trap.
4307                  */
4308                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
4309                 return 0;
4310         case RTN_UNICAST:
4311                 if (mlxsw_sp_fi_is_gateway(mlxsw_sp, fi))
4312                         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
4313                 else
4314                         fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
4315                 return 0;
4316         default:
4317                 return -EINVAL;
4318         }
4319 }
4320
4321 static struct mlxsw_sp_fib4_entry *
4322 mlxsw_sp_fib4_entry_create(struct mlxsw_sp *mlxsw_sp,
4323                            struct mlxsw_sp_fib_node *fib_node,
4324                            const struct fib_entry_notifier_info *fen_info)
4325 {
4326         struct mlxsw_sp_fib4_entry *fib4_entry;
4327         struct mlxsw_sp_fib_entry *fib_entry;
4328         int err;
4329
4330         fib4_entry = kzalloc(sizeof(*fib4_entry), GFP_KERNEL);
4331         if (!fib4_entry)
4332                 return ERR_PTR(-ENOMEM);
4333         fib_entry = &fib4_entry->common;
4334
4335         err = mlxsw_sp_fib4_entry_type_set(mlxsw_sp, fen_info, fib_entry);
4336         if (err)
4337                 goto err_fib4_entry_type_set;
4338
4339         err = mlxsw_sp_nexthop4_group_get(mlxsw_sp, fib_entry, fen_info->fi);
4340         if (err)
4341                 goto err_nexthop4_group_get;
4342
4343         fib4_entry->prio = fen_info->fi->fib_priority;
4344         fib4_entry->tb_id = fen_info->tb_id;
4345         fib4_entry->type = fen_info->type;
4346         fib4_entry->tos = fen_info->tos;
4347
4348         fib_entry->fib_node = fib_node;
4349
4350         return fib4_entry;
4351
4352 err_nexthop4_group_get:
4353 err_fib4_entry_type_set:
4354         kfree(fib4_entry);
4355         return ERR_PTR(err);
4356 }
4357
4358 static void mlxsw_sp_fib4_entry_destroy(struct mlxsw_sp *mlxsw_sp,
4359                                         struct mlxsw_sp_fib4_entry *fib4_entry)
4360 {
4361         mlxsw_sp_nexthop4_group_put(mlxsw_sp, &fib4_entry->common);
4362         kfree(fib4_entry);
4363 }
4364
4365 static struct mlxsw_sp_fib4_entry *
4366 mlxsw_sp_fib4_entry_lookup(struct mlxsw_sp *mlxsw_sp,
4367                            const struct fib_entry_notifier_info *fen_info)
4368 {
4369         struct mlxsw_sp_fib4_entry *fib4_entry;
4370         struct mlxsw_sp_fib_node *fib_node;
4371         struct mlxsw_sp_fib *fib;
4372         struct mlxsw_sp_vr *vr;
4373
4374         vr = mlxsw_sp_vr_find(mlxsw_sp, fen_info->tb_id);
4375         if (!vr)
4376                 return NULL;
4377         fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV4);
4378
4379         fib_node = mlxsw_sp_fib_node_lookup(fib, &fen_info->dst,
4380                                             sizeof(fen_info->dst),
4381                                             fen_info->dst_len);
4382         if (!fib_node)
4383                 return NULL;
4384
4385         list_for_each_entry(fib4_entry, &fib_node->entry_list, common.list) {
4386                 if (fib4_entry->tb_id == fen_info->tb_id &&
4387                     fib4_entry->tos == fen_info->tos &&
4388                     fib4_entry->type == fen_info->type &&
4389                     mlxsw_sp_nexthop4_group_fi(fib4_entry->common.nh_group) ==
4390                     fen_info->fi) {
4391                         return fib4_entry;
4392                 }
4393         }
4394
4395         return NULL;
4396 }
4397
4398 static const struct rhashtable_params mlxsw_sp_fib_ht_params = {
4399         .key_offset = offsetof(struct mlxsw_sp_fib_node, key),
4400         .head_offset = offsetof(struct mlxsw_sp_fib_node, ht_node),
4401         .key_len = sizeof(struct mlxsw_sp_fib_key),
4402         .automatic_shrinking = true,
4403 };
4404
4405 static int mlxsw_sp_fib_node_insert(struct mlxsw_sp_fib *fib,
4406                                     struct mlxsw_sp_fib_node *fib_node)
4407 {
4408         return rhashtable_insert_fast(&fib->ht, &fib_node->ht_node,
4409                                       mlxsw_sp_fib_ht_params);
4410 }
4411
4412 static void mlxsw_sp_fib_node_remove(struct mlxsw_sp_fib *fib,
4413                                      struct mlxsw_sp_fib_node *fib_node)
4414 {
4415         rhashtable_remove_fast(&fib->ht, &fib_node->ht_node,
4416                                mlxsw_sp_fib_ht_params);
4417 }
4418
4419 static struct mlxsw_sp_fib_node *
4420 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr,
4421                          size_t addr_len, unsigned char prefix_len)
4422 {
4423         struct mlxsw_sp_fib_key key;
4424
4425         memset(&key, 0, sizeof(key));
4426         memcpy(key.addr, addr, addr_len);
4427         key.prefix_len = prefix_len;
4428         return rhashtable_lookup_fast(&fib->ht, &key, mlxsw_sp_fib_ht_params);
4429 }
4430
4431 static struct mlxsw_sp_fib_node *
4432 mlxsw_sp_fib_node_create(struct mlxsw_sp_fib *fib, const void *addr,
4433                          size_t addr_len, unsigned char prefix_len)
4434 {
4435         struct mlxsw_sp_fib_node *fib_node;
4436
4437         fib_node = kzalloc(sizeof(*fib_node), GFP_KERNEL);
4438         if (!fib_node)
4439                 return NULL;
4440
4441         INIT_LIST_HEAD(&fib_node->entry_list);
4442         list_add(&fib_node->list, &fib->node_list);
4443         memcpy(fib_node->key.addr, addr, addr_len);
4444         fib_node->key.prefix_len = prefix_len;
4445
4446         return fib_node;
4447 }
4448
4449 static void mlxsw_sp_fib_node_destroy(struct mlxsw_sp_fib_node *fib_node)
4450 {
4451         list_del(&fib_node->list);
4452         WARN_ON(!list_empty(&fib_node->entry_list));
4453         kfree(fib_node);
4454 }
4455
4456 static bool
4457 mlxsw_sp_fib_node_entry_is_first(const struct mlxsw_sp_fib_node *fib_node,
4458                                  const struct mlxsw_sp_fib_entry *fib_entry)
4459 {
4460         return list_first_entry(&fib_node->entry_list,
4461                                 struct mlxsw_sp_fib_entry, list) == fib_entry;
4462 }
4463
4464 static int mlxsw_sp_fib_lpm_tree_link(struct mlxsw_sp *mlxsw_sp,
4465                                       struct mlxsw_sp_fib_node *fib_node)
4466 {
4467         struct mlxsw_sp_prefix_usage req_prefix_usage;
4468         struct mlxsw_sp_fib *fib = fib_node->fib;
4469         struct mlxsw_sp_lpm_tree *lpm_tree;
4470         int err;
4471
4472         lpm_tree = mlxsw_sp->router->lpm.proto_trees[fib->proto];
4473         if (lpm_tree->prefix_ref_count[fib_node->key.prefix_len] != 0)
4474                 goto out;
4475
4476         mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &lpm_tree->prefix_usage);
4477         mlxsw_sp_prefix_usage_set(&req_prefix_usage, fib_node->key.prefix_len);
4478         lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
4479                                          fib->proto);
4480         if (IS_ERR(lpm_tree))
4481                 return PTR_ERR(lpm_tree);
4482
4483         err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree);
4484         if (err)
4485                 goto err_lpm_tree_replace;
4486
4487 out:
4488         lpm_tree->prefix_ref_count[fib_node->key.prefix_len]++;
4489         return 0;
4490
4491 err_lpm_tree_replace:
4492         mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
4493         return err;
4494 }
4495
4496 static void mlxsw_sp_fib_lpm_tree_unlink(struct mlxsw_sp *mlxsw_sp,
4497                                          struct mlxsw_sp_fib_node *fib_node)
4498 {
4499         struct mlxsw_sp_lpm_tree *lpm_tree = fib_node->fib->lpm_tree;
4500         struct mlxsw_sp_prefix_usage req_prefix_usage;
4501         struct mlxsw_sp_fib *fib = fib_node->fib;
4502         int err;
4503
4504         if (--lpm_tree->prefix_ref_count[fib_node->key.prefix_len] != 0)
4505                 return;
4506         /* Try to construct a new LPM tree from the current prefix usage
4507          * minus the unused one. If we fail, continue using the old one.
4508          */
4509         mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &lpm_tree->prefix_usage);
4510         mlxsw_sp_prefix_usage_clear(&req_prefix_usage,
4511                                     fib_node->key.prefix_len);
4512         lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
4513                                          fib->proto);
4514         if (IS_ERR(lpm_tree))
4515                 return;
4516
4517         err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree);
4518         if (err)
4519                 goto err_lpm_tree_replace;
4520
4521         return;
4522
4523 err_lpm_tree_replace:
4524         mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
4525 }
4526
4527 static int mlxsw_sp_fib_node_init(struct mlxsw_sp *mlxsw_sp,
4528                                   struct mlxsw_sp_fib_node *fib_node,
4529                                   struct mlxsw_sp_fib *fib)
4530 {
4531         int err;
4532
4533         err = mlxsw_sp_fib_node_insert(fib, fib_node);
4534         if (err)
4535                 return err;
4536         fib_node->fib = fib;
4537
4538         err = mlxsw_sp_fib_lpm_tree_link(mlxsw_sp, fib_node);
4539         if (err)
4540                 goto err_fib_lpm_tree_link;
4541
4542         return 0;
4543
4544 err_fib_lpm_tree_link:
4545         fib_node->fib = NULL;
4546         mlxsw_sp_fib_node_remove(fib, fib_node);
4547         return err;
4548 }
4549
4550 static void mlxsw_sp_fib_node_fini(struct mlxsw_sp *mlxsw_sp,
4551                                    struct mlxsw_sp_fib_node *fib_node)
4552 {
4553         struct mlxsw_sp_fib *fib = fib_node->fib;
4554
4555         mlxsw_sp_fib_lpm_tree_unlink(mlxsw_sp, fib_node);
4556         fib_node->fib = NULL;
4557         mlxsw_sp_fib_node_remove(fib, fib_node);
4558 }
4559
4560 static struct mlxsw_sp_fib_node *
4561 mlxsw_sp_fib_node_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, const void *addr,
4562                       size_t addr_len, unsigned char prefix_len,
4563                       enum mlxsw_sp_l3proto proto)
4564 {
4565         struct mlxsw_sp_fib_node *fib_node;
4566         struct mlxsw_sp_fib *fib;
4567         struct mlxsw_sp_vr *vr;
4568         int err;
4569
4570         vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id, NULL);
4571         if (IS_ERR(vr))
4572                 return ERR_CAST(vr);
4573         fib = mlxsw_sp_vr_fib(vr, proto);
4574
4575         fib_node = mlxsw_sp_fib_node_lookup(fib, addr, addr_len, prefix_len);
4576         if (fib_node)
4577                 return fib_node;
4578
4579         fib_node = mlxsw_sp_fib_node_create(fib, addr, addr_len, prefix_len);
4580         if (!fib_node) {
4581                 err = -ENOMEM;
4582                 goto err_fib_node_create;
4583         }
4584
4585         err = mlxsw_sp_fib_node_init(mlxsw_sp, fib_node, fib);
4586         if (err)
4587                 goto err_fib_node_init;
4588
4589         return fib_node;
4590
4591 err_fib_node_init:
4592         mlxsw_sp_fib_node_destroy(fib_node);
4593 err_fib_node_create:
4594         mlxsw_sp_vr_put(mlxsw_sp, vr);
4595         return ERR_PTR(err);
4596 }
4597
4598 static void mlxsw_sp_fib_node_put(struct mlxsw_sp *mlxsw_sp,
4599                                   struct mlxsw_sp_fib_node *fib_node)
4600 {
4601         struct mlxsw_sp_vr *vr = fib_node->fib->vr;
4602
4603         if (!list_empty(&fib_node->entry_list))
4604                 return;
4605         mlxsw_sp_fib_node_fini(mlxsw_sp, fib_node);
4606         mlxsw_sp_fib_node_destroy(fib_node);
4607         mlxsw_sp_vr_put(mlxsw_sp, vr);
4608 }
4609
4610 static struct mlxsw_sp_fib4_entry *
4611 mlxsw_sp_fib4_node_entry_find(const struct mlxsw_sp_fib_node *fib_node,
4612                               const struct mlxsw_sp_fib4_entry *new4_entry)
4613 {
4614         struct mlxsw_sp_fib4_entry *fib4_entry;
4615
4616         list_for_each_entry(fib4_entry, &fib_node->entry_list, common.list) {
4617                 if (fib4_entry->tb_id > new4_entry->tb_id)
4618                         continue;
4619                 if (fib4_entry->tb_id != new4_entry->tb_id)
4620                         break;
4621                 if (fib4_entry->tos > new4_entry->tos)
4622                         continue;
4623                 if (fib4_entry->prio >= new4_entry->prio ||
4624                     fib4_entry->tos < new4_entry->tos)
4625                         return fib4_entry;
4626         }
4627
4628         return NULL;
4629 }
4630
4631 static int
4632 mlxsw_sp_fib4_node_list_append(struct mlxsw_sp_fib4_entry *fib4_entry,
4633                                struct mlxsw_sp_fib4_entry *new4_entry)
4634 {
4635         struct mlxsw_sp_fib_node *fib_node;
4636
4637         if (WARN_ON(!fib4_entry))
4638                 return -EINVAL;
4639
4640         fib_node = fib4_entry->common.fib_node;
4641         list_for_each_entry_from(fib4_entry, &fib_node->entry_list,
4642                                  common.list) {
4643                 if (fib4_entry->tb_id != new4_entry->tb_id ||
4644                     fib4_entry->tos != new4_entry->tos ||
4645                     fib4_entry->prio != new4_entry->prio)
4646                         break;
4647         }
4648
4649         list_add_tail(&new4_entry->common.list, &fib4_entry->common.list);
4650         return 0;
4651 }
4652
4653 static int
4654 mlxsw_sp_fib4_node_list_insert(struct mlxsw_sp_fib4_entry *new4_entry,
4655                                bool replace, bool append)
4656 {
4657         struct mlxsw_sp_fib_node *fib_node = new4_entry->common.fib_node;
4658         struct mlxsw_sp_fib4_entry *fib4_entry;
4659
4660         fib4_entry = mlxsw_sp_fib4_node_entry_find(fib_node, new4_entry);
4661
4662         if (append)
4663                 return mlxsw_sp_fib4_node_list_append(fib4_entry, new4_entry);
4664         if (replace && WARN_ON(!fib4_entry))
4665                 return -EINVAL;
4666
4667         /* Insert new entry before replaced one, so that we can later
4668          * remove the second.
4669          */
4670         if (fib4_entry) {
4671                 list_add_tail(&new4_entry->common.list,
4672                               &fib4_entry->common.list);
4673         } else {
4674                 struct mlxsw_sp_fib4_entry *last;
4675
4676                 list_for_each_entry(last, &fib_node->entry_list, common.list) {
4677                         if (new4_entry->tb_id > last->tb_id)
4678                                 break;
4679                         fib4_entry = last;
4680                 }
4681
4682                 if (fib4_entry)
4683                         list_add(&new4_entry->common.list,
4684                                  &fib4_entry->common.list);
4685                 else
4686                         list_add(&new4_entry->common.list,
4687                                  &fib_node->entry_list);
4688         }
4689
4690         return 0;
4691 }
4692
4693 static void
4694 mlxsw_sp_fib4_node_list_remove(struct mlxsw_sp_fib4_entry *fib4_entry)
4695 {
4696         list_del(&fib4_entry->common.list);
4697 }
4698
4699 static int mlxsw_sp_fib_node_entry_add(struct mlxsw_sp *mlxsw_sp,
4700                                        struct mlxsw_sp_fib_entry *fib_entry)
4701 {
4702         struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
4703
4704         if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry))
4705                 return 0;
4706
4707         /* To prevent packet loss, overwrite the previously offloaded
4708          * entry.
4709          */
4710         if (!list_is_singular(&fib_node->entry_list)) {
4711                 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_DELETE;
4712                 struct mlxsw_sp_fib_entry *n = list_next_entry(fib_entry, list);
4713
4714                 mlxsw_sp_fib_entry_offload_refresh(n, op, 0);
4715         }
4716
4717         return mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
4718 }
4719
4720 static void mlxsw_sp_fib_node_entry_del(struct mlxsw_sp *mlxsw_sp,
4721                                         struct mlxsw_sp_fib_entry *fib_entry)
4722 {
4723         struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
4724
4725         if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry))
4726                 return;
4727
4728         /* Promote the next entry by overwriting the deleted entry */
4729         if (!list_is_singular(&fib_node->entry_list)) {
4730                 struct mlxsw_sp_fib_entry *n = list_next_entry(fib_entry, list);
4731                 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_DELETE;
4732
4733                 mlxsw_sp_fib_entry_update(mlxsw_sp, n);
4734                 mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, 0);
4735                 return;
4736         }
4737
4738         mlxsw_sp_fib_entry_del(mlxsw_sp, fib_entry);
4739 }
4740
4741 static int mlxsw_sp_fib4_node_entry_link(struct mlxsw_sp *mlxsw_sp,
4742                                          struct mlxsw_sp_fib4_entry *fib4_entry,
4743                                          bool replace, bool append)
4744 {
4745         int err;
4746
4747         err = mlxsw_sp_fib4_node_list_insert(fib4_entry, replace, append);
4748         if (err)
4749                 return err;
4750
4751         err = mlxsw_sp_fib_node_entry_add(mlxsw_sp, &fib4_entry->common);
4752         if (err)
4753                 goto err_fib_node_entry_add;
4754
4755         return 0;
4756
4757 err_fib_node_entry_add:
4758         mlxsw_sp_fib4_node_list_remove(fib4_entry);
4759         return err;
4760 }
4761
4762 static void
4763 mlxsw_sp_fib4_node_entry_unlink(struct mlxsw_sp *mlxsw_sp,
4764                                 struct mlxsw_sp_fib4_entry *fib4_entry)
4765 {
4766         mlxsw_sp_fib_node_entry_del(mlxsw_sp, &fib4_entry->common);
4767         mlxsw_sp_fib4_node_list_remove(fib4_entry);
4768
4769         if (fib4_entry->common.type == MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP)
4770                 mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, &fib4_entry->common);
4771 }
4772
4773 static void mlxsw_sp_fib4_entry_replace(struct mlxsw_sp *mlxsw_sp,
4774                                         struct mlxsw_sp_fib4_entry *fib4_entry,
4775                                         bool replace)
4776 {
4777         struct mlxsw_sp_fib_node *fib_node = fib4_entry->common.fib_node;
4778         struct mlxsw_sp_fib4_entry *replaced;
4779
4780         if (!replace)
4781                 return;
4782
4783         /* We inserted the new entry before replaced one */
4784         replaced = list_next_entry(fib4_entry, common.list);
4785
4786         mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, replaced);
4787         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, replaced);
4788         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
4789 }
4790
4791 static int
4792 mlxsw_sp_router_fib4_add(struct mlxsw_sp *mlxsw_sp,
4793                          const struct fib_entry_notifier_info *fen_info,
4794                          bool replace, bool append)
4795 {
4796         struct mlxsw_sp_fib4_entry *fib4_entry;
4797         struct mlxsw_sp_fib_node *fib_node;
4798         int err;
4799
4800         if (mlxsw_sp->router->aborted)
4801                 return 0;
4802
4803         fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, fen_info->tb_id,
4804                                          &fen_info->dst, sizeof(fen_info->dst),
4805                                          fen_info->dst_len,
4806                                          MLXSW_SP_L3_PROTO_IPV4);
4807         if (IS_ERR(fib_node)) {
4808                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to get FIB node\n");
4809                 return PTR_ERR(fib_node);
4810         }
4811
4812         fib4_entry = mlxsw_sp_fib4_entry_create(mlxsw_sp, fib_node, fen_info);
4813         if (IS_ERR(fib4_entry)) {
4814                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to create FIB entry\n");
4815                 err = PTR_ERR(fib4_entry);
4816                 goto err_fib4_entry_create;
4817         }
4818
4819         err = mlxsw_sp_fib4_node_entry_link(mlxsw_sp, fib4_entry, replace,
4820                                             append);
4821         if (err) {
4822                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to link FIB entry to node\n");
4823                 goto err_fib4_node_entry_link;
4824         }
4825
4826         mlxsw_sp_fib4_entry_replace(mlxsw_sp, fib4_entry, replace);
4827
4828         return 0;
4829
4830 err_fib4_node_entry_link:
4831         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
4832 err_fib4_entry_create:
4833         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
4834         return err;
4835 }
4836
4837 static void mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp,
4838                                      struct fib_entry_notifier_info *fen_info)
4839 {
4840         struct mlxsw_sp_fib4_entry *fib4_entry;
4841         struct mlxsw_sp_fib_node *fib_node;
4842
4843         if (mlxsw_sp->router->aborted)
4844                 return;
4845
4846         fib4_entry = mlxsw_sp_fib4_entry_lookup(mlxsw_sp, fen_info);
4847         if (WARN_ON(!fib4_entry))
4848                 return;
4849         fib_node = fib4_entry->common.fib_node;
4850
4851         mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib4_entry);
4852         mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
4853         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
4854 }
4855
4856 static bool mlxsw_sp_fib6_rt_should_ignore(const struct fib6_info *rt)
4857 {
4858         /* Packets with link-local destination IP arriving to the router
4859          * are trapped to the CPU, so no need to program specific routes
4860          * for them.
4861          */
4862         if (ipv6_addr_type(&rt->fib6_dst.addr) & IPV6_ADDR_LINKLOCAL)
4863                 return true;
4864
4865         /* Multicast routes aren't supported, so ignore them. Neighbour
4866          * Discovery packets are specifically trapped.
4867          */
4868         if (ipv6_addr_type(&rt->fib6_dst.addr) & IPV6_ADDR_MULTICAST)
4869                 return true;
4870
4871         /* Cloned routes are irrelevant in the forwarding path. */
4872         if (rt->fib6_flags & RTF_CACHE)
4873                 return true;
4874
4875         return false;
4876 }
4877
4878 static struct mlxsw_sp_rt6 *mlxsw_sp_rt6_create(struct fib6_info *rt)
4879 {
4880         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
4881
4882         mlxsw_sp_rt6 = kzalloc(sizeof(*mlxsw_sp_rt6), GFP_KERNEL);
4883         if (!mlxsw_sp_rt6)
4884                 return ERR_PTR(-ENOMEM);
4885
4886         /* In case of route replace, replaced route is deleted with
4887          * no notification. Take reference to prevent accessing freed
4888          * memory.
4889          */
4890         mlxsw_sp_rt6->rt = rt;
4891         fib6_info_hold(rt);
4892
4893         return mlxsw_sp_rt6;
4894 }
4895
4896 #if IS_ENABLED(CONFIG_IPV6)
4897 static void mlxsw_sp_rt6_release(struct fib6_info *rt)
4898 {
4899         fib6_info_release(rt);
4900 }
4901 #else
4902 static void mlxsw_sp_rt6_release(struct fib6_info *rt)
4903 {
4904 }
4905 #endif
4906
4907 static void mlxsw_sp_rt6_destroy(struct mlxsw_sp_rt6 *mlxsw_sp_rt6)
4908 {
4909         mlxsw_sp_rt6_release(mlxsw_sp_rt6->rt);
4910         kfree(mlxsw_sp_rt6);
4911 }
4912
4913 static bool mlxsw_sp_fib6_rt_can_mp(const struct fib6_info *rt)
4914 {
4915         /* RTF_CACHE routes are ignored */
4916         return (rt->fib6_flags & (RTF_GATEWAY | RTF_ADDRCONF)) == RTF_GATEWAY;
4917 }
4918
4919 static struct fib6_info *
4920 mlxsw_sp_fib6_entry_rt(const struct mlxsw_sp_fib6_entry *fib6_entry)
4921 {
4922         return list_first_entry(&fib6_entry->rt6_list, struct mlxsw_sp_rt6,
4923                                 list)->rt;
4924 }
4925
4926 static struct mlxsw_sp_fib6_entry *
4927 mlxsw_sp_fib6_node_mp_entry_find(const struct mlxsw_sp_fib_node *fib_node,
4928                                  const struct fib6_info *nrt, bool replace)
4929 {
4930         struct mlxsw_sp_fib6_entry *fib6_entry;
4931
4932         if (!mlxsw_sp_fib6_rt_can_mp(nrt) || replace)
4933                 return NULL;
4934
4935         list_for_each_entry(fib6_entry, &fib_node->entry_list, common.list) {
4936                 struct fib6_info *rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
4937
4938                 /* RT6_TABLE_LOCAL and RT6_TABLE_MAIN share the same
4939                  * virtual router.
4940                  */
4941                 if (rt->fib6_table->tb6_id > nrt->fib6_table->tb6_id)
4942                         continue;
4943                 if (rt->fib6_table->tb6_id != nrt->fib6_table->tb6_id)
4944                         break;
4945                 if (rt->fib6_metric < nrt->fib6_metric)
4946                         continue;
4947                 if (rt->fib6_metric == nrt->fib6_metric &&
4948                     mlxsw_sp_fib6_rt_can_mp(rt))
4949                         return fib6_entry;
4950                 if (rt->fib6_metric > nrt->fib6_metric)
4951                         break;
4952         }
4953
4954         return NULL;
4955 }
4956
4957 static struct mlxsw_sp_rt6 *
4958 mlxsw_sp_fib6_entry_rt_find(const struct mlxsw_sp_fib6_entry *fib6_entry,
4959                             const struct fib6_info *rt)
4960 {
4961         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
4962
4963         list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
4964                 if (mlxsw_sp_rt6->rt == rt)
4965                         return mlxsw_sp_rt6;
4966         }
4967
4968         return NULL;
4969 }
4970
4971 static bool mlxsw_sp_nexthop6_ipip_type(const struct mlxsw_sp *mlxsw_sp,
4972                                         const struct fib6_info *rt,
4973                                         enum mlxsw_sp_ipip_type *ret)
4974 {
4975         return rt->fib6_nh.nh_dev &&
4976                mlxsw_sp_netdev_ipip_type(mlxsw_sp, rt->fib6_nh.nh_dev, ret);
4977 }
4978
4979 static int mlxsw_sp_nexthop6_type_init(struct mlxsw_sp *mlxsw_sp,
4980                                        struct mlxsw_sp_nexthop_group *nh_grp,
4981                                        struct mlxsw_sp_nexthop *nh,
4982                                        const struct fib6_info *rt)
4983 {
4984         const struct mlxsw_sp_ipip_ops *ipip_ops;
4985         struct mlxsw_sp_ipip_entry *ipip_entry;
4986         struct net_device *dev = rt->fib6_nh.nh_dev;
4987         struct mlxsw_sp_rif *rif;
4988         int err;
4989
4990         ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, dev);
4991         if (ipip_entry) {
4992                 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
4993                 if (ipip_ops->can_offload(mlxsw_sp, dev,
4994                                           MLXSW_SP_L3_PROTO_IPV6)) {
4995                         nh->type = MLXSW_SP_NEXTHOP_TYPE_IPIP;
4996                         mlxsw_sp_nexthop_ipip_init(mlxsw_sp, nh, ipip_entry);
4997                         return 0;
4998                 }
4999         }
5000
5001         nh->type = MLXSW_SP_NEXTHOP_TYPE_ETH;
5002         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
5003         if (!rif)
5004                 return 0;
5005         mlxsw_sp_nexthop_rif_init(nh, rif);
5006
5007         err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
5008         if (err)
5009                 goto err_nexthop_neigh_init;
5010
5011         return 0;
5012
5013 err_nexthop_neigh_init:
5014         mlxsw_sp_nexthop_rif_fini(nh);
5015         return err;
5016 }
5017
5018 static void mlxsw_sp_nexthop6_type_fini(struct mlxsw_sp *mlxsw_sp,
5019                                         struct mlxsw_sp_nexthop *nh)
5020 {
5021         mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
5022 }
5023
5024 static int mlxsw_sp_nexthop6_init(struct mlxsw_sp *mlxsw_sp,
5025                                   struct mlxsw_sp_nexthop_group *nh_grp,
5026                                   struct mlxsw_sp_nexthop *nh,
5027                                   const struct fib6_info *rt)
5028 {
5029         struct net_device *dev = rt->fib6_nh.nh_dev;
5030
5031         nh->nh_grp = nh_grp;
5032         nh->nh_weight = rt->fib6_nh.nh_weight;
5033         memcpy(&nh->gw_addr, &rt->fib6_nh.nh_gw, sizeof(nh->gw_addr));
5034         mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh);
5035
5036         list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list);
5037
5038         if (!dev)
5039                 return 0;
5040         nh->ifindex = dev->ifindex;
5041
5042         return mlxsw_sp_nexthop6_type_init(mlxsw_sp, nh_grp, nh, rt);
5043 }
5044
5045 static void mlxsw_sp_nexthop6_fini(struct mlxsw_sp *mlxsw_sp,
5046                                    struct mlxsw_sp_nexthop *nh)
5047 {
5048         mlxsw_sp_nexthop6_type_fini(mlxsw_sp, nh);
5049         list_del(&nh->router_list_node);
5050         mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh);
5051 }
5052
5053 static bool mlxsw_sp_rt6_is_gateway(const struct mlxsw_sp *mlxsw_sp,
5054                                     const struct fib6_info *rt)
5055 {
5056         return rt->fib6_flags & RTF_GATEWAY ||
5057                mlxsw_sp_nexthop6_ipip_type(mlxsw_sp, rt, NULL);
5058 }
5059
5060 static struct mlxsw_sp_nexthop_group *
5061 mlxsw_sp_nexthop6_group_create(struct mlxsw_sp *mlxsw_sp,
5062                                struct mlxsw_sp_fib6_entry *fib6_entry)
5063 {
5064         struct mlxsw_sp_nexthop_group *nh_grp;
5065         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
5066         struct mlxsw_sp_nexthop *nh;
5067         int i = 0;
5068         int err;
5069
5070         nh_grp = kzalloc(struct_size(nh_grp, nexthops, fib6_entry->nrt6),
5071                          GFP_KERNEL);
5072         if (!nh_grp)
5073                 return ERR_PTR(-ENOMEM);
5074         INIT_LIST_HEAD(&nh_grp->fib_list);
5075 #if IS_ENABLED(CONFIG_IPV6)
5076         nh_grp->neigh_tbl = &nd_tbl;
5077 #endif
5078         mlxsw_sp_rt6 = list_first_entry(&fib6_entry->rt6_list,
5079                                         struct mlxsw_sp_rt6, list);
5080         nh_grp->gateway = mlxsw_sp_rt6_is_gateway(mlxsw_sp, mlxsw_sp_rt6->rt);
5081         nh_grp->count = fib6_entry->nrt6;
5082         for (i = 0; i < nh_grp->count; i++) {
5083                 struct fib6_info *rt = mlxsw_sp_rt6->rt;
5084
5085                 nh = &nh_grp->nexthops[i];
5086                 err = mlxsw_sp_nexthop6_init(mlxsw_sp, nh_grp, nh, rt);
5087                 if (err)
5088                         goto err_nexthop6_init;
5089                 mlxsw_sp_rt6 = list_next_entry(mlxsw_sp_rt6, list);
5090         }
5091
5092         err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
5093         if (err)
5094                 goto err_nexthop_group_insert;
5095
5096         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
5097         return nh_grp;
5098
5099 err_nexthop_group_insert:
5100 err_nexthop6_init:
5101         for (i--; i >= 0; i--) {
5102                 nh = &nh_grp->nexthops[i];
5103                 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh);
5104         }
5105         kfree(nh_grp);
5106         return ERR_PTR(err);
5107 }
5108
5109 static void
5110 mlxsw_sp_nexthop6_group_destroy(struct mlxsw_sp *mlxsw_sp,
5111                                 struct mlxsw_sp_nexthop_group *nh_grp)
5112 {
5113         struct mlxsw_sp_nexthop *nh;
5114         int i = nh_grp->count;
5115
5116         mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
5117         for (i--; i >= 0; i--) {
5118                 nh = &nh_grp->nexthops[i];
5119                 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh);
5120         }
5121         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
5122         WARN_ON(nh_grp->adj_index_valid);
5123         kfree(nh_grp);
5124 }
5125
5126 static int mlxsw_sp_nexthop6_group_get(struct mlxsw_sp *mlxsw_sp,
5127                                        struct mlxsw_sp_fib6_entry *fib6_entry)
5128 {
5129         struct mlxsw_sp_nexthop_group *nh_grp;
5130
5131         nh_grp = mlxsw_sp_nexthop6_group_lookup(mlxsw_sp, fib6_entry);
5132         if (!nh_grp) {
5133                 nh_grp = mlxsw_sp_nexthop6_group_create(mlxsw_sp, fib6_entry);
5134                 if (IS_ERR(nh_grp))
5135                         return PTR_ERR(nh_grp);
5136         }
5137
5138         list_add_tail(&fib6_entry->common.nexthop_group_node,
5139                       &nh_grp->fib_list);
5140         fib6_entry->common.nh_group = nh_grp;
5141
5142         return 0;
5143 }
5144
5145 static void mlxsw_sp_nexthop6_group_put(struct mlxsw_sp *mlxsw_sp,
5146                                         struct mlxsw_sp_fib_entry *fib_entry)
5147 {
5148         struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
5149
5150         list_del(&fib_entry->nexthop_group_node);
5151         if (!list_empty(&nh_grp->fib_list))
5152                 return;
5153         mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, nh_grp);
5154 }
5155
5156 static int
5157 mlxsw_sp_nexthop6_group_update(struct mlxsw_sp *mlxsw_sp,
5158                                struct mlxsw_sp_fib6_entry *fib6_entry)
5159 {
5160         struct mlxsw_sp_nexthop_group *old_nh_grp = fib6_entry->common.nh_group;
5161         int err;
5162
5163         fib6_entry->common.nh_group = NULL;
5164         list_del(&fib6_entry->common.nexthop_group_node);
5165
5166         err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry);
5167         if (err)
5168                 goto err_nexthop6_group_get;
5169
5170         /* In case this entry is offloaded, then the adjacency index
5171          * currently associated with it in the device's table is that
5172          * of the old group. Start using the new one instead.
5173          */
5174         err = mlxsw_sp_fib_node_entry_add(mlxsw_sp, &fib6_entry->common);
5175         if (err)
5176                 goto err_fib_node_entry_add;
5177
5178         if (list_empty(&old_nh_grp->fib_list))
5179                 mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, old_nh_grp);
5180
5181         return 0;
5182
5183 err_fib_node_entry_add:
5184         mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common);
5185 err_nexthop6_group_get:
5186         list_add_tail(&fib6_entry->common.nexthop_group_node,
5187                       &old_nh_grp->fib_list);
5188         fib6_entry->common.nh_group = old_nh_grp;
5189         return err;
5190 }
5191
5192 static int
5193 mlxsw_sp_fib6_entry_nexthop_add(struct mlxsw_sp *mlxsw_sp,
5194                                 struct mlxsw_sp_fib6_entry *fib6_entry,
5195                                 struct fib6_info *rt)
5196 {
5197         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
5198         int err;
5199
5200         mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt);
5201         if (IS_ERR(mlxsw_sp_rt6))
5202                 return PTR_ERR(mlxsw_sp_rt6);
5203
5204         list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list);
5205         fib6_entry->nrt6++;
5206
5207         err = mlxsw_sp_nexthop6_group_update(mlxsw_sp, fib6_entry);
5208         if (err)
5209                 goto err_nexthop6_group_update;
5210
5211         return 0;
5212
5213 err_nexthop6_group_update:
5214         fib6_entry->nrt6--;
5215         list_del(&mlxsw_sp_rt6->list);
5216         mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
5217         return err;
5218 }
5219
5220 static void
5221 mlxsw_sp_fib6_entry_nexthop_del(struct mlxsw_sp *mlxsw_sp,
5222                                 struct mlxsw_sp_fib6_entry *fib6_entry,
5223                                 struct fib6_info *rt)
5224 {
5225         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
5226
5227         mlxsw_sp_rt6 = mlxsw_sp_fib6_entry_rt_find(fib6_entry, rt);
5228         if (WARN_ON(!mlxsw_sp_rt6))
5229                 return;
5230
5231         fib6_entry->nrt6--;
5232         list_del(&mlxsw_sp_rt6->list);
5233         mlxsw_sp_nexthop6_group_update(mlxsw_sp, fib6_entry);
5234         mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
5235 }
5236
5237 static void mlxsw_sp_fib6_entry_type_set(struct mlxsw_sp *mlxsw_sp,
5238                                          struct mlxsw_sp_fib_entry *fib_entry,
5239                                          const struct fib6_info *rt)
5240 {
5241         /* Packets hitting RTF_REJECT routes need to be discarded by the
5242          * stack. We can rely on their destination device not having a
5243          * RIF (it's the loopback device) and can thus use action type
5244          * local, which will cause them to be trapped with a lower
5245          * priority than packets that need to be locally received.
5246          */
5247         if (rt->fib6_flags & (RTF_LOCAL | RTF_ANYCAST))
5248                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
5249         else if (rt->fib6_type == RTN_BLACKHOLE)
5250                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_BLACKHOLE;
5251         else if (rt->fib6_flags & RTF_REJECT)
5252                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
5253         else if (mlxsw_sp_rt6_is_gateway(mlxsw_sp, rt))
5254                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
5255         else
5256                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
5257 }
5258
5259 static void
5260 mlxsw_sp_fib6_entry_rt_destroy_all(struct mlxsw_sp_fib6_entry *fib6_entry)
5261 {
5262         struct mlxsw_sp_rt6 *mlxsw_sp_rt6, *tmp;
5263
5264         list_for_each_entry_safe(mlxsw_sp_rt6, tmp, &fib6_entry->rt6_list,
5265                                  list) {
5266                 fib6_entry->nrt6--;
5267                 list_del(&mlxsw_sp_rt6->list);
5268                 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
5269         }
5270 }
5271
5272 static struct mlxsw_sp_fib6_entry *
5273 mlxsw_sp_fib6_entry_create(struct mlxsw_sp *mlxsw_sp,
5274                            struct mlxsw_sp_fib_node *fib_node,
5275                            struct fib6_info *rt)
5276 {
5277         struct mlxsw_sp_fib6_entry *fib6_entry;
5278         struct mlxsw_sp_fib_entry *fib_entry;
5279         struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
5280         int err;
5281
5282         fib6_entry = kzalloc(sizeof(*fib6_entry), GFP_KERNEL);
5283         if (!fib6_entry)
5284                 return ERR_PTR(-ENOMEM);
5285         fib_entry = &fib6_entry->common;
5286
5287         mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt);
5288         if (IS_ERR(mlxsw_sp_rt6)) {
5289                 err = PTR_ERR(mlxsw_sp_rt6);
5290                 goto err_rt6_create;
5291         }
5292
5293         mlxsw_sp_fib6_entry_type_set(mlxsw_sp, fib_entry, mlxsw_sp_rt6->rt);
5294
5295         INIT_LIST_HEAD(&fib6_entry->rt6_list);
5296         list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list);
5297         fib6_entry->nrt6 = 1;
5298         err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry);
5299         if (err)
5300                 goto err_nexthop6_group_get;
5301
5302         fib_entry->fib_node = fib_node;
5303
5304         return fib6_entry;
5305
5306 err_nexthop6_group_get:
5307         list_del(&mlxsw_sp_rt6->list);
5308         mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
5309 err_rt6_create:
5310         kfree(fib6_entry);
5311         return ERR_PTR(err);
5312 }
5313
5314 static void mlxsw_sp_fib6_entry_destroy(struct mlxsw_sp *mlxsw_sp,
5315                                         struct mlxsw_sp_fib6_entry *fib6_entry)
5316 {
5317         mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common);
5318         mlxsw_sp_fib6_entry_rt_destroy_all(fib6_entry);
5319         WARN_ON(fib6_entry->nrt6);
5320         kfree(fib6_entry);
5321 }
5322
5323 static struct mlxsw_sp_fib6_entry *
5324 mlxsw_sp_fib6_node_entry_find(const struct mlxsw_sp_fib_node *fib_node,
5325                               const struct fib6_info *nrt, bool replace)
5326 {
5327         struct mlxsw_sp_fib6_entry *fib6_entry, *fallback = NULL;
5328
5329         list_for_each_entry(fib6_entry, &fib_node->entry_list, common.list) {
5330                 struct fib6_info *rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
5331
5332                 if (rt->fib6_table->tb6_id > nrt->fib6_table->tb6_id)
5333                         continue;
5334                 if (rt->fib6_table->tb6_id != nrt->fib6_table->tb6_id)
5335                         break;
5336                 if (replace && rt->fib6_metric == nrt->fib6_metric) {
5337                         if (mlxsw_sp_fib6_rt_can_mp(rt) ==
5338                             mlxsw_sp_fib6_rt_can_mp(nrt))
5339                                 return fib6_entry;
5340                         if (mlxsw_sp_fib6_rt_can_mp(nrt))
5341                                 fallback = fallback ?: fib6_entry;
5342                 }
5343                 if (rt->fib6_metric > nrt->fib6_metric)
5344                         return fallback ?: fib6_entry;
5345         }
5346
5347         return fallback;
5348 }
5349
5350 static int
5351 mlxsw_sp_fib6_node_list_insert(struct mlxsw_sp_fib6_entry *new6_entry,
5352                                bool replace)
5353 {
5354         struct mlxsw_sp_fib_node *fib_node = new6_entry->common.fib_node;
5355         struct fib6_info *nrt = mlxsw_sp_fib6_entry_rt(new6_entry);
5356         struct mlxsw_sp_fib6_entry *fib6_entry;
5357
5358         fib6_entry = mlxsw_sp_fib6_node_entry_find(fib_node, nrt, replace);
5359
5360         if (replace && WARN_ON(!fib6_entry))
5361                 return -EINVAL;
5362
5363         if (fib6_entry) {
5364                 list_add_tail(&new6_entry->common.list,
5365                               &fib6_entry->common.list);
5366         } else {
5367                 struct mlxsw_sp_fib6_entry *last;
5368
5369                 list_for_each_entry(last, &fib_node->entry_list, common.list) {
5370                         struct fib6_info *rt = mlxsw_sp_fib6_entry_rt(last);
5371
5372                         if (nrt->fib6_table->tb6_id > rt->fib6_table->tb6_id)
5373                                 break;
5374                         fib6_entry = last;
5375                 }
5376
5377                 if (fib6_entry)
5378                         list_add(&new6_entry->common.list,
5379                                  &fib6_entry->common.list);
5380                 else
5381                         list_add(&new6_entry->common.list,
5382                                  &fib_node->entry_list);
5383         }
5384
5385         return 0;
5386 }
5387
5388 static void
5389 mlxsw_sp_fib6_node_list_remove(struct mlxsw_sp_fib6_entry *fib6_entry)
5390 {
5391         list_del(&fib6_entry->common.list);
5392 }
5393
5394 static int mlxsw_sp_fib6_node_entry_link(struct mlxsw_sp *mlxsw_sp,
5395                                          struct mlxsw_sp_fib6_entry *fib6_entry,
5396                                          bool replace)
5397 {
5398         int err;
5399
5400         err = mlxsw_sp_fib6_node_list_insert(fib6_entry, replace);
5401         if (err)
5402                 return err;
5403
5404         err = mlxsw_sp_fib_node_entry_add(mlxsw_sp, &fib6_entry->common);
5405         if (err)
5406                 goto err_fib_node_entry_add;
5407
5408         return 0;
5409
5410 err_fib_node_entry_add:
5411         mlxsw_sp_fib6_node_list_remove(fib6_entry);
5412         return err;
5413 }
5414
5415 static void
5416 mlxsw_sp_fib6_node_entry_unlink(struct mlxsw_sp *mlxsw_sp,
5417                                 struct mlxsw_sp_fib6_entry *fib6_entry)
5418 {
5419         mlxsw_sp_fib_node_entry_del(mlxsw_sp, &fib6_entry->common);
5420         mlxsw_sp_fib6_node_list_remove(fib6_entry);
5421 }
5422
5423 static struct mlxsw_sp_fib6_entry *
5424 mlxsw_sp_fib6_entry_lookup(struct mlxsw_sp *mlxsw_sp,
5425                            const struct fib6_info *rt)
5426 {
5427         struct mlxsw_sp_fib6_entry *fib6_entry;
5428         struct mlxsw_sp_fib_node *fib_node;
5429         struct mlxsw_sp_fib *fib;
5430         struct mlxsw_sp_vr *vr;
5431
5432         vr = mlxsw_sp_vr_find(mlxsw_sp, rt->fib6_table->tb6_id);
5433         if (!vr)
5434                 return NULL;
5435         fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV6);
5436
5437         fib_node = mlxsw_sp_fib_node_lookup(fib, &rt->fib6_dst.addr,
5438                                             sizeof(rt->fib6_dst.addr),
5439                                             rt->fib6_dst.plen);
5440         if (!fib_node)
5441                 return NULL;
5442
5443         list_for_each_entry(fib6_entry, &fib_node->entry_list, common.list) {
5444                 struct fib6_info *iter_rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
5445
5446                 if (rt->fib6_table->tb6_id == iter_rt->fib6_table->tb6_id &&
5447                     rt->fib6_metric == iter_rt->fib6_metric &&
5448                     mlxsw_sp_fib6_entry_rt_find(fib6_entry, rt))
5449                         return fib6_entry;
5450         }
5451
5452         return NULL;
5453 }
5454
5455 static void mlxsw_sp_fib6_entry_replace(struct mlxsw_sp *mlxsw_sp,
5456                                         struct mlxsw_sp_fib6_entry *fib6_entry,
5457                                         bool replace)
5458 {
5459         struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node;
5460         struct mlxsw_sp_fib6_entry *replaced;
5461
5462         if (!replace)
5463                 return;
5464
5465         replaced = list_next_entry(fib6_entry, common.list);
5466
5467         mlxsw_sp_fib6_node_entry_unlink(mlxsw_sp, replaced);
5468         mlxsw_sp_fib6_entry_destroy(mlxsw_sp, replaced);
5469         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
5470 }
5471
5472 static int mlxsw_sp_router_fib6_add(struct mlxsw_sp *mlxsw_sp,
5473                                     struct fib6_info *rt, bool replace)
5474 {
5475         struct mlxsw_sp_fib6_entry *fib6_entry;
5476         struct mlxsw_sp_fib_node *fib_node;
5477         int err;
5478
5479         if (mlxsw_sp->router->aborted)
5480                 return 0;
5481
5482         if (rt->fib6_src.plen)
5483                 return -EINVAL;
5484
5485         if (mlxsw_sp_fib6_rt_should_ignore(rt))
5486                 return 0;
5487
5488         fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, rt->fib6_table->tb6_id,
5489                                          &rt->fib6_dst.addr,
5490                                          sizeof(rt->fib6_dst.addr),
5491                                          rt->fib6_dst.plen,
5492                                          MLXSW_SP_L3_PROTO_IPV6);
5493         if (IS_ERR(fib_node))
5494                 return PTR_ERR(fib_node);
5495
5496         /* Before creating a new entry, try to append route to an existing
5497          * multipath entry.
5498          */
5499         fib6_entry = mlxsw_sp_fib6_node_mp_entry_find(fib_node, rt, replace);
5500         if (fib6_entry) {
5501                 err = mlxsw_sp_fib6_entry_nexthop_add(mlxsw_sp, fib6_entry, rt);
5502                 if (err)
5503                         goto err_fib6_entry_nexthop_add;
5504                 return 0;
5505         }
5506
5507         fib6_entry = mlxsw_sp_fib6_entry_create(mlxsw_sp, fib_node, rt);
5508         if (IS_ERR(fib6_entry)) {
5509                 err = PTR_ERR(fib6_entry);
5510                 goto err_fib6_entry_create;
5511         }
5512
5513         err = mlxsw_sp_fib6_node_entry_link(mlxsw_sp, fib6_entry, replace);
5514         if (err)
5515                 goto err_fib6_node_entry_link;
5516
5517         mlxsw_sp_fib6_entry_replace(mlxsw_sp, fib6_entry, replace);
5518
5519         return 0;
5520
5521 err_fib6_node_entry_link:
5522         mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
5523 err_fib6_entry_create:
5524 err_fib6_entry_nexthop_add:
5525         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
5526         return err;
5527 }
5528
5529 static void mlxsw_sp_router_fib6_del(struct mlxsw_sp *mlxsw_sp,
5530                                      struct fib6_info *rt)
5531 {
5532         struct mlxsw_sp_fib6_entry *fib6_entry;
5533         struct mlxsw_sp_fib_node *fib_node;
5534
5535         if (mlxsw_sp->router->aborted)
5536                 return;
5537
5538         if (mlxsw_sp_fib6_rt_should_ignore(rt))
5539                 return;
5540
5541         fib6_entry = mlxsw_sp_fib6_entry_lookup(mlxsw_sp, rt);
5542         if (WARN_ON(!fib6_entry))
5543                 return;
5544
5545         /* If route is part of a multipath entry, but not the last one
5546          * removed, then only reduce its nexthop group.
5547          */
5548         if (!list_is_singular(&fib6_entry->rt6_list)) {
5549                 mlxsw_sp_fib6_entry_nexthop_del(mlxsw_sp, fib6_entry, rt);
5550                 return;
5551         }
5552
5553         fib_node = fib6_entry->common.fib_node;
5554
5555         mlxsw_sp_fib6_node_entry_unlink(mlxsw_sp, fib6_entry);
5556         mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
5557         mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
5558 }
5559
5560 static int __mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp,
5561                                             enum mlxsw_reg_ralxx_protocol proto,
5562                                             u8 tree_id)
5563 {
5564         char ralta_pl[MLXSW_REG_RALTA_LEN];
5565         char ralst_pl[MLXSW_REG_RALST_LEN];
5566         int i, err;
5567
5568         mlxsw_reg_ralta_pack(ralta_pl, true, proto, tree_id);
5569         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
5570         if (err)
5571                 return err;
5572
5573         mlxsw_reg_ralst_pack(ralst_pl, 0xff, tree_id);
5574         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
5575         if (err)
5576                 return err;
5577
5578         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
5579                 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
5580                 char raltb_pl[MLXSW_REG_RALTB_LEN];
5581                 char ralue_pl[MLXSW_REG_RALUE_LEN];
5582
5583                 mlxsw_reg_raltb_pack(raltb_pl, vr->id, proto, tree_id);
5584                 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb),
5585                                       raltb_pl);
5586                 if (err)
5587                         return err;
5588
5589                 mlxsw_reg_ralue_pack(ralue_pl, proto,
5590                                      MLXSW_REG_RALUE_OP_WRITE_WRITE, vr->id, 0);
5591                 mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
5592                 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue),
5593                                       ralue_pl);
5594                 if (err)
5595                         return err;
5596         }
5597
5598         return 0;
5599 }
5600
5601 static struct mlxsw_sp_mr_table *
5602 mlxsw_sp_router_fibmr_family_to_table(struct mlxsw_sp_vr *vr, int family)
5603 {
5604         if (family == RTNL_FAMILY_IPMR)
5605                 return vr->mr_table[MLXSW_SP_L3_PROTO_IPV4];
5606         else
5607                 return vr->mr_table[MLXSW_SP_L3_PROTO_IPV6];
5608 }
5609
5610 static int mlxsw_sp_router_fibmr_add(struct mlxsw_sp *mlxsw_sp,
5611                                      struct mfc_entry_notifier_info *men_info,
5612                                      bool replace)
5613 {
5614         struct mlxsw_sp_mr_table *mrt;
5615         struct mlxsw_sp_vr *vr;
5616
5617         if (mlxsw_sp->router->aborted)
5618                 return 0;
5619
5620         vr = mlxsw_sp_vr_get(mlxsw_sp, men_info->tb_id, NULL);
5621         if (IS_ERR(vr))
5622                 return PTR_ERR(vr);
5623
5624         mrt = mlxsw_sp_router_fibmr_family_to_table(vr, men_info->info.family);
5625         return mlxsw_sp_mr_route_add(mrt, men_info->mfc, replace);
5626 }
5627
5628 static void mlxsw_sp_router_fibmr_del(struct mlxsw_sp *mlxsw_sp,
5629                                       struct mfc_entry_notifier_info *men_info)
5630 {
5631         struct mlxsw_sp_mr_table *mrt;
5632         struct mlxsw_sp_vr *vr;
5633
5634         if (mlxsw_sp->router->aborted)
5635                 return;
5636
5637         vr = mlxsw_sp_vr_find(mlxsw_sp, men_info->tb_id);
5638         if (WARN_ON(!vr))
5639                 return;
5640
5641         mrt = mlxsw_sp_router_fibmr_family_to_table(vr, men_info->info.family);
5642         mlxsw_sp_mr_route_del(mrt, men_info->mfc);
5643         mlxsw_sp_vr_put(mlxsw_sp, vr);
5644 }
5645
5646 static int
5647 mlxsw_sp_router_fibmr_vif_add(struct mlxsw_sp *mlxsw_sp,
5648                               struct vif_entry_notifier_info *ven_info)
5649 {
5650         struct mlxsw_sp_mr_table *mrt;
5651         struct mlxsw_sp_rif *rif;
5652         struct mlxsw_sp_vr *vr;
5653
5654         if (mlxsw_sp->router->aborted)
5655                 return 0;
5656
5657         vr = mlxsw_sp_vr_get(mlxsw_sp, ven_info->tb_id, NULL);
5658         if (IS_ERR(vr))
5659                 return PTR_ERR(vr);
5660
5661         mrt = mlxsw_sp_router_fibmr_family_to_table(vr, ven_info->info.family);
5662         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, ven_info->dev);
5663         return mlxsw_sp_mr_vif_add(mrt, ven_info->dev,
5664                                    ven_info->vif_index,
5665                                    ven_info->vif_flags, rif);
5666 }
5667
5668 static void
5669 mlxsw_sp_router_fibmr_vif_del(struct mlxsw_sp *mlxsw_sp,
5670                               struct vif_entry_notifier_info *ven_info)
5671 {
5672         struct mlxsw_sp_mr_table *mrt;
5673         struct mlxsw_sp_vr *vr;
5674
5675         if (mlxsw_sp->router->aborted)
5676                 return;
5677
5678         vr = mlxsw_sp_vr_find(mlxsw_sp, ven_info->tb_id);
5679         if (WARN_ON(!vr))
5680                 return;
5681
5682         mrt = mlxsw_sp_router_fibmr_family_to_table(vr, ven_info->info.family);
5683         mlxsw_sp_mr_vif_del(mrt, ven_info->vif_index);
5684         mlxsw_sp_vr_put(mlxsw_sp, vr);
5685 }
5686
5687 static int mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp)
5688 {
5689         enum mlxsw_reg_ralxx_protocol proto = MLXSW_REG_RALXX_PROTOCOL_IPV4;
5690         int err;
5691
5692         err = __mlxsw_sp_router_set_abort_trap(mlxsw_sp, proto,
5693                                                MLXSW_SP_LPM_TREE_MIN);
5694         if (err)
5695                 return err;
5696
5697         /* The multicast router code does not need an abort trap as by default,
5698          * packets that don't match any routes are trapped to the CPU.
5699          */
5700
5701         proto = MLXSW_REG_RALXX_PROTOCOL_IPV6;
5702         return __mlxsw_sp_router_set_abort_trap(mlxsw_sp, proto,
5703                                                 MLXSW_SP_LPM_TREE_MIN + 1);
5704 }
5705
5706 static void mlxsw_sp_fib4_node_flush(struct mlxsw_sp *mlxsw_sp,
5707                                      struct mlxsw_sp_fib_node *fib_node)
5708 {
5709         struct mlxsw_sp_fib4_entry *fib4_entry, *tmp;
5710
5711         list_for_each_entry_safe(fib4_entry, tmp, &fib_node->entry_list,
5712                                  common.list) {
5713                 bool do_break = &tmp->common.list == &fib_node->entry_list;
5714
5715                 mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib4_entry);
5716                 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
5717                 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
5718                 /* Break when entry list is empty and node was freed.
5719                  * Otherwise, we'll access freed memory in the next
5720                  * iteration.
5721                  */
5722                 if (do_break)
5723                         break;
5724         }
5725 }
5726
5727 static void mlxsw_sp_fib6_node_flush(struct mlxsw_sp *mlxsw_sp,
5728                                      struct mlxsw_sp_fib_node *fib_node)
5729 {
5730         struct mlxsw_sp_fib6_entry *fib6_entry, *tmp;
5731
5732         list_for_each_entry_safe(fib6_entry, tmp, &fib_node->entry_list,
5733                                  common.list) {
5734                 bool do_break = &tmp->common.list == &fib_node->entry_list;
5735
5736                 mlxsw_sp_fib6_node_entry_unlink(mlxsw_sp, fib6_entry);
5737                 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
5738                 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
5739                 if (do_break)
5740                         break;
5741         }
5742 }
5743
5744 static void mlxsw_sp_fib_node_flush(struct mlxsw_sp *mlxsw_sp,
5745                                     struct mlxsw_sp_fib_node *fib_node)
5746 {
5747         switch (fib_node->fib->proto) {
5748         case MLXSW_SP_L3_PROTO_IPV4:
5749                 mlxsw_sp_fib4_node_flush(mlxsw_sp, fib_node);
5750                 break;
5751         case MLXSW_SP_L3_PROTO_IPV6:
5752                 mlxsw_sp_fib6_node_flush(mlxsw_sp, fib_node);
5753                 break;
5754         }
5755 }
5756
5757 static void mlxsw_sp_vr_fib_flush(struct mlxsw_sp *mlxsw_sp,
5758                                   struct mlxsw_sp_vr *vr,
5759                                   enum mlxsw_sp_l3proto proto)
5760 {
5761         struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
5762         struct mlxsw_sp_fib_node *fib_node, *tmp;
5763
5764         list_for_each_entry_safe(fib_node, tmp, &fib->node_list, list) {
5765                 bool do_break = &tmp->list == &fib->node_list;
5766
5767                 mlxsw_sp_fib_node_flush(mlxsw_sp, fib_node);
5768                 if (do_break)
5769                         break;
5770         }
5771 }
5772
5773 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp)
5774 {
5775         int i, j;
5776
5777         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
5778                 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
5779
5780                 if (!mlxsw_sp_vr_is_used(vr))
5781                         continue;
5782
5783                 for (j = 0; j < MLXSW_SP_L3_PROTO_MAX; j++)
5784                         mlxsw_sp_mr_table_flush(vr->mr_table[j]);
5785                 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4);
5786
5787                 /* If virtual router was only used for IPv4, then it's no
5788                  * longer used.
5789                  */
5790                 if (!mlxsw_sp_vr_is_used(vr))
5791                         continue;
5792                 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6);
5793         }
5794 }
5795
5796 static void mlxsw_sp_router_fib_abort(struct mlxsw_sp *mlxsw_sp)
5797 {
5798         int err;
5799
5800         if (mlxsw_sp->router->aborted)
5801                 return;
5802         dev_warn(mlxsw_sp->bus_info->dev, "FIB abort triggered. Note that FIB entries are no longer being offloaded to this device.\n");
5803         mlxsw_sp_router_fib_flush(mlxsw_sp);
5804         mlxsw_sp->router->aborted = true;
5805         err = mlxsw_sp_router_set_abort_trap(mlxsw_sp);
5806         if (err)
5807                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set abort trap.\n");
5808 }
5809
5810 struct mlxsw_sp_fib_event_work {
5811         struct work_struct work;
5812         union {
5813                 struct fib6_entry_notifier_info fen6_info;
5814                 struct fib_entry_notifier_info fen_info;
5815                 struct fib_rule_notifier_info fr_info;
5816                 struct fib_nh_notifier_info fnh_info;
5817                 struct mfc_entry_notifier_info men_info;
5818                 struct vif_entry_notifier_info ven_info;
5819         };
5820         struct mlxsw_sp *mlxsw_sp;
5821         unsigned long event;
5822 };
5823
5824 static void mlxsw_sp_router_fib4_event_work(struct work_struct *work)
5825 {
5826         struct mlxsw_sp_fib_event_work *fib_work =
5827                 container_of(work, struct mlxsw_sp_fib_event_work, work);
5828         struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp;
5829         bool replace, append;
5830         int err;
5831
5832         /* Protect internal structures from changes */
5833         rtnl_lock();
5834         mlxsw_sp_span_respin(mlxsw_sp);
5835
5836         switch (fib_work->event) {
5837         case FIB_EVENT_ENTRY_REPLACE: /* fall through */
5838         case FIB_EVENT_ENTRY_APPEND: /* fall through */
5839         case FIB_EVENT_ENTRY_ADD:
5840                 replace = fib_work->event == FIB_EVENT_ENTRY_REPLACE;
5841                 append = fib_work->event == FIB_EVENT_ENTRY_APPEND;
5842                 err = mlxsw_sp_router_fib4_add(mlxsw_sp, &fib_work->fen_info,
5843                                                replace, append);
5844                 if (err)
5845                         mlxsw_sp_router_fib_abort(mlxsw_sp);
5846                 fib_info_put(fib_work->fen_info.fi);
5847                 break;
5848         case FIB_EVENT_ENTRY_DEL:
5849                 mlxsw_sp_router_fib4_del(mlxsw_sp, &fib_work->fen_info);
5850                 fib_info_put(fib_work->fen_info.fi);
5851                 break;
5852         case FIB_EVENT_RULE_ADD:
5853                 /* if we get here, a rule was added that we do not support.
5854                  * just do the fib_abort
5855                  */
5856                 mlxsw_sp_router_fib_abort(mlxsw_sp);
5857                 break;
5858         case FIB_EVENT_NH_ADD: /* fall through */
5859         case FIB_EVENT_NH_DEL:
5860                 mlxsw_sp_nexthop4_event(mlxsw_sp, fib_work->event,
5861                                         fib_work->fnh_info.fib_nh);
5862                 fib_info_put(fib_work->fnh_info.fib_nh->nh_parent);
5863                 break;
5864         }
5865         rtnl_unlock();
5866         kfree(fib_work);
5867 }
5868
5869 static void mlxsw_sp_router_fib6_event_work(struct work_struct *work)
5870 {
5871         struct mlxsw_sp_fib_event_work *fib_work =
5872                 container_of(work, struct mlxsw_sp_fib_event_work, work);
5873         struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp;
5874         bool replace;
5875         int err;
5876
5877         rtnl_lock();
5878         mlxsw_sp_span_respin(mlxsw_sp);
5879
5880         switch (fib_work->event) {
5881         case FIB_EVENT_ENTRY_REPLACE: /* fall through */
5882         case FIB_EVENT_ENTRY_APPEND: /* fall through */
5883         case FIB_EVENT_ENTRY_ADD:
5884                 replace = fib_work->event == FIB_EVENT_ENTRY_REPLACE;
5885                 err = mlxsw_sp_router_fib6_add(mlxsw_sp,
5886                                                fib_work->fen6_info.rt, replace);
5887                 if (err)
5888                         mlxsw_sp_router_fib_abort(mlxsw_sp);
5889                 mlxsw_sp_rt6_release(fib_work->fen6_info.rt);
5890                 break;
5891         case FIB_EVENT_ENTRY_DEL:
5892                 mlxsw_sp_router_fib6_del(mlxsw_sp, fib_work->fen6_info.rt);
5893                 mlxsw_sp_rt6_release(fib_work->fen6_info.rt);
5894                 break;
5895         case FIB_EVENT_RULE_ADD:
5896                 /* if we get here, a rule was added that we do not support.
5897                  * just do the fib_abort
5898                  */
5899                 mlxsw_sp_router_fib_abort(mlxsw_sp);
5900                 break;
5901         }
5902         rtnl_unlock();
5903         kfree(fib_work);
5904 }
5905
5906 static void mlxsw_sp_router_fibmr_event_work(struct work_struct *work)
5907 {
5908         struct mlxsw_sp_fib_event_work *fib_work =
5909                 container_of(work, struct mlxsw_sp_fib_event_work, work);
5910         struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp;
5911         bool replace;
5912         int err;
5913
5914         rtnl_lock();
5915         switch (fib_work->event) {
5916         case FIB_EVENT_ENTRY_REPLACE: /* fall through */
5917         case FIB_EVENT_ENTRY_ADD:
5918                 replace = fib_work->event == FIB_EVENT_ENTRY_REPLACE;
5919
5920                 err = mlxsw_sp_router_fibmr_add(mlxsw_sp, &fib_work->men_info,
5921                                                 replace);
5922                 if (err)
5923                         mlxsw_sp_router_fib_abort(mlxsw_sp);
5924                 mr_cache_put(fib_work->men_info.mfc);
5925                 break;
5926         case FIB_EVENT_ENTRY_DEL:
5927                 mlxsw_sp_router_fibmr_del(mlxsw_sp, &fib_work->men_info);
5928                 mr_cache_put(fib_work->men_info.mfc);
5929                 break;
5930         case FIB_EVENT_VIF_ADD:
5931                 err = mlxsw_sp_router_fibmr_vif_add(mlxsw_sp,
5932                                                     &fib_work->ven_info);
5933                 if (err)
5934                         mlxsw_sp_router_fib_abort(mlxsw_sp);
5935                 dev_put(fib_work->ven_info.dev);
5936                 break;
5937         case FIB_EVENT_VIF_DEL:
5938                 mlxsw_sp_router_fibmr_vif_del(mlxsw_sp,
5939                                               &fib_work->ven_info);
5940                 dev_put(fib_work->ven_info.dev);
5941                 break;
5942         case FIB_EVENT_RULE_ADD:
5943                 /* if we get here, a rule was added that we do not support.
5944                  * just do the fib_abort
5945                  */
5946                 mlxsw_sp_router_fib_abort(mlxsw_sp);
5947                 break;
5948         }
5949         rtnl_unlock();
5950         kfree(fib_work);
5951 }
5952
5953 static void mlxsw_sp_router_fib4_event(struct mlxsw_sp_fib_event_work *fib_work,
5954                                        struct fib_notifier_info *info)
5955 {
5956         struct fib_entry_notifier_info *fen_info;
5957         struct fib_nh_notifier_info *fnh_info;
5958
5959         switch (fib_work->event) {
5960         case FIB_EVENT_ENTRY_REPLACE: /* fall through */
5961         case FIB_EVENT_ENTRY_APPEND: /* fall through */
5962         case FIB_EVENT_ENTRY_ADD: /* fall through */
5963         case FIB_EVENT_ENTRY_DEL:
5964                 fen_info = container_of(info, struct fib_entry_notifier_info,
5965                                         info);
5966                 fib_work->fen_info = *fen_info;
5967                 /* Take reference on fib_info to prevent it from being
5968                  * freed while work is queued. Release it afterwards.
5969                  */
5970                 fib_info_hold(fib_work->fen_info.fi);
5971                 break;
5972         case FIB_EVENT_NH_ADD: /* fall through */
5973         case FIB_EVENT_NH_DEL:
5974                 fnh_info = container_of(info, struct fib_nh_notifier_info,
5975                                         info);
5976                 fib_work->fnh_info = *fnh_info;
5977                 fib_info_hold(fib_work->fnh_info.fib_nh->nh_parent);
5978                 break;
5979         }
5980 }
5981
5982 static void mlxsw_sp_router_fib6_event(struct mlxsw_sp_fib_event_work *fib_work,
5983                                        struct fib_notifier_info *info)
5984 {
5985         struct fib6_entry_notifier_info *fen6_info;
5986
5987         switch (fib_work->event) {
5988         case FIB_EVENT_ENTRY_REPLACE: /* fall through */
5989         case FIB_EVENT_ENTRY_APPEND: /* fall through */
5990         case FIB_EVENT_ENTRY_ADD: /* fall through */
5991         case FIB_EVENT_ENTRY_DEL:
5992                 fen6_info = container_of(info, struct fib6_entry_notifier_info,
5993                                          info);
5994                 fib_work->fen6_info = *fen6_info;
5995                 fib6_info_hold(fib_work->fen6_info.rt);
5996                 break;
5997         }
5998 }
5999
6000 static void
6001 mlxsw_sp_router_fibmr_event(struct mlxsw_sp_fib_event_work *fib_work,
6002                             struct fib_notifier_info *info)
6003 {
6004         switch (fib_work->event) {
6005         case FIB_EVENT_ENTRY_REPLACE: /* fall through */
6006         case FIB_EVENT_ENTRY_ADD: /* fall through */
6007         case FIB_EVENT_ENTRY_DEL:
6008                 memcpy(&fib_work->men_info, info, sizeof(fib_work->men_info));
6009                 mr_cache_hold(fib_work->men_info.mfc);
6010                 break;
6011         case FIB_EVENT_VIF_ADD: /* fall through */
6012         case FIB_EVENT_VIF_DEL:
6013                 memcpy(&fib_work->ven_info, info, sizeof(fib_work->ven_info));
6014                 dev_hold(fib_work->ven_info.dev);
6015                 break;
6016         }
6017 }
6018
6019 static int mlxsw_sp_router_fib_rule_event(unsigned long event,
6020                                           struct fib_notifier_info *info,
6021                                           struct mlxsw_sp *mlxsw_sp)
6022 {
6023         struct netlink_ext_ack *extack = info->extack;
6024         struct fib_rule_notifier_info *fr_info;
6025         struct fib_rule *rule;
6026         int err = 0;
6027
6028         /* nothing to do at the moment */
6029         if (event == FIB_EVENT_RULE_DEL)
6030                 return 0;
6031
6032         if (mlxsw_sp->router->aborted)
6033                 return 0;
6034
6035         fr_info = container_of(info, struct fib_rule_notifier_info, info);
6036         rule = fr_info->rule;
6037
6038         switch (info->family) {
6039         case AF_INET:
6040                 if (!fib4_rule_default(rule) && !rule->l3mdev)
6041                         err = -EOPNOTSUPP;
6042                 break;
6043         case AF_INET6:
6044                 if (!fib6_rule_default(rule) && !rule->l3mdev)
6045                         err = -EOPNOTSUPP;
6046                 break;
6047         case RTNL_FAMILY_IPMR:
6048                 if (!ipmr_rule_default(rule) && !rule->l3mdev)
6049                         err = -EOPNOTSUPP;
6050                 break;
6051         case RTNL_FAMILY_IP6MR:
6052                 if (!ip6mr_rule_default(rule) && !rule->l3mdev)
6053                         err = -EOPNOTSUPP;
6054                 break;
6055         }
6056
6057         if (err < 0)
6058                 NL_SET_ERR_MSG_MOD(extack, "FIB rules not supported");
6059
6060         return err;
6061 }
6062
6063 /* Called with rcu_read_lock() */
6064 static int mlxsw_sp_router_fib_event(struct notifier_block *nb,
6065                                      unsigned long event, void *ptr)
6066 {
6067         struct mlxsw_sp_fib_event_work *fib_work;
6068         struct fib_notifier_info *info = ptr;
6069         struct mlxsw_sp_router *router;
6070         int err;
6071
6072         if (!net_eq(info->net, &init_net) ||
6073             (info->family != AF_INET && info->family != AF_INET6 &&
6074              info->family != RTNL_FAMILY_IPMR &&
6075              info->family != RTNL_FAMILY_IP6MR))
6076                 return NOTIFY_DONE;
6077
6078         router = container_of(nb, struct mlxsw_sp_router, fib_nb);
6079
6080         switch (event) {
6081         case FIB_EVENT_RULE_ADD: /* fall through */
6082         case FIB_EVENT_RULE_DEL:
6083                 err = mlxsw_sp_router_fib_rule_event(event, info,
6084                                                      router->mlxsw_sp);
6085                 if (!err || info->extack)
6086                         return notifier_from_errno(err);
6087                 break;
6088         case FIB_EVENT_ENTRY_ADD:
6089                 if (router->aborted) {
6090                         NL_SET_ERR_MSG_MOD(info->extack, "FIB offload was aborted. Not configuring route");
6091                         return notifier_from_errno(-EINVAL);
6092                 }
6093                 break;
6094         }
6095
6096         fib_work = kzalloc(sizeof(*fib_work), GFP_ATOMIC);
6097         if (WARN_ON(!fib_work))
6098                 return NOTIFY_BAD;
6099
6100         fib_work->mlxsw_sp = router->mlxsw_sp;
6101         fib_work->event = event;
6102
6103         switch (info->family) {
6104         case AF_INET:
6105                 INIT_WORK(&fib_work->work, mlxsw_sp_router_fib4_event_work);
6106                 mlxsw_sp_router_fib4_event(fib_work, info);
6107                 break;
6108         case AF_INET6:
6109                 INIT_WORK(&fib_work->work, mlxsw_sp_router_fib6_event_work);
6110                 mlxsw_sp_router_fib6_event(fib_work, info);
6111                 break;
6112         case RTNL_FAMILY_IP6MR:
6113         case RTNL_FAMILY_IPMR:
6114                 INIT_WORK(&fib_work->work, mlxsw_sp_router_fibmr_event_work);
6115                 mlxsw_sp_router_fibmr_event(fib_work, info);
6116                 break;
6117         }
6118
6119         mlxsw_core_schedule_work(&fib_work->work);
6120
6121         return NOTIFY_DONE;
6122 }
6123
6124 struct mlxsw_sp_rif *
6125 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
6126                          const struct net_device *dev)
6127 {
6128         int i;
6129
6130         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
6131                 if (mlxsw_sp->router->rifs[i] &&
6132                     mlxsw_sp->router->rifs[i]->dev == dev)
6133                         return mlxsw_sp->router->rifs[i];
6134
6135         return NULL;
6136 }
6137
6138 static int mlxsw_sp_router_rif_disable(struct mlxsw_sp *mlxsw_sp, u16 rif)
6139 {
6140         char ritr_pl[MLXSW_REG_RITR_LEN];
6141         int err;
6142
6143         mlxsw_reg_ritr_rif_pack(ritr_pl, rif);
6144         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
6145         if (err)
6146                 return err;
6147
6148         mlxsw_reg_ritr_enable_set(ritr_pl, false);
6149         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
6150 }
6151
6152 static void mlxsw_sp_router_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
6153                                           struct mlxsw_sp_rif *rif)
6154 {
6155         mlxsw_sp_router_rif_disable(mlxsw_sp, rif->rif_index);
6156         mlxsw_sp_nexthop_rif_gone_sync(mlxsw_sp, rif);
6157         mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif);
6158 }
6159
6160 static bool
6161 mlxsw_sp_rif_should_config(struct mlxsw_sp_rif *rif, struct net_device *dev,
6162                            unsigned long event)
6163 {
6164         struct inet6_dev *inet6_dev;
6165         bool addr_list_empty = true;
6166         struct in_device *idev;
6167
6168         switch (event) {
6169         case NETDEV_UP:
6170                 return rif == NULL;
6171         case NETDEV_DOWN:
6172                 idev = __in_dev_get_rtnl(dev);
6173                 if (idev && idev->ifa_list)
6174                         addr_list_empty = false;
6175
6176                 inet6_dev = __in6_dev_get(dev);
6177                 if (addr_list_empty && inet6_dev &&
6178                     !list_empty(&inet6_dev->addr_list))
6179                         addr_list_empty = false;
6180
6181                 /* macvlans do not have a RIF, but rather piggy back on the
6182                  * RIF of their lower device.
6183                  */
6184                 if (netif_is_macvlan(dev) && addr_list_empty)
6185                         return true;
6186
6187                 if (rif && addr_list_empty &&
6188                     !netif_is_l3_slave(rif->dev))
6189                         return true;
6190                 /* It is possible we already removed the RIF ourselves
6191                  * if it was assigned to a netdev that is now a bridge
6192                  * or LAG slave.
6193                  */
6194                 return false;
6195         }
6196
6197         return false;
6198 }
6199
6200 static enum mlxsw_sp_rif_type
6201 mlxsw_sp_dev_rif_type(const struct mlxsw_sp *mlxsw_sp,
6202                       const struct net_device *dev)
6203 {
6204         enum mlxsw_sp_fid_type type;
6205
6206         if (mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL))
6207                 return MLXSW_SP_RIF_TYPE_IPIP_LB;
6208
6209         /* Otherwise RIF type is derived from the type of the underlying FID. */
6210         if (is_vlan_dev(dev) && netif_is_bridge_master(vlan_dev_real_dev(dev)))
6211                 type = MLXSW_SP_FID_TYPE_8021Q;
6212         else if (netif_is_bridge_master(dev) && br_vlan_enabled(dev))
6213                 type = MLXSW_SP_FID_TYPE_8021Q;
6214         else if (netif_is_bridge_master(dev))
6215                 type = MLXSW_SP_FID_TYPE_8021D;
6216         else
6217                 type = MLXSW_SP_FID_TYPE_RFID;
6218
6219         return mlxsw_sp_fid_type_rif_type(mlxsw_sp, type);
6220 }
6221
6222 static int mlxsw_sp_rif_index_alloc(struct mlxsw_sp *mlxsw_sp, u16 *p_rif_index)
6223 {
6224         int i;
6225
6226         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) {
6227                 if (!mlxsw_sp->router->rifs[i]) {
6228                         *p_rif_index = i;
6229                         return 0;
6230                 }
6231         }
6232
6233         return -ENOBUFS;
6234 }
6235
6236 static struct mlxsw_sp_rif *mlxsw_sp_rif_alloc(size_t rif_size, u16 rif_index,
6237                                                u16 vr_id,
6238                                                struct net_device *l3_dev)
6239 {
6240         struct mlxsw_sp_rif *rif;
6241
6242         rif = kzalloc(rif_size, GFP_KERNEL);
6243         if (!rif)
6244                 return NULL;
6245
6246         INIT_LIST_HEAD(&rif->nexthop_list);
6247         INIT_LIST_HEAD(&rif->neigh_list);
6248         if (l3_dev) {
6249                 ether_addr_copy(rif->addr, l3_dev->dev_addr);
6250                 rif->mtu = l3_dev->mtu;
6251                 rif->dev = l3_dev;
6252         }
6253         rif->vr_id = vr_id;
6254         rif->rif_index = rif_index;
6255
6256         return rif;
6257 }
6258
6259 struct mlxsw_sp_rif *mlxsw_sp_rif_by_index(const struct mlxsw_sp *mlxsw_sp,
6260                                            u16 rif_index)
6261 {
6262         return mlxsw_sp->router->rifs[rif_index];
6263 }
6264
6265 u16 mlxsw_sp_rif_index(const struct mlxsw_sp_rif *rif)
6266 {
6267         return rif->rif_index;
6268 }
6269
6270 u16 mlxsw_sp_ipip_lb_rif_index(const struct mlxsw_sp_rif_ipip_lb *lb_rif)
6271 {
6272         return lb_rif->common.rif_index;
6273 }
6274
6275 u16 mlxsw_sp_ipip_lb_ul_vr_id(const struct mlxsw_sp_rif_ipip_lb *lb_rif)
6276 {
6277         u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(lb_rif->common.dev);
6278         struct mlxsw_sp_vr *ul_vr;
6279
6280         ul_vr = mlxsw_sp_vr_get(lb_rif->common.mlxsw_sp, ul_tb_id, NULL);
6281         if (WARN_ON(IS_ERR(ul_vr)))
6282                 return 0;
6283
6284         return ul_vr->id;
6285 }
6286
6287 u16 mlxsw_sp_ipip_lb_ul_rif_id(const struct mlxsw_sp_rif_ipip_lb *lb_rif)
6288 {
6289         return lb_rif->ul_rif_id;
6290 }
6291
6292 int mlxsw_sp_rif_dev_ifindex(const struct mlxsw_sp_rif *rif)
6293 {
6294         return rif->dev->ifindex;
6295 }
6296
6297 const struct net_device *mlxsw_sp_rif_dev(const struct mlxsw_sp_rif *rif)
6298 {
6299         return rif->dev;
6300 }
6301
6302 struct mlxsw_sp_fid *mlxsw_sp_rif_fid(const struct mlxsw_sp_rif *rif)
6303 {
6304         return rif->fid;
6305 }
6306
6307 static struct mlxsw_sp_rif *
6308 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp,
6309                     const struct mlxsw_sp_rif_params *params,
6310                     struct netlink_ext_ack *extack)
6311 {
6312         u32 tb_id = l3mdev_fib_table(params->dev);
6313         const struct mlxsw_sp_rif_ops *ops;
6314         struct mlxsw_sp_fid *fid = NULL;
6315         enum mlxsw_sp_rif_type type;
6316         struct mlxsw_sp_rif *rif;
6317         struct mlxsw_sp_vr *vr;
6318         u16 rif_index;
6319         int i, err;
6320
6321         type = mlxsw_sp_dev_rif_type(mlxsw_sp, params->dev);
6322         ops = mlxsw_sp->rif_ops_arr[type];
6323
6324         vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id ? : RT_TABLE_MAIN, extack);
6325         if (IS_ERR(vr))
6326                 return ERR_CAST(vr);
6327         vr->rif_count++;
6328
6329         err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index);
6330         if (err) {
6331                 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interfaces");
6332                 goto err_rif_index_alloc;
6333         }
6334
6335         rif = mlxsw_sp_rif_alloc(ops->rif_size, rif_index, vr->id, params->dev);
6336         if (!rif) {
6337                 err = -ENOMEM;
6338                 goto err_rif_alloc;
6339         }
6340         dev_hold(rif->dev);
6341         mlxsw_sp->router->rifs[rif_index] = rif;
6342         rif->mlxsw_sp = mlxsw_sp;
6343         rif->ops = ops;
6344
6345         if (ops->fid_get) {
6346                 fid = ops->fid_get(rif, extack);
6347                 if (IS_ERR(fid)) {
6348                         err = PTR_ERR(fid);
6349                         goto err_fid_get;
6350                 }
6351                 rif->fid = fid;
6352         }
6353
6354         if (ops->setup)
6355                 ops->setup(rif, params);
6356
6357         err = ops->configure(rif);
6358         if (err)
6359                 goto err_configure;
6360
6361         for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) {
6362                 err = mlxsw_sp_mr_rif_add(vr->mr_table[i], rif);
6363                 if (err)
6364                         goto err_mr_rif_add;
6365         }
6366
6367         mlxsw_sp_rif_counters_alloc(rif);
6368
6369         return rif;
6370
6371 err_mr_rif_add:
6372         for (i--; i >= 0; i--)
6373                 mlxsw_sp_mr_rif_del(vr->mr_table[i], rif);
6374         ops->deconfigure(rif);
6375 err_configure:
6376         if (fid)
6377                 mlxsw_sp_fid_put(fid);
6378 err_fid_get:
6379         mlxsw_sp->router->rifs[rif_index] = NULL;
6380         dev_put(rif->dev);
6381         kfree(rif);
6382 err_rif_alloc:
6383 err_rif_index_alloc:
6384         vr->rif_count--;
6385         mlxsw_sp_vr_put(mlxsw_sp, vr);
6386         return ERR_PTR(err);
6387 }
6388
6389 static void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif)
6390 {
6391         const struct mlxsw_sp_rif_ops *ops = rif->ops;
6392         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
6393         struct mlxsw_sp_fid *fid = rif->fid;
6394         struct mlxsw_sp_vr *vr;
6395         int i;
6396
6397         mlxsw_sp_router_rif_gone_sync(mlxsw_sp, rif);
6398         vr = &mlxsw_sp->router->vrs[rif->vr_id];
6399
6400         mlxsw_sp_rif_counters_free(rif);
6401         for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++)
6402                 mlxsw_sp_mr_rif_del(vr->mr_table[i], rif);
6403         ops->deconfigure(rif);
6404         if (fid)
6405                 /* Loopback RIFs are not associated with a FID. */
6406                 mlxsw_sp_fid_put(fid);
6407         mlxsw_sp->router->rifs[rif->rif_index] = NULL;
6408         dev_put(rif->dev);
6409         kfree(rif);
6410         vr->rif_count--;
6411         mlxsw_sp_vr_put(mlxsw_sp, vr);
6412 }
6413
6414 void mlxsw_sp_rif_destroy_by_dev(struct mlxsw_sp *mlxsw_sp,
6415                                  struct net_device *dev)
6416 {
6417         struct mlxsw_sp_rif *rif;
6418
6419         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
6420         if (!rif)
6421                 return;
6422         mlxsw_sp_rif_destroy(rif);
6423 }
6424
6425 static void
6426 mlxsw_sp_rif_subport_params_init(struct mlxsw_sp_rif_params *params,
6427                                  struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
6428 {
6429         struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
6430
6431         params->vid = mlxsw_sp_port_vlan->vid;
6432         params->lag = mlxsw_sp_port->lagged;
6433         if (params->lag)
6434                 params->lag_id = mlxsw_sp_port->lag_id;
6435         else
6436                 params->system_port = mlxsw_sp_port->local_port;
6437 }
6438
6439 static struct mlxsw_sp_rif_subport *
6440 mlxsw_sp_rif_subport_rif(const struct mlxsw_sp_rif *rif)
6441 {
6442         return container_of(rif, struct mlxsw_sp_rif_subport, common);
6443 }
6444
6445 static struct mlxsw_sp_rif *
6446 mlxsw_sp_rif_subport_get(struct mlxsw_sp *mlxsw_sp,
6447                          const struct mlxsw_sp_rif_params *params,
6448                          struct netlink_ext_ack *extack)
6449 {
6450         struct mlxsw_sp_rif_subport *rif_subport;
6451         struct mlxsw_sp_rif *rif;
6452
6453         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, params->dev);
6454         if (!rif)
6455                 return mlxsw_sp_rif_create(mlxsw_sp, params, extack);
6456
6457         rif_subport = mlxsw_sp_rif_subport_rif(rif);
6458         refcount_inc(&rif_subport->ref_count);
6459         return rif;
6460 }
6461
6462 static void mlxsw_sp_rif_subport_put(struct mlxsw_sp_rif *rif)
6463 {
6464         struct mlxsw_sp_rif_subport *rif_subport;
6465
6466         rif_subport = mlxsw_sp_rif_subport_rif(rif);
6467         if (!refcount_dec_and_test(&rif_subport->ref_count))
6468                 return;
6469
6470         mlxsw_sp_rif_destroy(rif);
6471 }
6472
6473 static int
6474 mlxsw_sp_port_vlan_router_join(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan,
6475                                struct net_device *l3_dev,
6476                                struct netlink_ext_ack *extack)
6477 {
6478         struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
6479         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
6480         struct mlxsw_sp_rif_params params = {
6481                 .dev = l3_dev,
6482         };
6483         u16 vid = mlxsw_sp_port_vlan->vid;
6484         struct mlxsw_sp_rif *rif;
6485         struct mlxsw_sp_fid *fid;
6486         int err;
6487
6488         mlxsw_sp_rif_subport_params_init(&params, mlxsw_sp_port_vlan);
6489         rif = mlxsw_sp_rif_subport_get(mlxsw_sp, &params, extack);
6490         if (IS_ERR(rif))
6491                 return PTR_ERR(rif);
6492
6493         /* FID was already created, just take a reference */
6494         fid = rif->ops->fid_get(rif, extack);
6495         err = mlxsw_sp_fid_port_vid_map(fid, mlxsw_sp_port, vid);
6496         if (err)
6497                 goto err_fid_port_vid_map;
6498
6499         err = mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, false);
6500         if (err)
6501                 goto err_port_vid_learning_set;
6502
6503         err = mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid,
6504                                         BR_STATE_FORWARDING);
6505         if (err)
6506                 goto err_port_vid_stp_set;
6507
6508         mlxsw_sp_port_vlan->fid = fid;
6509
6510         return 0;
6511
6512 err_port_vid_stp_set:
6513         mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true);
6514 err_port_vid_learning_set:
6515         mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid);
6516 err_fid_port_vid_map:
6517         mlxsw_sp_fid_put(fid);
6518         mlxsw_sp_rif_subport_put(rif);
6519         return err;
6520 }
6521
6522 void
6523 mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
6524 {
6525         struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
6526         struct mlxsw_sp_fid *fid = mlxsw_sp_port_vlan->fid;
6527         struct mlxsw_sp_rif *rif = mlxsw_sp_fid_rif(fid);
6528         u16 vid = mlxsw_sp_port_vlan->vid;
6529
6530         if (WARN_ON(mlxsw_sp_fid_type(fid) != MLXSW_SP_FID_TYPE_RFID))
6531                 return;
6532
6533         mlxsw_sp_port_vlan->fid = NULL;
6534         mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid, BR_STATE_BLOCKING);
6535         mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true);
6536         mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid);
6537         mlxsw_sp_fid_put(fid);
6538         mlxsw_sp_rif_subport_put(rif);
6539 }
6540
6541 static int mlxsw_sp_inetaddr_port_vlan_event(struct net_device *l3_dev,
6542                                              struct net_device *port_dev,
6543                                              unsigned long event, u16 vid,
6544                                              struct netlink_ext_ack *extack)
6545 {
6546         struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(port_dev);
6547         struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan;
6548
6549         mlxsw_sp_port_vlan = mlxsw_sp_port_vlan_find_by_vid(mlxsw_sp_port, vid);
6550         if (WARN_ON(!mlxsw_sp_port_vlan))
6551                 return -EINVAL;
6552
6553         switch (event) {
6554         case NETDEV_UP:
6555                 return mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan,
6556                                                       l3_dev, extack);
6557         case NETDEV_DOWN:
6558                 mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan);
6559                 break;
6560         }
6561
6562         return 0;
6563 }
6564
6565 static int mlxsw_sp_inetaddr_port_event(struct net_device *port_dev,
6566                                         unsigned long event,
6567                                         struct netlink_ext_ack *extack)
6568 {
6569         if (netif_is_bridge_port(port_dev) ||
6570             netif_is_lag_port(port_dev) ||
6571             netif_is_ovs_port(port_dev))
6572                 return 0;
6573
6574         return mlxsw_sp_inetaddr_port_vlan_event(port_dev, port_dev, event,
6575                                                  MLXSW_SP_DEFAULT_VID, extack);
6576 }
6577
6578 static int __mlxsw_sp_inetaddr_lag_event(struct net_device *l3_dev,
6579                                          struct net_device *lag_dev,
6580                                          unsigned long event, u16 vid,
6581                                          struct netlink_ext_ack *extack)
6582 {
6583         struct net_device *port_dev;
6584         struct list_head *iter;
6585         int err;
6586
6587         netdev_for_each_lower_dev(lag_dev, port_dev, iter) {
6588                 if (mlxsw_sp_port_dev_check(port_dev)) {
6589                         err = mlxsw_sp_inetaddr_port_vlan_event(l3_dev,
6590                                                                 port_dev,
6591                                                                 event, vid,
6592                                                                 extack);
6593                         if (err)
6594                                 return err;
6595                 }
6596         }
6597
6598         return 0;
6599 }
6600
6601 static int mlxsw_sp_inetaddr_lag_event(struct net_device *lag_dev,
6602                                        unsigned long event,
6603                                        struct netlink_ext_ack *extack)
6604 {
6605         if (netif_is_bridge_port(lag_dev))
6606                 return 0;
6607
6608         return __mlxsw_sp_inetaddr_lag_event(lag_dev, lag_dev, event,
6609                                              MLXSW_SP_DEFAULT_VID, extack);
6610 }
6611
6612 static int mlxsw_sp_inetaddr_bridge_event(struct mlxsw_sp *mlxsw_sp,
6613                                           struct net_device *l3_dev,
6614                                           unsigned long event,
6615                                           struct netlink_ext_ack *extack)
6616 {
6617         struct mlxsw_sp_rif_params params = {
6618                 .dev = l3_dev,
6619         };
6620         struct mlxsw_sp_rif *rif;
6621
6622         switch (event) {
6623         case NETDEV_UP:
6624                 rif = mlxsw_sp_rif_create(mlxsw_sp, &params, extack);
6625                 if (IS_ERR(rif))
6626                         return PTR_ERR(rif);
6627                 break;
6628         case NETDEV_DOWN:
6629                 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
6630                 mlxsw_sp_rif_destroy(rif);
6631                 break;
6632         }
6633
6634         return 0;
6635 }
6636
6637 static int mlxsw_sp_inetaddr_vlan_event(struct mlxsw_sp *mlxsw_sp,
6638                                         struct net_device *vlan_dev,
6639                                         unsigned long event,
6640                                         struct netlink_ext_ack *extack)
6641 {
6642         struct net_device *real_dev = vlan_dev_real_dev(vlan_dev);
6643         u16 vid = vlan_dev_vlan_id(vlan_dev);
6644
6645         if (netif_is_bridge_port(vlan_dev))
6646                 return 0;
6647
6648         if (mlxsw_sp_port_dev_check(real_dev))
6649                 return mlxsw_sp_inetaddr_port_vlan_event(vlan_dev, real_dev,
6650                                                          event, vid, extack);
6651         else if (netif_is_lag_master(real_dev))
6652                 return __mlxsw_sp_inetaddr_lag_event(vlan_dev, real_dev, event,
6653                                                      vid, extack);
6654         else if (netif_is_bridge_master(real_dev) && br_vlan_enabled(real_dev))
6655                 return mlxsw_sp_inetaddr_bridge_event(mlxsw_sp, vlan_dev, event,
6656                                                       extack);
6657
6658         return 0;
6659 }
6660
6661 static bool mlxsw_sp_rif_macvlan_is_vrrp4(const u8 *mac)
6662 {
6663         u8 vrrp4[ETH_ALEN] = { 0x00, 0x00, 0x5e, 0x00, 0x01, 0x00 };
6664         u8 mask[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0x00 };
6665
6666         return ether_addr_equal_masked(mac, vrrp4, mask);
6667 }
6668
6669 static bool mlxsw_sp_rif_macvlan_is_vrrp6(const u8 *mac)
6670 {
6671         u8 vrrp6[ETH_ALEN] = { 0x00, 0x00, 0x5e, 0x00, 0x02, 0x00 };
6672         u8 mask[ETH_ALEN] = { 0xff, 0xff, 0xff, 0xff, 0xff, 0x00 };
6673
6674         return ether_addr_equal_masked(mac, vrrp6, mask);
6675 }
6676
6677 static int mlxsw_sp_rif_vrrp_op(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
6678                                 const u8 *mac, bool adding)
6679 {
6680         char ritr_pl[MLXSW_REG_RITR_LEN];
6681         u8 vrrp_id = adding ? mac[5] : 0;
6682         int err;
6683
6684         if (!mlxsw_sp_rif_macvlan_is_vrrp4(mac) &&
6685             !mlxsw_sp_rif_macvlan_is_vrrp6(mac))
6686                 return 0;
6687
6688         mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
6689         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
6690         if (err)
6691                 return err;
6692
6693         if (mlxsw_sp_rif_macvlan_is_vrrp4(mac))
6694                 mlxsw_reg_ritr_if_vrrp_id_ipv4_set(ritr_pl, vrrp_id);
6695         else
6696                 mlxsw_reg_ritr_if_vrrp_id_ipv6_set(ritr_pl, vrrp_id);
6697
6698         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
6699 }
6700
6701 static int mlxsw_sp_rif_macvlan_add(struct mlxsw_sp *mlxsw_sp,
6702                                     const struct net_device *macvlan_dev,
6703                                     struct netlink_ext_ack *extack)
6704 {
6705         struct macvlan_dev *vlan = netdev_priv(macvlan_dev);
6706         struct mlxsw_sp_rif *rif;
6707         int err;
6708
6709         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, vlan->lowerdev);
6710         if (!rif) {
6711                 NL_SET_ERR_MSG_MOD(extack, "macvlan is only supported on top of router interfaces");
6712                 return -EOPNOTSUPP;
6713         }
6714
6715         err = mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr,
6716                                   mlxsw_sp_fid_index(rif->fid), true);
6717         if (err)
6718                 return err;
6719
6720         err = mlxsw_sp_rif_vrrp_op(mlxsw_sp, rif->rif_index,
6721                                    macvlan_dev->dev_addr, true);
6722         if (err)
6723                 goto err_rif_vrrp_add;
6724
6725         /* Make sure the bridge driver does not have this MAC pointing at
6726          * some other port.
6727          */
6728         if (rif->ops->fdb_del)
6729                 rif->ops->fdb_del(rif, macvlan_dev->dev_addr);
6730
6731         return 0;
6732
6733 err_rif_vrrp_add:
6734         mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr,
6735                             mlxsw_sp_fid_index(rif->fid), false);
6736         return err;
6737 }
6738
6739 void mlxsw_sp_rif_macvlan_del(struct mlxsw_sp *mlxsw_sp,
6740                               const struct net_device *macvlan_dev)
6741 {
6742         struct macvlan_dev *vlan = netdev_priv(macvlan_dev);
6743         struct mlxsw_sp_rif *rif;
6744
6745         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, vlan->lowerdev);
6746         /* If we do not have a RIF, then we already took care of
6747          * removing the macvlan's MAC during RIF deletion.
6748          */
6749         if (!rif)
6750                 return;
6751         mlxsw_sp_rif_vrrp_op(mlxsw_sp, rif->rif_index, macvlan_dev->dev_addr,
6752                              false);
6753         mlxsw_sp_rif_fdb_op(mlxsw_sp, macvlan_dev->dev_addr,
6754                             mlxsw_sp_fid_index(rif->fid), false);
6755 }
6756
6757 static int mlxsw_sp_inetaddr_macvlan_event(struct mlxsw_sp *mlxsw_sp,
6758                                            struct net_device *macvlan_dev,
6759                                            unsigned long event,
6760                                            struct netlink_ext_ack *extack)
6761 {
6762         switch (event) {
6763         case NETDEV_UP:
6764                 return mlxsw_sp_rif_macvlan_add(mlxsw_sp, macvlan_dev, extack);
6765         case NETDEV_DOWN:
6766                 mlxsw_sp_rif_macvlan_del(mlxsw_sp, macvlan_dev);
6767                 break;
6768         }
6769
6770         return 0;
6771 }
6772
6773 static int mlxsw_sp_router_port_check_rif_addr(struct mlxsw_sp *mlxsw_sp,
6774                                                struct net_device *dev,
6775                                                const unsigned char *dev_addr,
6776                                                struct netlink_ext_ack *extack)
6777 {
6778         struct mlxsw_sp_rif *rif;
6779         int i;
6780
6781         /* A RIF is not created for macvlan netdevs. Their MAC is used to
6782          * populate the FDB
6783          */
6784         if (netif_is_macvlan(dev))
6785                 return 0;
6786
6787         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) {
6788                 rif = mlxsw_sp->router->rifs[i];
6789                 if (rif && rif->dev && rif->dev != dev &&
6790                     !ether_addr_equal_masked(rif->dev->dev_addr, dev_addr,
6791                                              mlxsw_sp->mac_mask)) {
6792                         NL_SET_ERR_MSG_MOD(extack, "All router interface MAC addresses must have the same prefix");
6793                         return -EINVAL;
6794                 }
6795         }
6796
6797         return 0;
6798 }
6799
6800 static int __mlxsw_sp_inetaddr_event(struct mlxsw_sp *mlxsw_sp,
6801                                      struct net_device *dev,
6802                                      unsigned long event,
6803                                      struct netlink_ext_ack *extack)
6804 {
6805         if (mlxsw_sp_port_dev_check(dev))
6806                 return mlxsw_sp_inetaddr_port_event(dev, event, extack);
6807         else if (netif_is_lag_master(dev))
6808                 return mlxsw_sp_inetaddr_lag_event(dev, event, extack);
6809         else if (netif_is_bridge_master(dev))
6810                 return mlxsw_sp_inetaddr_bridge_event(mlxsw_sp, dev, event,
6811                                                       extack);
6812         else if (is_vlan_dev(dev))
6813                 return mlxsw_sp_inetaddr_vlan_event(mlxsw_sp, dev, event,
6814                                                     extack);
6815         else if (netif_is_macvlan(dev))
6816                 return mlxsw_sp_inetaddr_macvlan_event(mlxsw_sp, dev, event,
6817                                                        extack);
6818         else
6819                 return 0;
6820 }
6821
6822 static int mlxsw_sp_inetaddr_event(struct notifier_block *nb,
6823                                    unsigned long event, void *ptr)
6824 {
6825         struct in_ifaddr *ifa = (struct in_ifaddr *) ptr;
6826         struct net_device *dev = ifa->ifa_dev->dev;
6827         struct mlxsw_sp_router *router;
6828         struct mlxsw_sp_rif *rif;
6829         int err = 0;
6830
6831         /* NETDEV_UP event is handled by mlxsw_sp_inetaddr_valid_event */
6832         if (event == NETDEV_UP)
6833                 goto out;
6834
6835         router = container_of(nb, struct mlxsw_sp_router, inetaddr_nb);
6836         rif = mlxsw_sp_rif_find_by_dev(router->mlxsw_sp, dev);
6837         if (!mlxsw_sp_rif_should_config(rif, dev, event))
6838                 goto out;
6839
6840         err = __mlxsw_sp_inetaddr_event(router->mlxsw_sp, dev, event, NULL);
6841 out:
6842         return notifier_from_errno(err);
6843 }
6844
6845 int mlxsw_sp_inetaddr_valid_event(struct notifier_block *unused,
6846                                   unsigned long event, void *ptr)
6847 {
6848         struct in_validator_info *ivi = (struct in_validator_info *) ptr;
6849         struct net_device *dev = ivi->ivi_dev->dev;
6850         struct mlxsw_sp *mlxsw_sp;
6851         struct mlxsw_sp_rif *rif;
6852         int err = 0;
6853
6854         mlxsw_sp = mlxsw_sp_lower_get(dev);
6855         if (!mlxsw_sp)
6856                 goto out;
6857
6858         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
6859         if (!mlxsw_sp_rif_should_config(rif, dev, event))
6860                 goto out;
6861
6862         err = mlxsw_sp_router_port_check_rif_addr(mlxsw_sp, dev, dev->dev_addr,
6863                                                   ivi->extack);
6864         if (err)
6865                 goto out;
6866
6867         err = __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, ivi->extack);
6868 out:
6869         return notifier_from_errno(err);
6870 }
6871
6872 struct mlxsw_sp_inet6addr_event_work {
6873         struct work_struct work;
6874         struct mlxsw_sp *mlxsw_sp;
6875         struct net_device *dev;
6876         unsigned long event;
6877 };
6878
6879 static void mlxsw_sp_inet6addr_event_work(struct work_struct *work)
6880 {
6881         struct mlxsw_sp_inet6addr_event_work *inet6addr_work =
6882                 container_of(work, struct mlxsw_sp_inet6addr_event_work, work);
6883         struct mlxsw_sp *mlxsw_sp = inet6addr_work->mlxsw_sp;
6884         struct net_device *dev = inet6addr_work->dev;
6885         unsigned long event = inet6addr_work->event;
6886         struct mlxsw_sp_rif *rif;
6887
6888         rtnl_lock();
6889
6890         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
6891         if (!mlxsw_sp_rif_should_config(rif, dev, event))
6892                 goto out;
6893
6894         __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, NULL);
6895 out:
6896         rtnl_unlock();
6897         dev_put(dev);
6898         kfree(inet6addr_work);
6899 }
6900
6901 /* Called with rcu_read_lock() */
6902 static int mlxsw_sp_inet6addr_event(struct notifier_block *nb,
6903                                     unsigned long event, void *ptr)
6904 {
6905         struct inet6_ifaddr *if6 = (struct inet6_ifaddr *) ptr;
6906         struct mlxsw_sp_inet6addr_event_work *inet6addr_work;
6907         struct net_device *dev = if6->idev->dev;
6908         struct mlxsw_sp_router *router;
6909
6910         /* NETDEV_UP event is handled by mlxsw_sp_inet6addr_valid_event */
6911         if (event == NETDEV_UP)
6912                 return NOTIFY_DONE;
6913
6914         inet6addr_work = kzalloc(sizeof(*inet6addr_work), GFP_ATOMIC);
6915         if (!inet6addr_work)
6916                 return NOTIFY_BAD;
6917
6918         router = container_of(nb, struct mlxsw_sp_router, inet6addr_nb);
6919         INIT_WORK(&inet6addr_work->work, mlxsw_sp_inet6addr_event_work);
6920         inet6addr_work->mlxsw_sp = router->mlxsw_sp;
6921         inet6addr_work->dev = dev;
6922         inet6addr_work->event = event;
6923         dev_hold(dev);
6924         mlxsw_core_schedule_work(&inet6addr_work->work);
6925
6926         return NOTIFY_DONE;
6927 }
6928
6929 int mlxsw_sp_inet6addr_valid_event(struct notifier_block *unused,
6930                                    unsigned long event, void *ptr)
6931 {
6932         struct in6_validator_info *i6vi = (struct in6_validator_info *) ptr;
6933         struct net_device *dev = i6vi->i6vi_dev->dev;
6934         struct mlxsw_sp *mlxsw_sp;
6935         struct mlxsw_sp_rif *rif;
6936         int err = 0;
6937
6938         mlxsw_sp = mlxsw_sp_lower_get(dev);
6939         if (!mlxsw_sp)
6940                 goto out;
6941
6942         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
6943         if (!mlxsw_sp_rif_should_config(rif, dev, event))
6944                 goto out;
6945
6946         err = mlxsw_sp_router_port_check_rif_addr(mlxsw_sp, dev, dev->dev_addr,
6947                                                   i6vi->extack);
6948         if (err)
6949                 goto out;
6950
6951         err = __mlxsw_sp_inetaddr_event(mlxsw_sp, dev, event, i6vi->extack);
6952 out:
6953         return notifier_from_errno(err);
6954 }
6955
6956 static int mlxsw_sp_rif_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
6957                              const char *mac, int mtu)
6958 {
6959         char ritr_pl[MLXSW_REG_RITR_LEN];
6960         int err;
6961
6962         mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
6963         err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
6964         if (err)
6965                 return err;
6966
6967         mlxsw_reg_ritr_mtu_set(ritr_pl, mtu);
6968         mlxsw_reg_ritr_if_mac_memcpy_to(ritr_pl, mac);
6969         mlxsw_reg_ritr_op_set(ritr_pl, MLXSW_REG_RITR_RIF_CREATE);
6970         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
6971 }
6972
6973 static int
6974 mlxsw_sp_router_port_change_event(struct mlxsw_sp *mlxsw_sp,
6975                                   struct mlxsw_sp_rif *rif)
6976 {
6977         struct net_device *dev = rif->dev;
6978         u16 fid_index;
6979         int err;
6980
6981         fid_index = mlxsw_sp_fid_index(rif->fid);
6982
6983         err = mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, false);
6984         if (err)
6985                 return err;
6986
6987         err = mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, dev->dev_addr,
6988                                 dev->mtu);
6989         if (err)
6990                 goto err_rif_edit;
6991
6992         err = mlxsw_sp_rif_fdb_op(mlxsw_sp, dev->dev_addr, fid_index, true);
6993         if (err)
6994                 goto err_rif_fdb_op;
6995
6996         if (rif->mtu != dev->mtu) {
6997                 struct mlxsw_sp_vr *vr;
6998                 int i;
6999
7000                 /* The RIF is relevant only to its mr_table instance, as unlike
7001                  * unicast routing, in multicast routing a RIF cannot be shared
7002                  * between several multicast routing tables.
7003                  */
7004                 vr = &mlxsw_sp->router->vrs[rif->vr_id];
7005                 for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++)
7006                         mlxsw_sp_mr_rif_mtu_update(vr->mr_table[i],
7007                                                    rif, dev->mtu);
7008         }
7009
7010         ether_addr_copy(rif->addr, dev->dev_addr);
7011         rif->mtu = dev->mtu;
7012
7013         netdev_dbg(dev, "Updated RIF=%d\n", rif->rif_index);
7014
7015         return 0;
7016
7017 err_rif_fdb_op:
7018         mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, rif->addr, rif->mtu);
7019 err_rif_edit:
7020         mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, true);
7021         return err;
7022 }
7023
7024 static int mlxsw_sp_router_port_pre_changeaddr_event(struct mlxsw_sp_rif *rif,
7025                             struct netdev_notifier_pre_changeaddr_info *info)
7026 {
7027         struct netlink_ext_ack *extack;
7028
7029         extack = netdev_notifier_info_to_extack(&info->info);
7030         return mlxsw_sp_router_port_check_rif_addr(rif->mlxsw_sp, rif->dev,
7031                                                    info->dev_addr, extack);
7032 }
7033
7034 int mlxsw_sp_netdevice_router_port_event(struct net_device *dev,
7035                                          unsigned long event, void *ptr)
7036 {
7037         struct mlxsw_sp *mlxsw_sp;
7038         struct mlxsw_sp_rif *rif;
7039
7040         mlxsw_sp = mlxsw_sp_lower_get(dev);
7041         if (!mlxsw_sp)
7042                 return 0;
7043
7044         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
7045         if (!rif)
7046                 return 0;
7047
7048         switch (event) {
7049         case NETDEV_CHANGEMTU: /* fall through */
7050         case NETDEV_CHANGEADDR:
7051                 return mlxsw_sp_router_port_change_event(mlxsw_sp, rif);
7052         case NETDEV_PRE_CHANGEADDR:
7053                 return mlxsw_sp_router_port_pre_changeaddr_event(rif, ptr);
7054         }
7055
7056         return 0;
7057 }
7058
7059 static int mlxsw_sp_port_vrf_join(struct mlxsw_sp *mlxsw_sp,
7060                                   struct net_device *l3_dev,
7061                                   struct netlink_ext_ack *extack)
7062 {
7063         struct mlxsw_sp_rif *rif;
7064
7065         /* If netdev is already associated with a RIF, then we need to
7066          * destroy it and create a new one with the new virtual router ID.
7067          */
7068         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
7069         if (rif)
7070                 __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_DOWN,
7071                                           extack);
7072
7073         return __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_UP, extack);
7074 }
7075
7076 static void mlxsw_sp_port_vrf_leave(struct mlxsw_sp *mlxsw_sp,
7077                                     struct net_device *l3_dev)
7078 {
7079         struct mlxsw_sp_rif *rif;
7080
7081         rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
7082         if (!rif)
7083                 return;
7084         __mlxsw_sp_inetaddr_event(mlxsw_sp, l3_dev, NETDEV_DOWN, NULL);
7085 }
7086
7087 int mlxsw_sp_netdevice_vrf_event(struct net_device *l3_dev, unsigned long event,
7088                                  struct netdev_notifier_changeupper_info *info)
7089 {
7090         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev);
7091         int err = 0;
7092
7093         /* We do not create a RIF for a macvlan, but only use it to
7094          * direct more MAC addresses to the router.
7095          */
7096         if (!mlxsw_sp || netif_is_macvlan(l3_dev))
7097                 return 0;
7098
7099         switch (event) {
7100         case NETDEV_PRECHANGEUPPER:
7101                 return 0;
7102         case NETDEV_CHANGEUPPER:
7103                 if (info->linking) {
7104                         struct netlink_ext_ack *extack;
7105
7106                         extack = netdev_notifier_info_to_extack(&info->info);
7107                         err = mlxsw_sp_port_vrf_join(mlxsw_sp, l3_dev, extack);
7108                 } else {
7109                         mlxsw_sp_port_vrf_leave(mlxsw_sp, l3_dev);
7110                 }
7111                 break;
7112         }
7113
7114         return err;
7115 }
7116
7117 static int __mlxsw_sp_rif_macvlan_flush(struct net_device *dev, void *data)
7118 {
7119         struct mlxsw_sp_rif *rif = data;
7120
7121         if (!netif_is_macvlan(dev))
7122                 return 0;
7123
7124         return mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, dev->dev_addr,
7125                                    mlxsw_sp_fid_index(rif->fid), false);
7126 }
7127
7128 static int mlxsw_sp_rif_macvlan_flush(struct mlxsw_sp_rif *rif)
7129 {
7130         if (!netif_is_macvlan_port(rif->dev))
7131                 return 0;
7132
7133         netdev_warn(rif->dev, "Router interface is deleted. Upper macvlans will not work\n");
7134         return netdev_walk_all_upper_dev_rcu(rif->dev,
7135                                              __mlxsw_sp_rif_macvlan_flush, rif);
7136 }
7137
7138 static void mlxsw_sp_rif_subport_setup(struct mlxsw_sp_rif *rif,
7139                                        const struct mlxsw_sp_rif_params *params)
7140 {
7141         struct mlxsw_sp_rif_subport *rif_subport;
7142
7143         rif_subport = mlxsw_sp_rif_subport_rif(rif);
7144         refcount_set(&rif_subport->ref_count, 1);
7145         rif_subport->vid = params->vid;
7146         rif_subport->lag = params->lag;
7147         if (params->lag)
7148                 rif_subport->lag_id = params->lag_id;
7149         else
7150                 rif_subport->system_port = params->system_port;
7151 }
7152
7153 static int mlxsw_sp_rif_subport_op(struct mlxsw_sp_rif *rif, bool enable)
7154 {
7155         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
7156         struct mlxsw_sp_rif_subport *rif_subport;
7157         char ritr_pl[MLXSW_REG_RITR_LEN];
7158
7159         rif_subport = mlxsw_sp_rif_subport_rif(rif);
7160         mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_SP_IF,
7161                             rif->rif_index, rif->vr_id, rif->dev->mtu);
7162         mlxsw_reg_ritr_mac_pack(ritr_pl, rif->dev->dev_addr);
7163         mlxsw_reg_ritr_sp_if_pack(ritr_pl, rif_subport->lag,
7164                                   rif_subport->lag ? rif_subport->lag_id :
7165                                                      rif_subport->system_port,
7166                                   rif_subport->vid);
7167
7168         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
7169 }
7170
7171 static int mlxsw_sp_rif_subport_configure(struct mlxsw_sp_rif *rif)
7172 {
7173         int err;
7174
7175         err = mlxsw_sp_rif_subport_op(rif, true);
7176         if (err)
7177                 return err;
7178
7179         err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
7180                                   mlxsw_sp_fid_index(rif->fid), true);
7181         if (err)
7182                 goto err_rif_fdb_op;
7183
7184         mlxsw_sp_fid_rif_set(rif->fid, rif);
7185         return 0;
7186
7187 err_rif_fdb_op:
7188         mlxsw_sp_rif_subport_op(rif, false);
7189         return err;
7190 }
7191
7192 static void mlxsw_sp_rif_subport_deconfigure(struct mlxsw_sp_rif *rif)
7193 {
7194         struct mlxsw_sp_fid *fid = rif->fid;
7195
7196         mlxsw_sp_fid_rif_set(fid, NULL);
7197         mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
7198                             mlxsw_sp_fid_index(fid), false);
7199         mlxsw_sp_rif_macvlan_flush(rif);
7200         mlxsw_sp_rif_subport_op(rif, false);
7201 }
7202
7203 static struct mlxsw_sp_fid *
7204 mlxsw_sp_rif_subport_fid_get(struct mlxsw_sp_rif *rif,
7205                              struct netlink_ext_ack *extack)
7206 {
7207         return mlxsw_sp_fid_rfid_get(rif->mlxsw_sp, rif->rif_index);
7208 }
7209
7210 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_subport_ops = {
7211         .type                   = MLXSW_SP_RIF_TYPE_SUBPORT,
7212         .rif_size               = sizeof(struct mlxsw_sp_rif_subport),
7213         .setup                  = mlxsw_sp_rif_subport_setup,
7214         .configure              = mlxsw_sp_rif_subport_configure,
7215         .deconfigure            = mlxsw_sp_rif_subport_deconfigure,
7216         .fid_get                = mlxsw_sp_rif_subport_fid_get,
7217 };
7218
7219 static int mlxsw_sp_rif_vlan_fid_op(struct mlxsw_sp_rif *rif,
7220                                     enum mlxsw_reg_ritr_if_type type,
7221                                     u16 vid_fid, bool enable)
7222 {
7223         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
7224         char ritr_pl[MLXSW_REG_RITR_LEN];
7225
7226         mlxsw_reg_ritr_pack(ritr_pl, enable, type, rif->rif_index, rif->vr_id,
7227                             rif->dev->mtu);
7228         mlxsw_reg_ritr_mac_pack(ritr_pl, rif->dev->dev_addr);
7229         mlxsw_reg_ritr_fid_set(ritr_pl, type, vid_fid);
7230
7231         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
7232 }
7233
7234 u8 mlxsw_sp_router_port(const struct mlxsw_sp *mlxsw_sp)
7235 {
7236         return mlxsw_core_max_ports(mlxsw_sp->core) + 1;
7237 }
7238
7239 static int mlxsw_sp_rif_vlan_configure(struct mlxsw_sp_rif *rif)
7240 {
7241         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
7242         u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid);
7243         int err;
7244
7245         err = mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, true);
7246         if (err)
7247                 return err;
7248
7249         err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
7250                                      mlxsw_sp_router_port(mlxsw_sp), true);
7251         if (err)
7252                 goto err_fid_mc_flood_set;
7253
7254         err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
7255                                      mlxsw_sp_router_port(mlxsw_sp), true);
7256         if (err)
7257                 goto err_fid_bc_flood_set;
7258
7259         err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
7260                                   mlxsw_sp_fid_index(rif->fid), true);
7261         if (err)
7262                 goto err_rif_fdb_op;
7263
7264         mlxsw_sp_fid_rif_set(rif->fid, rif);
7265         return 0;
7266
7267 err_rif_fdb_op:
7268         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
7269                                mlxsw_sp_router_port(mlxsw_sp), false);
7270 err_fid_bc_flood_set:
7271         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
7272                                mlxsw_sp_router_port(mlxsw_sp), false);
7273 err_fid_mc_flood_set:
7274         mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, false);
7275         return err;
7276 }
7277
7278 static void mlxsw_sp_rif_vlan_deconfigure(struct mlxsw_sp_rif *rif)
7279 {
7280         u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid);
7281         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
7282         struct mlxsw_sp_fid *fid = rif->fid;
7283
7284         mlxsw_sp_fid_rif_set(fid, NULL);
7285         mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
7286                             mlxsw_sp_fid_index(fid), false);
7287         mlxsw_sp_rif_macvlan_flush(rif);
7288         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
7289                                mlxsw_sp_router_port(mlxsw_sp), false);
7290         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
7291                                mlxsw_sp_router_port(mlxsw_sp), false);
7292         mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, false);
7293 }
7294
7295 static struct mlxsw_sp_fid *
7296 mlxsw_sp_rif_vlan_fid_get(struct mlxsw_sp_rif *rif,
7297                           struct netlink_ext_ack *extack)
7298 {
7299         struct net_device *br_dev = rif->dev;
7300         u16 vid;
7301         int err;
7302
7303         if (is_vlan_dev(rif->dev)) {
7304                 vid = vlan_dev_vlan_id(rif->dev);
7305                 br_dev = vlan_dev_real_dev(rif->dev);
7306                 if (WARN_ON(!netif_is_bridge_master(br_dev)))
7307                         return ERR_PTR(-EINVAL);
7308         } else {
7309                 err = br_vlan_get_pvid(rif->dev, &vid);
7310                 if (err < 0 || !vid) {
7311                         NL_SET_ERR_MSG_MOD(extack, "Couldn't determine bridge PVID");
7312                         return ERR_PTR(-EINVAL);
7313                 }
7314         }
7315
7316         return mlxsw_sp_bridge_fid_get(rif->mlxsw_sp, br_dev, vid, extack);
7317 }
7318
7319 static void mlxsw_sp_rif_vlan_fdb_del(struct mlxsw_sp_rif *rif, const char *mac)
7320 {
7321         u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid);
7322         struct switchdev_notifier_fdb_info info;
7323         struct net_device *br_dev;
7324         struct net_device *dev;
7325
7326         br_dev = is_vlan_dev(rif->dev) ? vlan_dev_real_dev(rif->dev) : rif->dev;
7327         dev = br_fdb_find_port(br_dev, mac, vid);
7328         if (!dev)
7329                 return;
7330
7331         info.addr = mac;
7332         info.vid = vid;
7333         call_switchdev_notifiers(SWITCHDEV_FDB_DEL_TO_BRIDGE, dev, &info.info,
7334                                  NULL);
7335 }
7336
7337 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_vlan_ops = {
7338         .type                   = MLXSW_SP_RIF_TYPE_VLAN,
7339         .rif_size               = sizeof(struct mlxsw_sp_rif),
7340         .configure              = mlxsw_sp_rif_vlan_configure,
7341         .deconfigure            = mlxsw_sp_rif_vlan_deconfigure,
7342         .fid_get                = mlxsw_sp_rif_vlan_fid_get,
7343         .fdb_del                = mlxsw_sp_rif_vlan_fdb_del,
7344 };
7345
7346 static int mlxsw_sp_rif_fid_configure(struct mlxsw_sp_rif *rif)
7347 {
7348         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
7349         u16 fid_index = mlxsw_sp_fid_index(rif->fid);
7350         int err;
7351
7352         err = mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index,
7353                                        true);
7354         if (err)
7355                 return err;
7356
7357         err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
7358                                      mlxsw_sp_router_port(mlxsw_sp), true);
7359         if (err)
7360                 goto err_fid_mc_flood_set;
7361
7362         err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
7363                                      mlxsw_sp_router_port(mlxsw_sp), true);
7364         if (err)
7365                 goto err_fid_bc_flood_set;
7366
7367         err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
7368                                   mlxsw_sp_fid_index(rif->fid), true);
7369         if (err)
7370                 goto err_rif_fdb_op;
7371
7372         mlxsw_sp_fid_rif_set(rif->fid, rif);
7373         return 0;
7374
7375 err_rif_fdb_op:
7376         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
7377                                mlxsw_sp_router_port(mlxsw_sp), false);
7378 err_fid_bc_flood_set:
7379         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
7380                                mlxsw_sp_router_port(mlxsw_sp), false);
7381 err_fid_mc_flood_set:
7382         mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false);
7383         return err;
7384 }
7385
7386 static void mlxsw_sp_rif_fid_deconfigure(struct mlxsw_sp_rif *rif)
7387 {
7388         u16 fid_index = mlxsw_sp_fid_index(rif->fid);
7389         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
7390         struct mlxsw_sp_fid *fid = rif->fid;
7391
7392         mlxsw_sp_fid_rif_set(fid, NULL);
7393         mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
7394                             mlxsw_sp_fid_index(fid), false);
7395         mlxsw_sp_rif_macvlan_flush(rif);
7396         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
7397                                mlxsw_sp_router_port(mlxsw_sp), false);
7398         mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
7399                                mlxsw_sp_router_port(mlxsw_sp), false);
7400         mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false);
7401 }
7402
7403 static struct mlxsw_sp_fid *
7404 mlxsw_sp_rif_fid_fid_get(struct mlxsw_sp_rif *rif,
7405                          struct netlink_ext_ack *extack)
7406 {
7407         return mlxsw_sp_bridge_fid_get(rif->mlxsw_sp, rif->dev, 0, extack);
7408 }
7409
7410 static void mlxsw_sp_rif_fid_fdb_del(struct mlxsw_sp_rif *rif, const char *mac)
7411 {
7412         struct switchdev_notifier_fdb_info info;
7413         struct net_device *dev;
7414
7415         dev = br_fdb_find_port(rif->dev, mac, 0);
7416         if (!dev)
7417                 return;
7418
7419         info.addr = mac;
7420         info.vid = 0;
7421         call_switchdev_notifiers(SWITCHDEV_FDB_DEL_TO_BRIDGE, dev, &info.info,
7422                                  NULL);
7423 }
7424
7425 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_fid_ops = {
7426         .type                   = MLXSW_SP_RIF_TYPE_FID,
7427         .rif_size               = sizeof(struct mlxsw_sp_rif),
7428         .configure              = mlxsw_sp_rif_fid_configure,
7429         .deconfigure            = mlxsw_sp_rif_fid_deconfigure,
7430         .fid_get                = mlxsw_sp_rif_fid_fid_get,
7431         .fdb_del                = mlxsw_sp_rif_fid_fdb_del,
7432 };
7433
7434 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_vlan_emu_ops = {
7435         .type                   = MLXSW_SP_RIF_TYPE_VLAN,
7436         .rif_size               = sizeof(struct mlxsw_sp_rif),
7437         .configure              = mlxsw_sp_rif_fid_configure,
7438         .deconfigure            = mlxsw_sp_rif_fid_deconfigure,
7439         .fid_get                = mlxsw_sp_rif_vlan_fid_get,
7440         .fdb_del                = mlxsw_sp_rif_vlan_fdb_del,
7441 };
7442
7443 static struct mlxsw_sp_rif_ipip_lb *
7444 mlxsw_sp_rif_ipip_lb_rif(struct mlxsw_sp_rif *rif)
7445 {
7446         return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common);
7447 }
7448
7449 static void
7450 mlxsw_sp_rif_ipip_lb_setup(struct mlxsw_sp_rif *rif,
7451                            const struct mlxsw_sp_rif_params *params)
7452 {
7453         struct mlxsw_sp_rif_params_ipip_lb *params_lb;
7454         struct mlxsw_sp_rif_ipip_lb *rif_lb;
7455
7456         params_lb = container_of(params, struct mlxsw_sp_rif_params_ipip_lb,
7457                                  common);
7458         rif_lb = mlxsw_sp_rif_ipip_lb_rif(rif);
7459         rif_lb->lb_config = params_lb->lb_config;
7460 }
7461
7462 static int
7463 mlxsw_sp1_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif)
7464 {
7465         struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
7466         u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(rif->dev);
7467         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
7468         struct mlxsw_sp_vr *ul_vr;
7469         int err;
7470
7471         ul_vr = mlxsw_sp_vr_get(mlxsw_sp, ul_tb_id, NULL);
7472         if (IS_ERR(ul_vr))
7473                 return PTR_ERR(ul_vr);
7474
7475         err = mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr->id, 0, true);
7476         if (err)
7477                 goto err_loopback_op;
7478
7479         lb_rif->ul_vr_id = ul_vr->id;
7480         lb_rif->ul_rif_id = 0;
7481         ++ul_vr->rif_count;
7482         return 0;
7483
7484 err_loopback_op:
7485         mlxsw_sp_vr_put(mlxsw_sp, ul_vr);
7486         return err;
7487 }
7488
7489 static void mlxsw_sp1_rif_ipip_lb_deconfigure(struct mlxsw_sp_rif *rif)
7490 {
7491         struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
7492         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
7493         struct mlxsw_sp_vr *ul_vr;
7494
7495         ul_vr = &mlxsw_sp->router->vrs[lb_rif->ul_vr_id];
7496         mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr->id, 0, false);
7497
7498         --ul_vr->rif_count;
7499         mlxsw_sp_vr_put(mlxsw_sp, ul_vr);
7500 }
7501
7502 static const struct mlxsw_sp_rif_ops mlxsw_sp1_rif_ipip_lb_ops = {
7503         .type                   = MLXSW_SP_RIF_TYPE_IPIP_LB,
7504         .rif_size               = sizeof(struct mlxsw_sp_rif_ipip_lb),
7505         .setup                  = mlxsw_sp_rif_ipip_lb_setup,
7506         .configure              = mlxsw_sp1_rif_ipip_lb_configure,
7507         .deconfigure            = mlxsw_sp1_rif_ipip_lb_deconfigure,
7508 };
7509
7510 const struct mlxsw_sp_rif_ops *mlxsw_sp1_rif_ops_arr[] = {
7511         [MLXSW_SP_RIF_TYPE_SUBPORT]     = &mlxsw_sp_rif_subport_ops,
7512         [MLXSW_SP_RIF_TYPE_VLAN]        = &mlxsw_sp_rif_vlan_emu_ops,
7513         [MLXSW_SP_RIF_TYPE_FID]         = &mlxsw_sp_rif_fid_ops,
7514         [MLXSW_SP_RIF_TYPE_IPIP_LB]     = &mlxsw_sp1_rif_ipip_lb_ops,
7515 };
7516
7517 static int
7518 mlxsw_sp_rif_ipip_lb_ul_rif_op(struct mlxsw_sp_rif *ul_rif, bool enable)
7519 {
7520         struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp;
7521         char ritr_pl[MLXSW_REG_RITR_LEN];
7522
7523         mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF,
7524                             ul_rif->rif_index, ul_rif->vr_id, IP_MAX_MTU);
7525         mlxsw_reg_ritr_loopback_protocol_set(ritr_pl,
7526                                              MLXSW_REG_RITR_LOOPBACK_GENERIC);
7527
7528         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
7529 }
7530
7531 static struct mlxsw_sp_rif *
7532 mlxsw_sp_ul_rif_create(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr,
7533                        struct netlink_ext_ack *extack)
7534 {
7535         struct mlxsw_sp_rif *ul_rif;
7536         u16 rif_index;
7537         int err;
7538
7539         err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index);
7540         if (err) {
7541                 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interfaces");
7542                 return ERR_PTR(err);
7543         }
7544
7545         ul_rif = mlxsw_sp_rif_alloc(sizeof(*ul_rif), rif_index, vr->id, NULL);
7546         if (!ul_rif)
7547                 return ERR_PTR(-ENOMEM);
7548
7549         mlxsw_sp->router->rifs[rif_index] = ul_rif;
7550         ul_rif->mlxsw_sp = mlxsw_sp;
7551         err = mlxsw_sp_rif_ipip_lb_ul_rif_op(ul_rif, true);
7552         if (err)
7553                 goto ul_rif_op_err;
7554
7555         return ul_rif;
7556
7557 ul_rif_op_err:
7558         mlxsw_sp->router->rifs[rif_index] = NULL;
7559         kfree(ul_rif);
7560         return ERR_PTR(err);
7561 }
7562
7563 static void mlxsw_sp_ul_rif_destroy(struct mlxsw_sp_rif *ul_rif)
7564 {
7565         struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp;
7566
7567         mlxsw_sp_rif_ipip_lb_ul_rif_op(ul_rif, false);
7568         mlxsw_sp->router->rifs[ul_rif->rif_index] = NULL;
7569         kfree(ul_rif);
7570 }
7571
7572 static struct mlxsw_sp_rif *
7573 mlxsw_sp_ul_rif_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id,
7574                     struct netlink_ext_ack *extack)
7575 {
7576         struct mlxsw_sp_vr *vr;
7577         int err;
7578
7579         vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id, extack);
7580         if (IS_ERR(vr))
7581                 return ERR_CAST(vr);
7582
7583         if (refcount_inc_not_zero(&vr->ul_rif_refcnt))
7584                 return vr->ul_rif;
7585
7586         vr->ul_rif = mlxsw_sp_ul_rif_create(mlxsw_sp, vr, extack);
7587         if (IS_ERR(vr->ul_rif)) {
7588                 err = PTR_ERR(vr->ul_rif);
7589                 goto err_ul_rif_create;
7590         }
7591
7592         vr->rif_count++;
7593         refcount_set(&vr->ul_rif_refcnt, 1);
7594
7595         return vr->ul_rif;
7596
7597 err_ul_rif_create:
7598         mlxsw_sp_vr_put(mlxsw_sp, vr);
7599         return ERR_PTR(err);
7600 }
7601
7602 static void mlxsw_sp_ul_rif_put(struct mlxsw_sp_rif *ul_rif)
7603 {
7604         struct mlxsw_sp *mlxsw_sp = ul_rif->mlxsw_sp;
7605         struct mlxsw_sp_vr *vr;
7606
7607         vr = &mlxsw_sp->router->vrs[ul_rif->vr_id];
7608
7609         if (!refcount_dec_and_test(&vr->ul_rif_refcnt))
7610                 return;
7611
7612         vr->rif_count--;
7613         mlxsw_sp_ul_rif_destroy(ul_rif);
7614         mlxsw_sp_vr_put(mlxsw_sp, vr);
7615 }
7616
7617 int mlxsw_sp_router_ul_rif_get(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id,
7618                                u16 *ul_rif_index)
7619 {
7620         struct mlxsw_sp_rif *ul_rif;
7621
7622         ASSERT_RTNL();
7623
7624         ul_rif = mlxsw_sp_ul_rif_get(mlxsw_sp, ul_tb_id, NULL);
7625         if (IS_ERR(ul_rif))
7626                 return PTR_ERR(ul_rif);
7627         *ul_rif_index = ul_rif->rif_index;
7628
7629         return 0;
7630 }
7631
7632 void mlxsw_sp_router_ul_rif_put(struct mlxsw_sp *mlxsw_sp, u16 ul_rif_index)
7633 {
7634         struct mlxsw_sp_rif *ul_rif;
7635
7636         ASSERT_RTNL();
7637
7638         ul_rif = mlxsw_sp->router->rifs[ul_rif_index];
7639         if (WARN_ON(!ul_rif))
7640                 return;
7641
7642         mlxsw_sp_ul_rif_put(ul_rif);
7643 }
7644
7645 static int
7646 mlxsw_sp2_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif)
7647 {
7648         struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
7649         u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(rif->dev);
7650         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
7651         struct mlxsw_sp_rif *ul_rif;
7652         int err;
7653
7654         ul_rif = mlxsw_sp_ul_rif_get(mlxsw_sp, ul_tb_id, NULL);
7655         if (IS_ERR(ul_rif))
7656                 return PTR_ERR(ul_rif);
7657
7658         err = mlxsw_sp_rif_ipip_lb_op(lb_rif, 0, ul_rif->rif_index, true);
7659         if (err)
7660                 goto err_loopback_op;
7661
7662         lb_rif->ul_vr_id = 0;
7663         lb_rif->ul_rif_id = ul_rif->rif_index;
7664
7665         return 0;
7666
7667 err_loopback_op:
7668         mlxsw_sp_ul_rif_put(ul_rif);
7669         return err;
7670 }
7671
7672 static void mlxsw_sp2_rif_ipip_lb_deconfigure(struct mlxsw_sp_rif *rif)
7673 {
7674         struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
7675         struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
7676         struct mlxsw_sp_rif *ul_rif;
7677
7678         ul_rif = mlxsw_sp_rif_by_index(mlxsw_sp, lb_rif->ul_rif_id);
7679         mlxsw_sp_rif_ipip_lb_op(lb_rif, 0, lb_rif->ul_rif_id, false);
7680         mlxsw_sp_ul_rif_put(ul_rif);
7681 }
7682
7683 static const struct mlxsw_sp_rif_ops mlxsw_sp2_rif_ipip_lb_ops = {
7684         .type                   = MLXSW_SP_RIF_TYPE_IPIP_LB,
7685         .rif_size               = sizeof(struct mlxsw_sp_rif_ipip_lb),
7686         .setup                  = mlxsw_sp_rif_ipip_lb_setup,
7687         .configure              = mlxsw_sp2_rif_ipip_lb_configure,
7688         .deconfigure            = mlxsw_sp2_rif_ipip_lb_deconfigure,
7689 };
7690
7691 const struct mlxsw_sp_rif_ops *mlxsw_sp2_rif_ops_arr[] = {
7692         [MLXSW_SP_RIF_TYPE_SUBPORT]     = &mlxsw_sp_rif_subport_ops,
7693         [MLXSW_SP_RIF_TYPE_VLAN]        = &mlxsw_sp_rif_vlan_emu_ops,
7694         [MLXSW_SP_RIF_TYPE_FID]         = &mlxsw_sp_rif_fid_ops,
7695         [MLXSW_SP_RIF_TYPE_IPIP_LB]     = &mlxsw_sp2_rif_ipip_lb_ops,
7696 };
7697
7698 static int mlxsw_sp_rifs_init(struct mlxsw_sp *mlxsw_sp)
7699 {
7700         u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
7701
7702         mlxsw_sp->router->rifs = kcalloc(max_rifs,
7703                                          sizeof(struct mlxsw_sp_rif *),
7704                                          GFP_KERNEL);
7705         if (!mlxsw_sp->router->rifs)
7706                 return -ENOMEM;
7707
7708         return 0;
7709 }
7710
7711 static void mlxsw_sp_rifs_fini(struct mlxsw_sp *mlxsw_sp)
7712 {
7713         int i;
7714
7715         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
7716                 WARN_ON_ONCE(mlxsw_sp->router->rifs[i]);
7717
7718         kfree(mlxsw_sp->router->rifs);
7719 }
7720
7721 static int
7722 mlxsw_sp_ipip_config_tigcr(struct mlxsw_sp *mlxsw_sp)
7723 {
7724         char tigcr_pl[MLXSW_REG_TIGCR_LEN];
7725
7726         mlxsw_reg_tigcr_pack(tigcr_pl, true, 0);
7727         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(tigcr), tigcr_pl);
7728 }
7729
7730 static int mlxsw_sp_ipips_init(struct mlxsw_sp *mlxsw_sp)
7731 {
7732         mlxsw_sp->router->ipip_ops_arr = mlxsw_sp_ipip_ops_arr;
7733         INIT_LIST_HEAD(&mlxsw_sp->router->ipip_list);
7734         return mlxsw_sp_ipip_config_tigcr(mlxsw_sp);
7735 }
7736
7737 static void mlxsw_sp_ipips_fini(struct mlxsw_sp *mlxsw_sp)
7738 {
7739         WARN_ON(!list_empty(&mlxsw_sp->router->ipip_list));
7740 }
7741
7742 static void mlxsw_sp_router_fib_dump_flush(struct notifier_block *nb)
7743 {
7744         struct mlxsw_sp_router *router;
7745
7746         /* Flush pending FIB notifications and then flush the device's
7747          * table before requesting another dump. The FIB notification
7748          * block is unregistered, so no need to take RTNL.
7749          */
7750         mlxsw_core_flush_owq();
7751         router = container_of(nb, struct mlxsw_sp_router, fib_nb);
7752         mlxsw_sp_router_fib_flush(router->mlxsw_sp);
7753 }
7754
7755 #ifdef CONFIG_IP_ROUTE_MULTIPATH
7756 static void mlxsw_sp_mp_hash_header_set(char *recr2_pl, int header)
7757 {
7758         mlxsw_reg_recr2_outer_header_enables_set(recr2_pl, header, true);
7759 }
7760
7761 static void mlxsw_sp_mp_hash_field_set(char *recr2_pl, int field)
7762 {
7763         mlxsw_reg_recr2_outer_header_fields_enable_set(recr2_pl, field, true);
7764 }
7765
7766 static void mlxsw_sp_mp4_hash_init(char *recr2_pl)
7767 {
7768         bool only_l3 = !init_net.ipv4.sysctl_fib_multipath_hash_policy;
7769
7770         mlxsw_sp_mp_hash_header_set(recr2_pl,
7771                                     MLXSW_REG_RECR2_IPV4_EN_NOT_TCP_NOT_UDP);
7772         mlxsw_sp_mp_hash_header_set(recr2_pl, MLXSW_REG_RECR2_IPV4_EN_TCP_UDP);
7773         mlxsw_reg_recr2_ipv4_sip_enable(recr2_pl);
7774         mlxsw_reg_recr2_ipv4_dip_enable(recr2_pl);
7775         if (only_l3)
7776                 return;
7777         mlxsw_sp_mp_hash_header_set(recr2_pl, MLXSW_REG_RECR2_TCP_UDP_EN_IPV4);
7778         mlxsw_sp_mp_hash_field_set(recr2_pl, MLXSW_REG_RECR2_IPV4_PROTOCOL);
7779         mlxsw_sp_mp_hash_field_set(recr2_pl, MLXSW_REG_RECR2_TCP_UDP_SPORT);
7780         mlxsw_sp_mp_hash_field_set(recr2_pl, MLXSW_REG_RECR2_TCP_UDP_DPORT);
7781 }
7782
7783 static void mlxsw_sp_mp6_hash_init(char *recr2_pl)
7784 {
7785         bool only_l3 = !ip6_multipath_hash_policy(&init_net);
7786
7787         mlxsw_sp_mp_hash_header_set(recr2_pl,
7788                                     MLXSW_REG_RECR2_IPV6_EN_NOT_TCP_NOT_UDP);
7789         mlxsw_sp_mp_hash_header_set(recr2_pl, MLXSW_REG_RECR2_IPV6_EN_TCP_UDP);
7790         mlxsw_reg_recr2_ipv6_sip_enable(recr2_pl);
7791         mlxsw_reg_recr2_ipv6_dip_enable(recr2_pl);
7792         mlxsw_sp_mp_hash_field_set(recr2_pl, MLXSW_REG_RECR2_IPV6_NEXT_HEADER);
7793         if (only_l3) {
7794                 mlxsw_sp_mp_hash_field_set(recr2_pl,
7795                                            MLXSW_REG_RECR2_IPV6_FLOW_LABEL);
7796         } else {
7797                 mlxsw_sp_mp_hash_header_set(recr2_pl,
7798                                             MLXSW_REG_RECR2_TCP_UDP_EN_IPV6);
7799                 mlxsw_sp_mp_hash_field_set(recr2_pl,
7800                                            MLXSW_REG_RECR2_TCP_UDP_SPORT);
7801                 mlxsw_sp_mp_hash_field_set(recr2_pl,
7802                                            MLXSW_REG_RECR2_TCP_UDP_DPORT);
7803         }
7804 }
7805
7806 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp)
7807 {
7808         char recr2_pl[MLXSW_REG_RECR2_LEN];
7809         u32 seed;
7810
7811         get_random_bytes(&seed, sizeof(seed));
7812         mlxsw_reg_recr2_pack(recr2_pl, seed);
7813         mlxsw_sp_mp4_hash_init(recr2_pl);
7814         mlxsw_sp_mp6_hash_init(recr2_pl);
7815
7816         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(recr2), recr2_pl);
7817 }
7818 #else
7819 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp)
7820 {
7821         return 0;
7822 }
7823 #endif
7824
7825 static int mlxsw_sp_dscp_init(struct mlxsw_sp *mlxsw_sp)
7826 {
7827         char rdpm_pl[MLXSW_REG_RDPM_LEN];
7828         unsigned int i;
7829
7830         MLXSW_REG_ZERO(rdpm, rdpm_pl);
7831
7832         /* HW is determining switch priority based on DSCP-bits, but the
7833          * kernel is still doing that based on the ToS. Since there's a
7834          * mismatch in bits we need to make sure to translate the right
7835          * value ToS would observe, skipping the 2 least-significant ECN bits.
7836          */
7837         for (i = 0; i < MLXSW_REG_RDPM_DSCP_ENTRY_REC_MAX_COUNT; i++)
7838                 mlxsw_reg_rdpm_pack(rdpm_pl, i, rt_tos2priority(i << 2));
7839
7840         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rdpm), rdpm_pl);
7841 }
7842
7843 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
7844 {
7845         bool usp = init_net.ipv4.sysctl_ip_fwd_update_priority;
7846         char rgcr_pl[MLXSW_REG_RGCR_LEN];
7847         u64 max_rifs;
7848         int err;
7849
7850         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_RIFS))
7851                 return -EIO;
7852         max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
7853
7854         mlxsw_reg_rgcr_pack(rgcr_pl, true, true);
7855         mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, max_rifs);
7856         mlxsw_reg_rgcr_usp_set(rgcr_pl, usp);
7857         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
7858         if (err)
7859                 return err;
7860         return 0;
7861 }
7862
7863 static void __mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
7864 {
7865         char rgcr_pl[MLXSW_REG_RGCR_LEN];
7866
7867         mlxsw_reg_rgcr_pack(rgcr_pl, false, false);
7868         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
7869 }
7870
7871 int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
7872 {
7873         struct mlxsw_sp_router *router;
7874         int err;
7875
7876         router = kzalloc(sizeof(*mlxsw_sp->router), GFP_KERNEL);
7877         if (!router)
7878                 return -ENOMEM;
7879         mlxsw_sp->router = router;
7880         router->mlxsw_sp = mlxsw_sp;
7881
7882         router->inetaddr_nb.notifier_call = mlxsw_sp_inetaddr_event;
7883         err = register_inetaddr_notifier(&router->inetaddr_nb);
7884         if (err)
7885                 goto err_register_inetaddr_notifier;
7886
7887         router->inet6addr_nb.notifier_call = mlxsw_sp_inet6addr_event;
7888         err = register_inet6addr_notifier(&router->inet6addr_nb);
7889         if (err)
7890                 goto err_register_inet6addr_notifier;
7891
7892         INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_neighs_list);
7893         err = __mlxsw_sp_router_init(mlxsw_sp);
7894         if (err)
7895                 goto err_router_init;
7896
7897         err = mlxsw_sp_rifs_init(mlxsw_sp);
7898         if (err)
7899                 goto err_rifs_init;
7900
7901         err = mlxsw_sp_ipips_init(mlxsw_sp);
7902         if (err)
7903                 goto err_ipips_init;
7904
7905         err = rhashtable_init(&mlxsw_sp->router->nexthop_ht,
7906                               &mlxsw_sp_nexthop_ht_params);
7907         if (err)
7908                 goto err_nexthop_ht_init;
7909
7910         err = rhashtable_init(&mlxsw_sp->router->nexthop_group_ht,
7911                               &mlxsw_sp_nexthop_group_ht_params);
7912         if (err)
7913                 goto err_nexthop_group_ht_init;
7914
7915         INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_list);
7916         err = mlxsw_sp_lpm_init(mlxsw_sp);
7917         if (err)
7918                 goto err_lpm_init;
7919
7920         err = mlxsw_sp_mr_init(mlxsw_sp, &mlxsw_sp_mr_tcam_ops);
7921         if (err)
7922                 goto err_mr_init;
7923
7924         err = mlxsw_sp_vrs_init(mlxsw_sp);
7925         if (err)
7926                 goto err_vrs_init;
7927
7928         err = mlxsw_sp_neigh_init(mlxsw_sp);
7929         if (err)
7930                 goto err_neigh_init;
7931
7932         mlxsw_sp->router->netevent_nb.notifier_call =
7933                 mlxsw_sp_router_netevent_event;
7934         err = register_netevent_notifier(&mlxsw_sp->router->netevent_nb);
7935         if (err)
7936                 goto err_register_netevent_notifier;
7937
7938         err = mlxsw_sp_mp_hash_init(mlxsw_sp);
7939         if (err)
7940                 goto err_mp_hash_init;
7941
7942         err = mlxsw_sp_dscp_init(mlxsw_sp);
7943         if (err)
7944                 goto err_dscp_init;
7945
7946         mlxsw_sp->router->fib_nb.notifier_call = mlxsw_sp_router_fib_event;
7947         err = register_fib_notifier(&mlxsw_sp->router->fib_nb,
7948                                     mlxsw_sp_router_fib_dump_flush);
7949         if (err)
7950                 goto err_register_fib_notifier;
7951
7952         return 0;
7953
7954 err_register_fib_notifier:
7955 err_dscp_init:
7956 err_mp_hash_init:
7957         unregister_netevent_notifier(&mlxsw_sp->router->netevent_nb);
7958 err_register_netevent_notifier:
7959         mlxsw_sp_neigh_fini(mlxsw_sp);
7960 err_neigh_init:
7961         mlxsw_sp_vrs_fini(mlxsw_sp);
7962 err_vrs_init:
7963         mlxsw_sp_mr_fini(mlxsw_sp);
7964 err_mr_init:
7965         mlxsw_sp_lpm_fini(mlxsw_sp);
7966 err_lpm_init:
7967         rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht);
7968 err_nexthop_group_ht_init:
7969         rhashtable_destroy(&mlxsw_sp->router->nexthop_ht);
7970 err_nexthop_ht_init:
7971         mlxsw_sp_ipips_fini(mlxsw_sp);
7972 err_ipips_init:
7973         mlxsw_sp_rifs_fini(mlxsw_sp);
7974 err_rifs_init:
7975         __mlxsw_sp_router_fini(mlxsw_sp);
7976 err_router_init:
7977         unregister_inet6addr_notifier(&router->inet6addr_nb);
7978 err_register_inet6addr_notifier:
7979         unregister_inetaddr_notifier(&router->inetaddr_nb);
7980 err_register_inetaddr_notifier:
7981         kfree(mlxsw_sp->router);
7982         return err;
7983 }
7984
7985 void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
7986 {
7987         unregister_fib_notifier(&mlxsw_sp->router->fib_nb);
7988         unregister_netevent_notifier(&mlxsw_sp->router->netevent_nb);
7989         mlxsw_sp_neigh_fini(mlxsw_sp);
7990         mlxsw_sp_vrs_fini(mlxsw_sp);
7991         mlxsw_sp_mr_fini(mlxsw_sp);
7992         mlxsw_sp_lpm_fini(mlxsw_sp);
7993         rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht);
7994         rhashtable_destroy(&mlxsw_sp->router->nexthop_ht);
7995         mlxsw_sp_ipips_fini(mlxsw_sp);
7996         mlxsw_sp_rifs_fini(mlxsw_sp);
7997         __mlxsw_sp_router_fini(mlxsw_sp);
7998         unregister_inet6addr_notifier(&mlxsw_sp->router->inet6addr_nb);
7999         unregister_inetaddr_notifier(&mlxsw_sp->router->inetaddr_nb);
8000         kfree(mlxsw_sp->router);
8001 }