2 * drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
3 * Copyright (c) 2016 Mellanox Technologies. All rights reserved.
4 * Copyright (c) 2016 Jiri Pirko <jiri@mellanox.com>
5 * Copyright (c) 2016 Ido Schimmel <idosch@mellanox.com>
6 * Copyright (c) 2016 Yotam Gigi <yotamg@mellanox.com>
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 * 3. Neither the names of the copyright holders nor the names of its
17 * contributors may be used to endorse or promote products derived from
18 * this software without specific prior written permission.
20 * Alternatively, this software may be distributed under the terms of the
21 * GNU General Public License ("GPL") version 2 as published by the Free
22 * Software Foundation.
24 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
25 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
27 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
28 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
31 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
32 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
33 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
34 * POSSIBILITY OF SUCH DAMAGE.
37 #include <linux/kernel.h>
38 #include <linux/types.h>
39 #include <linux/rhashtable.h>
40 #include <linux/bitops.h>
41 #include <linux/in6.h>
42 #include <linux/notifier.h>
43 #include <linux/inetdevice.h>
44 #include <linux/netdevice.h>
45 #include <linux/if_bridge.h>
46 #include <net/netevent.h>
47 #include <net/neighbour.h>
49 #include <net/ip_fib.h>
50 #include <net/fib_rules.h>
51 #include <net/l3mdev.h>
56 #include "spectrum_cnt.h"
57 #include "spectrum_dpipe.h"
58 #include "spectrum_router.h"
61 struct mlxsw_sp_lpm_tree;
62 struct mlxsw_sp_rif_ops;
64 struct mlxsw_sp_router {
65 struct mlxsw_sp *mlxsw_sp;
66 struct mlxsw_sp_rif **rifs;
67 struct mlxsw_sp_vr *vrs;
68 struct rhashtable neigh_ht;
69 struct rhashtable nexthop_group_ht;
70 struct rhashtable nexthop_ht;
72 struct mlxsw_sp_lpm_tree *trees;
73 unsigned int tree_count;
76 struct delayed_work dw;
77 unsigned long interval; /* ms */
79 struct delayed_work nexthop_probe_dw;
80 #define MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL 5000 /* ms */
81 struct list_head nexthop_neighs_list;
83 struct notifier_block fib_nb;
84 const struct mlxsw_sp_rif_ops **rif_ops_arr;
88 struct list_head nexthop_list;
89 struct list_head neigh_list;
90 struct net_device *dev;
91 struct mlxsw_sp_fid *fid;
92 unsigned char addr[ETH_ALEN];
96 const struct mlxsw_sp_rif_ops *ops;
97 struct mlxsw_sp *mlxsw_sp;
99 unsigned int counter_ingress;
100 bool counter_ingress_valid;
101 unsigned int counter_egress;
102 bool counter_egress_valid;
105 struct mlxsw_sp_rif_params {
106 struct net_device *dev;
115 struct mlxsw_sp_rif_subport {
116 struct mlxsw_sp_rif common;
125 struct mlxsw_sp_rif_ops {
126 enum mlxsw_sp_rif_type type;
129 void (*setup)(struct mlxsw_sp_rif *rif,
130 const struct mlxsw_sp_rif_params *params);
131 int (*configure)(struct mlxsw_sp_rif *rif);
132 void (*deconfigure)(struct mlxsw_sp_rif *rif);
133 struct mlxsw_sp_fid * (*fid_get)(struct mlxsw_sp_rif *rif);
136 static unsigned int *
137 mlxsw_sp_rif_p_counter_get(struct mlxsw_sp_rif *rif,
138 enum mlxsw_sp_rif_counter_dir dir)
141 case MLXSW_SP_RIF_COUNTER_EGRESS:
142 return &rif->counter_egress;
143 case MLXSW_SP_RIF_COUNTER_INGRESS:
144 return &rif->counter_ingress;
150 mlxsw_sp_rif_counter_valid_get(struct mlxsw_sp_rif *rif,
151 enum mlxsw_sp_rif_counter_dir dir)
154 case MLXSW_SP_RIF_COUNTER_EGRESS:
155 return rif->counter_egress_valid;
156 case MLXSW_SP_RIF_COUNTER_INGRESS:
157 return rif->counter_ingress_valid;
163 mlxsw_sp_rif_counter_valid_set(struct mlxsw_sp_rif *rif,
164 enum mlxsw_sp_rif_counter_dir dir,
168 case MLXSW_SP_RIF_COUNTER_EGRESS:
169 rif->counter_egress_valid = valid;
171 case MLXSW_SP_RIF_COUNTER_INGRESS:
172 rif->counter_ingress_valid = valid;
177 static int mlxsw_sp_rif_counter_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
178 unsigned int counter_index, bool enable,
179 enum mlxsw_sp_rif_counter_dir dir)
181 char ritr_pl[MLXSW_REG_RITR_LEN];
182 bool is_egress = false;
185 if (dir == MLXSW_SP_RIF_COUNTER_EGRESS)
187 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
188 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
192 mlxsw_reg_ritr_counter_pack(ritr_pl, counter_index, enable,
194 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
197 int mlxsw_sp_rif_counter_value_get(struct mlxsw_sp *mlxsw_sp,
198 struct mlxsw_sp_rif *rif,
199 enum mlxsw_sp_rif_counter_dir dir, u64 *cnt)
201 char ricnt_pl[MLXSW_REG_RICNT_LEN];
202 unsigned int *p_counter_index;
206 valid = mlxsw_sp_rif_counter_valid_get(rif, dir);
210 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
211 if (!p_counter_index)
213 mlxsw_reg_ricnt_pack(ricnt_pl, *p_counter_index,
214 MLXSW_REG_RICNT_OPCODE_NOP);
215 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl);
218 *cnt = mlxsw_reg_ricnt_good_unicast_packets_get(ricnt_pl);
222 static int mlxsw_sp_rif_counter_clear(struct mlxsw_sp *mlxsw_sp,
223 unsigned int counter_index)
225 char ricnt_pl[MLXSW_REG_RICNT_LEN];
227 mlxsw_reg_ricnt_pack(ricnt_pl, counter_index,
228 MLXSW_REG_RICNT_OPCODE_CLEAR);
229 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl);
232 int mlxsw_sp_rif_counter_alloc(struct mlxsw_sp *mlxsw_sp,
233 struct mlxsw_sp_rif *rif,
234 enum mlxsw_sp_rif_counter_dir dir)
236 unsigned int *p_counter_index;
239 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
240 if (!p_counter_index)
242 err = mlxsw_sp_counter_alloc(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
247 err = mlxsw_sp_rif_counter_clear(mlxsw_sp, *p_counter_index);
249 goto err_counter_clear;
251 err = mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index,
252 *p_counter_index, true, dir);
254 goto err_counter_edit;
255 mlxsw_sp_rif_counter_valid_set(rif, dir, true);
260 mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
265 void mlxsw_sp_rif_counter_free(struct mlxsw_sp *mlxsw_sp,
266 struct mlxsw_sp_rif *rif,
267 enum mlxsw_sp_rif_counter_dir dir)
269 unsigned int *p_counter_index;
271 if (!mlxsw_sp_rif_counter_valid_get(rif, dir))
274 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
275 if (WARN_ON(!p_counter_index))
277 mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index,
278 *p_counter_index, false, dir);
279 mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
281 mlxsw_sp_rif_counter_valid_set(rif, dir, false);
284 static void mlxsw_sp_rif_counters_alloc(struct mlxsw_sp_rif *rif)
286 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
287 struct devlink *devlink;
289 devlink = priv_to_devlink(mlxsw_sp->core);
290 if (!devlink_dpipe_table_counter_enabled(devlink,
291 MLXSW_SP_DPIPE_TABLE_NAME_ERIF))
293 mlxsw_sp_rif_counter_alloc(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS);
296 static void mlxsw_sp_rif_counters_free(struct mlxsw_sp_rif *rif)
298 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
300 mlxsw_sp_rif_counter_free(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS);
303 static struct mlxsw_sp_rif *
304 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
305 const struct net_device *dev);
307 #define MLXSW_SP_PREFIX_COUNT (sizeof(struct in6_addr) * BITS_PER_BYTE)
309 struct mlxsw_sp_prefix_usage {
310 DECLARE_BITMAP(b, MLXSW_SP_PREFIX_COUNT);
313 #define mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) \
314 for_each_set_bit(prefix, (prefix_usage)->b, MLXSW_SP_PREFIX_COUNT)
317 mlxsw_sp_prefix_usage_subset(struct mlxsw_sp_prefix_usage *prefix_usage1,
318 struct mlxsw_sp_prefix_usage *prefix_usage2)
320 unsigned char prefix;
322 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage1) {
323 if (!test_bit(prefix, prefix_usage2->b))
330 mlxsw_sp_prefix_usage_eq(struct mlxsw_sp_prefix_usage *prefix_usage1,
331 struct mlxsw_sp_prefix_usage *prefix_usage2)
333 return !memcmp(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
337 mlxsw_sp_prefix_usage_none(struct mlxsw_sp_prefix_usage *prefix_usage)
339 struct mlxsw_sp_prefix_usage prefix_usage_none = {{ 0 } };
341 return mlxsw_sp_prefix_usage_eq(prefix_usage, &prefix_usage_none);
345 mlxsw_sp_prefix_usage_cpy(struct mlxsw_sp_prefix_usage *prefix_usage1,
346 struct mlxsw_sp_prefix_usage *prefix_usage2)
348 memcpy(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
352 mlxsw_sp_prefix_usage_set(struct mlxsw_sp_prefix_usage *prefix_usage,
353 unsigned char prefix_len)
355 set_bit(prefix_len, prefix_usage->b);
359 mlxsw_sp_prefix_usage_clear(struct mlxsw_sp_prefix_usage *prefix_usage,
360 unsigned char prefix_len)
362 clear_bit(prefix_len, prefix_usage->b);
365 struct mlxsw_sp_fib_key {
366 unsigned char addr[sizeof(struct in6_addr)];
367 unsigned char prefix_len;
370 enum mlxsw_sp_fib_entry_type {
371 MLXSW_SP_FIB_ENTRY_TYPE_REMOTE,
372 MLXSW_SP_FIB_ENTRY_TYPE_LOCAL,
373 MLXSW_SP_FIB_ENTRY_TYPE_TRAP,
376 struct mlxsw_sp_nexthop_group;
379 struct mlxsw_sp_fib_node {
380 struct list_head entry_list;
381 struct list_head list;
382 struct rhash_head ht_node;
383 struct mlxsw_sp_fib *fib;
384 struct mlxsw_sp_fib_key key;
387 struct mlxsw_sp_fib_entry_params {
394 struct mlxsw_sp_fib_entry {
395 struct list_head list;
396 struct mlxsw_sp_fib_node *fib_node;
397 enum mlxsw_sp_fib_entry_type type;
398 struct list_head nexthop_group_node;
399 struct mlxsw_sp_nexthop_group *nh_group;
400 struct mlxsw_sp_fib_entry_params params;
404 enum mlxsw_sp_l3proto {
405 MLXSW_SP_L3_PROTO_IPV4,
406 MLXSW_SP_L3_PROTO_IPV6,
409 struct mlxsw_sp_lpm_tree {
411 unsigned int ref_count;
412 enum mlxsw_sp_l3proto proto;
413 struct mlxsw_sp_prefix_usage prefix_usage;
416 struct mlxsw_sp_fib {
417 struct rhashtable ht;
418 struct list_head node_list;
419 struct mlxsw_sp_vr *vr;
420 struct mlxsw_sp_lpm_tree *lpm_tree;
421 unsigned long prefix_ref_count[MLXSW_SP_PREFIX_COUNT];
422 struct mlxsw_sp_prefix_usage prefix_usage;
423 enum mlxsw_sp_l3proto proto;
427 u16 id; /* virtual router ID */
428 u32 tb_id; /* kernel fib table id */
429 unsigned int rif_count;
430 struct mlxsw_sp_fib *fib4;
433 static const struct rhashtable_params mlxsw_sp_fib_ht_params;
435 static struct mlxsw_sp_fib *mlxsw_sp_fib_create(struct mlxsw_sp_vr *vr,
436 enum mlxsw_sp_l3proto proto)
438 struct mlxsw_sp_fib *fib;
441 fib = kzalloc(sizeof(*fib), GFP_KERNEL);
443 return ERR_PTR(-ENOMEM);
444 err = rhashtable_init(&fib->ht, &mlxsw_sp_fib_ht_params);
446 goto err_rhashtable_init;
447 INIT_LIST_HEAD(&fib->node_list);
457 static void mlxsw_sp_fib_destroy(struct mlxsw_sp_fib *fib)
459 WARN_ON(!list_empty(&fib->node_list));
460 WARN_ON(fib->lpm_tree);
461 rhashtable_destroy(&fib->ht);
465 static struct mlxsw_sp_lpm_tree *
466 mlxsw_sp_lpm_tree_find_unused(struct mlxsw_sp *mlxsw_sp)
468 static struct mlxsw_sp_lpm_tree *lpm_tree;
471 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
472 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
473 if (lpm_tree->ref_count == 0)
479 static int mlxsw_sp_lpm_tree_alloc(struct mlxsw_sp *mlxsw_sp,
480 struct mlxsw_sp_lpm_tree *lpm_tree)
482 char ralta_pl[MLXSW_REG_RALTA_LEN];
484 mlxsw_reg_ralta_pack(ralta_pl, true,
485 (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
487 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
490 static int mlxsw_sp_lpm_tree_free(struct mlxsw_sp *mlxsw_sp,
491 struct mlxsw_sp_lpm_tree *lpm_tree)
493 char ralta_pl[MLXSW_REG_RALTA_LEN];
495 mlxsw_reg_ralta_pack(ralta_pl, false,
496 (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
498 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
502 mlxsw_sp_lpm_tree_left_struct_set(struct mlxsw_sp *mlxsw_sp,
503 struct mlxsw_sp_prefix_usage *prefix_usage,
504 struct mlxsw_sp_lpm_tree *lpm_tree)
506 char ralst_pl[MLXSW_REG_RALST_LEN];
509 u8 last_prefix = MLXSW_REG_RALST_BIN_NO_CHILD;
511 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage)
514 mlxsw_reg_ralst_pack(ralst_pl, root_bin, lpm_tree->id);
515 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) {
518 mlxsw_reg_ralst_bin_pack(ralst_pl, prefix, last_prefix,
519 MLXSW_REG_RALST_BIN_NO_CHILD);
520 last_prefix = prefix;
522 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
525 static struct mlxsw_sp_lpm_tree *
526 mlxsw_sp_lpm_tree_create(struct mlxsw_sp *mlxsw_sp,
527 struct mlxsw_sp_prefix_usage *prefix_usage,
528 enum mlxsw_sp_l3proto proto)
530 struct mlxsw_sp_lpm_tree *lpm_tree;
533 lpm_tree = mlxsw_sp_lpm_tree_find_unused(mlxsw_sp);
535 return ERR_PTR(-EBUSY);
536 lpm_tree->proto = proto;
537 err = mlxsw_sp_lpm_tree_alloc(mlxsw_sp, lpm_tree);
541 err = mlxsw_sp_lpm_tree_left_struct_set(mlxsw_sp, prefix_usage,
544 goto err_left_struct_set;
545 memcpy(&lpm_tree->prefix_usage, prefix_usage,
546 sizeof(lpm_tree->prefix_usage));
550 mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
554 static int mlxsw_sp_lpm_tree_destroy(struct mlxsw_sp *mlxsw_sp,
555 struct mlxsw_sp_lpm_tree *lpm_tree)
557 return mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
560 static struct mlxsw_sp_lpm_tree *
561 mlxsw_sp_lpm_tree_get(struct mlxsw_sp *mlxsw_sp,
562 struct mlxsw_sp_prefix_usage *prefix_usage,
563 enum mlxsw_sp_l3proto proto)
565 struct mlxsw_sp_lpm_tree *lpm_tree;
568 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
569 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
570 if (lpm_tree->ref_count != 0 &&
571 lpm_tree->proto == proto &&
572 mlxsw_sp_prefix_usage_eq(&lpm_tree->prefix_usage,
576 lpm_tree = mlxsw_sp_lpm_tree_create(mlxsw_sp, prefix_usage,
578 if (IS_ERR(lpm_tree))
582 lpm_tree->ref_count++;
586 static int mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp,
587 struct mlxsw_sp_lpm_tree *lpm_tree)
589 if (--lpm_tree->ref_count == 0)
590 return mlxsw_sp_lpm_tree_destroy(mlxsw_sp, lpm_tree);
594 #define MLXSW_SP_LPM_TREE_MIN 1 /* tree 0 is reserved */
596 static int mlxsw_sp_lpm_init(struct mlxsw_sp *mlxsw_sp)
598 struct mlxsw_sp_lpm_tree *lpm_tree;
602 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_LPM_TREES))
605 max_trees = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_LPM_TREES);
606 mlxsw_sp->router->lpm.tree_count = max_trees - MLXSW_SP_LPM_TREE_MIN;
607 mlxsw_sp->router->lpm.trees = kcalloc(mlxsw_sp->router->lpm.tree_count,
608 sizeof(struct mlxsw_sp_lpm_tree),
610 if (!mlxsw_sp->router->lpm.trees)
613 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
614 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
615 lpm_tree->id = i + MLXSW_SP_LPM_TREE_MIN;
621 static void mlxsw_sp_lpm_fini(struct mlxsw_sp *mlxsw_sp)
623 kfree(mlxsw_sp->router->lpm.trees);
626 static bool mlxsw_sp_vr_is_used(const struct mlxsw_sp_vr *vr)
631 static struct mlxsw_sp_vr *mlxsw_sp_vr_find_unused(struct mlxsw_sp *mlxsw_sp)
633 struct mlxsw_sp_vr *vr;
636 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
637 vr = &mlxsw_sp->router->vrs[i];
638 if (!mlxsw_sp_vr_is_used(vr))
644 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp,
645 const struct mlxsw_sp_fib *fib)
647 char raltb_pl[MLXSW_REG_RALTB_LEN];
649 mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id,
650 (enum mlxsw_reg_ralxx_protocol) fib->proto,
652 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
655 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp,
656 const struct mlxsw_sp_fib *fib)
658 char raltb_pl[MLXSW_REG_RALTB_LEN];
660 /* Bind to tree 0 which is default */
661 mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id,
662 (enum mlxsw_reg_ralxx_protocol) fib->proto, 0);
663 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
666 static u32 mlxsw_sp_fix_tb_id(u32 tb_id)
668 /* For our purpose, squash main and local table into one */
669 if (tb_id == RT_TABLE_LOCAL)
670 tb_id = RT_TABLE_MAIN;
674 static struct mlxsw_sp_vr *mlxsw_sp_vr_find(struct mlxsw_sp *mlxsw_sp,
677 struct mlxsw_sp_vr *vr;
680 tb_id = mlxsw_sp_fix_tb_id(tb_id);
682 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
683 vr = &mlxsw_sp->router->vrs[i];
684 if (mlxsw_sp_vr_is_used(vr) && vr->tb_id == tb_id)
690 static struct mlxsw_sp_fib *mlxsw_sp_vr_fib(const struct mlxsw_sp_vr *vr,
691 enum mlxsw_sp_l3proto proto)
694 case MLXSW_SP_L3_PROTO_IPV4:
696 case MLXSW_SP_L3_PROTO_IPV6:
702 static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp,
705 struct mlxsw_sp_vr *vr;
707 vr = mlxsw_sp_vr_find_unused(mlxsw_sp);
709 return ERR_PTR(-EBUSY);
710 vr->fib4 = mlxsw_sp_fib_create(vr, MLXSW_SP_L3_PROTO_IPV4);
711 if (IS_ERR(vr->fib4))
712 return ERR_CAST(vr->fib4);
717 static void mlxsw_sp_vr_destroy(struct mlxsw_sp_vr *vr)
719 mlxsw_sp_fib_destroy(vr->fib4);
724 mlxsw_sp_vr_lpm_tree_check(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_fib *fib,
725 struct mlxsw_sp_prefix_usage *req_prefix_usage)
727 struct mlxsw_sp_lpm_tree *lpm_tree = fib->lpm_tree;
728 struct mlxsw_sp_lpm_tree *new_tree;
731 if (mlxsw_sp_prefix_usage_eq(req_prefix_usage, &lpm_tree->prefix_usage))
734 new_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, req_prefix_usage,
736 if (IS_ERR(new_tree)) {
737 /* We failed to get a tree according to the required
738 * prefix usage. However, the current tree might be still good
739 * for us if our requirement is subset of the prefixes used
742 if (mlxsw_sp_prefix_usage_subset(req_prefix_usage,
743 &lpm_tree->prefix_usage))
745 return PTR_ERR(new_tree);
748 /* Prevent packet loss by overwriting existing binding */
749 fib->lpm_tree = new_tree;
750 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib);
753 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
758 fib->lpm_tree = lpm_tree;
759 mlxsw_sp_lpm_tree_put(mlxsw_sp, new_tree);
763 static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id)
765 struct mlxsw_sp_vr *vr;
767 tb_id = mlxsw_sp_fix_tb_id(tb_id);
768 vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id);
770 vr = mlxsw_sp_vr_create(mlxsw_sp, tb_id);
774 static void mlxsw_sp_vr_put(struct mlxsw_sp_vr *vr)
776 if (!vr->rif_count && list_empty(&vr->fib4->node_list))
777 mlxsw_sp_vr_destroy(vr);
780 static int mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp)
782 struct mlxsw_sp_vr *vr;
786 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_VRS))
789 max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS);
790 mlxsw_sp->router->vrs = kcalloc(max_vrs, sizeof(struct mlxsw_sp_vr),
792 if (!mlxsw_sp->router->vrs)
795 for (i = 0; i < max_vrs; i++) {
796 vr = &mlxsw_sp->router->vrs[i];
803 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp);
805 static void mlxsw_sp_vrs_fini(struct mlxsw_sp *mlxsw_sp)
807 /* At this stage we're guaranteed not to have new incoming
808 * FIB notifications and the work queue is free from FIBs
809 * sitting on top of mlxsw netdevs. However, we can still
810 * have other FIBs queued. Flush the queue before flushing
811 * the device's tables. No need for locks, as we're the only
814 mlxsw_core_flush_owq();
815 mlxsw_sp_router_fib_flush(mlxsw_sp);
816 kfree(mlxsw_sp->router->vrs);
819 struct mlxsw_sp_neigh_key {
823 struct mlxsw_sp_neigh_entry {
824 struct list_head rif_list_node;
825 struct rhash_head ht_node;
826 struct mlxsw_sp_neigh_key key;
829 unsigned char ha[ETH_ALEN];
830 struct list_head nexthop_list; /* list of nexthops using
833 struct list_head nexthop_neighs_list_node;
836 static const struct rhashtable_params mlxsw_sp_neigh_ht_params = {
837 .key_offset = offsetof(struct mlxsw_sp_neigh_entry, key),
838 .head_offset = offsetof(struct mlxsw_sp_neigh_entry, ht_node),
839 .key_len = sizeof(struct mlxsw_sp_neigh_key),
842 static struct mlxsw_sp_neigh_entry *
843 mlxsw_sp_neigh_entry_alloc(struct mlxsw_sp *mlxsw_sp, struct neighbour *n,
846 struct mlxsw_sp_neigh_entry *neigh_entry;
848 neigh_entry = kzalloc(sizeof(*neigh_entry), GFP_KERNEL);
852 neigh_entry->key.n = n;
853 neigh_entry->rif = rif;
854 INIT_LIST_HEAD(&neigh_entry->nexthop_list);
859 static void mlxsw_sp_neigh_entry_free(struct mlxsw_sp_neigh_entry *neigh_entry)
865 mlxsw_sp_neigh_entry_insert(struct mlxsw_sp *mlxsw_sp,
866 struct mlxsw_sp_neigh_entry *neigh_entry)
868 return rhashtable_insert_fast(&mlxsw_sp->router->neigh_ht,
869 &neigh_entry->ht_node,
870 mlxsw_sp_neigh_ht_params);
874 mlxsw_sp_neigh_entry_remove(struct mlxsw_sp *mlxsw_sp,
875 struct mlxsw_sp_neigh_entry *neigh_entry)
877 rhashtable_remove_fast(&mlxsw_sp->router->neigh_ht,
878 &neigh_entry->ht_node,
879 mlxsw_sp_neigh_ht_params);
882 static struct mlxsw_sp_neigh_entry *
883 mlxsw_sp_neigh_entry_create(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
885 struct mlxsw_sp_neigh_entry *neigh_entry;
886 struct mlxsw_sp_rif *rif;
889 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, n->dev);
891 return ERR_PTR(-EINVAL);
893 neigh_entry = mlxsw_sp_neigh_entry_alloc(mlxsw_sp, n, rif->rif_index);
895 return ERR_PTR(-ENOMEM);
897 err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
899 goto err_neigh_entry_insert;
901 list_add(&neigh_entry->rif_list_node, &rif->neigh_list);
905 err_neigh_entry_insert:
906 mlxsw_sp_neigh_entry_free(neigh_entry);
911 mlxsw_sp_neigh_entry_destroy(struct mlxsw_sp *mlxsw_sp,
912 struct mlxsw_sp_neigh_entry *neigh_entry)
914 list_del(&neigh_entry->rif_list_node);
915 mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry);
916 mlxsw_sp_neigh_entry_free(neigh_entry);
919 static struct mlxsw_sp_neigh_entry *
920 mlxsw_sp_neigh_entry_lookup(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
922 struct mlxsw_sp_neigh_key key;
925 return rhashtable_lookup_fast(&mlxsw_sp->router->neigh_ht,
926 &key, mlxsw_sp_neigh_ht_params);
930 mlxsw_sp_router_neighs_update_interval_init(struct mlxsw_sp *mlxsw_sp)
932 unsigned long interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME);
934 mlxsw_sp->router->neighs_update.interval = jiffies_to_msecs(interval);
937 static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp,
941 struct net_device *dev;
947 mlxsw_reg_rauhtd_ent_ipv4_unpack(rauhtd_pl, ent_index, &rif, &dip);
949 if (!mlxsw_sp->router->rifs[rif]) {
950 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
955 dev = mlxsw_sp->router->rifs[rif]->dev;
956 n = neigh_lookup(&arp_tbl, &dipn, dev);
958 netdev_err(dev, "Failed to find matching neighbour for IP=%pI4h\n",
963 netdev_dbg(dev, "Updating neighbour with IP=%pI4h\n", &dip);
964 neigh_event_send(n, NULL);
968 static void mlxsw_sp_router_neigh_rec_ipv4_process(struct mlxsw_sp *mlxsw_sp,
975 num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
977 /* Hardware starts counting at 0, so add 1. */
980 /* Each record consists of several neighbour entries. */
981 for (i = 0; i < num_entries; i++) {
984 ent_index = rec_index * MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC + i;
985 mlxsw_sp_router_neigh_ent_ipv4_process(mlxsw_sp, rauhtd_pl,
991 static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp,
992 char *rauhtd_pl, int rec_index)
994 switch (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, rec_index)) {
995 case MLXSW_REG_RAUHTD_TYPE_IPV4:
996 mlxsw_sp_router_neigh_rec_ipv4_process(mlxsw_sp, rauhtd_pl,
999 case MLXSW_REG_RAUHTD_TYPE_IPV6:
1005 static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl)
1007 u8 num_rec, last_rec_index, num_entries;
1009 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
1010 last_rec_index = num_rec - 1;
1012 if (num_rec < MLXSW_REG_RAUHTD_REC_MAX_NUM)
1014 if (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, last_rec_index) ==
1015 MLXSW_REG_RAUHTD_TYPE_IPV6)
1018 num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
1020 if (++num_entries == MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC)
1025 static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp)
1031 rauhtd_pl = kmalloc(MLXSW_REG_RAUHTD_LEN, GFP_KERNEL);
1035 /* Make sure the neighbour's netdev isn't removed in the
1040 mlxsw_reg_rauhtd_pack(rauhtd_pl, MLXSW_REG_RAUHTD_TYPE_IPV4);
1041 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(rauhtd),
1044 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to dump neighbour talbe\n");
1047 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
1048 for (i = 0; i < num_rec; i++)
1049 mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl,
1051 } while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl));
1058 static void mlxsw_sp_router_neighs_update_nh(struct mlxsw_sp *mlxsw_sp)
1060 struct mlxsw_sp_neigh_entry *neigh_entry;
1062 /* Take RTNL mutex here to prevent lists from changes */
1064 list_for_each_entry(neigh_entry, &mlxsw_sp->router->nexthop_neighs_list,
1065 nexthop_neighs_list_node)
1066 /* If this neigh have nexthops, make the kernel think this neigh
1067 * is active regardless of the traffic.
1069 neigh_event_send(neigh_entry->key.n, NULL);
1074 mlxsw_sp_router_neighs_update_work_schedule(struct mlxsw_sp *mlxsw_sp)
1076 unsigned long interval = mlxsw_sp->router->neighs_update.interval;
1078 mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw,
1079 msecs_to_jiffies(interval));
1082 static void mlxsw_sp_router_neighs_update_work(struct work_struct *work)
1084 struct mlxsw_sp_router *router;
1087 router = container_of(work, struct mlxsw_sp_router,
1088 neighs_update.dw.work);
1089 err = mlxsw_sp_router_neighs_update_rauhtd(router->mlxsw_sp);
1091 dev_err(router->mlxsw_sp->bus_info->dev, "Could not update kernel for neigh activity");
1093 mlxsw_sp_router_neighs_update_nh(router->mlxsw_sp);
1095 mlxsw_sp_router_neighs_update_work_schedule(router->mlxsw_sp);
1098 static void mlxsw_sp_router_probe_unresolved_nexthops(struct work_struct *work)
1100 struct mlxsw_sp_neigh_entry *neigh_entry;
1101 struct mlxsw_sp_router *router;
1103 router = container_of(work, struct mlxsw_sp_router,
1104 nexthop_probe_dw.work);
1105 /* Iterate over nexthop neighbours, find those who are unresolved and
1106 * send arp on them. This solves the chicken-egg problem when
1107 * the nexthop wouldn't get offloaded until the neighbor is resolved
1108 * but it wouldn't get resolved ever in case traffic is flowing in HW
1109 * using different nexthop.
1111 * Take RTNL mutex here to prevent lists from changes.
1114 list_for_each_entry(neigh_entry, &router->nexthop_neighs_list,
1115 nexthop_neighs_list_node)
1116 if (!neigh_entry->connected)
1117 neigh_event_send(neigh_entry->key.n, NULL);
1120 mlxsw_core_schedule_dw(&router->nexthop_probe_dw,
1121 MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL);
1125 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
1126 struct mlxsw_sp_neigh_entry *neigh_entry,
1129 static enum mlxsw_reg_rauht_op mlxsw_sp_rauht_op(bool adding)
1131 return adding ? MLXSW_REG_RAUHT_OP_WRITE_ADD :
1132 MLXSW_REG_RAUHT_OP_WRITE_DELETE;
1136 mlxsw_sp_router_neigh_entry_op4(struct mlxsw_sp *mlxsw_sp,
1137 struct mlxsw_sp_neigh_entry *neigh_entry,
1138 enum mlxsw_reg_rauht_op op)
1140 struct neighbour *n = neigh_entry->key.n;
1141 u32 dip = ntohl(*((__be32 *) n->primary_key));
1142 char rauht_pl[MLXSW_REG_RAUHT_LEN];
1144 mlxsw_reg_rauht_pack4(rauht_pl, op, neigh_entry->rif, neigh_entry->ha,
1146 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
1150 mlxsw_sp_neigh_entry_update(struct mlxsw_sp *mlxsw_sp,
1151 struct mlxsw_sp_neigh_entry *neigh_entry,
1154 if (!adding && !neigh_entry->connected)
1156 neigh_entry->connected = adding;
1157 if (neigh_entry->key.n->tbl == &arp_tbl)
1158 mlxsw_sp_router_neigh_entry_op4(mlxsw_sp, neigh_entry,
1159 mlxsw_sp_rauht_op(adding));
1164 struct mlxsw_sp_neigh_event_work {
1165 struct work_struct work;
1166 struct mlxsw_sp *mlxsw_sp;
1167 struct neighbour *n;
1170 static void mlxsw_sp_router_neigh_event_work(struct work_struct *work)
1172 struct mlxsw_sp_neigh_event_work *neigh_work =
1173 container_of(work, struct mlxsw_sp_neigh_event_work, work);
1174 struct mlxsw_sp *mlxsw_sp = neigh_work->mlxsw_sp;
1175 struct mlxsw_sp_neigh_entry *neigh_entry;
1176 struct neighbour *n = neigh_work->n;
1177 unsigned char ha[ETH_ALEN];
1178 bool entry_connected;
1181 /* If these parameters are changed after we release the lock,
1182 * then we are guaranteed to receive another event letting us
1185 read_lock_bh(&n->lock);
1186 memcpy(ha, n->ha, ETH_ALEN);
1187 nud_state = n->nud_state;
1189 read_unlock_bh(&n->lock);
1192 entry_connected = nud_state & NUD_VALID && !dead;
1193 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
1194 if (!entry_connected && !neigh_entry)
1197 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
1198 if (IS_ERR(neigh_entry))
1202 memcpy(neigh_entry->ha, ha, ETH_ALEN);
1203 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, entry_connected);
1204 mlxsw_sp_nexthop_neigh_update(mlxsw_sp, neigh_entry, !entry_connected);
1206 if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
1207 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
1215 int mlxsw_sp_router_netevent_event(struct notifier_block *unused,
1216 unsigned long event, void *ptr)
1218 struct mlxsw_sp_neigh_event_work *neigh_work;
1219 struct mlxsw_sp_port *mlxsw_sp_port;
1220 struct mlxsw_sp *mlxsw_sp;
1221 unsigned long interval;
1222 struct neigh_parms *p;
1223 struct neighbour *n;
1226 case NETEVENT_DELAY_PROBE_TIME_UPDATE:
1229 /* We don't care about changes in the default table. */
1230 if (!p->dev || p->tbl != &arp_tbl)
1233 /* We are in atomic context and can't take RTNL mutex,
1234 * so use RCU variant to walk the device chain.
1236 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(p->dev);
1240 mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1241 interval = jiffies_to_msecs(NEIGH_VAR(p, DELAY_PROBE_TIME));
1242 mlxsw_sp->router->neighs_update.interval = interval;
1244 mlxsw_sp_port_dev_put(mlxsw_sp_port);
1246 case NETEVENT_NEIGH_UPDATE:
1249 if (n->tbl != &arp_tbl)
1252 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(n->dev);
1256 neigh_work = kzalloc(sizeof(*neigh_work), GFP_ATOMIC);
1258 mlxsw_sp_port_dev_put(mlxsw_sp_port);
1262 INIT_WORK(&neigh_work->work, mlxsw_sp_router_neigh_event_work);
1263 neigh_work->mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1266 /* Take a reference to ensure the neighbour won't be
1267 * destructed until we drop the reference in delayed
1271 mlxsw_core_schedule_work(&neigh_work->work);
1272 mlxsw_sp_port_dev_put(mlxsw_sp_port);
1279 static int mlxsw_sp_neigh_init(struct mlxsw_sp *mlxsw_sp)
1283 err = rhashtable_init(&mlxsw_sp->router->neigh_ht,
1284 &mlxsw_sp_neigh_ht_params);
1288 /* Initialize the polling interval according to the default
1291 mlxsw_sp_router_neighs_update_interval_init(mlxsw_sp);
1293 /* Create the delayed works for the activity_update */
1294 INIT_DELAYED_WORK(&mlxsw_sp->router->neighs_update.dw,
1295 mlxsw_sp_router_neighs_update_work);
1296 INIT_DELAYED_WORK(&mlxsw_sp->router->nexthop_probe_dw,
1297 mlxsw_sp_router_probe_unresolved_nexthops);
1298 mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw, 0);
1299 mlxsw_core_schedule_dw(&mlxsw_sp->router->nexthop_probe_dw, 0);
1303 static void mlxsw_sp_neigh_fini(struct mlxsw_sp *mlxsw_sp)
1305 cancel_delayed_work_sync(&mlxsw_sp->router->neighs_update.dw);
1306 cancel_delayed_work_sync(&mlxsw_sp->router->nexthop_probe_dw);
1307 rhashtable_destroy(&mlxsw_sp->router->neigh_ht);
1310 static int mlxsw_sp_neigh_rif_flush(struct mlxsw_sp *mlxsw_sp,
1311 const struct mlxsw_sp_rif *rif)
1313 char rauht_pl[MLXSW_REG_RAUHT_LEN];
1315 mlxsw_reg_rauht_pack(rauht_pl, MLXSW_REG_RAUHT_OP_WRITE_DELETE_ALL,
1316 rif->rif_index, rif->addr);
1317 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
1320 static void mlxsw_sp_neigh_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
1321 struct mlxsw_sp_rif *rif)
1323 struct mlxsw_sp_neigh_entry *neigh_entry, *tmp;
1325 mlxsw_sp_neigh_rif_flush(mlxsw_sp, rif);
1326 list_for_each_entry_safe(neigh_entry, tmp, &rif->neigh_list,
1328 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
1331 struct mlxsw_sp_nexthop_key {
1332 struct fib_nh *fib_nh;
1335 struct mlxsw_sp_nexthop {
1336 struct list_head neigh_list_node; /* member of neigh entry list */
1337 struct list_head rif_list_node;
1338 struct mlxsw_sp_nexthop_group *nh_grp; /* pointer back to the group
1341 struct rhash_head ht_node;
1342 struct mlxsw_sp_nexthop_key key;
1343 struct mlxsw_sp_rif *rif;
1344 u8 should_offload:1, /* set indicates this neigh is connected and
1345 * should be put to KVD linear area of this group.
1347 offloaded:1, /* set in case the neigh is actually put into
1348 * KVD linear area of this group.
1350 update:1; /* set indicates that MAC of this neigh should be
1353 struct mlxsw_sp_neigh_entry *neigh_entry;
1356 struct mlxsw_sp_nexthop_group_key {
1357 struct fib_info *fi;
1360 struct mlxsw_sp_nexthop_group {
1361 struct rhash_head ht_node;
1362 struct list_head fib_list; /* list of fib entries that use this group */
1363 struct mlxsw_sp_nexthop_group_key key;
1364 u8 adj_index_valid:1,
1365 gateway:1; /* routes using the group use a gateway */
1369 struct mlxsw_sp_nexthop nexthops[0];
1370 #define nh_rif nexthops[0].rif
1373 static const struct rhashtable_params mlxsw_sp_nexthop_group_ht_params = {
1374 .key_offset = offsetof(struct mlxsw_sp_nexthop_group, key),
1375 .head_offset = offsetof(struct mlxsw_sp_nexthop_group, ht_node),
1376 .key_len = sizeof(struct mlxsw_sp_nexthop_group_key),
1379 static int mlxsw_sp_nexthop_group_insert(struct mlxsw_sp *mlxsw_sp,
1380 struct mlxsw_sp_nexthop_group *nh_grp)
1382 return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_group_ht,
1384 mlxsw_sp_nexthop_group_ht_params);
1387 static void mlxsw_sp_nexthop_group_remove(struct mlxsw_sp *mlxsw_sp,
1388 struct mlxsw_sp_nexthop_group *nh_grp)
1390 rhashtable_remove_fast(&mlxsw_sp->router->nexthop_group_ht,
1392 mlxsw_sp_nexthop_group_ht_params);
1395 static struct mlxsw_sp_nexthop_group *
1396 mlxsw_sp_nexthop_group_lookup(struct mlxsw_sp *mlxsw_sp,
1397 struct mlxsw_sp_nexthop_group_key key)
1399 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht, &key,
1400 mlxsw_sp_nexthop_group_ht_params);
1403 static const struct rhashtable_params mlxsw_sp_nexthop_ht_params = {
1404 .key_offset = offsetof(struct mlxsw_sp_nexthop, key),
1405 .head_offset = offsetof(struct mlxsw_sp_nexthop, ht_node),
1406 .key_len = sizeof(struct mlxsw_sp_nexthop_key),
1409 static int mlxsw_sp_nexthop_insert(struct mlxsw_sp *mlxsw_sp,
1410 struct mlxsw_sp_nexthop *nh)
1412 return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_ht,
1413 &nh->ht_node, mlxsw_sp_nexthop_ht_params);
1416 static void mlxsw_sp_nexthop_remove(struct mlxsw_sp *mlxsw_sp,
1417 struct mlxsw_sp_nexthop *nh)
1419 rhashtable_remove_fast(&mlxsw_sp->router->nexthop_ht, &nh->ht_node,
1420 mlxsw_sp_nexthop_ht_params);
1423 static struct mlxsw_sp_nexthop *
1424 mlxsw_sp_nexthop_lookup(struct mlxsw_sp *mlxsw_sp,
1425 struct mlxsw_sp_nexthop_key key)
1427 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_ht, &key,
1428 mlxsw_sp_nexthop_ht_params);
1431 static int mlxsw_sp_adj_index_mass_update_vr(struct mlxsw_sp *mlxsw_sp,
1432 const struct mlxsw_sp_fib *fib,
1433 u32 adj_index, u16 ecmp_size,
1437 char raleu_pl[MLXSW_REG_RALEU_LEN];
1439 mlxsw_reg_raleu_pack(raleu_pl,
1440 (enum mlxsw_reg_ralxx_protocol) fib->proto,
1441 fib->vr->id, adj_index, ecmp_size, new_adj_index,
1443 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raleu), raleu_pl);
1446 static int mlxsw_sp_adj_index_mass_update(struct mlxsw_sp *mlxsw_sp,
1447 struct mlxsw_sp_nexthop_group *nh_grp,
1448 u32 old_adj_index, u16 old_ecmp_size)
1450 struct mlxsw_sp_fib_entry *fib_entry;
1451 struct mlxsw_sp_fib *fib = NULL;
1454 list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
1455 if (fib == fib_entry->fib_node->fib)
1457 fib = fib_entry->fib_node->fib;
1458 err = mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, fib,
1469 static int mlxsw_sp_nexthop_mac_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index,
1470 struct mlxsw_sp_nexthop *nh)
1472 struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
1473 char ratr_pl[MLXSW_REG_RATR_LEN];
1475 mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY,
1476 true, adj_index, neigh_entry->rif);
1477 mlxsw_reg_ratr_eth_entry_pack(ratr_pl, neigh_entry->ha);
1478 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl);
1482 mlxsw_sp_nexthop_group_mac_update(struct mlxsw_sp *mlxsw_sp,
1483 struct mlxsw_sp_nexthop_group *nh_grp,
1486 u32 adj_index = nh_grp->adj_index; /* base */
1487 struct mlxsw_sp_nexthop *nh;
1491 for (i = 0; i < nh_grp->count; i++) {
1492 nh = &nh_grp->nexthops[i];
1494 if (!nh->should_offload) {
1499 if (nh->update || reallocate) {
1500 err = mlxsw_sp_nexthop_mac_update(mlxsw_sp,
1512 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
1513 struct mlxsw_sp_fib_entry *fib_entry);
1516 mlxsw_sp_nexthop_fib_entries_update(struct mlxsw_sp *mlxsw_sp,
1517 struct mlxsw_sp_nexthop_group *nh_grp)
1519 struct mlxsw_sp_fib_entry *fib_entry;
1522 list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
1523 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
1531 mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp,
1532 struct mlxsw_sp_nexthop_group *nh_grp)
1534 struct mlxsw_sp_nexthop *nh;
1535 bool offload_change = false;
1538 bool old_adj_index_valid;
1544 if (!nh_grp->gateway) {
1545 mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
1549 for (i = 0; i < nh_grp->count; i++) {
1550 nh = &nh_grp->nexthops[i];
1552 if (nh->should_offload ^ nh->offloaded) {
1553 offload_change = true;
1554 if (nh->should_offload)
1557 if (nh->should_offload)
1560 if (!offload_change) {
1561 /* Nothing was added or removed, so no need to reallocate. Just
1562 * update MAC on existing adjacency indexes.
1564 err = mlxsw_sp_nexthop_group_mac_update(mlxsw_sp, nh_grp,
1567 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
1573 /* No neigh of this group is connected so we just set
1574 * the trap and let everthing flow through kernel.
1578 err = mlxsw_sp_kvdl_alloc(mlxsw_sp, ecmp_size, &adj_index);
1580 /* We ran out of KVD linear space, just set the
1581 * trap and let everything flow through kernel.
1583 dev_warn(mlxsw_sp->bus_info->dev, "Failed to allocate KVD linear area for nexthop group.\n");
1586 old_adj_index_valid = nh_grp->adj_index_valid;
1587 old_adj_index = nh_grp->adj_index;
1588 old_ecmp_size = nh_grp->ecmp_size;
1589 nh_grp->adj_index_valid = 1;
1590 nh_grp->adj_index = adj_index;
1591 nh_grp->ecmp_size = ecmp_size;
1592 err = mlxsw_sp_nexthop_group_mac_update(mlxsw_sp, nh_grp, true);
1594 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
1598 if (!old_adj_index_valid) {
1599 /* The trap was set for fib entries, so we have to call
1600 * fib entry update to unset it and use adjacency index.
1602 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
1604 dev_warn(mlxsw_sp->bus_info->dev, "Failed to add adjacency index to fib entries.\n");
1610 err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, nh_grp,
1611 old_adj_index, old_ecmp_size);
1612 mlxsw_sp_kvdl_free(mlxsw_sp, old_adj_index);
1614 dev_warn(mlxsw_sp->bus_info->dev, "Failed to mass-update adjacency index for nexthop group.\n");
1620 old_adj_index_valid = nh_grp->adj_index_valid;
1621 nh_grp->adj_index_valid = 0;
1622 for (i = 0; i < nh_grp->count; i++) {
1623 nh = &nh_grp->nexthops[i];
1626 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
1628 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set traps for fib entries.\n");
1629 if (old_adj_index_valid)
1630 mlxsw_sp_kvdl_free(mlxsw_sp, nh_grp->adj_index);
1633 static void __mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp_nexthop *nh,
1636 if (!removing && !nh->should_offload)
1637 nh->should_offload = 1;
1638 else if (removing && nh->offloaded)
1639 nh->should_offload = 0;
1644 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
1645 struct mlxsw_sp_neigh_entry *neigh_entry,
1648 struct mlxsw_sp_nexthop *nh;
1650 list_for_each_entry(nh, &neigh_entry->nexthop_list,
1652 __mlxsw_sp_nexthop_neigh_update(nh, removing);
1653 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
1657 static void mlxsw_sp_nexthop_rif_init(struct mlxsw_sp_nexthop *nh,
1658 struct mlxsw_sp_rif *rif)
1664 list_add(&nh->rif_list_node, &rif->nexthop_list);
1667 static void mlxsw_sp_nexthop_rif_fini(struct mlxsw_sp_nexthop *nh)
1672 list_del(&nh->rif_list_node);
1676 static int mlxsw_sp_nexthop_neigh_init(struct mlxsw_sp *mlxsw_sp,
1677 struct mlxsw_sp_nexthop *nh)
1679 struct mlxsw_sp_neigh_entry *neigh_entry;
1680 struct fib_nh *fib_nh = nh->key.fib_nh;
1681 struct neighbour *n;
1685 if (!nh->nh_grp->gateway || nh->neigh_entry)
1688 /* Take a reference of neigh here ensuring that neigh would
1689 * not be detructed before the nexthop entry is finished.
1690 * The reference is taken either in neigh_lookup() or
1691 * in neigh_create() in case n is not found.
1693 n = neigh_lookup(&arp_tbl, &fib_nh->nh_gw, fib_nh->nh_dev);
1695 n = neigh_create(&arp_tbl, &fib_nh->nh_gw, fib_nh->nh_dev);
1698 neigh_event_send(n, NULL);
1700 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
1702 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
1703 if (IS_ERR(neigh_entry)) {
1705 goto err_neigh_entry_create;
1709 /* If that is the first nexthop connected to that neigh, add to
1710 * nexthop_neighs_list
1712 if (list_empty(&neigh_entry->nexthop_list))
1713 list_add_tail(&neigh_entry->nexthop_neighs_list_node,
1714 &mlxsw_sp->router->nexthop_neighs_list);
1716 nh->neigh_entry = neigh_entry;
1717 list_add_tail(&nh->neigh_list_node, &neigh_entry->nexthop_list);
1718 read_lock_bh(&n->lock);
1719 nud_state = n->nud_state;
1721 read_unlock_bh(&n->lock);
1722 __mlxsw_sp_nexthop_neigh_update(nh, !(nud_state & NUD_VALID && !dead));
1726 err_neigh_entry_create:
1731 static void mlxsw_sp_nexthop_neigh_fini(struct mlxsw_sp *mlxsw_sp,
1732 struct mlxsw_sp_nexthop *nh)
1734 struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
1735 struct neighbour *n;
1739 n = neigh_entry->key.n;
1741 __mlxsw_sp_nexthop_neigh_update(nh, true);
1742 list_del(&nh->neigh_list_node);
1743 nh->neigh_entry = NULL;
1745 /* If that is the last nexthop connected to that neigh, remove from
1746 * nexthop_neighs_list
1748 if (list_empty(&neigh_entry->nexthop_list))
1749 list_del(&neigh_entry->nexthop_neighs_list_node);
1751 if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
1752 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
1757 static int mlxsw_sp_nexthop_init(struct mlxsw_sp *mlxsw_sp,
1758 struct mlxsw_sp_nexthop_group *nh_grp,
1759 struct mlxsw_sp_nexthop *nh,
1760 struct fib_nh *fib_nh)
1762 struct net_device *dev = fib_nh->nh_dev;
1763 struct in_device *in_dev;
1764 struct mlxsw_sp_rif *rif;
1767 nh->nh_grp = nh_grp;
1768 nh->key.fib_nh = fib_nh;
1769 err = mlxsw_sp_nexthop_insert(mlxsw_sp, nh);
1776 in_dev = __in_dev_get_rtnl(dev);
1777 if (in_dev && IN_DEV_IGNORE_ROUTES_WITH_LINKDOWN(in_dev) &&
1778 fib_nh->nh_flags & RTNH_F_LINKDOWN)
1781 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
1784 mlxsw_sp_nexthop_rif_init(nh, rif);
1786 err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
1788 goto err_nexthop_neigh_init;
1792 err_nexthop_neigh_init:
1793 mlxsw_sp_nexthop_rif_fini(nh);
1794 mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
1798 static void mlxsw_sp_nexthop_fini(struct mlxsw_sp *mlxsw_sp,
1799 struct mlxsw_sp_nexthop *nh)
1801 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
1802 mlxsw_sp_nexthop_rif_fini(nh);
1803 mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
1806 static void mlxsw_sp_nexthop_event(struct mlxsw_sp *mlxsw_sp,
1807 unsigned long event, struct fib_nh *fib_nh)
1809 struct mlxsw_sp_nexthop_key key;
1810 struct mlxsw_sp_nexthop *nh;
1811 struct mlxsw_sp_rif *rif;
1813 if (mlxsw_sp->router->aborted)
1816 key.fib_nh = fib_nh;
1817 nh = mlxsw_sp_nexthop_lookup(mlxsw_sp, key);
1818 if (WARN_ON_ONCE(!nh))
1821 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, fib_nh->nh_dev);
1826 case FIB_EVENT_NH_ADD:
1827 mlxsw_sp_nexthop_rif_init(nh, rif);
1828 mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
1830 case FIB_EVENT_NH_DEL:
1831 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
1832 mlxsw_sp_nexthop_rif_fini(nh);
1836 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
1839 static void mlxsw_sp_nexthop_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
1840 struct mlxsw_sp_rif *rif)
1842 struct mlxsw_sp_nexthop *nh, *tmp;
1844 list_for_each_entry_safe(nh, tmp, &rif->nexthop_list, rif_list_node) {
1845 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
1846 mlxsw_sp_nexthop_rif_fini(nh);
1847 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
1851 static struct mlxsw_sp_nexthop_group *
1852 mlxsw_sp_nexthop_group_create(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi)
1854 struct mlxsw_sp_nexthop_group *nh_grp;
1855 struct mlxsw_sp_nexthop *nh;
1856 struct fib_nh *fib_nh;
1861 alloc_size = sizeof(*nh_grp) +
1862 fi->fib_nhs * sizeof(struct mlxsw_sp_nexthop);
1863 nh_grp = kzalloc(alloc_size, GFP_KERNEL);
1865 return ERR_PTR(-ENOMEM);
1866 INIT_LIST_HEAD(&nh_grp->fib_list);
1867 nh_grp->gateway = fi->fib_nh->nh_scope == RT_SCOPE_LINK;
1868 nh_grp->count = fi->fib_nhs;
1869 nh_grp->key.fi = fi;
1871 for (i = 0; i < nh_grp->count; i++) {
1872 nh = &nh_grp->nexthops[i];
1873 fib_nh = &fi->fib_nh[i];
1874 err = mlxsw_sp_nexthop_init(mlxsw_sp, nh_grp, nh, fib_nh);
1876 goto err_nexthop_init;
1878 err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
1880 goto err_nexthop_group_insert;
1881 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
1884 err_nexthop_group_insert:
1886 for (i--; i >= 0; i--) {
1887 nh = &nh_grp->nexthops[i];
1888 mlxsw_sp_nexthop_fini(mlxsw_sp, nh);
1890 fib_info_put(nh_grp->key.fi);
1892 return ERR_PTR(err);
1896 mlxsw_sp_nexthop_group_destroy(struct mlxsw_sp *mlxsw_sp,
1897 struct mlxsw_sp_nexthop_group *nh_grp)
1899 struct mlxsw_sp_nexthop *nh;
1902 mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
1903 for (i = 0; i < nh_grp->count; i++) {
1904 nh = &nh_grp->nexthops[i];
1905 mlxsw_sp_nexthop_fini(mlxsw_sp, nh);
1907 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
1908 WARN_ON_ONCE(nh_grp->adj_index_valid);
1909 fib_info_put(nh_grp->key.fi);
1913 static int mlxsw_sp_nexthop_group_get(struct mlxsw_sp *mlxsw_sp,
1914 struct mlxsw_sp_fib_entry *fib_entry,
1915 struct fib_info *fi)
1917 struct mlxsw_sp_nexthop_group_key key;
1918 struct mlxsw_sp_nexthop_group *nh_grp;
1921 nh_grp = mlxsw_sp_nexthop_group_lookup(mlxsw_sp, key);
1923 nh_grp = mlxsw_sp_nexthop_group_create(mlxsw_sp, fi);
1925 return PTR_ERR(nh_grp);
1927 list_add_tail(&fib_entry->nexthop_group_node, &nh_grp->fib_list);
1928 fib_entry->nh_group = nh_grp;
1932 static void mlxsw_sp_nexthop_group_put(struct mlxsw_sp *mlxsw_sp,
1933 struct mlxsw_sp_fib_entry *fib_entry)
1935 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
1937 list_del(&fib_entry->nexthop_group_node);
1938 if (!list_empty(&nh_grp->fib_list))
1940 mlxsw_sp_nexthop_group_destroy(mlxsw_sp, nh_grp);
1944 mlxsw_sp_fib_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry)
1946 struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group;
1948 if (fib_entry->params.tos)
1951 switch (fib_entry->type) {
1952 case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
1953 return !!nh_group->adj_index_valid;
1954 case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
1955 return !!nh_group->nh_rif;
1961 static void mlxsw_sp_fib_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry)
1963 fib_entry->offloaded = true;
1965 switch (fib_entry->fib_node->fib->proto) {
1966 case MLXSW_SP_L3_PROTO_IPV4:
1967 fib_info_offload_inc(fib_entry->nh_group->key.fi);
1969 case MLXSW_SP_L3_PROTO_IPV6:
1975 mlxsw_sp_fib_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry)
1977 switch (fib_entry->fib_node->fib->proto) {
1978 case MLXSW_SP_L3_PROTO_IPV4:
1979 fib_info_offload_dec(fib_entry->nh_group->key.fi);
1981 case MLXSW_SP_L3_PROTO_IPV6:
1985 fib_entry->offloaded = false;
1989 mlxsw_sp_fib_entry_offload_refresh(struct mlxsw_sp_fib_entry *fib_entry,
1990 enum mlxsw_reg_ralue_op op, int err)
1993 case MLXSW_REG_RALUE_OP_WRITE_DELETE:
1994 if (!fib_entry->offloaded)
1996 return mlxsw_sp_fib_entry_offload_unset(fib_entry);
1997 case MLXSW_REG_RALUE_OP_WRITE_WRITE:
2000 if (mlxsw_sp_fib_entry_should_offload(fib_entry) &&
2001 !fib_entry->offloaded)
2002 mlxsw_sp_fib_entry_offload_set(fib_entry);
2003 else if (!mlxsw_sp_fib_entry_should_offload(fib_entry) &&
2004 fib_entry->offloaded)
2005 mlxsw_sp_fib_entry_offload_unset(fib_entry);
2012 static int mlxsw_sp_fib_entry_op4_remote(struct mlxsw_sp *mlxsw_sp,
2013 struct mlxsw_sp_fib_entry *fib_entry,
2014 enum mlxsw_reg_ralue_op op)
2016 char ralue_pl[MLXSW_REG_RALUE_LEN];
2017 struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
2018 u32 *p_dip = (u32 *) fib_entry->fib_node->key.addr;
2019 enum mlxsw_reg_ralue_trap_action trap_action;
2021 u32 adjacency_index = 0;
2024 /* In case the nexthop group adjacency index is valid, use it
2025 * with provided ECMP size. Otherwise, setup trap and pass
2026 * traffic to kernel.
2028 if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
2029 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
2030 adjacency_index = fib_entry->nh_group->adj_index;
2031 ecmp_size = fib_entry->nh_group->ecmp_size;
2033 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
2034 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
2037 mlxsw_reg_ralue_pack4(ralue_pl,
2038 (enum mlxsw_reg_ralxx_protocol) fib->proto, op,
2039 fib->vr->id, fib_entry->fib_node->key.prefix_len,
2041 mlxsw_reg_ralue_act_remote_pack(ralue_pl, trap_action, trap_id,
2042 adjacency_index, ecmp_size);
2043 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
2046 static int mlxsw_sp_fib_entry_op4_local(struct mlxsw_sp *mlxsw_sp,
2047 struct mlxsw_sp_fib_entry *fib_entry,
2048 enum mlxsw_reg_ralue_op op)
2050 struct mlxsw_sp_rif *rif = fib_entry->nh_group->nh_rif;
2051 struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
2052 enum mlxsw_reg_ralue_trap_action trap_action;
2053 char ralue_pl[MLXSW_REG_RALUE_LEN];
2054 u32 *p_dip = (u32 *) fib_entry->fib_node->key.addr;
2058 if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
2059 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
2060 rif_index = rif->rif_index;
2062 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
2063 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
2066 mlxsw_reg_ralue_pack4(ralue_pl,
2067 (enum mlxsw_reg_ralxx_protocol) fib->proto, op,
2068 fib->vr->id, fib_entry->fib_node->key.prefix_len,
2070 mlxsw_reg_ralue_act_local_pack(ralue_pl, trap_action, trap_id,
2072 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
2075 static int mlxsw_sp_fib_entry_op4_trap(struct mlxsw_sp *mlxsw_sp,
2076 struct mlxsw_sp_fib_entry *fib_entry,
2077 enum mlxsw_reg_ralue_op op)
2079 struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
2080 char ralue_pl[MLXSW_REG_RALUE_LEN];
2081 u32 *p_dip = (u32 *) fib_entry->fib_node->key.addr;
2083 mlxsw_reg_ralue_pack4(ralue_pl,
2084 (enum mlxsw_reg_ralxx_protocol) fib->proto, op,
2085 fib->vr->id, fib_entry->fib_node->key.prefix_len,
2087 mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
2088 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
2091 static int mlxsw_sp_fib_entry_op4(struct mlxsw_sp *mlxsw_sp,
2092 struct mlxsw_sp_fib_entry *fib_entry,
2093 enum mlxsw_reg_ralue_op op)
2095 switch (fib_entry->type) {
2096 case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
2097 return mlxsw_sp_fib_entry_op4_remote(mlxsw_sp, fib_entry, op);
2098 case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
2099 return mlxsw_sp_fib_entry_op4_local(mlxsw_sp, fib_entry, op);
2100 case MLXSW_SP_FIB_ENTRY_TYPE_TRAP:
2101 return mlxsw_sp_fib_entry_op4_trap(mlxsw_sp, fib_entry, op);
2106 static int mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
2107 struct mlxsw_sp_fib_entry *fib_entry,
2108 enum mlxsw_reg_ralue_op op)
2112 switch (fib_entry->fib_node->fib->proto) {
2113 case MLXSW_SP_L3_PROTO_IPV4:
2114 err = mlxsw_sp_fib_entry_op4(mlxsw_sp, fib_entry, op);
2116 case MLXSW_SP_L3_PROTO_IPV6:
2119 mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, err);
2123 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
2124 struct mlxsw_sp_fib_entry *fib_entry)
2126 return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
2127 MLXSW_REG_RALUE_OP_WRITE_WRITE);
2130 static int mlxsw_sp_fib_entry_del(struct mlxsw_sp *mlxsw_sp,
2131 struct mlxsw_sp_fib_entry *fib_entry)
2133 return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
2134 MLXSW_REG_RALUE_OP_WRITE_DELETE);
2138 mlxsw_sp_fib4_entry_type_set(struct mlxsw_sp *mlxsw_sp,
2139 const struct fib_entry_notifier_info *fen_info,
2140 struct mlxsw_sp_fib_entry *fib_entry)
2142 struct fib_info *fi = fen_info->fi;
2144 switch (fen_info->type) {
2145 case RTN_BROADCAST: /* fall through */
2147 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
2149 case RTN_UNREACHABLE: /* fall through */
2150 case RTN_BLACKHOLE: /* fall through */
2152 /* Packets hitting these routes need to be trapped, but
2153 * can do so with a lower priority than packets directed
2154 * at the host, so use action type local instead of trap.
2156 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
2159 if (fi->fib_nh->nh_scope != RT_SCOPE_LINK)
2160 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
2162 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
2169 static struct mlxsw_sp_fib_entry *
2170 mlxsw_sp_fib4_entry_create(struct mlxsw_sp *mlxsw_sp,
2171 struct mlxsw_sp_fib_node *fib_node,
2172 const struct fib_entry_notifier_info *fen_info)
2174 struct mlxsw_sp_fib_entry *fib_entry;
2177 fib_entry = kzalloc(sizeof(*fib_entry), GFP_KERNEL);
2180 goto err_fib_entry_alloc;
2183 err = mlxsw_sp_fib4_entry_type_set(mlxsw_sp, fen_info, fib_entry);
2185 goto err_fib4_entry_type_set;
2187 err = mlxsw_sp_nexthop_group_get(mlxsw_sp, fib_entry, fen_info->fi);
2189 goto err_nexthop_group_get;
2191 fib_entry->params.prio = fen_info->fi->fib_priority;
2192 fib_entry->params.tb_id = fen_info->tb_id;
2193 fib_entry->params.type = fen_info->type;
2194 fib_entry->params.tos = fen_info->tos;
2196 fib_entry->fib_node = fib_node;
2200 err_nexthop_group_get:
2201 err_fib4_entry_type_set:
2203 err_fib_entry_alloc:
2204 return ERR_PTR(err);
2207 static void mlxsw_sp_fib4_entry_destroy(struct mlxsw_sp *mlxsw_sp,
2208 struct mlxsw_sp_fib_entry *fib_entry)
2210 mlxsw_sp_nexthop_group_put(mlxsw_sp, fib_entry);
2214 static struct mlxsw_sp_fib_node *
2215 mlxsw_sp_fib4_node_get(struct mlxsw_sp *mlxsw_sp,
2216 const struct fib_entry_notifier_info *fen_info);
2218 static struct mlxsw_sp_fib_entry *
2219 mlxsw_sp_fib4_entry_lookup(struct mlxsw_sp *mlxsw_sp,
2220 const struct fib_entry_notifier_info *fen_info)
2222 struct mlxsw_sp_fib_entry *fib_entry;
2223 struct mlxsw_sp_fib_node *fib_node;
2225 fib_node = mlxsw_sp_fib4_node_get(mlxsw_sp, fen_info);
2226 if (IS_ERR(fib_node))
2229 list_for_each_entry(fib_entry, &fib_node->entry_list, list) {
2230 if (fib_entry->params.tb_id == fen_info->tb_id &&
2231 fib_entry->params.tos == fen_info->tos &&
2232 fib_entry->params.type == fen_info->type &&
2233 fib_entry->nh_group->key.fi == fen_info->fi) {
2241 static const struct rhashtable_params mlxsw_sp_fib_ht_params = {
2242 .key_offset = offsetof(struct mlxsw_sp_fib_node, key),
2243 .head_offset = offsetof(struct mlxsw_sp_fib_node, ht_node),
2244 .key_len = sizeof(struct mlxsw_sp_fib_key),
2245 .automatic_shrinking = true,
2248 static int mlxsw_sp_fib_node_insert(struct mlxsw_sp_fib *fib,
2249 struct mlxsw_sp_fib_node *fib_node)
2251 return rhashtable_insert_fast(&fib->ht, &fib_node->ht_node,
2252 mlxsw_sp_fib_ht_params);
2255 static void mlxsw_sp_fib_node_remove(struct mlxsw_sp_fib *fib,
2256 struct mlxsw_sp_fib_node *fib_node)
2258 rhashtable_remove_fast(&fib->ht, &fib_node->ht_node,
2259 mlxsw_sp_fib_ht_params);
2262 static struct mlxsw_sp_fib_node *
2263 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr,
2264 size_t addr_len, unsigned char prefix_len)
2266 struct mlxsw_sp_fib_key key;
2268 memset(&key, 0, sizeof(key));
2269 memcpy(key.addr, addr, addr_len);
2270 key.prefix_len = prefix_len;
2271 return rhashtable_lookup_fast(&fib->ht, &key, mlxsw_sp_fib_ht_params);
2274 static struct mlxsw_sp_fib_node *
2275 mlxsw_sp_fib_node_create(struct mlxsw_sp_fib *fib, const void *addr,
2276 size_t addr_len, unsigned char prefix_len)
2278 struct mlxsw_sp_fib_node *fib_node;
2280 fib_node = kzalloc(sizeof(*fib_node), GFP_KERNEL);
2284 INIT_LIST_HEAD(&fib_node->entry_list);
2285 list_add(&fib_node->list, &fib->node_list);
2286 memcpy(fib_node->key.addr, addr, addr_len);
2287 fib_node->key.prefix_len = prefix_len;
2292 static void mlxsw_sp_fib_node_destroy(struct mlxsw_sp_fib_node *fib_node)
2294 list_del(&fib_node->list);
2295 WARN_ON(!list_empty(&fib_node->entry_list));
2300 mlxsw_sp_fib_node_entry_is_first(const struct mlxsw_sp_fib_node *fib_node,
2301 const struct mlxsw_sp_fib_entry *fib_entry)
2303 return list_first_entry(&fib_node->entry_list,
2304 struct mlxsw_sp_fib_entry, list) == fib_entry;
2307 static void mlxsw_sp_fib_node_prefix_inc(struct mlxsw_sp_fib_node *fib_node)
2309 unsigned char prefix_len = fib_node->key.prefix_len;
2310 struct mlxsw_sp_fib *fib = fib_node->fib;
2312 if (fib->prefix_ref_count[prefix_len]++ == 0)
2313 mlxsw_sp_prefix_usage_set(&fib->prefix_usage, prefix_len);
2316 static void mlxsw_sp_fib_node_prefix_dec(struct mlxsw_sp_fib_node *fib_node)
2318 unsigned char prefix_len = fib_node->key.prefix_len;
2319 struct mlxsw_sp_fib *fib = fib_node->fib;
2321 if (--fib->prefix_ref_count[prefix_len] == 0)
2322 mlxsw_sp_prefix_usage_clear(&fib->prefix_usage, prefix_len);
2325 static int mlxsw_sp_fib_node_init(struct mlxsw_sp *mlxsw_sp,
2326 struct mlxsw_sp_fib_node *fib_node,
2327 struct mlxsw_sp_fib *fib)
2329 struct mlxsw_sp_prefix_usage req_prefix_usage;
2330 struct mlxsw_sp_lpm_tree *lpm_tree;
2333 err = mlxsw_sp_fib_node_insert(fib, fib_node);
2336 fib_node->fib = fib;
2338 mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &fib->prefix_usage);
2339 mlxsw_sp_prefix_usage_set(&req_prefix_usage, fib_node->key.prefix_len);
2341 if (!mlxsw_sp_prefix_usage_none(&fib->prefix_usage)) {
2342 err = mlxsw_sp_vr_lpm_tree_check(mlxsw_sp, fib,
2345 goto err_tree_check;
2347 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
2349 if (IS_ERR(lpm_tree))
2350 return PTR_ERR(lpm_tree);
2351 fib->lpm_tree = lpm_tree;
2352 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib);
2357 mlxsw_sp_fib_node_prefix_inc(fib_node);
2362 fib->lpm_tree = NULL;
2363 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
2365 fib_node->fib = NULL;
2366 mlxsw_sp_fib_node_remove(fib, fib_node);
2370 static void mlxsw_sp_fib_node_fini(struct mlxsw_sp *mlxsw_sp,
2371 struct mlxsw_sp_fib_node *fib_node)
2373 struct mlxsw_sp_lpm_tree *lpm_tree = fib_node->fib->lpm_tree;
2374 struct mlxsw_sp_fib *fib = fib_node->fib;
2376 mlxsw_sp_fib_node_prefix_dec(fib_node);
2378 if (mlxsw_sp_prefix_usage_none(&fib->prefix_usage)) {
2379 mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, fib);
2380 fib->lpm_tree = NULL;
2381 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
2383 mlxsw_sp_vr_lpm_tree_check(mlxsw_sp, fib, &fib->prefix_usage);
2386 fib_node->fib = NULL;
2387 mlxsw_sp_fib_node_remove(fib, fib_node);
2390 static struct mlxsw_sp_fib_node *
2391 mlxsw_sp_fib4_node_get(struct mlxsw_sp *mlxsw_sp,
2392 const struct fib_entry_notifier_info *fen_info)
2394 struct mlxsw_sp_fib_node *fib_node;
2395 struct mlxsw_sp_fib *fib;
2396 struct mlxsw_sp_vr *vr;
2399 vr = mlxsw_sp_vr_get(mlxsw_sp, fen_info->tb_id);
2401 return ERR_CAST(vr);
2402 fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV4);
2404 fib_node = mlxsw_sp_fib_node_lookup(fib, &fen_info->dst,
2405 sizeof(fen_info->dst),
2410 fib_node = mlxsw_sp_fib_node_create(fib, &fen_info->dst,
2411 sizeof(fen_info->dst),
2415 goto err_fib_node_create;
2418 err = mlxsw_sp_fib_node_init(mlxsw_sp, fib_node, fib);
2420 goto err_fib_node_init;
2425 mlxsw_sp_fib_node_destroy(fib_node);
2426 err_fib_node_create:
2427 mlxsw_sp_vr_put(vr);
2428 return ERR_PTR(err);
2431 static void mlxsw_sp_fib4_node_put(struct mlxsw_sp *mlxsw_sp,
2432 struct mlxsw_sp_fib_node *fib_node)
2434 struct mlxsw_sp_vr *vr = fib_node->fib->vr;
2436 if (!list_empty(&fib_node->entry_list))
2438 mlxsw_sp_fib_node_fini(mlxsw_sp, fib_node);
2439 mlxsw_sp_fib_node_destroy(fib_node);
2440 mlxsw_sp_vr_put(vr);
2443 static struct mlxsw_sp_fib_entry *
2444 mlxsw_sp_fib4_node_entry_find(const struct mlxsw_sp_fib_node *fib_node,
2445 const struct mlxsw_sp_fib_entry_params *params)
2447 struct mlxsw_sp_fib_entry *fib_entry;
2449 list_for_each_entry(fib_entry, &fib_node->entry_list, list) {
2450 if (fib_entry->params.tb_id > params->tb_id)
2452 if (fib_entry->params.tb_id != params->tb_id)
2454 if (fib_entry->params.tos > params->tos)
2456 if (fib_entry->params.prio >= params->prio ||
2457 fib_entry->params.tos < params->tos)
2464 static int mlxsw_sp_fib4_node_list_append(struct mlxsw_sp_fib_entry *fib_entry,
2465 struct mlxsw_sp_fib_entry *new_entry)
2467 struct mlxsw_sp_fib_node *fib_node;
2469 if (WARN_ON(!fib_entry))
2472 fib_node = fib_entry->fib_node;
2473 list_for_each_entry_from(fib_entry, &fib_node->entry_list, list) {
2474 if (fib_entry->params.tb_id != new_entry->params.tb_id ||
2475 fib_entry->params.tos != new_entry->params.tos ||
2476 fib_entry->params.prio != new_entry->params.prio)
2480 list_add_tail(&new_entry->list, &fib_entry->list);
2485 mlxsw_sp_fib4_node_list_insert(struct mlxsw_sp_fib_node *fib_node,
2486 struct mlxsw_sp_fib_entry *new_entry,
2487 bool replace, bool append)
2489 struct mlxsw_sp_fib_entry *fib_entry;
2491 fib_entry = mlxsw_sp_fib4_node_entry_find(fib_node, &new_entry->params);
2494 return mlxsw_sp_fib4_node_list_append(fib_entry, new_entry);
2495 if (replace && WARN_ON(!fib_entry))
2498 /* Insert new entry before replaced one, so that we can later
2499 * remove the second.
2502 list_add_tail(&new_entry->list, &fib_entry->list);
2504 struct mlxsw_sp_fib_entry *last;
2506 list_for_each_entry(last, &fib_node->entry_list, list) {
2507 if (new_entry->params.tb_id > last->params.tb_id)
2513 list_add(&new_entry->list, &fib_entry->list);
2515 list_add(&new_entry->list, &fib_node->entry_list);
2522 mlxsw_sp_fib4_node_list_remove(struct mlxsw_sp_fib_entry *fib_entry)
2524 list_del(&fib_entry->list);
2528 mlxsw_sp_fib4_node_entry_add(struct mlxsw_sp *mlxsw_sp,
2529 const struct mlxsw_sp_fib_node *fib_node,
2530 struct mlxsw_sp_fib_entry *fib_entry)
2532 if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry))
2535 /* To prevent packet loss, overwrite the previously offloaded
2538 if (!list_is_singular(&fib_node->entry_list)) {
2539 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_DELETE;
2540 struct mlxsw_sp_fib_entry *n = list_next_entry(fib_entry, list);
2542 mlxsw_sp_fib_entry_offload_refresh(n, op, 0);
2545 return mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
2549 mlxsw_sp_fib4_node_entry_del(struct mlxsw_sp *mlxsw_sp,
2550 const struct mlxsw_sp_fib_node *fib_node,
2551 struct mlxsw_sp_fib_entry *fib_entry)
2553 if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry))
2556 /* Promote the next entry by overwriting the deleted entry */
2557 if (!list_is_singular(&fib_node->entry_list)) {
2558 struct mlxsw_sp_fib_entry *n = list_next_entry(fib_entry, list);
2559 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_DELETE;
2561 mlxsw_sp_fib_entry_update(mlxsw_sp, n);
2562 mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, 0);
2566 mlxsw_sp_fib_entry_del(mlxsw_sp, fib_entry);
2569 static int mlxsw_sp_fib4_node_entry_link(struct mlxsw_sp *mlxsw_sp,
2570 struct mlxsw_sp_fib_entry *fib_entry,
2571 bool replace, bool append)
2573 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
2576 err = mlxsw_sp_fib4_node_list_insert(fib_node, fib_entry, replace,
2581 err = mlxsw_sp_fib4_node_entry_add(mlxsw_sp, fib_node, fib_entry);
2583 goto err_fib4_node_entry_add;
2587 err_fib4_node_entry_add:
2588 mlxsw_sp_fib4_node_list_remove(fib_entry);
2593 mlxsw_sp_fib4_node_entry_unlink(struct mlxsw_sp *mlxsw_sp,
2594 struct mlxsw_sp_fib_entry *fib_entry)
2596 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
2598 mlxsw_sp_fib4_node_entry_del(mlxsw_sp, fib_node, fib_entry);
2599 mlxsw_sp_fib4_node_list_remove(fib_entry);
2602 static void mlxsw_sp_fib4_entry_replace(struct mlxsw_sp *mlxsw_sp,
2603 struct mlxsw_sp_fib_entry *fib_entry,
2606 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
2607 struct mlxsw_sp_fib_entry *replaced;
2612 /* We inserted the new entry before replaced one */
2613 replaced = list_next_entry(fib_entry, list);
2615 mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, replaced);
2616 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, replaced);
2617 mlxsw_sp_fib4_node_put(mlxsw_sp, fib_node);
2621 mlxsw_sp_router_fib4_add(struct mlxsw_sp *mlxsw_sp,
2622 const struct fib_entry_notifier_info *fen_info,
2623 bool replace, bool append)
2625 struct mlxsw_sp_fib_entry *fib_entry;
2626 struct mlxsw_sp_fib_node *fib_node;
2629 if (mlxsw_sp->router->aborted)
2632 fib_node = mlxsw_sp_fib4_node_get(mlxsw_sp, fen_info);
2633 if (IS_ERR(fib_node)) {
2634 dev_warn(mlxsw_sp->bus_info->dev, "Failed to get FIB node\n");
2635 return PTR_ERR(fib_node);
2638 fib_entry = mlxsw_sp_fib4_entry_create(mlxsw_sp, fib_node, fen_info);
2639 if (IS_ERR(fib_entry)) {
2640 dev_warn(mlxsw_sp->bus_info->dev, "Failed to create FIB entry\n");
2641 err = PTR_ERR(fib_entry);
2642 goto err_fib4_entry_create;
2645 err = mlxsw_sp_fib4_node_entry_link(mlxsw_sp, fib_entry, replace,
2648 dev_warn(mlxsw_sp->bus_info->dev, "Failed to link FIB entry to node\n");
2649 goto err_fib4_node_entry_link;
2652 mlxsw_sp_fib4_entry_replace(mlxsw_sp, fib_entry, replace);
2656 err_fib4_node_entry_link:
2657 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib_entry);
2658 err_fib4_entry_create:
2659 mlxsw_sp_fib4_node_put(mlxsw_sp, fib_node);
2663 static void mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp,
2664 struct fib_entry_notifier_info *fen_info)
2666 struct mlxsw_sp_fib_entry *fib_entry;
2667 struct mlxsw_sp_fib_node *fib_node;
2669 if (mlxsw_sp->router->aborted)
2672 fib_entry = mlxsw_sp_fib4_entry_lookup(mlxsw_sp, fen_info);
2673 if (WARN_ON(!fib_entry))
2675 fib_node = fib_entry->fib_node;
2677 mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib_entry);
2678 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib_entry);
2679 mlxsw_sp_fib4_node_put(mlxsw_sp, fib_node);
2682 static int mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp)
2684 char ralta_pl[MLXSW_REG_RALTA_LEN];
2685 char ralst_pl[MLXSW_REG_RALST_LEN];
2688 mlxsw_reg_ralta_pack(ralta_pl, true, MLXSW_REG_RALXX_PROTOCOL_IPV4,
2689 MLXSW_SP_LPM_TREE_MIN);
2690 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
2694 mlxsw_reg_ralst_pack(ralst_pl, 0xff, MLXSW_SP_LPM_TREE_MIN);
2695 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
2699 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
2700 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
2701 char raltb_pl[MLXSW_REG_RALTB_LEN];
2702 char ralue_pl[MLXSW_REG_RALUE_LEN];
2704 if (!mlxsw_sp_vr_is_used(vr))
2707 mlxsw_reg_raltb_pack(raltb_pl, vr->id,
2708 MLXSW_REG_RALXX_PROTOCOL_IPV4,
2709 MLXSW_SP_LPM_TREE_MIN);
2710 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb),
2715 mlxsw_reg_ralue_pack4(ralue_pl, MLXSW_SP_L3_PROTO_IPV4,
2716 MLXSW_REG_RALUE_OP_WRITE_WRITE, vr->id, 0,
2718 mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
2719 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue),
2728 static void mlxsw_sp_fib4_node_flush(struct mlxsw_sp *mlxsw_sp,
2729 struct mlxsw_sp_fib_node *fib_node)
2731 struct mlxsw_sp_fib_entry *fib_entry, *tmp;
2733 list_for_each_entry_safe(fib_entry, tmp, &fib_node->entry_list, list) {
2734 bool do_break = &tmp->list == &fib_node->entry_list;
2736 mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib_entry);
2737 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib_entry);
2738 mlxsw_sp_fib4_node_put(mlxsw_sp, fib_node);
2739 /* Break when entry list is empty and node was freed.
2740 * Otherwise, we'll access freed memory in the next
2748 static void mlxsw_sp_fib_node_flush(struct mlxsw_sp *mlxsw_sp,
2749 struct mlxsw_sp_fib_node *fib_node)
2751 switch (fib_node->fib->proto) {
2752 case MLXSW_SP_L3_PROTO_IPV4:
2753 mlxsw_sp_fib4_node_flush(mlxsw_sp, fib_node);
2755 case MLXSW_SP_L3_PROTO_IPV6:
2761 static void mlxsw_sp_vr_fib_flush(struct mlxsw_sp *mlxsw_sp,
2762 struct mlxsw_sp_vr *vr,
2763 enum mlxsw_sp_l3proto proto)
2765 struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
2766 struct mlxsw_sp_fib_node *fib_node, *tmp;
2768 list_for_each_entry_safe(fib_node, tmp, &fib->node_list, list) {
2769 bool do_break = &tmp->list == &fib->node_list;
2771 mlxsw_sp_fib_node_flush(mlxsw_sp, fib_node);
2777 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp)
2781 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
2782 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
2784 if (!mlxsw_sp_vr_is_used(vr))
2786 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4);
2790 static void mlxsw_sp_router_fib4_abort(struct mlxsw_sp *mlxsw_sp)
2794 if (mlxsw_sp->router->aborted)
2796 dev_warn(mlxsw_sp->bus_info->dev, "FIB abort triggered. Note that FIB entries are no longer being offloaded to this device.\n");
2797 mlxsw_sp_router_fib_flush(mlxsw_sp);
2798 mlxsw_sp->router->aborted = true;
2799 err = mlxsw_sp_router_set_abort_trap(mlxsw_sp);
2801 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set abort trap.\n");
2804 struct mlxsw_sp_fib_event_work {
2805 struct work_struct work;
2807 struct fib_entry_notifier_info fen_info;
2808 struct fib_rule_notifier_info fr_info;
2809 struct fib_nh_notifier_info fnh_info;
2811 struct mlxsw_sp *mlxsw_sp;
2812 unsigned long event;
2815 static void mlxsw_sp_router_fib_event_work(struct work_struct *work)
2817 struct mlxsw_sp_fib_event_work *fib_work =
2818 container_of(work, struct mlxsw_sp_fib_event_work, work);
2819 struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp;
2820 struct fib_rule *rule;
2821 bool replace, append;
2824 /* Protect internal structures from changes */
2826 switch (fib_work->event) {
2827 case FIB_EVENT_ENTRY_REPLACE: /* fall through */
2828 case FIB_EVENT_ENTRY_APPEND: /* fall through */
2829 case FIB_EVENT_ENTRY_ADD:
2830 replace = fib_work->event == FIB_EVENT_ENTRY_REPLACE;
2831 append = fib_work->event == FIB_EVENT_ENTRY_APPEND;
2832 err = mlxsw_sp_router_fib4_add(mlxsw_sp, &fib_work->fen_info,
2835 mlxsw_sp_router_fib4_abort(mlxsw_sp);
2836 fib_info_put(fib_work->fen_info.fi);
2838 case FIB_EVENT_ENTRY_DEL:
2839 mlxsw_sp_router_fib4_del(mlxsw_sp, &fib_work->fen_info);
2840 fib_info_put(fib_work->fen_info.fi);
2842 case FIB_EVENT_RULE_ADD: /* fall through */
2843 case FIB_EVENT_RULE_DEL:
2844 rule = fib_work->fr_info.rule;
2845 if (!fib4_rule_default(rule) && !rule->l3mdev)
2846 mlxsw_sp_router_fib4_abort(mlxsw_sp);
2849 case FIB_EVENT_NH_ADD: /* fall through */
2850 case FIB_EVENT_NH_DEL:
2851 mlxsw_sp_nexthop_event(mlxsw_sp, fib_work->event,
2852 fib_work->fnh_info.fib_nh);
2853 fib_info_put(fib_work->fnh_info.fib_nh->nh_parent);
2860 /* Called with rcu_read_lock() */
2861 static int mlxsw_sp_router_fib_event(struct notifier_block *nb,
2862 unsigned long event, void *ptr)
2864 struct mlxsw_sp_fib_event_work *fib_work;
2865 struct fib_notifier_info *info = ptr;
2866 struct mlxsw_sp_router *router;
2868 if (!net_eq(info->net, &init_net))
2871 fib_work = kzalloc(sizeof(*fib_work), GFP_ATOMIC);
2872 if (WARN_ON(!fib_work))
2875 INIT_WORK(&fib_work->work, mlxsw_sp_router_fib_event_work);
2876 router = container_of(nb, struct mlxsw_sp_router, fib_nb);
2877 fib_work->mlxsw_sp = router->mlxsw_sp;
2878 fib_work->event = event;
2881 case FIB_EVENT_ENTRY_REPLACE: /* fall through */
2882 case FIB_EVENT_ENTRY_APPEND: /* fall through */
2883 case FIB_EVENT_ENTRY_ADD: /* fall through */
2884 case FIB_EVENT_ENTRY_DEL:
2885 memcpy(&fib_work->fen_info, ptr, sizeof(fib_work->fen_info));
2886 /* Take referece on fib_info to prevent it from being
2887 * freed while work is queued. Release it afterwards.
2889 fib_info_hold(fib_work->fen_info.fi);
2891 case FIB_EVENT_RULE_ADD: /* fall through */
2892 case FIB_EVENT_RULE_DEL:
2893 memcpy(&fib_work->fr_info, ptr, sizeof(fib_work->fr_info));
2894 fib_rule_get(fib_work->fr_info.rule);
2896 case FIB_EVENT_NH_ADD: /* fall through */
2897 case FIB_EVENT_NH_DEL:
2898 memcpy(&fib_work->fnh_info, ptr, sizeof(fib_work->fnh_info));
2899 fib_info_hold(fib_work->fnh_info.fib_nh->nh_parent);
2903 mlxsw_core_schedule_work(&fib_work->work);
2908 static struct mlxsw_sp_rif *
2909 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
2910 const struct net_device *dev)
2914 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
2915 if (mlxsw_sp->router->rifs[i] &&
2916 mlxsw_sp->router->rifs[i]->dev == dev)
2917 return mlxsw_sp->router->rifs[i];
2922 static int mlxsw_sp_router_rif_disable(struct mlxsw_sp *mlxsw_sp, u16 rif)
2924 char ritr_pl[MLXSW_REG_RITR_LEN];
2927 mlxsw_reg_ritr_rif_pack(ritr_pl, rif);
2928 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
2929 if (WARN_ON_ONCE(err))
2932 mlxsw_reg_ritr_enable_set(ritr_pl, false);
2933 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
2936 static void mlxsw_sp_router_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
2937 struct mlxsw_sp_rif *rif)
2939 mlxsw_sp_router_rif_disable(mlxsw_sp, rif->rif_index);
2940 mlxsw_sp_nexthop_rif_gone_sync(mlxsw_sp, rif);
2941 mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif);
2944 static bool mlxsw_sp_rif_should_config(struct mlxsw_sp_rif *rif,
2945 const struct in_device *in_dev,
2946 unsigned long event)
2954 if (rif && !in_dev->ifa_list &&
2955 !netif_is_l3_slave(rif->dev))
2957 /* It is possible we already removed the RIF ourselves
2958 * if it was assigned to a netdev that is now a bridge
2967 static enum mlxsw_sp_rif_type
2968 mlxsw_sp_dev_rif_type(const struct mlxsw_sp *mlxsw_sp,
2969 const struct net_device *dev)
2971 enum mlxsw_sp_fid_type type;
2973 /* RIF type is derived from the type of the underlying FID */
2974 if (is_vlan_dev(dev) && netif_is_bridge_master(vlan_dev_real_dev(dev)))
2975 type = MLXSW_SP_FID_TYPE_8021Q;
2976 else if (netif_is_bridge_master(dev) && br_vlan_enabled(dev))
2977 type = MLXSW_SP_FID_TYPE_8021Q;
2978 else if (netif_is_bridge_master(dev))
2979 type = MLXSW_SP_FID_TYPE_8021D;
2981 type = MLXSW_SP_FID_TYPE_RFID;
2983 return mlxsw_sp_fid_type_rif_type(mlxsw_sp, type);
2986 static int mlxsw_sp_rif_index_alloc(struct mlxsw_sp *mlxsw_sp, u16 *p_rif_index)
2990 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) {
2991 if (!mlxsw_sp->router->rifs[i]) {
3000 static struct mlxsw_sp_rif *mlxsw_sp_rif_alloc(size_t rif_size, u16 rif_index,
3002 struct net_device *l3_dev)
3004 struct mlxsw_sp_rif *rif;
3006 rif = kzalloc(rif_size, GFP_KERNEL);
3010 INIT_LIST_HEAD(&rif->nexthop_list);
3011 INIT_LIST_HEAD(&rif->neigh_list);
3012 ether_addr_copy(rif->addr, l3_dev->dev_addr);
3013 rif->mtu = l3_dev->mtu;
3016 rif->rif_index = rif_index;
3021 struct mlxsw_sp_rif *mlxsw_sp_rif_by_index(const struct mlxsw_sp *mlxsw_sp,
3024 return mlxsw_sp->router->rifs[rif_index];
3027 u16 mlxsw_sp_rif_index(const struct mlxsw_sp_rif *rif)
3029 return rif->rif_index;
3032 int mlxsw_sp_rif_dev_ifindex(const struct mlxsw_sp_rif *rif)
3034 return rif->dev->ifindex;
3037 static struct mlxsw_sp_rif *
3038 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp,
3039 const struct mlxsw_sp_rif_params *params)
3041 u32 tb_id = l3mdev_fib_table(params->dev);
3042 const struct mlxsw_sp_rif_ops *ops;
3043 enum mlxsw_sp_rif_type type;
3044 struct mlxsw_sp_rif *rif;
3045 struct mlxsw_sp_fid *fid;
3046 struct mlxsw_sp_vr *vr;
3050 type = mlxsw_sp_dev_rif_type(mlxsw_sp, params->dev);
3051 ops = mlxsw_sp->router->rif_ops_arr[type];
3053 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id ? : RT_TABLE_MAIN);
3055 return ERR_CAST(vr);
3057 err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index);
3059 goto err_rif_index_alloc;
3061 rif = mlxsw_sp_rif_alloc(ops->rif_size, rif_index, vr->id, params->dev);
3066 rif->mlxsw_sp = mlxsw_sp;
3069 fid = ops->fid_get(rif);
3077 ops->setup(rif, params);
3079 err = ops->configure(rif);
3083 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, params->dev->dev_addr,
3084 mlxsw_sp_fid_index(fid), true);
3086 goto err_rif_fdb_op;
3088 mlxsw_sp_rif_counters_alloc(rif);
3089 mlxsw_sp_fid_rif_set(fid, rif);
3090 mlxsw_sp->router->rifs[rif_index] = rif;
3096 ops->deconfigure(rif);
3098 mlxsw_sp_fid_put(fid);
3102 err_rif_index_alloc:
3103 mlxsw_sp_vr_put(vr);
3104 return ERR_PTR(err);
3107 void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif)
3109 const struct mlxsw_sp_rif_ops *ops = rif->ops;
3110 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
3111 struct mlxsw_sp_fid *fid = rif->fid;
3112 struct mlxsw_sp_vr *vr;
3114 mlxsw_sp_router_rif_gone_sync(mlxsw_sp, rif);
3115 vr = &mlxsw_sp->router->vrs[rif->vr_id];
3118 mlxsw_sp->router->rifs[rif->rif_index] = NULL;
3119 mlxsw_sp_fid_rif_set(fid, NULL);
3120 mlxsw_sp_rif_counters_free(rif);
3121 mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->dev->dev_addr,
3122 mlxsw_sp_fid_index(fid), false);
3123 ops->deconfigure(rif);
3124 mlxsw_sp_fid_put(fid);
3126 mlxsw_sp_vr_put(vr);
3130 mlxsw_sp_rif_subport_params_init(struct mlxsw_sp_rif_params *params,
3131 struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
3133 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
3135 params->vid = mlxsw_sp_port_vlan->vid;
3136 params->lag = mlxsw_sp_port->lagged;
3138 params->lag_id = mlxsw_sp_port->lag_id;
3140 params->system_port = mlxsw_sp_port->local_port;
3144 mlxsw_sp_port_vlan_router_join(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan,
3145 struct net_device *l3_dev)
3147 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
3148 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
3149 u16 vid = mlxsw_sp_port_vlan->vid;
3150 struct mlxsw_sp_rif *rif;
3151 struct mlxsw_sp_fid *fid;
3154 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
3156 struct mlxsw_sp_rif_params params = {
3160 mlxsw_sp_rif_subport_params_init(¶ms, mlxsw_sp_port_vlan);
3161 rif = mlxsw_sp_rif_create(mlxsw_sp, ¶ms);
3163 return PTR_ERR(rif);
3166 /* FID was already created, just take a reference */
3167 fid = rif->ops->fid_get(rif);
3168 err = mlxsw_sp_fid_port_vid_map(fid, mlxsw_sp_port, vid);
3170 goto err_fid_port_vid_map;
3172 err = mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, false);
3174 goto err_port_vid_learning_set;
3176 err = mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid,
3177 BR_STATE_FORWARDING);
3179 goto err_port_vid_stp_set;
3181 mlxsw_sp_port_vlan->fid = fid;
3185 err_port_vid_stp_set:
3186 mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true);
3187 err_port_vid_learning_set:
3188 mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid);
3189 err_fid_port_vid_map:
3190 mlxsw_sp_fid_put(fid);
3195 mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
3197 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
3198 struct mlxsw_sp_fid *fid = mlxsw_sp_port_vlan->fid;
3199 u16 vid = mlxsw_sp_port_vlan->vid;
3201 if (WARN_ON(mlxsw_sp_fid_type(fid) != MLXSW_SP_FID_TYPE_RFID))
3204 mlxsw_sp_port_vlan->fid = NULL;
3205 mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid, BR_STATE_BLOCKING);
3206 mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true);
3207 mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid);
3208 /* If router port holds the last reference on the rFID, then the
3209 * associated Sub-port RIF will be destroyed.
3211 mlxsw_sp_fid_put(fid);
3214 static int mlxsw_sp_inetaddr_port_vlan_event(struct net_device *l3_dev,
3215 struct net_device *port_dev,
3216 unsigned long event, u16 vid)
3218 struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(port_dev);
3219 struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan;
3221 mlxsw_sp_port_vlan = mlxsw_sp_port_vlan_find_by_vid(mlxsw_sp_port, vid);
3222 if (WARN_ON(!mlxsw_sp_port_vlan))
3227 return mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan,
3230 mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan);
3237 static int mlxsw_sp_inetaddr_port_event(struct net_device *port_dev,
3238 unsigned long event)
3240 if (netif_is_bridge_port(port_dev) ||
3241 netif_is_lag_port(port_dev) ||
3242 netif_is_ovs_port(port_dev))
3245 return mlxsw_sp_inetaddr_port_vlan_event(port_dev, port_dev, event, 1);
3248 static int __mlxsw_sp_inetaddr_lag_event(struct net_device *l3_dev,
3249 struct net_device *lag_dev,
3250 unsigned long event, u16 vid)
3252 struct net_device *port_dev;
3253 struct list_head *iter;
3256 netdev_for_each_lower_dev(lag_dev, port_dev, iter) {
3257 if (mlxsw_sp_port_dev_check(port_dev)) {
3258 err = mlxsw_sp_inetaddr_port_vlan_event(l3_dev,
3269 static int mlxsw_sp_inetaddr_lag_event(struct net_device *lag_dev,
3270 unsigned long event)
3272 if (netif_is_bridge_port(lag_dev))
3275 return __mlxsw_sp_inetaddr_lag_event(lag_dev, lag_dev, event, 1);
3278 static int mlxsw_sp_inetaddr_bridge_event(struct net_device *l3_dev,
3279 unsigned long event)
3281 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev);
3282 struct mlxsw_sp_rif_params params = {
3285 struct mlxsw_sp_rif *rif;
3289 rif = mlxsw_sp_rif_create(mlxsw_sp, ¶ms);
3291 return PTR_ERR(rif);
3294 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
3295 mlxsw_sp_rif_destroy(rif);
3302 static int mlxsw_sp_inetaddr_vlan_event(struct net_device *vlan_dev,
3303 unsigned long event)
3305 struct net_device *real_dev = vlan_dev_real_dev(vlan_dev);
3306 u16 vid = vlan_dev_vlan_id(vlan_dev);
3308 if (netif_is_bridge_port(vlan_dev))
3311 if (mlxsw_sp_port_dev_check(real_dev))
3312 return mlxsw_sp_inetaddr_port_vlan_event(vlan_dev, real_dev,
3314 else if (netif_is_lag_master(real_dev))
3315 return __mlxsw_sp_inetaddr_lag_event(vlan_dev, real_dev, event,
3317 else if (netif_is_bridge_master(real_dev) && br_vlan_enabled(real_dev))
3318 return mlxsw_sp_inetaddr_bridge_event(vlan_dev, event);
3323 static int __mlxsw_sp_inetaddr_event(struct net_device *dev,
3324 unsigned long event)
3326 if (mlxsw_sp_port_dev_check(dev))
3327 return mlxsw_sp_inetaddr_port_event(dev, event);
3328 else if (netif_is_lag_master(dev))
3329 return mlxsw_sp_inetaddr_lag_event(dev, event);
3330 else if (netif_is_bridge_master(dev))
3331 return mlxsw_sp_inetaddr_bridge_event(dev, event);
3332 else if (is_vlan_dev(dev))
3333 return mlxsw_sp_inetaddr_vlan_event(dev, event);
3338 int mlxsw_sp_inetaddr_event(struct notifier_block *unused,
3339 unsigned long event, void *ptr)
3341 struct in_ifaddr *ifa = (struct in_ifaddr *) ptr;
3342 struct net_device *dev = ifa->ifa_dev->dev;
3343 struct mlxsw_sp *mlxsw_sp;
3344 struct mlxsw_sp_rif *rif;
3347 mlxsw_sp = mlxsw_sp_lower_get(dev);
3351 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
3352 if (!mlxsw_sp_rif_should_config(rif, ifa->ifa_dev, event))
3355 err = __mlxsw_sp_inetaddr_event(dev, event);
3357 return notifier_from_errno(err);
3360 static int mlxsw_sp_rif_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
3361 const char *mac, int mtu)
3363 char ritr_pl[MLXSW_REG_RITR_LEN];
3366 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
3367 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
3371 mlxsw_reg_ritr_mtu_set(ritr_pl, mtu);
3372 mlxsw_reg_ritr_if_mac_memcpy_to(ritr_pl, mac);
3373 mlxsw_reg_ritr_op_set(ritr_pl, MLXSW_REG_RITR_RIF_CREATE);
3374 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
3377 int mlxsw_sp_netdevice_router_port_event(struct net_device *dev)
3379 struct mlxsw_sp *mlxsw_sp;
3380 struct mlxsw_sp_rif *rif;
3384 mlxsw_sp = mlxsw_sp_lower_get(dev);
3388 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
3391 fid_index = mlxsw_sp_fid_index(rif->fid);
3393 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, false);
3397 err = mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, dev->dev_addr,
3402 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, dev->dev_addr, fid_index, true);
3404 goto err_rif_fdb_op;
3406 ether_addr_copy(rif->addr, dev->dev_addr);
3407 rif->mtu = dev->mtu;
3409 netdev_dbg(dev, "Updated RIF=%d\n", rif->rif_index);
3414 mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, rif->addr, rif->mtu);
3416 mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, true);
3420 static int mlxsw_sp_port_vrf_join(struct mlxsw_sp *mlxsw_sp,
3421 struct net_device *l3_dev)
3423 struct mlxsw_sp_rif *rif;
3425 /* If netdev is already associated with a RIF, then we need to
3426 * destroy it and create a new one with the new virtual router ID.
3428 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
3430 __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_DOWN);
3432 return __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_UP);
3435 static void mlxsw_sp_port_vrf_leave(struct mlxsw_sp *mlxsw_sp,
3436 struct net_device *l3_dev)
3438 struct mlxsw_sp_rif *rif;
3440 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
3443 __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_DOWN);
3446 int mlxsw_sp_netdevice_vrf_event(struct net_device *l3_dev, unsigned long event,
3447 struct netdev_notifier_changeupper_info *info)
3449 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev);
3456 case NETDEV_PRECHANGEUPPER:
3458 case NETDEV_CHANGEUPPER:
3460 err = mlxsw_sp_port_vrf_join(mlxsw_sp, l3_dev);
3462 mlxsw_sp_port_vrf_leave(mlxsw_sp, l3_dev);
3469 static struct mlxsw_sp_rif_subport *
3470 mlxsw_sp_rif_subport_rif(const struct mlxsw_sp_rif *rif)
3472 return container_of(rif, struct mlxsw_sp_rif_subport, common);
3475 static void mlxsw_sp_rif_subport_setup(struct mlxsw_sp_rif *rif,
3476 const struct mlxsw_sp_rif_params *params)
3478 struct mlxsw_sp_rif_subport *rif_subport;
3480 rif_subport = mlxsw_sp_rif_subport_rif(rif);
3481 rif_subport->vid = params->vid;
3482 rif_subport->lag = params->lag;
3484 rif_subport->lag_id = params->lag_id;
3486 rif_subport->system_port = params->system_port;
3489 static int mlxsw_sp_rif_subport_op(struct mlxsw_sp_rif *rif, bool enable)
3491 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
3492 struct mlxsw_sp_rif_subport *rif_subport;
3493 char ritr_pl[MLXSW_REG_RITR_LEN];
3495 rif_subport = mlxsw_sp_rif_subport_rif(rif);
3496 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_SP_IF,
3497 rif->rif_index, rif->vr_id, rif->dev->mtu,
3498 rif->dev->dev_addr);
3499 mlxsw_reg_ritr_sp_if_pack(ritr_pl, rif_subport->lag,
3500 rif_subport->lag ? rif_subport->lag_id :
3501 rif_subport->system_port,
3504 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
3507 static int mlxsw_sp_rif_subport_configure(struct mlxsw_sp_rif *rif)
3509 return mlxsw_sp_rif_subport_op(rif, true);
3512 static void mlxsw_sp_rif_subport_deconfigure(struct mlxsw_sp_rif *rif)
3514 mlxsw_sp_rif_subport_op(rif, false);
3517 static struct mlxsw_sp_fid *
3518 mlxsw_sp_rif_subport_fid_get(struct mlxsw_sp_rif *rif)
3520 return mlxsw_sp_fid_rfid_get(rif->mlxsw_sp, rif->rif_index);
3523 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_subport_ops = {
3524 .type = MLXSW_SP_RIF_TYPE_SUBPORT,
3525 .rif_size = sizeof(struct mlxsw_sp_rif_subport),
3526 .setup = mlxsw_sp_rif_subport_setup,
3527 .configure = mlxsw_sp_rif_subport_configure,
3528 .deconfigure = mlxsw_sp_rif_subport_deconfigure,
3529 .fid_get = mlxsw_sp_rif_subport_fid_get,
3532 static int mlxsw_sp_rif_vlan_fid_op(struct mlxsw_sp_rif *rif,
3533 enum mlxsw_reg_ritr_if_type type,
3534 u16 vid_fid, bool enable)
3536 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
3537 char ritr_pl[MLXSW_REG_RITR_LEN];
3539 mlxsw_reg_ritr_pack(ritr_pl, enable, type, rif->rif_index, rif->vr_id,
3540 rif->dev->mtu, rif->dev->dev_addr);
3541 mlxsw_reg_ritr_fid_set(ritr_pl, type, vid_fid);
3543 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
3546 static u8 mlxsw_sp_router_port(const struct mlxsw_sp *mlxsw_sp)
3548 return mlxsw_core_max_ports(mlxsw_sp->core) + 1;
3551 static int mlxsw_sp_rif_vlan_configure(struct mlxsw_sp_rif *rif)
3553 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
3554 u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid);
3557 err = mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, true);
3561 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
3562 mlxsw_sp_router_port(mlxsw_sp), true);
3564 goto err_fid_bc_flood_set;
3568 err_fid_bc_flood_set:
3569 mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, false);
3573 static void mlxsw_sp_rif_vlan_deconfigure(struct mlxsw_sp_rif *rif)
3575 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
3576 u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid);
3578 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
3579 mlxsw_sp_router_port(mlxsw_sp), false);
3580 mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, false);
3583 static struct mlxsw_sp_fid *
3584 mlxsw_sp_rif_vlan_fid_get(struct mlxsw_sp_rif *rif)
3586 u16 vid = is_vlan_dev(rif->dev) ? vlan_dev_vlan_id(rif->dev) : 1;
3588 return mlxsw_sp_fid_8021q_get(rif->mlxsw_sp, vid);
3591 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_vlan_ops = {
3592 .type = MLXSW_SP_RIF_TYPE_VLAN,
3593 .rif_size = sizeof(struct mlxsw_sp_rif),
3594 .configure = mlxsw_sp_rif_vlan_configure,
3595 .deconfigure = mlxsw_sp_rif_vlan_deconfigure,
3596 .fid_get = mlxsw_sp_rif_vlan_fid_get,
3599 static int mlxsw_sp_rif_fid_configure(struct mlxsw_sp_rif *rif)
3601 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
3602 u16 fid_index = mlxsw_sp_fid_index(rif->fid);
3605 err = mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index,
3610 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
3611 mlxsw_sp_router_port(mlxsw_sp), true);
3613 goto err_fid_bc_flood_set;
3617 err_fid_bc_flood_set:
3618 mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false);
3622 static void mlxsw_sp_rif_fid_deconfigure(struct mlxsw_sp_rif *rif)
3624 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
3625 u16 fid_index = mlxsw_sp_fid_index(rif->fid);
3627 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
3628 mlxsw_sp_router_port(mlxsw_sp), false);
3629 mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false);
3632 static struct mlxsw_sp_fid *
3633 mlxsw_sp_rif_fid_fid_get(struct mlxsw_sp_rif *rif)
3635 return mlxsw_sp_fid_8021d_get(rif->mlxsw_sp, rif->dev->ifindex);
3638 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_fid_ops = {
3639 .type = MLXSW_SP_RIF_TYPE_FID,
3640 .rif_size = sizeof(struct mlxsw_sp_rif),
3641 .configure = mlxsw_sp_rif_fid_configure,
3642 .deconfigure = mlxsw_sp_rif_fid_deconfigure,
3643 .fid_get = mlxsw_sp_rif_fid_fid_get,
3646 static const struct mlxsw_sp_rif_ops *mlxsw_sp_rif_ops_arr[] = {
3647 [MLXSW_SP_RIF_TYPE_SUBPORT] = &mlxsw_sp_rif_subport_ops,
3648 [MLXSW_SP_RIF_TYPE_VLAN] = &mlxsw_sp_rif_vlan_ops,
3649 [MLXSW_SP_RIF_TYPE_FID] = &mlxsw_sp_rif_fid_ops,
3652 static int mlxsw_sp_rifs_init(struct mlxsw_sp *mlxsw_sp)
3654 u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
3656 mlxsw_sp->router->rifs = kcalloc(max_rifs,
3657 sizeof(struct mlxsw_sp_rif *),
3659 if (!mlxsw_sp->router->rifs)
3662 mlxsw_sp->router->rif_ops_arr = mlxsw_sp_rif_ops_arr;
3667 static void mlxsw_sp_rifs_fini(struct mlxsw_sp *mlxsw_sp)
3671 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
3672 WARN_ON_ONCE(mlxsw_sp->router->rifs[i]);
3674 kfree(mlxsw_sp->router->rifs);
3677 static void mlxsw_sp_router_fib_dump_flush(struct notifier_block *nb)
3679 struct mlxsw_sp_router *router;
3681 /* Flush pending FIB notifications and then flush the device's
3682 * table before requesting another dump. The FIB notification
3683 * block is unregistered, so no need to take RTNL.
3685 mlxsw_core_flush_owq();
3686 router = container_of(nb, struct mlxsw_sp_router, fib_nb);
3687 mlxsw_sp_router_fib_flush(router->mlxsw_sp);
3690 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
3692 char rgcr_pl[MLXSW_REG_RGCR_LEN];
3696 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_RIFS))
3698 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
3700 mlxsw_reg_rgcr_pack(rgcr_pl, true);
3701 mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, max_rifs);
3702 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
3708 static void __mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
3710 char rgcr_pl[MLXSW_REG_RGCR_LEN];
3712 mlxsw_reg_rgcr_pack(rgcr_pl, false);
3713 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
3716 int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
3718 struct mlxsw_sp_router *router;
3721 router = kzalloc(sizeof(*mlxsw_sp->router), GFP_KERNEL);
3724 mlxsw_sp->router = router;
3725 router->mlxsw_sp = mlxsw_sp;
3727 INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_neighs_list);
3728 err = __mlxsw_sp_router_init(mlxsw_sp);
3730 goto err_router_init;
3732 err = mlxsw_sp_rifs_init(mlxsw_sp);
3736 err = rhashtable_init(&mlxsw_sp->router->nexthop_ht,
3737 &mlxsw_sp_nexthop_ht_params);
3739 goto err_nexthop_ht_init;
3741 err = rhashtable_init(&mlxsw_sp->router->nexthop_group_ht,
3742 &mlxsw_sp_nexthop_group_ht_params);
3744 goto err_nexthop_group_ht_init;
3746 err = mlxsw_sp_lpm_init(mlxsw_sp);
3750 err = mlxsw_sp_vrs_init(mlxsw_sp);
3754 err = mlxsw_sp_neigh_init(mlxsw_sp);
3756 goto err_neigh_init;
3758 mlxsw_sp->router->fib_nb.notifier_call = mlxsw_sp_router_fib_event;
3759 err = register_fib_notifier(&mlxsw_sp->router->fib_nb,
3760 mlxsw_sp_router_fib_dump_flush);
3762 goto err_register_fib_notifier;
3766 err_register_fib_notifier:
3767 mlxsw_sp_neigh_fini(mlxsw_sp);
3769 mlxsw_sp_vrs_fini(mlxsw_sp);
3771 mlxsw_sp_lpm_fini(mlxsw_sp);
3773 rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht);
3774 err_nexthop_group_ht_init:
3775 rhashtable_destroy(&mlxsw_sp->router->nexthop_ht);
3776 err_nexthop_ht_init:
3777 mlxsw_sp_rifs_fini(mlxsw_sp);
3779 __mlxsw_sp_router_fini(mlxsw_sp);
3781 kfree(mlxsw_sp->router);
3785 void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
3787 unregister_fib_notifier(&mlxsw_sp->router->fib_nb);
3788 mlxsw_sp_neigh_fini(mlxsw_sp);
3789 mlxsw_sp_vrs_fini(mlxsw_sp);
3790 mlxsw_sp_lpm_fini(mlxsw_sp);
3791 rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht);
3792 rhashtable_destroy(&mlxsw_sp->router->nexthop_ht);
3793 mlxsw_sp_rifs_fini(mlxsw_sp);
3794 __mlxsw_sp_router_fini(mlxsw_sp);
3795 kfree(mlxsw_sp->router);