2 * drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
3 * Copyright (c) 2016-2018 Mellanox Technologies. All rights reserved.
4 * Copyright (c) 2016 Jiri Pirko <jiri@mellanox.com>
5 * Copyright (c) 2016 Ido Schimmel <idosch@mellanox.com>
6 * Copyright (c) 2016 Yotam Gigi <yotamg@mellanox.com>
7 * Copyright (c) 2017-2018 Petr Machata <petrm@mellanox.com>
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 * 3. Neither the names of the copyright holders nor the names of its
18 * contributors may be used to endorse or promote products derived from
19 * this software without specific prior written permission.
21 * Alternatively, this software may be distributed under the terms of the
22 * GNU General Public License ("GPL") version 2 as published by the Free
23 * Software Foundation.
25 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
26 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
27 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
28 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
29 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
35 * POSSIBILITY OF SUCH DAMAGE.
38 #include <linux/kernel.h>
39 #include <linux/types.h>
40 #include <linux/rhashtable.h>
41 #include <linux/bitops.h>
42 #include <linux/in6.h>
43 #include <linux/notifier.h>
44 #include <linux/inetdevice.h>
45 #include <linux/netdevice.h>
46 #include <linux/if_bridge.h>
47 #include <linux/socket.h>
48 #include <linux/route.h>
49 #include <linux/gcd.h>
50 #include <linux/random.h>
51 #include <net/netevent.h>
52 #include <net/neighbour.h>
54 #include <net/ip_fib.h>
55 #include <net/ip6_fib.h>
56 #include <net/fib_rules.h>
57 #include <net/ip_tunnels.h>
58 #include <net/l3mdev.h>
59 #include <net/addrconf.h>
60 #include <net/ndisc.h>
62 #include <net/fib_notifier.h>
67 #include "spectrum_cnt.h"
68 #include "spectrum_dpipe.h"
69 #include "spectrum_ipip.h"
70 #include "spectrum_mr.h"
71 #include "spectrum_mr_tcam.h"
72 #include "spectrum_router.h"
73 #include "spectrum_span.h"
77 struct mlxsw_sp_lpm_tree;
78 struct mlxsw_sp_rif_ops;
80 struct mlxsw_sp_router {
81 struct mlxsw_sp *mlxsw_sp;
82 struct mlxsw_sp_rif **rifs;
83 struct mlxsw_sp_vr *vrs;
84 struct rhashtable neigh_ht;
85 struct rhashtable nexthop_group_ht;
86 struct rhashtable nexthop_ht;
87 struct list_head nexthop_list;
89 /* One tree for each protocol: IPv4 and IPv6 */
90 struct mlxsw_sp_lpm_tree *proto_trees[2];
91 struct mlxsw_sp_lpm_tree *trees;
92 unsigned int tree_count;
95 struct delayed_work dw;
96 unsigned long interval; /* ms */
98 struct delayed_work nexthop_probe_dw;
99 #define MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL 5000 /* ms */
100 struct list_head nexthop_neighs_list;
101 struct list_head ipip_list;
103 struct notifier_block fib_nb;
104 struct notifier_block netevent_nb;
105 const struct mlxsw_sp_rif_ops **rif_ops_arr;
106 const struct mlxsw_sp_ipip_ops **ipip_ops_arr;
109 struct mlxsw_sp_rif {
110 struct list_head nexthop_list;
111 struct list_head neigh_list;
112 struct net_device *dev;
113 struct mlxsw_sp_fid *fid;
114 unsigned char addr[ETH_ALEN];
118 const struct mlxsw_sp_rif_ops *ops;
119 struct mlxsw_sp *mlxsw_sp;
121 unsigned int counter_ingress;
122 bool counter_ingress_valid;
123 unsigned int counter_egress;
124 bool counter_egress_valid;
127 struct mlxsw_sp_rif_params {
128 struct net_device *dev;
137 struct mlxsw_sp_rif_subport {
138 struct mlxsw_sp_rif common;
147 struct mlxsw_sp_rif_ipip_lb {
148 struct mlxsw_sp_rif common;
149 struct mlxsw_sp_rif_ipip_lb_config lb_config;
150 u16 ul_vr_id; /* Reserved for Spectrum-2. */
153 struct mlxsw_sp_rif_params_ipip_lb {
154 struct mlxsw_sp_rif_params common;
155 struct mlxsw_sp_rif_ipip_lb_config lb_config;
158 struct mlxsw_sp_rif_ops {
159 enum mlxsw_sp_rif_type type;
162 void (*setup)(struct mlxsw_sp_rif *rif,
163 const struct mlxsw_sp_rif_params *params);
164 int (*configure)(struct mlxsw_sp_rif *rif);
165 void (*deconfigure)(struct mlxsw_sp_rif *rif);
166 struct mlxsw_sp_fid * (*fid_get)(struct mlxsw_sp_rif *rif);
169 static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree);
170 static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp,
171 struct mlxsw_sp_lpm_tree *lpm_tree);
172 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp,
173 const struct mlxsw_sp_fib *fib,
175 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp,
176 const struct mlxsw_sp_fib *fib);
178 static unsigned int *
179 mlxsw_sp_rif_p_counter_get(struct mlxsw_sp_rif *rif,
180 enum mlxsw_sp_rif_counter_dir dir)
183 case MLXSW_SP_RIF_COUNTER_EGRESS:
184 return &rif->counter_egress;
185 case MLXSW_SP_RIF_COUNTER_INGRESS:
186 return &rif->counter_ingress;
192 mlxsw_sp_rif_counter_valid_get(struct mlxsw_sp_rif *rif,
193 enum mlxsw_sp_rif_counter_dir dir)
196 case MLXSW_SP_RIF_COUNTER_EGRESS:
197 return rif->counter_egress_valid;
198 case MLXSW_SP_RIF_COUNTER_INGRESS:
199 return rif->counter_ingress_valid;
205 mlxsw_sp_rif_counter_valid_set(struct mlxsw_sp_rif *rif,
206 enum mlxsw_sp_rif_counter_dir dir,
210 case MLXSW_SP_RIF_COUNTER_EGRESS:
211 rif->counter_egress_valid = valid;
213 case MLXSW_SP_RIF_COUNTER_INGRESS:
214 rif->counter_ingress_valid = valid;
219 static int mlxsw_sp_rif_counter_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
220 unsigned int counter_index, bool enable,
221 enum mlxsw_sp_rif_counter_dir dir)
223 char ritr_pl[MLXSW_REG_RITR_LEN];
224 bool is_egress = false;
227 if (dir == MLXSW_SP_RIF_COUNTER_EGRESS)
229 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
230 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
234 mlxsw_reg_ritr_counter_pack(ritr_pl, counter_index, enable,
236 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
239 int mlxsw_sp_rif_counter_value_get(struct mlxsw_sp *mlxsw_sp,
240 struct mlxsw_sp_rif *rif,
241 enum mlxsw_sp_rif_counter_dir dir, u64 *cnt)
243 char ricnt_pl[MLXSW_REG_RICNT_LEN];
244 unsigned int *p_counter_index;
248 valid = mlxsw_sp_rif_counter_valid_get(rif, dir);
252 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
253 if (!p_counter_index)
255 mlxsw_reg_ricnt_pack(ricnt_pl, *p_counter_index,
256 MLXSW_REG_RICNT_OPCODE_NOP);
257 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl);
260 *cnt = mlxsw_reg_ricnt_good_unicast_packets_get(ricnt_pl);
264 static int mlxsw_sp_rif_counter_clear(struct mlxsw_sp *mlxsw_sp,
265 unsigned int counter_index)
267 char ricnt_pl[MLXSW_REG_RICNT_LEN];
269 mlxsw_reg_ricnt_pack(ricnt_pl, counter_index,
270 MLXSW_REG_RICNT_OPCODE_CLEAR);
271 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ricnt), ricnt_pl);
274 int mlxsw_sp_rif_counter_alloc(struct mlxsw_sp *mlxsw_sp,
275 struct mlxsw_sp_rif *rif,
276 enum mlxsw_sp_rif_counter_dir dir)
278 unsigned int *p_counter_index;
281 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
282 if (!p_counter_index)
284 err = mlxsw_sp_counter_alloc(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
289 err = mlxsw_sp_rif_counter_clear(mlxsw_sp, *p_counter_index);
291 goto err_counter_clear;
293 err = mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index,
294 *p_counter_index, true, dir);
296 goto err_counter_edit;
297 mlxsw_sp_rif_counter_valid_set(rif, dir, true);
302 mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
307 void mlxsw_sp_rif_counter_free(struct mlxsw_sp *mlxsw_sp,
308 struct mlxsw_sp_rif *rif,
309 enum mlxsw_sp_rif_counter_dir dir)
311 unsigned int *p_counter_index;
313 if (!mlxsw_sp_rif_counter_valid_get(rif, dir))
316 p_counter_index = mlxsw_sp_rif_p_counter_get(rif, dir);
317 if (WARN_ON(!p_counter_index))
319 mlxsw_sp_rif_counter_edit(mlxsw_sp, rif->rif_index,
320 *p_counter_index, false, dir);
321 mlxsw_sp_counter_free(mlxsw_sp, MLXSW_SP_COUNTER_SUB_POOL_RIF,
323 mlxsw_sp_rif_counter_valid_set(rif, dir, false);
326 static void mlxsw_sp_rif_counters_alloc(struct mlxsw_sp_rif *rif)
328 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
329 struct devlink *devlink;
331 devlink = priv_to_devlink(mlxsw_sp->core);
332 if (!devlink_dpipe_table_counter_enabled(devlink,
333 MLXSW_SP_DPIPE_TABLE_NAME_ERIF))
335 mlxsw_sp_rif_counter_alloc(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS);
338 static void mlxsw_sp_rif_counters_free(struct mlxsw_sp_rif *rif)
340 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
342 mlxsw_sp_rif_counter_free(mlxsw_sp, rif, MLXSW_SP_RIF_COUNTER_EGRESS);
345 static struct mlxsw_sp_rif *
346 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
347 const struct net_device *dev);
349 #define MLXSW_SP_PREFIX_COUNT (sizeof(struct in6_addr) * BITS_PER_BYTE + 1)
351 struct mlxsw_sp_prefix_usage {
352 DECLARE_BITMAP(b, MLXSW_SP_PREFIX_COUNT);
355 #define mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) \
356 for_each_set_bit(prefix, (prefix_usage)->b, MLXSW_SP_PREFIX_COUNT)
359 mlxsw_sp_prefix_usage_eq(struct mlxsw_sp_prefix_usage *prefix_usage1,
360 struct mlxsw_sp_prefix_usage *prefix_usage2)
362 return !memcmp(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
366 mlxsw_sp_prefix_usage_cpy(struct mlxsw_sp_prefix_usage *prefix_usage1,
367 struct mlxsw_sp_prefix_usage *prefix_usage2)
369 memcpy(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
373 mlxsw_sp_prefix_usage_set(struct mlxsw_sp_prefix_usage *prefix_usage,
374 unsigned char prefix_len)
376 set_bit(prefix_len, prefix_usage->b);
380 mlxsw_sp_prefix_usage_clear(struct mlxsw_sp_prefix_usage *prefix_usage,
381 unsigned char prefix_len)
383 clear_bit(prefix_len, prefix_usage->b);
386 struct mlxsw_sp_fib_key {
387 unsigned char addr[sizeof(struct in6_addr)];
388 unsigned char prefix_len;
391 enum mlxsw_sp_fib_entry_type {
392 MLXSW_SP_FIB_ENTRY_TYPE_REMOTE,
393 MLXSW_SP_FIB_ENTRY_TYPE_LOCAL,
394 MLXSW_SP_FIB_ENTRY_TYPE_TRAP,
396 /* This is a special case of local delivery, where a packet should be
397 * decapsulated on reception. Note that there is no corresponding ENCAP,
398 * because that's a type of next hop, not of FIB entry. (There can be
399 * several next hops in a REMOTE entry, and some of them may be
400 * encapsulating entries.)
402 MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP,
405 struct mlxsw_sp_nexthop_group;
407 struct mlxsw_sp_fib_node {
408 struct list_head entry_list;
409 struct list_head list;
410 struct rhash_head ht_node;
411 struct mlxsw_sp_fib *fib;
412 struct mlxsw_sp_fib_key key;
415 struct mlxsw_sp_fib_entry_decap {
416 struct mlxsw_sp_ipip_entry *ipip_entry;
420 struct mlxsw_sp_fib_entry {
421 struct list_head list;
422 struct mlxsw_sp_fib_node *fib_node;
423 enum mlxsw_sp_fib_entry_type type;
424 struct list_head nexthop_group_node;
425 struct mlxsw_sp_nexthop_group *nh_group;
426 struct mlxsw_sp_fib_entry_decap decap; /* Valid for decap entries. */
429 struct mlxsw_sp_fib4_entry {
430 struct mlxsw_sp_fib_entry common;
437 struct mlxsw_sp_fib6_entry {
438 struct mlxsw_sp_fib_entry common;
439 struct list_head rt6_list;
443 struct mlxsw_sp_rt6 {
444 struct list_head list;
445 struct fib6_info *rt;
448 struct mlxsw_sp_lpm_tree {
450 unsigned int ref_count;
451 enum mlxsw_sp_l3proto proto;
452 unsigned long prefix_ref_count[MLXSW_SP_PREFIX_COUNT];
453 struct mlxsw_sp_prefix_usage prefix_usage;
456 struct mlxsw_sp_fib {
457 struct rhashtable ht;
458 struct list_head node_list;
459 struct mlxsw_sp_vr *vr;
460 struct mlxsw_sp_lpm_tree *lpm_tree;
461 enum mlxsw_sp_l3proto proto;
465 u16 id; /* virtual router ID */
466 u32 tb_id; /* kernel fib table id */
467 unsigned int rif_count;
468 struct mlxsw_sp_fib *fib4;
469 struct mlxsw_sp_fib *fib6;
470 struct mlxsw_sp_mr_table *mr_table[MLXSW_SP_L3_PROTO_MAX];
473 static const struct rhashtable_params mlxsw_sp_fib_ht_params;
475 static struct mlxsw_sp_fib *mlxsw_sp_fib_create(struct mlxsw_sp *mlxsw_sp,
476 struct mlxsw_sp_vr *vr,
477 enum mlxsw_sp_l3proto proto)
479 struct mlxsw_sp_lpm_tree *lpm_tree;
480 struct mlxsw_sp_fib *fib;
483 lpm_tree = mlxsw_sp->router->lpm.proto_trees[proto];
484 fib = kzalloc(sizeof(*fib), GFP_KERNEL);
486 return ERR_PTR(-ENOMEM);
487 err = rhashtable_init(&fib->ht, &mlxsw_sp_fib_ht_params);
489 goto err_rhashtable_init;
490 INIT_LIST_HEAD(&fib->node_list);
493 fib->lpm_tree = lpm_tree;
494 mlxsw_sp_lpm_tree_hold(lpm_tree);
495 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, lpm_tree->id);
497 goto err_lpm_tree_bind;
501 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
507 static void mlxsw_sp_fib_destroy(struct mlxsw_sp *mlxsw_sp,
508 struct mlxsw_sp_fib *fib)
510 mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, fib);
511 mlxsw_sp_lpm_tree_put(mlxsw_sp, fib->lpm_tree);
512 WARN_ON(!list_empty(&fib->node_list));
513 rhashtable_destroy(&fib->ht);
517 static struct mlxsw_sp_lpm_tree *
518 mlxsw_sp_lpm_tree_find_unused(struct mlxsw_sp *mlxsw_sp)
520 static struct mlxsw_sp_lpm_tree *lpm_tree;
523 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
524 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
525 if (lpm_tree->ref_count == 0)
531 static int mlxsw_sp_lpm_tree_alloc(struct mlxsw_sp *mlxsw_sp,
532 struct mlxsw_sp_lpm_tree *lpm_tree)
534 char ralta_pl[MLXSW_REG_RALTA_LEN];
536 mlxsw_reg_ralta_pack(ralta_pl, true,
537 (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
539 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
542 static void mlxsw_sp_lpm_tree_free(struct mlxsw_sp *mlxsw_sp,
543 struct mlxsw_sp_lpm_tree *lpm_tree)
545 char ralta_pl[MLXSW_REG_RALTA_LEN];
547 mlxsw_reg_ralta_pack(ralta_pl, false,
548 (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
550 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
554 mlxsw_sp_lpm_tree_left_struct_set(struct mlxsw_sp *mlxsw_sp,
555 struct mlxsw_sp_prefix_usage *prefix_usage,
556 struct mlxsw_sp_lpm_tree *lpm_tree)
558 char ralst_pl[MLXSW_REG_RALST_LEN];
561 u8 last_prefix = MLXSW_REG_RALST_BIN_NO_CHILD;
563 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage)
566 mlxsw_reg_ralst_pack(ralst_pl, root_bin, lpm_tree->id);
567 mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) {
570 mlxsw_reg_ralst_bin_pack(ralst_pl, prefix, last_prefix,
571 MLXSW_REG_RALST_BIN_NO_CHILD);
572 last_prefix = prefix;
574 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
577 static struct mlxsw_sp_lpm_tree *
578 mlxsw_sp_lpm_tree_create(struct mlxsw_sp *mlxsw_sp,
579 struct mlxsw_sp_prefix_usage *prefix_usage,
580 enum mlxsw_sp_l3proto proto)
582 struct mlxsw_sp_lpm_tree *lpm_tree;
585 lpm_tree = mlxsw_sp_lpm_tree_find_unused(mlxsw_sp);
587 return ERR_PTR(-EBUSY);
588 lpm_tree->proto = proto;
589 err = mlxsw_sp_lpm_tree_alloc(mlxsw_sp, lpm_tree);
593 err = mlxsw_sp_lpm_tree_left_struct_set(mlxsw_sp, prefix_usage,
596 goto err_left_struct_set;
597 memcpy(&lpm_tree->prefix_usage, prefix_usage,
598 sizeof(lpm_tree->prefix_usage));
599 memset(&lpm_tree->prefix_ref_count, 0,
600 sizeof(lpm_tree->prefix_ref_count));
601 lpm_tree->ref_count = 1;
605 mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
609 static void mlxsw_sp_lpm_tree_destroy(struct mlxsw_sp *mlxsw_sp,
610 struct mlxsw_sp_lpm_tree *lpm_tree)
612 mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
615 static struct mlxsw_sp_lpm_tree *
616 mlxsw_sp_lpm_tree_get(struct mlxsw_sp *mlxsw_sp,
617 struct mlxsw_sp_prefix_usage *prefix_usage,
618 enum mlxsw_sp_l3proto proto)
620 struct mlxsw_sp_lpm_tree *lpm_tree;
623 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
624 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
625 if (lpm_tree->ref_count != 0 &&
626 lpm_tree->proto == proto &&
627 mlxsw_sp_prefix_usage_eq(&lpm_tree->prefix_usage,
629 mlxsw_sp_lpm_tree_hold(lpm_tree);
633 return mlxsw_sp_lpm_tree_create(mlxsw_sp, prefix_usage, proto);
636 static void mlxsw_sp_lpm_tree_hold(struct mlxsw_sp_lpm_tree *lpm_tree)
638 lpm_tree->ref_count++;
641 static void mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp,
642 struct mlxsw_sp_lpm_tree *lpm_tree)
644 if (--lpm_tree->ref_count == 0)
645 mlxsw_sp_lpm_tree_destroy(mlxsw_sp, lpm_tree);
648 #define MLXSW_SP_LPM_TREE_MIN 1 /* tree 0 is reserved */
650 static int mlxsw_sp_lpm_init(struct mlxsw_sp *mlxsw_sp)
652 struct mlxsw_sp_prefix_usage req_prefix_usage = {{ 0 } };
653 struct mlxsw_sp_lpm_tree *lpm_tree;
657 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_LPM_TREES))
660 max_trees = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_LPM_TREES);
661 mlxsw_sp->router->lpm.tree_count = max_trees - MLXSW_SP_LPM_TREE_MIN;
662 mlxsw_sp->router->lpm.trees = kcalloc(mlxsw_sp->router->lpm.tree_count,
663 sizeof(struct mlxsw_sp_lpm_tree),
665 if (!mlxsw_sp->router->lpm.trees)
668 for (i = 0; i < mlxsw_sp->router->lpm.tree_count; i++) {
669 lpm_tree = &mlxsw_sp->router->lpm.trees[i];
670 lpm_tree->id = i + MLXSW_SP_LPM_TREE_MIN;
673 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
674 MLXSW_SP_L3_PROTO_IPV4);
675 if (IS_ERR(lpm_tree)) {
676 err = PTR_ERR(lpm_tree);
677 goto err_ipv4_tree_get;
679 mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4] = lpm_tree;
681 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
682 MLXSW_SP_L3_PROTO_IPV6);
683 if (IS_ERR(lpm_tree)) {
684 err = PTR_ERR(lpm_tree);
685 goto err_ipv6_tree_get;
687 mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV6] = lpm_tree;
692 lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4];
693 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
695 kfree(mlxsw_sp->router->lpm.trees);
699 static void mlxsw_sp_lpm_fini(struct mlxsw_sp *mlxsw_sp)
701 struct mlxsw_sp_lpm_tree *lpm_tree;
703 lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV6];
704 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
706 lpm_tree = mlxsw_sp->router->lpm.proto_trees[MLXSW_SP_L3_PROTO_IPV4];
707 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
709 kfree(mlxsw_sp->router->lpm.trees);
712 static bool mlxsw_sp_vr_is_used(const struct mlxsw_sp_vr *vr)
714 return !!vr->fib4 || !!vr->fib6 ||
715 !!vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] ||
716 !!vr->mr_table[MLXSW_SP_L3_PROTO_IPV6];
719 static struct mlxsw_sp_vr *mlxsw_sp_vr_find_unused(struct mlxsw_sp *mlxsw_sp)
721 struct mlxsw_sp_vr *vr;
724 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
725 vr = &mlxsw_sp->router->vrs[i];
726 if (!mlxsw_sp_vr_is_used(vr))
732 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp,
733 const struct mlxsw_sp_fib *fib, u8 tree_id)
735 char raltb_pl[MLXSW_REG_RALTB_LEN];
737 mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id,
738 (enum mlxsw_reg_ralxx_protocol) fib->proto,
740 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
743 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp,
744 const struct mlxsw_sp_fib *fib)
746 char raltb_pl[MLXSW_REG_RALTB_LEN];
748 /* Bind to tree 0 which is default */
749 mlxsw_reg_raltb_pack(raltb_pl, fib->vr->id,
750 (enum mlxsw_reg_ralxx_protocol) fib->proto, 0);
751 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
754 static u32 mlxsw_sp_fix_tb_id(u32 tb_id)
756 /* For our purpose, squash main, default and local tables into one */
757 if (tb_id == RT_TABLE_LOCAL || tb_id == RT_TABLE_DEFAULT)
758 tb_id = RT_TABLE_MAIN;
762 static struct mlxsw_sp_vr *mlxsw_sp_vr_find(struct mlxsw_sp *mlxsw_sp,
765 struct mlxsw_sp_vr *vr;
768 tb_id = mlxsw_sp_fix_tb_id(tb_id);
770 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
771 vr = &mlxsw_sp->router->vrs[i];
772 if (mlxsw_sp_vr_is_used(vr) && vr->tb_id == tb_id)
778 static struct mlxsw_sp_fib *mlxsw_sp_vr_fib(const struct mlxsw_sp_vr *vr,
779 enum mlxsw_sp_l3proto proto)
782 case MLXSW_SP_L3_PROTO_IPV4:
784 case MLXSW_SP_L3_PROTO_IPV6:
790 static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp,
792 struct netlink_ext_ack *extack)
794 struct mlxsw_sp_mr_table *mr4_table, *mr6_table;
795 struct mlxsw_sp_fib *fib4;
796 struct mlxsw_sp_fib *fib6;
797 struct mlxsw_sp_vr *vr;
800 vr = mlxsw_sp_vr_find_unused(mlxsw_sp);
802 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported virtual routers");
803 return ERR_PTR(-EBUSY);
805 fib4 = mlxsw_sp_fib_create(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4);
807 return ERR_CAST(fib4);
808 fib6 = mlxsw_sp_fib_create(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6);
811 goto err_fib6_create;
813 mr4_table = mlxsw_sp_mr_table_create(mlxsw_sp, vr->id,
814 MLXSW_SP_L3_PROTO_IPV4);
815 if (IS_ERR(mr4_table)) {
816 err = PTR_ERR(mr4_table);
817 goto err_mr4_table_create;
819 mr6_table = mlxsw_sp_mr_table_create(mlxsw_sp, vr->id,
820 MLXSW_SP_L3_PROTO_IPV6);
821 if (IS_ERR(mr6_table)) {
822 err = PTR_ERR(mr6_table);
823 goto err_mr6_table_create;
828 vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] = mr4_table;
829 vr->mr_table[MLXSW_SP_L3_PROTO_IPV6] = mr6_table;
833 err_mr6_table_create:
834 mlxsw_sp_mr_table_destroy(mr4_table);
835 err_mr4_table_create:
836 mlxsw_sp_fib_destroy(mlxsw_sp, fib6);
838 mlxsw_sp_fib_destroy(mlxsw_sp, fib4);
842 static void mlxsw_sp_vr_destroy(struct mlxsw_sp *mlxsw_sp,
843 struct mlxsw_sp_vr *vr)
845 mlxsw_sp_mr_table_destroy(vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]);
846 vr->mr_table[MLXSW_SP_L3_PROTO_IPV6] = NULL;
847 mlxsw_sp_mr_table_destroy(vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]);
848 vr->mr_table[MLXSW_SP_L3_PROTO_IPV4] = NULL;
849 mlxsw_sp_fib_destroy(mlxsw_sp, vr->fib6);
851 mlxsw_sp_fib_destroy(mlxsw_sp, vr->fib4);
855 static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id,
856 struct netlink_ext_ack *extack)
858 struct mlxsw_sp_vr *vr;
860 tb_id = mlxsw_sp_fix_tb_id(tb_id);
861 vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id);
863 vr = mlxsw_sp_vr_create(mlxsw_sp, tb_id, extack);
867 static void mlxsw_sp_vr_put(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr)
869 if (!vr->rif_count && list_empty(&vr->fib4->node_list) &&
870 list_empty(&vr->fib6->node_list) &&
871 mlxsw_sp_mr_table_empty(vr->mr_table[MLXSW_SP_L3_PROTO_IPV4]) &&
872 mlxsw_sp_mr_table_empty(vr->mr_table[MLXSW_SP_L3_PROTO_IPV6]))
873 mlxsw_sp_vr_destroy(mlxsw_sp, vr);
877 mlxsw_sp_vr_lpm_tree_should_replace(struct mlxsw_sp_vr *vr,
878 enum mlxsw_sp_l3proto proto, u8 tree_id)
880 struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
882 if (!mlxsw_sp_vr_is_used(vr))
884 if (fib->lpm_tree->id == tree_id)
889 static int mlxsw_sp_vr_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp,
890 struct mlxsw_sp_fib *fib,
891 struct mlxsw_sp_lpm_tree *new_tree)
893 struct mlxsw_sp_lpm_tree *old_tree = fib->lpm_tree;
896 fib->lpm_tree = new_tree;
897 mlxsw_sp_lpm_tree_hold(new_tree);
898 err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, fib, new_tree->id);
901 mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree);
905 mlxsw_sp_lpm_tree_put(mlxsw_sp, new_tree);
906 fib->lpm_tree = old_tree;
910 static int mlxsw_sp_vrs_lpm_tree_replace(struct mlxsw_sp *mlxsw_sp,
911 struct mlxsw_sp_fib *fib,
912 struct mlxsw_sp_lpm_tree *new_tree)
914 enum mlxsw_sp_l3proto proto = fib->proto;
915 struct mlxsw_sp_lpm_tree *old_tree;
916 u8 old_id, new_id = new_tree->id;
917 struct mlxsw_sp_vr *vr;
920 old_tree = mlxsw_sp->router->lpm.proto_trees[proto];
921 old_id = old_tree->id;
923 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
924 vr = &mlxsw_sp->router->vrs[i];
925 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, old_id))
927 err = mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp,
928 mlxsw_sp_vr_fib(vr, proto),
931 goto err_tree_replace;
934 memcpy(new_tree->prefix_ref_count, old_tree->prefix_ref_count,
935 sizeof(new_tree->prefix_ref_count));
936 mlxsw_sp->router->lpm.proto_trees[proto] = new_tree;
937 mlxsw_sp_lpm_tree_put(mlxsw_sp, old_tree);
942 for (i--; i >= 0; i--) {
943 if (!mlxsw_sp_vr_lpm_tree_should_replace(vr, proto, new_id))
945 mlxsw_sp_vr_lpm_tree_replace(mlxsw_sp,
946 mlxsw_sp_vr_fib(vr, proto),
952 static int mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp)
954 struct mlxsw_sp_vr *vr;
958 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_VRS))
961 max_vrs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS);
962 mlxsw_sp->router->vrs = kcalloc(max_vrs, sizeof(struct mlxsw_sp_vr),
964 if (!mlxsw_sp->router->vrs)
967 for (i = 0; i < max_vrs; i++) {
968 vr = &mlxsw_sp->router->vrs[i];
975 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp);
977 static void mlxsw_sp_vrs_fini(struct mlxsw_sp *mlxsw_sp)
979 /* At this stage we're guaranteed not to have new incoming
980 * FIB notifications and the work queue is free from FIBs
981 * sitting on top of mlxsw netdevs. However, we can still
982 * have other FIBs queued. Flush the queue before flushing
983 * the device's tables. No need for locks, as we're the only
986 mlxsw_core_flush_owq();
987 mlxsw_sp_router_fib_flush(mlxsw_sp);
988 kfree(mlxsw_sp->router->vrs);
991 static struct net_device *
992 __mlxsw_sp_ipip_netdev_ul_dev_get(const struct net_device *ol_dev)
994 struct ip_tunnel *tun = netdev_priv(ol_dev);
995 struct net *net = dev_net(ol_dev);
997 return __dev_get_by_index(net, tun->parms.link);
1000 u32 mlxsw_sp_ipip_dev_ul_tb_id(const struct net_device *ol_dev)
1002 struct net_device *d = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev);
1005 return l3mdev_fib_table(d) ? : RT_TABLE_MAIN;
1007 return l3mdev_fib_table(ol_dev) ? : RT_TABLE_MAIN;
1010 static struct mlxsw_sp_rif *
1011 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp,
1012 const struct mlxsw_sp_rif_params *params,
1013 struct netlink_ext_ack *extack);
1015 static struct mlxsw_sp_rif_ipip_lb *
1016 mlxsw_sp_ipip_ol_ipip_lb_create(struct mlxsw_sp *mlxsw_sp,
1017 enum mlxsw_sp_ipip_type ipipt,
1018 struct net_device *ol_dev,
1019 struct netlink_ext_ack *extack)
1021 struct mlxsw_sp_rif_params_ipip_lb lb_params;
1022 const struct mlxsw_sp_ipip_ops *ipip_ops;
1023 struct mlxsw_sp_rif *rif;
1025 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt];
1026 lb_params = (struct mlxsw_sp_rif_params_ipip_lb) {
1027 .common.dev = ol_dev,
1028 .common.lag = false,
1029 .lb_config = ipip_ops->ol_loopback_config(mlxsw_sp, ol_dev),
1032 rif = mlxsw_sp_rif_create(mlxsw_sp, &lb_params.common, extack);
1034 return ERR_CAST(rif);
1035 return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common);
1038 static struct mlxsw_sp_ipip_entry *
1039 mlxsw_sp_ipip_entry_alloc(struct mlxsw_sp *mlxsw_sp,
1040 enum mlxsw_sp_ipip_type ipipt,
1041 struct net_device *ol_dev)
1043 const struct mlxsw_sp_ipip_ops *ipip_ops;
1044 struct mlxsw_sp_ipip_entry *ipip_entry;
1045 struct mlxsw_sp_ipip_entry *ret = NULL;
1047 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipipt];
1048 ipip_entry = kzalloc(sizeof(*ipip_entry), GFP_KERNEL);
1050 return ERR_PTR(-ENOMEM);
1052 ipip_entry->ol_lb = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp, ipipt,
1054 if (IS_ERR(ipip_entry->ol_lb)) {
1055 ret = ERR_CAST(ipip_entry->ol_lb);
1056 goto err_ol_ipip_lb_create;
1059 ipip_entry->ipipt = ipipt;
1060 ipip_entry->ol_dev = ol_dev;
1062 switch (ipip_ops->ul_proto) {
1063 case MLXSW_SP_L3_PROTO_IPV4:
1064 ipip_entry->parms4 = mlxsw_sp_ipip_netdev_parms4(ol_dev);
1066 case MLXSW_SP_L3_PROTO_IPV6:
1073 err_ol_ipip_lb_create:
1079 mlxsw_sp_ipip_entry_dealloc(struct mlxsw_sp_ipip_entry *ipip_entry)
1081 mlxsw_sp_rif_destroy(&ipip_entry->ol_lb->common);
1086 mlxsw_sp_ipip_entry_saddr_matches(struct mlxsw_sp *mlxsw_sp,
1087 const enum mlxsw_sp_l3proto ul_proto,
1088 union mlxsw_sp_l3addr saddr,
1090 struct mlxsw_sp_ipip_entry *ipip_entry)
1092 u32 tun_ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev);
1093 enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt;
1094 union mlxsw_sp_l3addr tun_saddr;
1096 if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto)
1099 tun_saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ipip_entry->ol_dev);
1100 return tun_ul_tb_id == ul_tb_id &&
1101 mlxsw_sp_l3addr_eq(&tun_saddr, &saddr);
1105 mlxsw_sp_fib_entry_decap_init(struct mlxsw_sp *mlxsw_sp,
1106 struct mlxsw_sp_fib_entry *fib_entry,
1107 struct mlxsw_sp_ipip_entry *ipip_entry)
1112 err = mlxsw_sp_kvdl_alloc(mlxsw_sp, 1, &tunnel_index);
1116 ipip_entry->decap_fib_entry = fib_entry;
1117 fib_entry->decap.ipip_entry = ipip_entry;
1118 fib_entry->decap.tunnel_index = tunnel_index;
1122 static void mlxsw_sp_fib_entry_decap_fini(struct mlxsw_sp *mlxsw_sp,
1123 struct mlxsw_sp_fib_entry *fib_entry)
1125 /* Unlink this node from the IPIP entry that it's the decap entry of. */
1126 fib_entry->decap.ipip_entry->decap_fib_entry = NULL;
1127 fib_entry->decap.ipip_entry = NULL;
1128 mlxsw_sp_kvdl_free(mlxsw_sp, fib_entry->decap.tunnel_index);
1131 static struct mlxsw_sp_fib_node *
1132 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr,
1133 size_t addr_len, unsigned char prefix_len);
1134 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
1135 struct mlxsw_sp_fib_entry *fib_entry);
1138 mlxsw_sp_ipip_entry_demote_decap(struct mlxsw_sp *mlxsw_sp,
1139 struct mlxsw_sp_ipip_entry *ipip_entry)
1141 struct mlxsw_sp_fib_entry *fib_entry = ipip_entry->decap_fib_entry;
1143 mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, fib_entry);
1144 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
1146 mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
1150 mlxsw_sp_ipip_entry_promote_decap(struct mlxsw_sp *mlxsw_sp,
1151 struct mlxsw_sp_ipip_entry *ipip_entry,
1152 struct mlxsw_sp_fib_entry *decap_fib_entry)
1154 if (mlxsw_sp_fib_entry_decap_init(mlxsw_sp, decap_fib_entry,
1157 decap_fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP;
1159 if (mlxsw_sp_fib_entry_update(mlxsw_sp, decap_fib_entry))
1160 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry);
1163 /* Given an IPIP entry, find the corresponding decap route. */
1164 static struct mlxsw_sp_fib_entry *
1165 mlxsw_sp_ipip_entry_find_decap(struct mlxsw_sp *mlxsw_sp,
1166 struct mlxsw_sp_ipip_entry *ipip_entry)
1168 static struct mlxsw_sp_fib_node *fib_node;
1169 const struct mlxsw_sp_ipip_ops *ipip_ops;
1170 struct mlxsw_sp_fib_entry *fib_entry;
1171 unsigned char saddr_prefix_len;
1172 union mlxsw_sp_l3addr saddr;
1173 struct mlxsw_sp_fib *ul_fib;
1174 struct mlxsw_sp_vr *ul_vr;
1180 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
1182 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ipip_entry->ol_dev);
1183 ul_vr = mlxsw_sp_vr_find(mlxsw_sp, ul_tb_id);
1187 ul_fib = mlxsw_sp_vr_fib(ul_vr, ipip_ops->ul_proto);
1188 saddr = mlxsw_sp_ipip_netdev_saddr(ipip_ops->ul_proto,
1189 ipip_entry->ol_dev);
1191 switch (ipip_ops->ul_proto) {
1192 case MLXSW_SP_L3_PROTO_IPV4:
1193 saddr4 = be32_to_cpu(saddr.addr4);
1196 saddr_prefix_len = 32;
1198 case MLXSW_SP_L3_PROTO_IPV6:
1203 fib_node = mlxsw_sp_fib_node_lookup(ul_fib, saddrp, saddr_len,
1205 if (!fib_node || list_empty(&fib_node->entry_list))
1208 fib_entry = list_first_entry(&fib_node->entry_list,
1209 struct mlxsw_sp_fib_entry, list);
1210 if (fib_entry->type != MLXSW_SP_FIB_ENTRY_TYPE_TRAP)
1216 static struct mlxsw_sp_ipip_entry *
1217 mlxsw_sp_ipip_entry_create(struct mlxsw_sp *mlxsw_sp,
1218 enum mlxsw_sp_ipip_type ipipt,
1219 struct net_device *ol_dev)
1221 struct mlxsw_sp_ipip_entry *ipip_entry;
1223 ipip_entry = mlxsw_sp_ipip_entry_alloc(mlxsw_sp, ipipt, ol_dev);
1224 if (IS_ERR(ipip_entry))
1227 list_add_tail(&ipip_entry->ipip_list_node,
1228 &mlxsw_sp->router->ipip_list);
1234 mlxsw_sp_ipip_entry_destroy(struct mlxsw_sp *mlxsw_sp,
1235 struct mlxsw_sp_ipip_entry *ipip_entry)
1237 list_del(&ipip_entry->ipip_list_node);
1238 mlxsw_sp_ipip_entry_dealloc(ipip_entry);
1242 mlxsw_sp_ipip_entry_matches_decap(struct mlxsw_sp *mlxsw_sp,
1243 const struct net_device *ul_dev,
1244 enum mlxsw_sp_l3proto ul_proto,
1245 union mlxsw_sp_l3addr ul_dip,
1246 struct mlxsw_sp_ipip_entry *ipip_entry)
1248 u32 ul_tb_id = l3mdev_fib_table(ul_dev) ? : RT_TABLE_MAIN;
1249 enum mlxsw_sp_ipip_type ipipt = ipip_entry->ipipt;
1250 struct net_device *ipip_ul_dev;
1252 if (mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto != ul_proto)
1255 ipip_ul_dev = __mlxsw_sp_ipip_netdev_ul_dev_get(ipip_entry->ol_dev);
1256 return mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, ul_dip,
1257 ul_tb_id, ipip_entry) &&
1258 (!ipip_ul_dev || ipip_ul_dev == ul_dev);
1261 /* Given decap parameters, find the corresponding IPIP entry. */
1262 static struct mlxsw_sp_ipip_entry *
1263 mlxsw_sp_ipip_entry_find_by_decap(struct mlxsw_sp *mlxsw_sp,
1264 const struct net_device *ul_dev,
1265 enum mlxsw_sp_l3proto ul_proto,
1266 union mlxsw_sp_l3addr ul_dip)
1268 struct mlxsw_sp_ipip_entry *ipip_entry;
1270 list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list,
1272 if (mlxsw_sp_ipip_entry_matches_decap(mlxsw_sp, ul_dev,
1280 static bool mlxsw_sp_netdev_ipip_type(const struct mlxsw_sp *mlxsw_sp,
1281 const struct net_device *dev,
1282 enum mlxsw_sp_ipip_type *p_type)
1284 struct mlxsw_sp_router *router = mlxsw_sp->router;
1285 const struct mlxsw_sp_ipip_ops *ipip_ops;
1286 enum mlxsw_sp_ipip_type ipipt;
1288 for (ipipt = 0; ipipt < MLXSW_SP_IPIP_TYPE_MAX; ++ipipt) {
1289 ipip_ops = router->ipip_ops_arr[ipipt];
1290 if (dev->type == ipip_ops->dev_type) {
1299 bool mlxsw_sp_netdev_is_ipip_ol(const struct mlxsw_sp *mlxsw_sp,
1300 const struct net_device *dev)
1302 return mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL);
1305 static struct mlxsw_sp_ipip_entry *
1306 mlxsw_sp_ipip_entry_find_by_ol_dev(struct mlxsw_sp *mlxsw_sp,
1307 const struct net_device *ol_dev)
1309 struct mlxsw_sp_ipip_entry *ipip_entry;
1311 list_for_each_entry(ipip_entry, &mlxsw_sp->router->ipip_list,
1313 if (ipip_entry->ol_dev == ol_dev)
1319 static struct mlxsw_sp_ipip_entry *
1320 mlxsw_sp_ipip_entry_find_by_ul_dev(const struct mlxsw_sp *mlxsw_sp,
1321 const struct net_device *ul_dev,
1322 struct mlxsw_sp_ipip_entry *start)
1324 struct mlxsw_sp_ipip_entry *ipip_entry;
1326 ipip_entry = list_prepare_entry(start, &mlxsw_sp->router->ipip_list,
1328 list_for_each_entry_continue(ipip_entry, &mlxsw_sp->router->ipip_list,
1330 struct net_device *ipip_ul_dev =
1331 __mlxsw_sp_ipip_netdev_ul_dev_get(ipip_entry->ol_dev);
1333 if (ipip_ul_dev == ul_dev)
1340 bool mlxsw_sp_netdev_is_ipip_ul(const struct mlxsw_sp *mlxsw_sp,
1341 const struct net_device *dev)
1343 return mlxsw_sp_ipip_entry_find_by_ul_dev(mlxsw_sp, dev, NULL);
1346 static bool mlxsw_sp_netdevice_ipip_can_offload(struct mlxsw_sp *mlxsw_sp,
1347 const struct net_device *ol_dev,
1348 enum mlxsw_sp_ipip_type ipipt)
1350 const struct mlxsw_sp_ipip_ops *ops
1351 = mlxsw_sp->router->ipip_ops_arr[ipipt];
1353 /* For deciding whether decap should be offloaded, we don't care about
1354 * overlay protocol, so ask whether either one is supported.
1356 return ops->can_offload(mlxsw_sp, ol_dev, MLXSW_SP_L3_PROTO_IPV4) ||
1357 ops->can_offload(mlxsw_sp, ol_dev, MLXSW_SP_L3_PROTO_IPV6);
1360 static int mlxsw_sp_netdevice_ipip_ol_reg_event(struct mlxsw_sp *mlxsw_sp,
1361 struct net_device *ol_dev)
1363 struct mlxsw_sp_ipip_entry *ipip_entry;
1364 enum mlxsw_sp_l3proto ul_proto;
1365 enum mlxsw_sp_ipip_type ipipt;
1366 union mlxsw_sp_l3addr saddr;
1369 mlxsw_sp_netdev_ipip_type(mlxsw_sp, ol_dev, &ipipt);
1370 if (mlxsw_sp_netdevice_ipip_can_offload(mlxsw_sp, ol_dev, ipipt)) {
1371 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ol_dev);
1372 ul_proto = mlxsw_sp->router->ipip_ops_arr[ipipt]->ul_proto;
1373 saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ol_dev);
1374 if (!mlxsw_sp_ipip_demote_tunnel_by_saddr(mlxsw_sp, ul_proto,
1377 ipip_entry = mlxsw_sp_ipip_entry_create(mlxsw_sp, ipipt,
1379 if (IS_ERR(ipip_entry))
1380 return PTR_ERR(ipip_entry);
1387 static void mlxsw_sp_netdevice_ipip_ol_unreg_event(struct mlxsw_sp *mlxsw_sp,
1388 struct net_device *ol_dev)
1390 struct mlxsw_sp_ipip_entry *ipip_entry;
1392 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1394 mlxsw_sp_ipip_entry_destroy(mlxsw_sp, ipip_entry);
1398 mlxsw_sp_ipip_entry_ol_up_event(struct mlxsw_sp *mlxsw_sp,
1399 struct mlxsw_sp_ipip_entry *ipip_entry)
1401 struct mlxsw_sp_fib_entry *decap_fib_entry;
1403 decap_fib_entry = mlxsw_sp_ipip_entry_find_decap(mlxsw_sp, ipip_entry);
1404 if (decap_fib_entry)
1405 mlxsw_sp_ipip_entry_promote_decap(mlxsw_sp, ipip_entry,
1410 mlxsw_sp_rif_ipip_lb_op(struct mlxsw_sp_rif_ipip_lb *lb_rif,
1411 struct mlxsw_sp_vr *ul_vr, bool enable)
1413 struct mlxsw_sp_rif_ipip_lb_config lb_cf = lb_rif->lb_config;
1414 struct mlxsw_sp_rif *rif = &lb_rif->common;
1415 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
1416 char ritr_pl[MLXSW_REG_RITR_LEN];
1419 switch (lb_cf.ul_protocol) {
1420 case MLXSW_SP_L3_PROTO_IPV4:
1421 saddr4 = be32_to_cpu(lb_cf.saddr.addr4);
1422 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_LOOPBACK_IF,
1423 rif->rif_index, rif->vr_id, rif->dev->mtu);
1424 mlxsw_reg_ritr_loopback_ipip4_pack(ritr_pl, lb_cf.lb_ipipt,
1425 MLXSW_REG_RITR_LOOPBACK_IPIP_OPTIONS_GRE_KEY_PRESET,
1426 ul_vr->id, saddr4, lb_cf.okey);
1429 case MLXSW_SP_L3_PROTO_IPV6:
1430 return -EAFNOSUPPORT;
1433 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
1436 static int mlxsw_sp_netdevice_ipip_ol_update_mtu(struct mlxsw_sp *mlxsw_sp,
1437 struct net_device *ol_dev)
1439 struct mlxsw_sp_ipip_entry *ipip_entry;
1440 struct mlxsw_sp_rif_ipip_lb *lb_rif;
1441 struct mlxsw_sp_vr *ul_vr;
1444 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1446 lb_rif = ipip_entry->ol_lb;
1447 ul_vr = &mlxsw_sp->router->vrs[lb_rif->ul_vr_id];
1448 err = mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr, true);
1451 lb_rif->common.mtu = ol_dev->mtu;
1458 static void mlxsw_sp_netdevice_ipip_ol_up_event(struct mlxsw_sp *mlxsw_sp,
1459 struct net_device *ol_dev)
1461 struct mlxsw_sp_ipip_entry *ipip_entry;
1463 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1465 mlxsw_sp_ipip_entry_ol_up_event(mlxsw_sp, ipip_entry);
1469 mlxsw_sp_ipip_entry_ol_down_event(struct mlxsw_sp *mlxsw_sp,
1470 struct mlxsw_sp_ipip_entry *ipip_entry)
1472 if (ipip_entry->decap_fib_entry)
1473 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry);
1476 static void mlxsw_sp_netdevice_ipip_ol_down_event(struct mlxsw_sp *mlxsw_sp,
1477 struct net_device *ol_dev)
1479 struct mlxsw_sp_ipip_entry *ipip_entry;
1481 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1483 mlxsw_sp_ipip_entry_ol_down_event(mlxsw_sp, ipip_entry);
1486 static void mlxsw_sp_nexthop_rif_migrate(struct mlxsw_sp *mlxsw_sp,
1487 struct mlxsw_sp_rif *old_rif,
1488 struct mlxsw_sp_rif *new_rif);
1490 mlxsw_sp_ipip_entry_ol_lb_update(struct mlxsw_sp *mlxsw_sp,
1491 struct mlxsw_sp_ipip_entry *ipip_entry,
1493 struct netlink_ext_ack *extack)
1495 struct mlxsw_sp_rif_ipip_lb *old_lb_rif = ipip_entry->ol_lb;
1496 struct mlxsw_sp_rif_ipip_lb *new_lb_rif;
1498 new_lb_rif = mlxsw_sp_ipip_ol_ipip_lb_create(mlxsw_sp,
1502 if (IS_ERR(new_lb_rif))
1503 return PTR_ERR(new_lb_rif);
1504 ipip_entry->ol_lb = new_lb_rif;
1507 mlxsw_sp_nexthop_rif_migrate(mlxsw_sp, &old_lb_rif->common,
1508 &new_lb_rif->common);
1510 mlxsw_sp_rif_destroy(&old_lb_rif->common);
1515 static void mlxsw_sp_nexthop_rif_update(struct mlxsw_sp *mlxsw_sp,
1516 struct mlxsw_sp_rif *rif);
1519 * Update the offload related to an IPIP entry. This always updates decap, and
1520 * in addition to that it also:
1521 * @recreate_loopback: recreates the associated loopback RIF
1522 * @keep_encap: updates next hops that use the tunnel netdevice. This is only
1523 * relevant when recreate_loopback is true.
1524 * @update_nexthops: updates next hops, keeping the current loopback RIF. This
1525 * is only relevant when recreate_loopback is false.
1527 int __mlxsw_sp_ipip_entry_update_tunnel(struct mlxsw_sp *mlxsw_sp,
1528 struct mlxsw_sp_ipip_entry *ipip_entry,
1529 bool recreate_loopback,
1531 bool update_nexthops,
1532 struct netlink_ext_ack *extack)
1536 /* RIFs can't be edited, so to update loopback, we need to destroy and
1537 * recreate it. That creates a window of opportunity where RALUE and
1538 * RATR registers end up referencing a RIF that's already gone. RATRs
1539 * are handled in mlxsw_sp_ipip_entry_ol_lb_update(), and to take care
1540 * of RALUE, demote the decap route back.
1542 if (ipip_entry->decap_fib_entry)
1543 mlxsw_sp_ipip_entry_demote_decap(mlxsw_sp, ipip_entry);
1545 if (recreate_loopback) {
1546 err = mlxsw_sp_ipip_entry_ol_lb_update(mlxsw_sp, ipip_entry,
1547 keep_encap, extack);
1550 } else if (update_nexthops) {
1551 mlxsw_sp_nexthop_rif_update(mlxsw_sp,
1552 &ipip_entry->ol_lb->common);
1555 if (ipip_entry->ol_dev->flags & IFF_UP)
1556 mlxsw_sp_ipip_entry_ol_up_event(mlxsw_sp, ipip_entry);
1561 static int mlxsw_sp_netdevice_ipip_ol_vrf_event(struct mlxsw_sp *mlxsw_sp,
1562 struct net_device *ol_dev,
1563 struct netlink_ext_ack *extack)
1565 struct mlxsw_sp_ipip_entry *ipip_entry =
1566 mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1567 enum mlxsw_sp_l3proto ul_proto;
1568 union mlxsw_sp_l3addr saddr;
1574 /* For flat configuration cases, moving overlay to a different VRF might
1575 * cause local address conflict, and the conflicting tunnels need to be
1578 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(ol_dev);
1579 ul_proto = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt]->ul_proto;
1580 saddr = mlxsw_sp_ipip_netdev_saddr(ul_proto, ol_dev);
1581 if (mlxsw_sp_ipip_demote_tunnel_by_saddr(mlxsw_sp, ul_proto,
1584 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry);
1588 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry,
1589 true, false, false, extack);
1593 mlxsw_sp_netdevice_ipip_ul_vrf_event(struct mlxsw_sp *mlxsw_sp,
1594 struct mlxsw_sp_ipip_entry *ipip_entry,
1595 struct net_device *ul_dev,
1596 struct netlink_ext_ack *extack)
1598 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry,
1599 true, true, false, extack);
1603 mlxsw_sp_netdevice_ipip_ul_up_event(struct mlxsw_sp *mlxsw_sp,
1604 struct mlxsw_sp_ipip_entry *ipip_entry,
1605 struct net_device *ul_dev)
1607 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry,
1608 false, false, true, NULL);
1612 mlxsw_sp_netdevice_ipip_ul_down_event(struct mlxsw_sp *mlxsw_sp,
1613 struct mlxsw_sp_ipip_entry *ipip_entry,
1614 struct net_device *ul_dev)
1616 /* A down underlay device causes encapsulated packets to not be
1617 * forwarded, but decap still works. So refresh next hops without
1618 * touching anything else.
1620 return __mlxsw_sp_ipip_entry_update_tunnel(mlxsw_sp, ipip_entry,
1621 false, false, true, NULL);
1625 mlxsw_sp_netdevice_ipip_ol_change_event(struct mlxsw_sp *mlxsw_sp,
1626 struct net_device *ol_dev,
1627 struct netlink_ext_ack *extack)
1629 const struct mlxsw_sp_ipip_ops *ipip_ops;
1630 struct mlxsw_sp_ipip_entry *ipip_entry;
1633 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, ol_dev);
1635 /* A change might make a tunnel eligible for offloading, but
1636 * that is currently not implemented. What falls to slow path
1641 /* A change might make a tunnel not eligible for offloading. */
1642 if (!mlxsw_sp_netdevice_ipip_can_offload(mlxsw_sp, ol_dev,
1643 ipip_entry->ipipt)) {
1644 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry);
1648 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
1649 err = ipip_ops->ol_netdev_change(mlxsw_sp, ipip_entry, extack);
1653 void mlxsw_sp_ipip_entry_demote_tunnel(struct mlxsw_sp *mlxsw_sp,
1654 struct mlxsw_sp_ipip_entry *ipip_entry)
1656 struct net_device *ol_dev = ipip_entry->ol_dev;
1658 if (ol_dev->flags & IFF_UP)
1659 mlxsw_sp_ipip_entry_ol_down_event(mlxsw_sp, ipip_entry);
1660 mlxsw_sp_ipip_entry_destroy(mlxsw_sp, ipip_entry);
1663 /* The configuration where several tunnels have the same local address in the
1664 * same underlay table needs special treatment in the HW. That is currently not
1665 * implemented in the driver. This function finds and demotes the first tunnel
1666 * with a given source address, except the one passed in in the argument
1670 mlxsw_sp_ipip_demote_tunnel_by_saddr(struct mlxsw_sp *mlxsw_sp,
1671 enum mlxsw_sp_l3proto ul_proto,
1672 union mlxsw_sp_l3addr saddr,
1674 const struct mlxsw_sp_ipip_entry *except)
1676 struct mlxsw_sp_ipip_entry *ipip_entry, *tmp;
1678 list_for_each_entry_safe(ipip_entry, tmp, &mlxsw_sp->router->ipip_list,
1680 if (ipip_entry != except &&
1681 mlxsw_sp_ipip_entry_saddr_matches(mlxsw_sp, ul_proto, saddr,
1682 ul_tb_id, ipip_entry)) {
1683 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry);
1691 static void mlxsw_sp_ipip_demote_tunnel_by_ul_netdev(struct mlxsw_sp *mlxsw_sp,
1692 struct net_device *ul_dev)
1694 struct mlxsw_sp_ipip_entry *ipip_entry, *tmp;
1696 list_for_each_entry_safe(ipip_entry, tmp, &mlxsw_sp->router->ipip_list,
1698 struct net_device *ipip_ul_dev =
1699 __mlxsw_sp_ipip_netdev_ul_dev_get(ipip_entry->ol_dev);
1701 if (ipip_ul_dev == ul_dev)
1702 mlxsw_sp_ipip_entry_demote_tunnel(mlxsw_sp, ipip_entry);
1706 int mlxsw_sp_netdevice_ipip_ol_event(struct mlxsw_sp *mlxsw_sp,
1707 struct net_device *ol_dev,
1708 unsigned long event,
1709 struct netdev_notifier_info *info)
1711 struct netdev_notifier_changeupper_info *chup;
1712 struct netlink_ext_ack *extack;
1715 case NETDEV_REGISTER:
1716 return mlxsw_sp_netdevice_ipip_ol_reg_event(mlxsw_sp, ol_dev);
1717 case NETDEV_UNREGISTER:
1718 mlxsw_sp_netdevice_ipip_ol_unreg_event(mlxsw_sp, ol_dev);
1721 mlxsw_sp_netdevice_ipip_ol_up_event(mlxsw_sp, ol_dev);
1724 mlxsw_sp_netdevice_ipip_ol_down_event(mlxsw_sp, ol_dev);
1726 case NETDEV_CHANGEUPPER:
1727 chup = container_of(info, typeof(*chup), info);
1728 extack = info->extack;
1729 if (netif_is_l3_master(chup->upper_dev))
1730 return mlxsw_sp_netdevice_ipip_ol_vrf_event(mlxsw_sp,
1735 extack = info->extack;
1736 return mlxsw_sp_netdevice_ipip_ol_change_event(mlxsw_sp,
1738 case NETDEV_CHANGEMTU:
1739 return mlxsw_sp_netdevice_ipip_ol_update_mtu(mlxsw_sp, ol_dev);
1745 __mlxsw_sp_netdevice_ipip_ul_event(struct mlxsw_sp *mlxsw_sp,
1746 struct mlxsw_sp_ipip_entry *ipip_entry,
1747 struct net_device *ul_dev,
1748 unsigned long event,
1749 struct netdev_notifier_info *info)
1751 struct netdev_notifier_changeupper_info *chup;
1752 struct netlink_ext_ack *extack;
1755 case NETDEV_CHANGEUPPER:
1756 chup = container_of(info, typeof(*chup), info);
1757 extack = info->extack;
1758 if (netif_is_l3_master(chup->upper_dev))
1759 return mlxsw_sp_netdevice_ipip_ul_vrf_event(mlxsw_sp,
1766 return mlxsw_sp_netdevice_ipip_ul_up_event(mlxsw_sp, ipip_entry,
1769 return mlxsw_sp_netdevice_ipip_ul_down_event(mlxsw_sp,
1777 mlxsw_sp_netdevice_ipip_ul_event(struct mlxsw_sp *mlxsw_sp,
1778 struct net_device *ul_dev,
1779 unsigned long event,
1780 struct netdev_notifier_info *info)
1782 struct mlxsw_sp_ipip_entry *ipip_entry = NULL;
1785 while ((ipip_entry = mlxsw_sp_ipip_entry_find_by_ul_dev(mlxsw_sp,
1788 err = __mlxsw_sp_netdevice_ipip_ul_event(mlxsw_sp, ipip_entry,
1789 ul_dev, event, info);
1791 mlxsw_sp_ipip_demote_tunnel_by_ul_netdev(mlxsw_sp,
1800 struct mlxsw_sp_neigh_key {
1801 struct neighbour *n;
1804 struct mlxsw_sp_neigh_entry {
1805 struct list_head rif_list_node;
1806 struct rhash_head ht_node;
1807 struct mlxsw_sp_neigh_key key;
1810 unsigned char ha[ETH_ALEN];
1811 struct list_head nexthop_list; /* list of nexthops using
1814 struct list_head nexthop_neighs_list_node;
1815 unsigned int counter_index;
1819 static const struct rhashtable_params mlxsw_sp_neigh_ht_params = {
1820 .key_offset = offsetof(struct mlxsw_sp_neigh_entry, key),
1821 .head_offset = offsetof(struct mlxsw_sp_neigh_entry, ht_node),
1822 .key_len = sizeof(struct mlxsw_sp_neigh_key),
1825 struct mlxsw_sp_neigh_entry *
1826 mlxsw_sp_rif_neigh_next(struct mlxsw_sp_rif *rif,
1827 struct mlxsw_sp_neigh_entry *neigh_entry)
1830 if (list_empty(&rif->neigh_list))
1833 return list_first_entry(&rif->neigh_list,
1834 typeof(*neigh_entry),
1837 if (list_is_last(&neigh_entry->rif_list_node, &rif->neigh_list))
1839 return list_next_entry(neigh_entry, rif_list_node);
1842 int mlxsw_sp_neigh_entry_type(struct mlxsw_sp_neigh_entry *neigh_entry)
1844 return neigh_entry->key.n->tbl->family;
1848 mlxsw_sp_neigh_entry_ha(struct mlxsw_sp_neigh_entry *neigh_entry)
1850 return neigh_entry->ha;
1853 u32 mlxsw_sp_neigh4_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry)
1855 struct neighbour *n;
1857 n = neigh_entry->key.n;
1858 return ntohl(*((__be32 *) n->primary_key));
1862 mlxsw_sp_neigh6_entry_dip(struct mlxsw_sp_neigh_entry *neigh_entry)
1864 struct neighbour *n;
1866 n = neigh_entry->key.n;
1867 return (struct in6_addr *) &n->primary_key;
1870 int mlxsw_sp_neigh_counter_get(struct mlxsw_sp *mlxsw_sp,
1871 struct mlxsw_sp_neigh_entry *neigh_entry,
1874 if (!neigh_entry->counter_valid)
1877 return mlxsw_sp_flow_counter_get(mlxsw_sp, neigh_entry->counter_index,
1881 static struct mlxsw_sp_neigh_entry *
1882 mlxsw_sp_neigh_entry_alloc(struct mlxsw_sp *mlxsw_sp, struct neighbour *n,
1885 struct mlxsw_sp_neigh_entry *neigh_entry;
1887 neigh_entry = kzalloc(sizeof(*neigh_entry), GFP_KERNEL);
1891 neigh_entry->key.n = n;
1892 neigh_entry->rif = rif;
1893 INIT_LIST_HEAD(&neigh_entry->nexthop_list);
1898 static void mlxsw_sp_neigh_entry_free(struct mlxsw_sp_neigh_entry *neigh_entry)
1904 mlxsw_sp_neigh_entry_insert(struct mlxsw_sp *mlxsw_sp,
1905 struct mlxsw_sp_neigh_entry *neigh_entry)
1907 return rhashtable_insert_fast(&mlxsw_sp->router->neigh_ht,
1908 &neigh_entry->ht_node,
1909 mlxsw_sp_neigh_ht_params);
1913 mlxsw_sp_neigh_entry_remove(struct mlxsw_sp *mlxsw_sp,
1914 struct mlxsw_sp_neigh_entry *neigh_entry)
1916 rhashtable_remove_fast(&mlxsw_sp->router->neigh_ht,
1917 &neigh_entry->ht_node,
1918 mlxsw_sp_neigh_ht_params);
1922 mlxsw_sp_neigh_counter_should_alloc(struct mlxsw_sp *mlxsw_sp,
1923 struct mlxsw_sp_neigh_entry *neigh_entry)
1925 struct devlink *devlink;
1926 const char *table_name;
1928 switch (mlxsw_sp_neigh_entry_type(neigh_entry)) {
1930 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST4;
1933 table_name = MLXSW_SP_DPIPE_TABLE_NAME_HOST6;
1940 devlink = priv_to_devlink(mlxsw_sp->core);
1941 return devlink_dpipe_table_counter_enabled(devlink, table_name);
1945 mlxsw_sp_neigh_counter_alloc(struct mlxsw_sp *mlxsw_sp,
1946 struct mlxsw_sp_neigh_entry *neigh_entry)
1948 if (!mlxsw_sp_neigh_counter_should_alloc(mlxsw_sp, neigh_entry))
1951 if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &neigh_entry->counter_index))
1954 neigh_entry->counter_valid = true;
1958 mlxsw_sp_neigh_counter_free(struct mlxsw_sp *mlxsw_sp,
1959 struct mlxsw_sp_neigh_entry *neigh_entry)
1961 if (!neigh_entry->counter_valid)
1963 mlxsw_sp_flow_counter_free(mlxsw_sp,
1964 neigh_entry->counter_index);
1965 neigh_entry->counter_valid = false;
1968 static struct mlxsw_sp_neigh_entry *
1969 mlxsw_sp_neigh_entry_create(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
1971 struct mlxsw_sp_neigh_entry *neigh_entry;
1972 struct mlxsw_sp_rif *rif;
1975 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, n->dev);
1977 return ERR_PTR(-EINVAL);
1979 neigh_entry = mlxsw_sp_neigh_entry_alloc(mlxsw_sp, n, rif->rif_index);
1981 return ERR_PTR(-ENOMEM);
1983 err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
1985 goto err_neigh_entry_insert;
1987 mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry);
1988 list_add(&neigh_entry->rif_list_node, &rif->neigh_list);
1992 err_neigh_entry_insert:
1993 mlxsw_sp_neigh_entry_free(neigh_entry);
1994 return ERR_PTR(err);
1998 mlxsw_sp_neigh_entry_destroy(struct mlxsw_sp *mlxsw_sp,
1999 struct mlxsw_sp_neigh_entry *neigh_entry)
2001 list_del(&neigh_entry->rif_list_node);
2002 mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry);
2003 mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry);
2004 mlxsw_sp_neigh_entry_free(neigh_entry);
2007 static struct mlxsw_sp_neigh_entry *
2008 mlxsw_sp_neigh_entry_lookup(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
2010 struct mlxsw_sp_neigh_key key;
2013 return rhashtable_lookup_fast(&mlxsw_sp->router->neigh_ht,
2014 &key, mlxsw_sp_neigh_ht_params);
2018 mlxsw_sp_router_neighs_update_interval_init(struct mlxsw_sp *mlxsw_sp)
2020 unsigned long interval;
2022 #if IS_ENABLED(CONFIG_IPV6)
2023 interval = min_t(unsigned long,
2024 NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME),
2025 NEIGH_VAR(&nd_tbl.parms, DELAY_PROBE_TIME));
2027 interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME);
2029 mlxsw_sp->router->neighs_update.interval = jiffies_to_msecs(interval);
2032 static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp,
2036 struct net_device *dev;
2037 struct neighbour *n;
2042 mlxsw_reg_rauhtd_ent_ipv4_unpack(rauhtd_pl, ent_index, &rif, &dip);
2044 if (!mlxsw_sp->router->rifs[rif]) {
2045 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
2050 dev = mlxsw_sp->router->rifs[rif]->dev;
2051 n = neigh_lookup(&arp_tbl, &dipn, dev);
2055 netdev_dbg(dev, "Updating neighbour with IP=%pI4h\n", &dip);
2056 neigh_event_send(n, NULL);
2060 #if IS_ENABLED(CONFIG_IPV6)
2061 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp,
2065 struct net_device *dev;
2066 struct neighbour *n;
2067 struct in6_addr dip;
2070 mlxsw_reg_rauhtd_ent_ipv6_unpack(rauhtd_pl, rec_index, &rif,
2073 if (!mlxsw_sp->router->rifs[rif]) {
2074 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
2078 dev = mlxsw_sp->router->rifs[rif]->dev;
2079 n = neigh_lookup(&nd_tbl, &dip, dev);
2083 netdev_dbg(dev, "Updating neighbour with IP=%pI6c\n", &dip);
2084 neigh_event_send(n, NULL);
2088 static void mlxsw_sp_router_neigh_ent_ipv6_process(struct mlxsw_sp *mlxsw_sp,
2095 static void mlxsw_sp_router_neigh_rec_ipv4_process(struct mlxsw_sp *mlxsw_sp,
2102 num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
2104 /* Hardware starts counting at 0, so add 1. */
2107 /* Each record consists of several neighbour entries. */
2108 for (i = 0; i < num_entries; i++) {
2111 ent_index = rec_index * MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC + i;
2112 mlxsw_sp_router_neigh_ent_ipv4_process(mlxsw_sp, rauhtd_pl,
2118 static void mlxsw_sp_router_neigh_rec_ipv6_process(struct mlxsw_sp *mlxsw_sp,
2122 /* One record contains one entry. */
2123 mlxsw_sp_router_neigh_ent_ipv6_process(mlxsw_sp, rauhtd_pl,
2127 static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp,
2128 char *rauhtd_pl, int rec_index)
2130 switch (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, rec_index)) {
2131 case MLXSW_REG_RAUHTD_TYPE_IPV4:
2132 mlxsw_sp_router_neigh_rec_ipv4_process(mlxsw_sp, rauhtd_pl,
2135 case MLXSW_REG_RAUHTD_TYPE_IPV6:
2136 mlxsw_sp_router_neigh_rec_ipv6_process(mlxsw_sp, rauhtd_pl,
2142 static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl)
2144 u8 num_rec, last_rec_index, num_entries;
2146 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
2147 last_rec_index = num_rec - 1;
2149 if (num_rec < MLXSW_REG_RAUHTD_REC_MAX_NUM)
2151 if (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, last_rec_index) ==
2152 MLXSW_REG_RAUHTD_TYPE_IPV6)
2155 num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
2157 if (++num_entries == MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC)
2163 __mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp,
2165 enum mlxsw_reg_rauhtd_type type)
2170 /* Make sure the neighbour's netdev isn't removed in the
2175 mlxsw_reg_rauhtd_pack(rauhtd_pl, type);
2176 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(rauhtd),
2179 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to dump neighbour table\n");
2182 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
2183 for (i = 0; i < num_rec; i++)
2184 mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl,
2186 } while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl));
2192 static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp)
2194 enum mlxsw_reg_rauhtd_type type;
2198 rauhtd_pl = kmalloc(MLXSW_REG_RAUHTD_LEN, GFP_KERNEL);
2202 type = MLXSW_REG_RAUHTD_TYPE_IPV4;
2203 err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type);
2207 type = MLXSW_REG_RAUHTD_TYPE_IPV6;
2208 err = __mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp, rauhtd_pl, type);
2214 static void mlxsw_sp_router_neighs_update_nh(struct mlxsw_sp *mlxsw_sp)
2216 struct mlxsw_sp_neigh_entry *neigh_entry;
2218 /* Take RTNL mutex here to prevent lists from changes */
2220 list_for_each_entry(neigh_entry, &mlxsw_sp->router->nexthop_neighs_list,
2221 nexthop_neighs_list_node)
2222 /* If this neigh have nexthops, make the kernel think this neigh
2223 * is active regardless of the traffic.
2225 neigh_event_send(neigh_entry->key.n, NULL);
2230 mlxsw_sp_router_neighs_update_work_schedule(struct mlxsw_sp *mlxsw_sp)
2232 unsigned long interval = mlxsw_sp->router->neighs_update.interval;
2234 mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw,
2235 msecs_to_jiffies(interval));
2238 static void mlxsw_sp_router_neighs_update_work(struct work_struct *work)
2240 struct mlxsw_sp_router *router;
2243 router = container_of(work, struct mlxsw_sp_router,
2244 neighs_update.dw.work);
2245 err = mlxsw_sp_router_neighs_update_rauhtd(router->mlxsw_sp);
2247 dev_err(router->mlxsw_sp->bus_info->dev, "Could not update kernel for neigh activity");
2249 mlxsw_sp_router_neighs_update_nh(router->mlxsw_sp);
2251 mlxsw_sp_router_neighs_update_work_schedule(router->mlxsw_sp);
2254 static void mlxsw_sp_router_probe_unresolved_nexthops(struct work_struct *work)
2256 struct mlxsw_sp_neigh_entry *neigh_entry;
2257 struct mlxsw_sp_router *router;
2259 router = container_of(work, struct mlxsw_sp_router,
2260 nexthop_probe_dw.work);
2261 /* Iterate over nexthop neighbours, find those who are unresolved and
2262 * send arp on them. This solves the chicken-egg problem when
2263 * the nexthop wouldn't get offloaded until the neighbor is resolved
2264 * but it wouldn't get resolved ever in case traffic is flowing in HW
2265 * using different nexthop.
2267 * Take RTNL mutex here to prevent lists from changes.
2270 list_for_each_entry(neigh_entry, &router->nexthop_neighs_list,
2271 nexthop_neighs_list_node)
2272 if (!neigh_entry->connected)
2273 neigh_event_send(neigh_entry->key.n, NULL);
2276 mlxsw_core_schedule_dw(&router->nexthop_probe_dw,
2277 MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL);
2281 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
2282 struct mlxsw_sp_neigh_entry *neigh_entry,
2285 static enum mlxsw_reg_rauht_op mlxsw_sp_rauht_op(bool adding)
2287 return adding ? MLXSW_REG_RAUHT_OP_WRITE_ADD :
2288 MLXSW_REG_RAUHT_OP_WRITE_DELETE;
2292 mlxsw_sp_router_neigh_entry_op4(struct mlxsw_sp *mlxsw_sp,
2293 struct mlxsw_sp_neigh_entry *neigh_entry,
2294 enum mlxsw_reg_rauht_op op)
2296 struct neighbour *n = neigh_entry->key.n;
2297 u32 dip = ntohl(*((__be32 *) n->primary_key));
2298 char rauht_pl[MLXSW_REG_RAUHT_LEN];
2300 mlxsw_reg_rauht_pack4(rauht_pl, op, neigh_entry->rif, neigh_entry->ha,
2302 if (neigh_entry->counter_valid)
2303 mlxsw_reg_rauht_pack_counter(rauht_pl,
2304 neigh_entry->counter_index);
2305 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
2309 mlxsw_sp_router_neigh_entry_op6(struct mlxsw_sp *mlxsw_sp,
2310 struct mlxsw_sp_neigh_entry *neigh_entry,
2311 enum mlxsw_reg_rauht_op op)
2313 struct neighbour *n = neigh_entry->key.n;
2314 char rauht_pl[MLXSW_REG_RAUHT_LEN];
2315 const char *dip = n->primary_key;
2317 mlxsw_reg_rauht_pack6(rauht_pl, op, neigh_entry->rif, neigh_entry->ha,
2319 if (neigh_entry->counter_valid)
2320 mlxsw_reg_rauht_pack_counter(rauht_pl,
2321 neigh_entry->counter_index);
2322 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht), rauht_pl);
2325 bool mlxsw_sp_neigh_ipv6_ignore(struct mlxsw_sp_neigh_entry *neigh_entry)
2327 struct neighbour *n = neigh_entry->key.n;
2329 /* Packets with a link-local destination address are trapped
2330 * after LPM lookup and never reach the neighbour table, so
2331 * there is no need to program such neighbours to the device.
2333 if (ipv6_addr_type((struct in6_addr *) &n->primary_key) &
2334 IPV6_ADDR_LINKLOCAL)
2340 mlxsw_sp_neigh_entry_update(struct mlxsw_sp *mlxsw_sp,
2341 struct mlxsw_sp_neigh_entry *neigh_entry,
2344 if (!adding && !neigh_entry->connected)
2346 neigh_entry->connected = adding;
2347 if (neigh_entry->key.n->tbl->family == AF_INET) {
2348 mlxsw_sp_router_neigh_entry_op4(mlxsw_sp, neigh_entry,
2349 mlxsw_sp_rauht_op(adding));
2350 } else if (neigh_entry->key.n->tbl->family == AF_INET6) {
2351 if (mlxsw_sp_neigh_ipv6_ignore(neigh_entry))
2353 mlxsw_sp_router_neigh_entry_op6(mlxsw_sp, neigh_entry,
2354 mlxsw_sp_rauht_op(adding));
2361 mlxsw_sp_neigh_entry_counter_update(struct mlxsw_sp *mlxsw_sp,
2362 struct mlxsw_sp_neigh_entry *neigh_entry,
2366 mlxsw_sp_neigh_counter_alloc(mlxsw_sp, neigh_entry);
2368 mlxsw_sp_neigh_counter_free(mlxsw_sp, neigh_entry);
2369 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, true);
2372 struct mlxsw_sp_netevent_work {
2373 struct work_struct work;
2374 struct mlxsw_sp *mlxsw_sp;
2375 struct neighbour *n;
2378 static void mlxsw_sp_router_neigh_event_work(struct work_struct *work)
2380 struct mlxsw_sp_netevent_work *net_work =
2381 container_of(work, struct mlxsw_sp_netevent_work, work);
2382 struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp;
2383 struct mlxsw_sp_neigh_entry *neigh_entry;
2384 struct neighbour *n = net_work->n;
2385 unsigned char ha[ETH_ALEN];
2386 bool entry_connected;
2389 /* If these parameters are changed after we release the lock,
2390 * then we are guaranteed to receive another event letting us
2393 read_lock_bh(&n->lock);
2394 memcpy(ha, n->ha, ETH_ALEN);
2395 nud_state = n->nud_state;
2397 read_unlock_bh(&n->lock);
2400 mlxsw_sp_span_respin(mlxsw_sp);
2402 entry_connected = nud_state & NUD_VALID && !dead;
2403 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
2404 if (!entry_connected && !neigh_entry)
2407 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
2408 if (IS_ERR(neigh_entry))
2412 memcpy(neigh_entry->ha, ha, ETH_ALEN);
2413 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, entry_connected);
2414 mlxsw_sp_nexthop_neigh_update(mlxsw_sp, neigh_entry, !entry_connected);
2416 if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
2417 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
2425 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp);
2427 static void mlxsw_sp_router_mp_hash_event_work(struct work_struct *work)
2429 struct mlxsw_sp_netevent_work *net_work =
2430 container_of(work, struct mlxsw_sp_netevent_work, work);
2431 struct mlxsw_sp *mlxsw_sp = net_work->mlxsw_sp;
2433 mlxsw_sp_mp_hash_init(mlxsw_sp);
2437 static int mlxsw_sp_router_netevent_event(struct notifier_block *nb,
2438 unsigned long event, void *ptr)
2440 struct mlxsw_sp_netevent_work *net_work;
2441 struct mlxsw_sp_port *mlxsw_sp_port;
2442 struct mlxsw_sp_router *router;
2443 struct mlxsw_sp *mlxsw_sp;
2444 unsigned long interval;
2445 struct neigh_parms *p;
2446 struct neighbour *n;
2450 case NETEVENT_DELAY_PROBE_TIME_UPDATE:
2453 /* We don't care about changes in the default table. */
2454 if (!p->dev || (p->tbl->family != AF_INET &&
2455 p->tbl->family != AF_INET6))
2458 /* We are in atomic context and can't take RTNL mutex,
2459 * so use RCU variant to walk the device chain.
2461 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(p->dev);
2465 mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
2466 interval = jiffies_to_msecs(NEIGH_VAR(p, DELAY_PROBE_TIME));
2467 mlxsw_sp->router->neighs_update.interval = interval;
2469 mlxsw_sp_port_dev_put(mlxsw_sp_port);
2471 case NETEVENT_NEIGH_UPDATE:
2474 if (n->tbl->family != AF_INET && n->tbl->family != AF_INET6)
2477 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(n->dev);
2481 net_work = kzalloc(sizeof(*net_work), GFP_ATOMIC);
2483 mlxsw_sp_port_dev_put(mlxsw_sp_port);
2487 INIT_WORK(&net_work->work, mlxsw_sp_router_neigh_event_work);
2488 net_work->mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
2491 /* Take a reference to ensure the neighbour won't be
2492 * destructed until we drop the reference in delayed
2496 mlxsw_core_schedule_work(&net_work->work);
2497 mlxsw_sp_port_dev_put(mlxsw_sp_port);
2499 case NETEVENT_IPV4_MPATH_HASH_UPDATE:
2500 case NETEVENT_IPV6_MPATH_HASH_UPDATE:
2503 if (!net_eq(net, &init_net))
2506 net_work = kzalloc(sizeof(*net_work), GFP_ATOMIC);
2510 router = container_of(nb, struct mlxsw_sp_router, netevent_nb);
2511 INIT_WORK(&net_work->work, mlxsw_sp_router_mp_hash_event_work);
2512 net_work->mlxsw_sp = router->mlxsw_sp;
2513 mlxsw_core_schedule_work(&net_work->work);
2520 static int mlxsw_sp_neigh_init(struct mlxsw_sp *mlxsw_sp)
2524 err = rhashtable_init(&mlxsw_sp->router->neigh_ht,
2525 &mlxsw_sp_neigh_ht_params);
2529 /* Initialize the polling interval according to the default
2532 mlxsw_sp_router_neighs_update_interval_init(mlxsw_sp);
2534 /* Create the delayed works for the activity_update */
2535 INIT_DELAYED_WORK(&mlxsw_sp->router->neighs_update.dw,
2536 mlxsw_sp_router_neighs_update_work);
2537 INIT_DELAYED_WORK(&mlxsw_sp->router->nexthop_probe_dw,
2538 mlxsw_sp_router_probe_unresolved_nexthops);
2539 mlxsw_core_schedule_dw(&mlxsw_sp->router->neighs_update.dw, 0);
2540 mlxsw_core_schedule_dw(&mlxsw_sp->router->nexthop_probe_dw, 0);
2544 static void mlxsw_sp_neigh_fini(struct mlxsw_sp *mlxsw_sp)
2546 cancel_delayed_work_sync(&mlxsw_sp->router->neighs_update.dw);
2547 cancel_delayed_work_sync(&mlxsw_sp->router->nexthop_probe_dw);
2548 rhashtable_destroy(&mlxsw_sp->router->neigh_ht);
2551 static void mlxsw_sp_neigh_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
2552 struct mlxsw_sp_rif *rif)
2554 struct mlxsw_sp_neigh_entry *neigh_entry, *tmp;
2556 list_for_each_entry_safe(neigh_entry, tmp, &rif->neigh_list,
2558 mlxsw_sp_neigh_entry_update(mlxsw_sp, neigh_entry, false);
2559 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
2563 enum mlxsw_sp_nexthop_type {
2564 MLXSW_SP_NEXTHOP_TYPE_ETH,
2565 MLXSW_SP_NEXTHOP_TYPE_IPIP,
2568 struct mlxsw_sp_nexthop_key {
2569 struct fib_nh *fib_nh;
2572 struct mlxsw_sp_nexthop {
2573 struct list_head neigh_list_node; /* member of neigh entry list */
2574 struct list_head rif_list_node;
2575 struct list_head router_list_node;
2576 struct mlxsw_sp_nexthop_group *nh_grp; /* pointer back to the group
2579 struct rhash_head ht_node;
2580 struct mlxsw_sp_nexthop_key key;
2581 unsigned char gw_addr[sizeof(struct in6_addr)];
2585 int num_adj_entries;
2586 struct mlxsw_sp_rif *rif;
2587 u8 should_offload:1, /* set indicates this neigh is connected and
2588 * should be put to KVD linear area of this group.
2590 offloaded:1, /* set in case the neigh is actually put into
2591 * KVD linear area of this group.
2593 update:1; /* set indicates that MAC of this neigh should be
2596 enum mlxsw_sp_nexthop_type type;
2598 struct mlxsw_sp_neigh_entry *neigh_entry;
2599 struct mlxsw_sp_ipip_entry *ipip_entry;
2601 unsigned int counter_index;
2605 struct mlxsw_sp_nexthop_group {
2607 struct rhash_head ht_node;
2608 struct list_head fib_list; /* list of fib entries that use this group */
2609 struct neigh_table *neigh_tbl;
2610 u8 adj_index_valid:1,
2611 gateway:1; /* routes using the group use a gateway */
2615 int sum_norm_weight;
2616 struct mlxsw_sp_nexthop nexthops[0];
2617 #define nh_rif nexthops[0].rif
2620 void mlxsw_sp_nexthop_counter_alloc(struct mlxsw_sp *mlxsw_sp,
2621 struct mlxsw_sp_nexthop *nh)
2623 struct devlink *devlink;
2625 devlink = priv_to_devlink(mlxsw_sp->core);
2626 if (!devlink_dpipe_table_counter_enabled(devlink,
2627 MLXSW_SP_DPIPE_TABLE_NAME_ADJ))
2630 if (mlxsw_sp_flow_counter_alloc(mlxsw_sp, &nh->counter_index))
2633 nh->counter_valid = true;
2636 void mlxsw_sp_nexthop_counter_free(struct mlxsw_sp *mlxsw_sp,
2637 struct mlxsw_sp_nexthop *nh)
2639 if (!nh->counter_valid)
2641 mlxsw_sp_flow_counter_free(mlxsw_sp, nh->counter_index);
2642 nh->counter_valid = false;
2645 int mlxsw_sp_nexthop_counter_get(struct mlxsw_sp *mlxsw_sp,
2646 struct mlxsw_sp_nexthop *nh, u64 *p_counter)
2648 if (!nh->counter_valid)
2651 return mlxsw_sp_flow_counter_get(mlxsw_sp, nh->counter_index,
2655 struct mlxsw_sp_nexthop *mlxsw_sp_nexthop_next(struct mlxsw_sp_router *router,
2656 struct mlxsw_sp_nexthop *nh)
2659 if (list_empty(&router->nexthop_list))
2662 return list_first_entry(&router->nexthop_list,
2663 typeof(*nh), router_list_node);
2665 if (list_is_last(&nh->router_list_node, &router->nexthop_list))
2667 return list_next_entry(nh, router_list_node);
2670 bool mlxsw_sp_nexthop_offload(struct mlxsw_sp_nexthop *nh)
2672 return nh->offloaded;
2675 unsigned char *mlxsw_sp_nexthop_ha(struct mlxsw_sp_nexthop *nh)
2679 return nh->neigh_entry->ha;
2682 int mlxsw_sp_nexthop_indexes(struct mlxsw_sp_nexthop *nh, u32 *p_adj_index,
2683 u32 *p_adj_size, u32 *p_adj_hash_index)
2685 struct mlxsw_sp_nexthop_group *nh_grp = nh->nh_grp;
2686 u32 adj_hash_index = 0;
2689 if (!nh->offloaded || !nh_grp->adj_index_valid)
2692 *p_adj_index = nh_grp->adj_index;
2693 *p_adj_size = nh_grp->ecmp_size;
2695 for (i = 0; i < nh_grp->count; i++) {
2696 struct mlxsw_sp_nexthop *nh_iter = &nh_grp->nexthops[i];
2700 if (nh_iter->offloaded)
2701 adj_hash_index += nh_iter->num_adj_entries;
2704 *p_adj_hash_index = adj_hash_index;
2708 struct mlxsw_sp_rif *mlxsw_sp_nexthop_rif(struct mlxsw_sp_nexthop *nh)
2713 bool mlxsw_sp_nexthop_group_has_ipip(struct mlxsw_sp_nexthop *nh)
2715 struct mlxsw_sp_nexthop_group *nh_grp = nh->nh_grp;
2718 for (i = 0; i < nh_grp->count; i++) {
2719 struct mlxsw_sp_nexthop *nh_iter = &nh_grp->nexthops[i];
2721 if (nh_iter->type == MLXSW_SP_NEXTHOP_TYPE_IPIP)
2727 static struct fib_info *
2728 mlxsw_sp_nexthop4_group_fi(const struct mlxsw_sp_nexthop_group *nh_grp)
2730 return nh_grp->priv;
2733 struct mlxsw_sp_nexthop_group_cmp_arg {
2734 enum mlxsw_sp_l3proto proto;
2736 struct fib_info *fi;
2737 struct mlxsw_sp_fib6_entry *fib6_entry;
2742 mlxsw_sp_nexthop6_group_has_nexthop(const struct mlxsw_sp_nexthop_group *nh_grp,
2743 const struct in6_addr *gw, int ifindex,
2748 for (i = 0; i < nh_grp->count; i++) {
2749 const struct mlxsw_sp_nexthop *nh;
2751 nh = &nh_grp->nexthops[i];
2752 if (nh->ifindex == ifindex && nh->nh_weight == weight &&
2753 ipv6_addr_equal(gw, (struct in6_addr *) nh->gw_addr))
2761 mlxsw_sp_nexthop6_group_cmp(const struct mlxsw_sp_nexthop_group *nh_grp,
2762 const struct mlxsw_sp_fib6_entry *fib6_entry)
2764 struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
2766 if (nh_grp->count != fib6_entry->nrt6)
2769 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
2770 struct in6_addr *gw;
2771 int ifindex, weight;
2773 ifindex = mlxsw_sp_rt6->rt->fib6_nh.nh_dev->ifindex;
2774 weight = mlxsw_sp_rt6->rt->fib6_nh.nh_weight;
2775 gw = &mlxsw_sp_rt6->rt->fib6_nh.nh_gw;
2776 if (!mlxsw_sp_nexthop6_group_has_nexthop(nh_grp, gw, ifindex,
2785 mlxsw_sp_nexthop_group_cmp(struct rhashtable_compare_arg *arg, const void *ptr)
2787 const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = arg->key;
2788 const struct mlxsw_sp_nexthop_group *nh_grp = ptr;
2790 switch (cmp_arg->proto) {
2791 case MLXSW_SP_L3_PROTO_IPV4:
2792 return cmp_arg->fi != mlxsw_sp_nexthop4_group_fi(nh_grp);
2793 case MLXSW_SP_L3_PROTO_IPV6:
2794 return !mlxsw_sp_nexthop6_group_cmp(nh_grp,
2795 cmp_arg->fib6_entry);
2803 mlxsw_sp_nexthop_group_type(const struct mlxsw_sp_nexthop_group *nh_grp)
2805 return nh_grp->neigh_tbl->family;
2808 static u32 mlxsw_sp_nexthop_group_hash_obj(const void *data, u32 len, u32 seed)
2810 const struct mlxsw_sp_nexthop_group *nh_grp = data;
2811 const struct mlxsw_sp_nexthop *nh;
2812 struct fib_info *fi;
2816 switch (mlxsw_sp_nexthop_group_type(nh_grp)) {
2818 fi = mlxsw_sp_nexthop4_group_fi(nh_grp);
2819 return jhash(&fi, sizeof(fi), seed);
2821 val = nh_grp->count;
2822 for (i = 0; i < nh_grp->count; i++) {
2823 nh = &nh_grp->nexthops[i];
2826 return jhash(&val, sizeof(val), seed);
2834 mlxsw_sp_nexthop6_group_hash(struct mlxsw_sp_fib6_entry *fib6_entry, u32 seed)
2836 unsigned int val = fib6_entry->nrt6;
2837 struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
2838 struct net_device *dev;
2840 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
2841 dev = mlxsw_sp_rt6->rt->fib6_nh.nh_dev;
2842 val ^= dev->ifindex;
2845 return jhash(&val, sizeof(val), seed);
2849 mlxsw_sp_nexthop_group_hash(const void *data, u32 len, u32 seed)
2851 const struct mlxsw_sp_nexthop_group_cmp_arg *cmp_arg = data;
2853 switch (cmp_arg->proto) {
2854 case MLXSW_SP_L3_PROTO_IPV4:
2855 return jhash(&cmp_arg->fi, sizeof(cmp_arg->fi), seed);
2856 case MLXSW_SP_L3_PROTO_IPV6:
2857 return mlxsw_sp_nexthop6_group_hash(cmp_arg->fib6_entry, seed);
2864 static const struct rhashtable_params mlxsw_sp_nexthop_group_ht_params = {
2865 .head_offset = offsetof(struct mlxsw_sp_nexthop_group, ht_node),
2866 .hashfn = mlxsw_sp_nexthop_group_hash,
2867 .obj_hashfn = mlxsw_sp_nexthop_group_hash_obj,
2868 .obj_cmpfn = mlxsw_sp_nexthop_group_cmp,
2871 static int mlxsw_sp_nexthop_group_insert(struct mlxsw_sp *mlxsw_sp,
2872 struct mlxsw_sp_nexthop_group *nh_grp)
2874 if (mlxsw_sp_nexthop_group_type(nh_grp) == AF_INET6 &&
2878 return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_group_ht,
2880 mlxsw_sp_nexthop_group_ht_params);
2883 static void mlxsw_sp_nexthop_group_remove(struct mlxsw_sp *mlxsw_sp,
2884 struct mlxsw_sp_nexthop_group *nh_grp)
2886 if (mlxsw_sp_nexthop_group_type(nh_grp) == AF_INET6 &&
2890 rhashtable_remove_fast(&mlxsw_sp->router->nexthop_group_ht,
2892 mlxsw_sp_nexthop_group_ht_params);
2895 static struct mlxsw_sp_nexthop_group *
2896 mlxsw_sp_nexthop4_group_lookup(struct mlxsw_sp *mlxsw_sp,
2897 struct fib_info *fi)
2899 struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg;
2901 cmp_arg.proto = MLXSW_SP_L3_PROTO_IPV4;
2903 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht,
2905 mlxsw_sp_nexthop_group_ht_params);
2908 static struct mlxsw_sp_nexthop_group *
2909 mlxsw_sp_nexthop6_group_lookup(struct mlxsw_sp *mlxsw_sp,
2910 struct mlxsw_sp_fib6_entry *fib6_entry)
2912 struct mlxsw_sp_nexthop_group_cmp_arg cmp_arg;
2914 cmp_arg.proto = MLXSW_SP_L3_PROTO_IPV6;
2915 cmp_arg.fib6_entry = fib6_entry;
2916 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_group_ht,
2918 mlxsw_sp_nexthop_group_ht_params);
2921 static const struct rhashtable_params mlxsw_sp_nexthop_ht_params = {
2922 .key_offset = offsetof(struct mlxsw_sp_nexthop, key),
2923 .head_offset = offsetof(struct mlxsw_sp_nexthop, ht_node),
2924 .key_len = sizeof(struct mlxsw_sp_nexthop_key),
2927 static int mlxsw_sp_nexthop_insert(struct mlxsw_sp *mlxsw_sp,
2928 struct mlxsw_sp_nexthop *nh)
2930 return rhashtable_insert_fast(&mlxsw_sp->router->nexthop_ht,
2931 &nh->ht_node, mlxsw_sp_nexthop_ht_params);
2934 static void mlxsw_sp_nexthop_remove(struct mlxsw_sp *mlxsw_sp,
2935 struct mlxsw_sp_nexthop *nh)
2937 rhashtable_remove_fast(&mlxsw_sp->router->nexthop_ht, &nh->ht_node,
2938 mlxsw_sp_nexthop_ht_params);
2941 static struct mlxsw_sp_nexthop *
2942 mlxsw_sp_nexthop_lookup(struct mlxsw_sp *mlxsw_sp,
2943 struct mlxsw_sp_nexthop_key key)
2945 return rhashtable_lookup_fast(&mlxsw_sp->router->nexthop_ht, &key,
2946 mlxsw_sp_nexthop_ht_params);
2949 static int mlxsw_sp_adj_index_mass_update_vr(struct mlxsw_sp *mlxsw_sp,
2950 const struct mlxsw_sp_fib *fib,
2951 u32 adj_index, u16 ecmp_size,
2955 char raleu_pl[MLXSW_REG_RALEU_LEN];
2957 mlxsw_reg_raleu_pack(raleu_pl,
2958 (enum mlxsw_reg_ralxx_protocol) fib->proto,
2959 fib->vr->id, adj_index, ecmp_size, new_adj_index,
2961 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raleu), raleu_pl);
2964 static int mlxsw_sp_adj_index_mass_update(struct mlxsw_sp *mlxsw_sp,
2965 struct mlxsw_sp_nexthop_group *nh_grp,
2966 u32 old_adj_index, u16 old_ecmp_size)
2968 struct mlxsw_sp_fib_entry *fib_entry;
2969 struct mlxsw_sp_fib *fib = NULL;
2972 list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
2973 if (fib == fib_entry->fib_node->fib)
2975 fib = fib_entry->fib_node->fib;
2976 err = mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, fib,
2987 static int __mlxsw_sp_nexthop_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index,
2988 struct mlxsw_sp_nexthop *nh)
2990 struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
2991 char ratr_pl[MLXSW_REG_RATR_LEN];
2993 mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY,
2994 true, MLXSW_REG_RATR_TYPE_ETHERNET,
2995 adj_index, neigh_entry->rif);
2996 mlxsw_reg_ratr_eth_entry_pack(ratr_pl, neigh_entry->ha);
2997 if (nh->counter_valid)
2998 mlxsw_reg_ratr_counter_pack(ratr_pl, nh->counter_index, true);
3000 mlxsw_reg_ratr_counter_pack(ratr_pl, 0, false);
3002 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl);
3005 int mlxsw_sp_nexthop_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index,
3006 struct mlxsw_sp_nexthop *nh)
3010 for (i = 0; i < nh->num_adj_entries; i++) {
3013 err = __mlxsw_sp_nexthop_update(mlxsw_sp, adj_index + i, nh);
3021 static int __mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp,
3023 struct mlxsw_sp_nexthop *nh)
3025 const struct mlxsw_sp_ipip_ops *ipip_ops;
3027 ipip_ops = mlxsw_sp->router->ipip_ops_arr[nh->ipip_entry->ipipt];
3028 return ipip_ops->nexthop_update(mlxsw_sp, adj_index, nh->ipip_entry);
3031 static int mlxsw_sp_nexthop_ipip_update(struct mlxsw_sp *mlxsw_sp,
3033 struct mlxsw_sp_nexthop *nh)
3037 for (i = 0; i < nh->num_adj_entries; i++) {
3040 err = __mlxsw_sp_nexthop_ipip_update(mlxsw_sp, adj_index + i,
3050 mlxsw_sp_nexthop_group_update(struct mlxsw_sp *mlxsw_sp,
3051 struct mlxsw_sp_nexthop_group *nh_grp,
3054 u32 adj_index = nh_grp->adj_index; /* base */
3055 struct mlxsw_sp_nexthop *nh;
3059 for (i = 0; i < nh_grp->count; i++) {
3060 nh = &nh_grp->nexthops[i];
3062 if (!nh->should_offload) {
3067 if (nh->update || reallocate) {
3069 case MLXSW_SP_NEXTHOP_TYPE_ETH:
3070 err = mlxsw_sp_nexthop_update
3071 (mlxsw_sp, adj_index, nh);
3073 case MLXSW_SP_NEXTHOP_TYPE_IPIP:
3074 err = mlxsw_sp_nexthop_ipip_update
3075 (mlxsw_sp, adj_index, nh);
3083 adj_index += nh->num_adj_entries;
3089 mlxsw_sp_fib_node_entry_is_first(const struct mlxsw_sp_fib_node *fib_node,
3090 const struct mlxsw_sp_fib_entry *fib_entry);
3093 mlxsw_sp_nexthop_fib_entries_update(struct mlxsw_sp *mlxsw_sp,
3094 struct mlxsw_sp_nexthop_group *nh_grp)
3096 struct mlxsw_sp_fib_entry *fib_entry;
3099 list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
3100 if (!mlxsw_sp_fib_node_entry_is_first(fib_entry->fib_node,
3103 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
3111 mlxsw_sp_fib_entry_offload_refresh(struct mlxsw_sp_fib_entry *fib_entry,
3112 enum mlxsw_reg_ralue_op op, int err);
3115 mlxsw_sp_nexthop_fib_entries_refresh(struct mlxsw_sp_nexthop_group *nh_grp)
3117 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_WRITE;
3118 struct mlxsw_sp_fib_entry *fib_entry;
3120 list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
3121 if (!mlxsw_sp_fib_node_entry_is_first(fib_entry->fib_node,
3124 mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, 0);
3128 static void mlxsw_sp_adj_grp_size_round_up(u16 *p_adj_grp_size)
3130 /* Valid sizes for an adjacency group are:
3131 * 1-64, 512, 1024, 2048 and 4096.
3133 if (*p_adj_grp_size <= 64)
3135 else if (*p_adj_grp_size <= 512)
3136 *p_adj_grp_size = 512;
3137 else if (*p_adj_grp_size <= 1024)
3138 *p_adj_grp_size = 1024;
3139 else if (*p_adj_grp_size <= 2048)
3140 *p_adj_grp_size = 2048;
3142 *p_adj_grp_size = 4096;
3145 static void mlxsw_sp_adj_grp_size_round_down(u16 *p_adj_grp_size,
3146 unsigned int alloc_size)
3148 if (alloc_size >= 4096)
3149 *p_adj_grp_size = 4096;
3150 else if (alloc_size >= 2048)
3151 *p_adj_grp_size = 2048;
3152 else if (alloc_size >= 1024)
3153 *p_adj_grp_size = 1024;
3154 else if (alloc_size >= 512)
3155 *p_adj_grp_size = 512;
3158 static int mlxsw_sp_fix_adj_grp_size(struct mlxsw_sp *mlxsw_sp,
3159 u16 *p_adj_grp_size)
3161 unsigned int alloc_size;
3164 /* Round up the requested group size to the next size supported
3165 * by the device and make sure the request can be satisfied.
3167 mlxsw_sp_adj_grp_size_round_up(p_adj_grp_size);
3168 err = mlxsw_sp_kvdl_alloc_size_query(mlxsw_sp, *p_adj_grp_size,
3172 /* It is possible the allocation results in more allocated
3173 * entries than requested. Try to use as much of them as
3176 mlxsw_sp_adj_grp_size_round_down(p_adj_grp_size, alloc_size);
3182 mlxsw_sp_nexthop_group_normalize(struct mlxsw_sp_nexthop_group *nh_grp)
3184 int i, g = 0, sum_norm_weight = 0;
3185 struct mlxsw_sp_nexthop *nh;
3187 for (i = 0; i < nh_grp->count; i++) {
3188 nh = &nh_grp->nexthops[i];
3190 if (!nh->should_offload)
3193 g = gcd(nh->nh_weight, g);
3198 for (i = 0; i < nh_grp->count; i++) {
3199 nh = &nh_grp->nexthops[i];
3201 if (!nh->should_offload)
3203 nh->norm_nh_weight = nh->nh_weight / g;
3204 sum_norm_weight += nh->norm_nh_weight;
3207 nh_grp->sum_norm_weight = sum_norm_weight;
3211 mlxsw_sp_nexthop_group_rebalance(struct mlxsw_sp_nexthop_group *nh_grp)
3213 int total = nh_grp->sum_norm_weight;
3214 u16 ecmp_size = nh_grp->ecmp_size;
3215 int i, weight = 0, lower_bound = 0;
3217 for (i = 0; i < nh_grp->count; i++) {
3218 struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i];
3221 if (!nh->should_offload)
3223 weight += nh->norm_nh_weight;
3224 upper_bound = DIV_ROUND_CLOSEST(ecmp_size * weight, total);
3225 nh->num_adj_entries = upper_bound - lower_bound;
3226 lower_bound = upper_bound;
3231 mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp,
3232 struct mlxsw_sp_nexthop_group *nh_grp)
3234 u16 ecmp_size, old_ecmp_size;
3235 struct mlxsw_sp_nexthop *nh;
3236 bool offload_change = false;
3238 bool old_adj_index_valid;
3243 if (!nh_grp->gateway) {
3244 mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
3248 for (i = 0; i < nh_grp->count; i++) {
3249 nh = &nh_grp->nexthops[i];
3251 if (nh->should_offload != nh->offloaded) {
3252 offload_change = true;
3253 if (nh->should_offload)
3257 if (!offload_change) {
3258 /* Nothing was added or removed, so no need to reallocate. Just
3259 * update MAC on existing adjacency indexes.
3261 err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nh_grp, false);
3263 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
3268 mlxsw_sp_nexthop_group_normalize(nh_grp);
3269 if (!nh_grp->sum_norm_weight)
3270 /* No neigh of this group is connected so we just set
3271 * the trap and let everthing flow through kernel.
3275 ecmp_size = nh_grp->sum_norm_weight;
3276 err = mlxsw_sp_fix_adj_grp_size(mlxsw_sp, &ecmp_size);
3278 /* No valid allocation size available. */
3281 err = mlxsw_sp_kvdl_alloc(mlxsw_sp, ecmp_size, &adj_index);
3283 /* We ran out of KVD linear space, just set the
3284 * trap and let everything flow through kernel.
3286 dev_warn(mlxsw_sp->bus_info->dev, "Failed to allocate KVD linear area for nexthop group.\n");
3289 old_adj_index_valid = nh_grp->adj_index_valid;
3290 old_adj_index = nh_grp->adj_index;
3291 old_ecmp_size = nh_grp->ecmp_size;
3292 nh_grp->adj_index_valid = 1;
3293 nh_grp->adj_index = adj_index;
3294 nh_grp->ecmp_size = ecmp_size;
3295 mlxsw_sp_nexthop_group_rebalance(nh_grp);
3296 err = mlxsw_sp_nexthop_group_update(mlxsw_sp, nh_grp, true);
3298 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
3302 if (!old_adj_index_valid) {
3303 /* The trap was set for fib entries, so we have to call
3304 * fib entry update to unset it and use adjacency index.
3306 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
3308 dev_warn(mlxsw_sp->bus_info->dev, "Failed to add adjacency index to fib entries.\n");
3314 err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, nh_grp,
3315 old_adj_index, old_ecmp_size);
3316 mlxsw_sp_kvdl_free(mlxsw_sp, old_adj_index);
3318 dev_warn(mlxsw_sp->bus_info->dev, "Failed to mass-update adjacency index for nexthop group.\n");
3322 /* Offload state within the group changed, so update the flags. */
3323 mlxsw_sp_nexthop_fib_entries_refresh(nh_grp);
3328 old_adj_index_valid = nh_grp->adj_index_valid;
3329 nh_grp->adj_index_valid = 0;
3330 for (i = 0; i < nh_grp->count; i++) {
3331 nh = &nh_grp->nexthops[i];
3334 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
3336 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set traps for fib entries.\n");
3337 if (old_adj_index_valid)
3338 mlxsw_sp_kvdl_free(mlxsw_sp, nh_grp->adj_index);
3341 static void __mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp_nexthop *nh,
3345 nh->should_offload = 1;
3347 nh->should_offload = 0;
3352 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
3353 struct mlxsw_sp_neigh_entry *neigh_entry,
3356 struct mlxsw_sp_nexthop *nh;
3358 list_for_each_entry(nh, &neigh_entry->nexthop_list,
3360 __mlxsw_sp_nexthop_neigh_update(nh, removing);
3361 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
3365 static void mlxsw_sp_nexthop_rif_init(struct mlxsw_sp_nexthop *nh,
3366 struct mlxsw_sp_rif *rif)
3372 list_add(&nh->rif_list_node, &rif->nexthop_list);
3375 static void mlxsw_sp_nexthop_rif_fini(struct mlxsw_sp_nexthop *nh)
3380 list_del(&nh->rif_list_node);
3384 static int mlxsw_sp_nexthop_neigh_init(struct mlxsw_sp *mlxsw_sp,
3385 struct mlxsw_sp_nexthop *nh)
3387 struct mlxsw_sp_neigh_entry *neigh_entry;
3388 struct neighbour *n;
3392 if (!nh->nh_grp->gateway || nh->neigh_entry)
3395 /* Take a reference of neigh here ensuring that neigh would
3396 * not be destructed before the nexthop entry is finished.
3397 * The reference is taken either in neigh_lookup() or
3398 * in neigh_create() in case n is not found.
3400 n = neigh_lookup(nh->nh_grp->neigh_tbl, &nh->gw_addr, nh->rif->dev);
3402 n = neigh_create(nh->nh_grp->neigh_tbl, &nh->gw_addr,
3406 neigh_event_send(n, NULL);
3408 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
3410 neigh_entry = mlxsw_sp_neigh_entry_create(mlxsw_sp, n);
3411 if (IS_ERR(neigh_entry)) {
3413 goto err_neigh_entry_create;
3417 /* If that is the first nexthop connected to that neigh, add to
3418 * nexthop_neighs_list
3420 if (list_empty(&neigh_entry->nexthop_list))
3421 list_add_tail(&neigh_entry->nexthop_neighs_list_node,
3422 &mlxsw_sp->router->nexthop_neighs_list);
3424 nh->neigh_entry = neigh_entry;
3425 list_add_tail(&nh->neigh_list_node, &neigh_entry->nexthop_list);
3426 read_lock_bh(&n->lock);
3427 nud_state = n->nud_state;
3429 read_unlock_bh(&n->lock);
3430 __mlxsw_sp_nexthop_neigh_update(nh, !(nud_state & NUD_VALID && !dead));
3434 err_neigh_entry_create:
3439 static void mlxsw_sp_nexthop_neigh_fini(struct mlxsw_sp *mlxsw_sp,
3440 struct mlxsw_sp_nexthop *nh)
3442 struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
3443 struct neighbour *n;
3447 n = neigh_entry->key.n;
3449 __mlxsw_sp_nexthop_neigh_update(nh, true);
3450 list_del(&nh->neigh_list_node);
3451 nh->neigh_entry = NULL;
3453 /* If that is the last nexthop connected to that neigh, remove from
3454 * nexthop_neighs_list
3456 if (list_empty(&neigh_entry->nexthop_list))
3457 list_del(&neigh_entry->nexthop_neighs_list_node);
3459 if (!neigh_entry->connected && list_empty(&neigh_entry->nexthop_list))
3460 mlxsw_sp_neigh_entry_destroy(mlxsw_sp, neigh_entry);
3465 static bool mlxsw_sp_ipip_netdev_ul_up(struct net_device *ol_dev)
3467 struct net_device *ul_dev = __mlxsw_sp_ipip_netdev_ul_dev_get(ol_dev);
3469 return ul_dev ? (ul_dev->flags & IFF_UP) : true;
3472 static void mlxsw_sp_nexthop_ipip_init(struct mlxsw_sp *mlxsw_sp,
3473 struct mlxsw_sp_nexthop *nh,
3474 struct mlxsw_sp_ipip_entry *ipip_entry)
3478 if (!nh->nh_grp->gateway || nh->ipip_entry)
3481 nh->ipip_entry = ipip_entry;
3482 removing = !mlxsw_sp_ipip_netdev_ul_up(ipip_entry->ol_dev);
3483 __mlxsw_sp_nexthop_neigh_update(nh, removing);
3484 mlxsw_sp_nexthop_rif_init(nh, &ipip_entry->ol_lb->common);
3487 static void mlxsw_sp_nexthop_ipip_fini(struct mlxsw_sp *mlxsw_sp,
3488 struct mlxsw_sp_nexthop *nh)
3490 struct mlxsw_sp_ipip_entry *ipip_entry = nh->ipip_entry;
3495 __mlxsw_sp_nexthop_neigh_update(nh, true);
3496 nh->ipip_entry = NULL;
3499 static bool mlxsw_sp_nexthop4_ipip_type(const struct mlxsw_sp *mlxsw_sp,
3500 const struct fib_nh *fib_nh,
3501 enum mlxsw_sp_ipip_type *p_ipipt)
3503 struct net_device *dev = fib_nh->nh_dev;
3506 fib_nh->nh_parent->fib_type == RTN_UNICAST &&
3507 mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, p_ipipt);
3510 static void mlxsw_sp_nexthop_type_fini(struct mlxsw_sp *mlxsw_sp,
3511 struct mlxsw_sp_nexthop *nh)
3514 case MLXSW_SP_NEXTHOP_TYPE_ETH:
3515 mlxsw_sp_nexthop_neigh_fini(mlxsw_sp, nh);
3516 mlxsw_sp_nexthop_rif_fini(nh);
3518 case MLXSW_SP_NEXTHOP_TYPE_IPIP:
3519 mlxsw_sp_nexthop_rif_fini(nh);
3520 mlxsw_sp_nexthop_ipip_fini(mlxsw_sp, nh);
3525 static int mlxsw_sp_nexthop4_type_init(struct mlxsw_sp *mlxsw_sp,
3526 struct mlxsw_sp_nexthop *nh,
3527 struct fib_nh *fib_nh)
3529 const struct mlxsw_sp_ipip_ops *ipip_ops;
3530 struct net_device *dev = fib_nh->nh_dev;
3531 struct mlxsw_sp_ipip_entry *ipip_entry;
3532 struct mlxsw_sp_rif *rif;
3535 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, dev);
3537 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
3538 if (ipip_ops->can_offload(mlxsw_sp, dev,
3539 MLXSW_SP_L3_PROTO_IPV4)) {
3540 nh->type = MLXSW_SP_NEXTHOP_TYPE_IPIP;
3541 mlxsw_sp_nexthop_ipip_init(mlxsw_sp, nh, ipip_entry);
3546 nh->type = MLXSW_SP_NEXTHOP_TYPE_ETH;
3547 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
3551 mlxsw_sp_nexthop_rif_init(nh, rif);
3552 err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
3554 goto err_neigh_init;
3559 mlxsw_sp_nexthop_rif_fini(nh);
3563 static void mlxsw_sp_nexthop4_type_fini(struct mlxsw_sp *mlxsw_sp,
3564 struct mlxsw_sp_nexthop *nh)
3566 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
3569 static int mlxsw_sp_nexthop4_init(struct mlxsw_sp *mlxsw_sp,
3570 struct mlxsw_sp_nexthop_group *nh_grp,
3571 struct mlxsw_sp_nexthop *nh,
3572 struct fib_nh *fib_nh)
3574 struct net_device *dev = fib_nh->nh_dev;
3575 struct in_device *in_dev;
3578 nh->nh_grp = nh_grp;
3579 nh->key.fib_nh = fib_nh;
3580 #ifdef CONFIG_IP_ROUTE_MULTIPATH
3581 nh->nh_weight = fib_nh->nh_weight;
3585 memcpy(&nh->gw_addr, &fib_nh->nh_gw, sizeof(fib_nh->nh_gw));
3586 err = mlxsw_sp_nexthop_insert(mlxsw_sp, nh);
3590 mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh);
3591 list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list);
3596 in_dev = __in_dev_get_rtnl(dev);
3597 if (in_dev && IN_DEV_IGNORE_ROUTES_WITH_LINKDOWN(in_dev) &&
3598 fib_nh->nh_flags & RTNH_F_LINKDOWN)
3601 err = mlxsw_sp_nexthop4_type_init(mlxsw_sp, nh, fib_nh);
3603 goto err_nexthop_neigh_init;
3607 err_nexthop_neigh_init:
3608 mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
3612 static void mlxsw_sp_nexthop4_fini(struct mlxsw_sp *mlxsw_sp,
3613 struct mlxsw_sp_nexthop *nh)
3615 mlxsw_sp_nexthop4_type_fini(mlxsw_sp, nh);
3616 list_del(&nh->router_list_node);
3617 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh);
3618 mlxsw_sp_nexthop_remove(mlxsw_sp, nh);
3621 static void mlxsw_sp_nexthop4_event(struct mlxsw_sp *mlxsw_sp,
3622 unsigned long event, struct fib_nh *fib_nh)
3624 struct mlxsw_sp_nexthop_key key;
3625 struct mlxsw_sp_nexthop *nh;
3627 if (mlxsw_sp->router->aborted)
3630 key.fib_nh = fib_nh;
3631 nh = mlxsw_sp_nexthop_lookup(mlxsw_sp, key);
3632 if (WARN_ON_ONCE(!nh))
3636 case FIB_EVENT_NH_ADD:
3637 mlxsw_sp_nexthop4_type_init(mlxsw_sp, nh, fib_nh);
3639 case FIB_EVENT_NH_DEL:
3640 mlxsw_sp_nexthop4_type_fini(mlxsw_sp, nh);
3644 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
3647 static void mlxsw_sp_nexthop_rif_update(struct mlxsw_sp *mlxsw_sp,
3648 struct mlxsw_sp_rif *rif)
3650 struct mlxsw_sp_nexthop *nh;
3653 list_for_each_entry(nh, &rif->nexthop_list, rif_list_node) {
3655 case MLXSW_SP_NEXTHOP_TYPE_ETH:
3658 case MLXSW_SP_NEXTHOP_TYPE_IPIP:
3659 removing = !mlxsw_sp_ipip_netdev_ul_up(rif->dev);
3666 __mlxsw_sp_nexthop_neigh_update(nh, removing);
3667 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
3671 static void mlxsw_sp_nexthop_rif_migrate(struct mlxsw_sp *mlxsw_sp,
3672 struct mlxsw_sp_rif *old_rif,
3673 struct mlxsw_sp_rif *new_rif)
3675 struct mlxsw_sp_nexthop *nh;
3677 list_splice_init(&old_rif->nexthop_list, &new_rif->nexthop_list);
3678 list_for_each_entry(nh, &new_rif->nexthop_list, rif_list_node)
3680 mlxsw_sp_nexthop_rif_update(mlxsw_sp, new_rif);
3683 static void mlxsw_sp_nexthop_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
3684 struct mlxsw_sp_rif *rif)
3686 struct mlxsw_sp_nexthop *nh, *tmp;
3688 list_for_each_entry_safe(nh, tmp, &rif->nexthop_list, rif_list_node) {
3689 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
3690 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
3694 static bool mlxsw_sp_fi_is_gateway(const struct mlxsw_sp *mlxsw_sp,
3695 const struct fib_info *fi)
3697 return fi->fib_nh->nh_scope == RT_SCOPE_LINK ||
3698 mlxsw_sp_nexthop4_ipip_type(mlxsw_sp, fi->fib_nh, NULL);
3701 static struct mlxsw_sp_nexthop_group *
3702 mlxsw_sp_nexthop4_group_create(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi)
3704 struct mlxsw_sp_nexthop_group *nh_grp;
3705 struct mlxsw_sp_nexthop *nh;
3706 struct fib_nh *fib_nh;
3711 alloc_size = sizeof(*nh_grp) +
3712 fi->fib_nhs * sizeof(struct mlxsw_sp_nexthop);
3713 nh_grp = kzalloc(alloc_size, GFP_KERNEL);
3715 return ERR_PTR(-ENOMEM);
3717 INIT_LIST_HEAD(&nh_grp->fib_list);
3718 nh_grp->neigh_tbl = &arp_tbl;
3720 nh_grp->gateway = mlxsw_sp_fi_is_gateway(mlxsw_sp, fi);
3721 nh_grp->count = fi->fib_nhs;
3723 for (i = 0; i < nh_grp->count; i++) {
3724 nh = &nh_grp->nexthops[i];
3725 fib_nh = &fi->fib_nh[i];
3726 err = mlxsw_sp_nexthop4_init(mlxsw_sp, nh_grp, nh, fib_nh);
3728 goto err_nexthop4_init;
3730 err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
3732 goto err_nexthop_group_insert;
3733 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
3736 err_nexthop_group_insert:
3738 for (i--; i >= 0; i--) {
3739 nh = &nh_grp->nexthops[i];
3740 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh);
3744 return ERR_PTR(err);
3748 mlxsw_sp_nexthop4_group_destroy(struct mlxsw_sp *mlxsw_sp,
3749 struct mlxsw_sp_nexthop_group *nh_grp)
3751 struct mlxsw_sp_nexthop *nh;
3754 mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
3755 for (i = 0; i < nh_grp->count; i++) {
3756 nh = &nh_grp->nexthops[i];
3757 mlxsw_sp_nexthop4_fini(mlxsw_sp, nh);
3759 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
3760 WARN_ON_ONCE(nh_grp->adj_index_valid);
3761 fib_info_put(mlxsw_sp_nexthop4_group_fi(nh_grp));
3765 static int mlxsw_sp_nexthop4_group_get(struct mlxsw_sp *mlxsw_sp,
3766 struct mlxsw_sp_fib_entry *fib_entry,
3767 struct fib_info *fi)
3769 struct mlxsw_sp_nexthop_group *nh_grp;
3771 nh_grp = mlxsw_sp_nexthop4_group_lookup(mlxsw_sp, fi);
3773 nh_grp = mlxsw_sp_nexthop4_group_create(mlxsw_sp, fi);
3775 return PTR_ERR(nh_grp);
3777 list_add_tail(&fib_entry->nexthop_group_node, &nh_grp->fib_list);
3778 fib_entry->nh_group = nh_grp;
3782 static void mlxsw_sp_nexthop4_group_put(struct mlxsw_sp *mlxsw_sp,
3783 struct mlxsw_sp_fib_entry *fib_entry)
3785 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
3787 list_del(&fib_entry->nexthop_group_node);
3788 if (!list_empty(&nh_grp->fib_list))
3790 mlxsw_sp_nexthop4_group_destroy(mlxsw_sp, nh_grp);
3794 mlxsw_sp_fib4_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry)
3796 struct mlxsw_sp_fib4_entry *fib4_entry;
3798 fib4_entry = container_of(fib_entry, struct mlxsw_sp_fib4_entry,
3800 return !fib4_entry->tos;
3804 mlxsw_sp_fib_entry_should_offload(const struct mlxsw_sp_fib_entry *fib_entry)
3806 struct mlxsw_sp_nexthop_group *nh_group = fib_entry->nh_group;
3808 switch (fib_entry->fib_node->fib->proto) {
3809 case MLXSW_SP_L3_PROTO_IPV4:
3810 if (!mlxsw_sp_fib4_entry_should_offload(fib_entry))
3813 case MLXSW_SP_L3_PROTO_IPV6:
3817 switch (fib_entry->type) {
3818 case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
3819 return !!nh_group->adj_index_valid;
3820 case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
3821 return !!nh_group->nh_rif;
3822 case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP:
3829 static struct mlxsw_sp_nexthop *
3830 mlxsw_sp_rt6_nexthop(struct mlxsw_sp_nexthop_group *nh_grp,
3831 const struct mlxsw_sp_rt6 *mlxsw_sp_rt6)
3835 for (i = 0; i < nh_grp->count; i++) {
3836 struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i];
3837 struct fib6_info *rt = mlxsw_sp_rt6->rt;
3839 if (nh->rif && nh->rif->dev == rt->fib6_nh.nh_dev &&
3840 ipv6_addr_equal((const struct in6_addr *) &nh->gw_addr,
3841 &rt->fib6_nh.nh_gw))
3850 mlxsw_sp_fib4_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry)
3852 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
3855 if (fib_entry->type == MLXSW_SP_FIB_ENTRY_TYPE_LOCAL ||
3856 fib_entry->type == MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP) {
3857 nh_grp->nexthops->key.fib_nh->nh_flags |= RTNH_F_OFFLOAD;
3861 for (i = 0; i < nh_grp->count; i++) {
3862 struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i];
3865 nh->key.fib_nh->nh_flags |= RTNH_F_OFFLOAD;
3867 nh->key.fib_nh->nh_flags &= ~RTNH_F_OFFLOAD;
3872 mlxsw_sp_fib4_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry)
3874 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
3877 if (!list_is_singular(&nh_grp->fib_list))
3880 for (i = 0; i < nh_grp->count; i++) {
3881 struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i];
3883 nh->key.fib_nh->nh_flags &= ~RTNH_F_OFFLOAD;
3888 mlxsw_sp_fib6_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry)
3890 struct mlxsw_sp_fib6_entry *fib6_entry;
3891 struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3893 fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry,
3896 if (fib_entry->type == MLXSW_SP_FIB_ENTRY_TYPE_LOCAL) {
3897 list_first_entry(&fib6_entry->rt6_list, struct mlxsw_sp_rt6,
3898 list)->rt->fib6_nh.nh_flags |= RTNH_F_OFFLOAD;
3902 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
3903 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
3904 struct mlxsw_sp_nexthop *nh;
3906 nh = mlxsw_sp_rt6_nexthop(nh_grp, mlxsw_sp_rt6);
3907 if (nh && nh->offloaded)
3908 mlxsw_sp_rt6->rt->fib6_nh.nh_flags |= RTNH_F_OFFLOAD;
3910 mlxsw_sp_rt6->rt->fib6_nh.nh_flags &= ~RTNH_F_OFFLOAD;
3915 mlxsw_sp_fib6_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry)
3917 struct mlxsw_sp_fib6_entry *fib6_entry;
3918 struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
3920 fib6_entry = container_of(fib_entry, struct mlxsw_sp_fib6_entry,
3922 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
3923 struct fib6_info *rt = mlxsw_sp_rt6->rt;
3925 rt->fib6_nh.nh_flags &= ~RTNH_F_OFFLOAD;
3929 static void mlxsw_sp_fib_entry_offload_set(struct mlxsw_sp_fib_entry *fib_entry)
3931 switch (fib_entry->fib_node->fib->proto) {
3932 case MLXSW_SP_L3_PROTO_IPV4:
3933 mlxsw_sp_fib4_entry_offload_set(fib_entry);
3935 case MLXSW_SP_L3_PROTO_IPV6:
3936 mlxsw_sp_fib6_entry_offload_set(fib_entry);
3942 mlxsw_sp_fib_entry_offload_unset(struct mlxsw_sp_fib_entry *fib_entry)
3944 switch (fib_entry->fib_node->fib->proto) {
3945 case MLXSW_SP_L3_PROTO_IPV4:
3946 mlxsw_sp_fib4_entry_offload_unset(fib_entry);
3948 case MLXSW_SP_L3_PROTO_IPV6:
3949 mlxsw_sp_fib6_entry_offload_unset(fib_entry);
3955 mlxsw_sp_fib_entry_offload_refresh(struct mlxsw_sp_fib_entry *fib_entry,
3956 enum mlxsw_reg_ralue_op op, int err)
3959 case MLXSW_REG_RALUE_OP_WRITE_DELETE:
3960 return mlxsw_sp_fib_entry_offload_unset(fib_entry);
3961 case MLXSW_REG_RALUE_OP_WRITE_WRITE:
3964 if (mlxsw_sp_fib_entry_should_offload(fib_entry))
3965 mlxsw_sp_fib_entry_offload_set(fib_entry);
3967 mlxsw_sp_fib_entry_offload_unset(fib_entry);
3975 mlxsw_sp_fib_entry_ralue_pack(char *ralue_pl,
3976 const struct mlxsw_sp_fib_entry *fib_entry,
3977 enum mlxsw_reg_ralue_op op)
3979 struct mlxsw_sp_fib *fib = fib_entry->fib_node->fib;
3980 enum mlxsw_reg_ralxx_protocol proto;
3983 proto = (enum mlxsw_reg_ralxx_protocol) fib->proto;
3985 switch (fib->proto) {
3986 case MLXSW_SP_L3_PROTO_IPV4:
3987 p_dip = (u32 *) fib_entry->fib_node->key.addr;
3988 mlxsw_reg_ralue_pack4(ralue_pl, proto, op, fib->vr->id,
3989 fib_entry->fib_node->key.prefix_len,
3992 case MLXSW_SP_L3_PROTO_IPV6:
3993 mlxsw_reg_ralue_pack6(ralue_pl, proto, op, fib->vr->id,
3994 fib_entry->fib_node->key.prefix_len,
3995 fib_entry->fib_node->key.addr);
4000 static int mlxsw_sp_fib_entry_op_remote(struct mlxsw_sp *mlxsw_sp,
4001 struct mlxsw_sp_fib_entry *fib_entry,
4002 enum mlxsw_reg_ralue_op op)
4004 char ralue_pl[MLXSW_REG_RALUE_LEN];
4005 enum mlxsw_reg_ralue_trap_action trap_action;
4007 u32 adjacency_index = 0;
4010 /* In case the nexthop group adjacency index is valid, use it
4011 * with provided ECMP size. Otherwise, setup trap and pass
4012 * traffic to kernel.
4014 if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
4015 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
4016 adjacency_index = fib_entry->nh_group->adj_index;
4017 ecmp_size = fib_entry->nh_group->ecmp_size;
4019 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
4020 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
4023 mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
4024 mlxsw_reg_ralue_act_remote_pack(ralue_pl, trap_action, trap_id,
4025 adjacency_index, ecmp_size);
4026 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
4029 static int mlxsw_sp_fib_entry_op_local(struct mlxsw_sp *mlxsw_sp,
4030 struct mlxsw_sp_fib_entry *fib_entry,
4031 enum mlxsw_reg_ralue_op op)
4033 struct mlxsw_sp_rif *rif = fib_entry->nh_group->nh_rif;
4034 enum mlxsw_reg_ralue_trap_action trap_action;
4035 char ralue_pl[MLXSW_REG_RALUE_LEN];
4039 if (mlxsw_sp_fib_entry_should_offload(fib_entry)) {
4040 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
4041 rif_index = rif->rif_index;
4043 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
4044 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
4047 mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
4048 mlxsw_reg_ralue_act_local_pack(ralue_pl, trap_action, trap_id,
4050 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
4053 static int mlxsw_sp_fib_entry_op_trap(struct mlxsw_sp *mlxsw_sp,
4054 struct mlxsw_sp_fib_entry *fib_entry,
4055 enum mlxsw_reg_ralue_op op)
4057 char ralue_pl[MLXSW_REG_RALUE_LEN];
4059 mlxsw_sp_fib_entry_ralue_pack(ralue_pl, fib_entry, op);
4060 mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
4061 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
4065 mlxsw_sp_fib_entry_op_ipip_decap(struct mlxsw_sp *mlxsw_sp,
4066 struct mlxsw_sp_fib_entry *fib_entry,
4067 enum mlxsw_reg_ralue_op op)
4069 struct mlxsw_sp_ipip_entry *ipip_entry = fib_entry->decap.ipip_entry;
4070 const struct mlxsw_sp_ipip_ops *ipip_ops;
4072 if (WARN_ON(!ipip_entry))
4075 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
4076 return ipip_ops->fib_entry_op(mlxsw_sp, ipip_entry, op,
4077 fib_entry->decap.tunnel_index);
4080 static int __mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
4081 struct mlxsw_sp_fib_entry *fib_entry,
4082 enum mlxsw_reg_ralue_op op)
4084 switch (fib_entry->type) {
4085 case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
4086 return mlxsw_sp_fib_entry_op_remote(mlxsw_sp, fib_entry, op);
4087 case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
4088 return mlxsw_sp_fib_entry_op_local(mlxsw_sp, fib_entry, op);
4089 case MLXSW_SP_FIB_ENTRY_TYPE_TRAP:
4090 return mlxsw_sp_fib_entry_op_trap(mlxsw_sp, fib_entry, op);
4091 case MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP:
4092 return mlxsw_sp_fib_entry_op_ipip_decap(mlxsw_sp,
4098 static int mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
4099 struct mlxsw_sp_fib_entry *fib_entry,
4100 enum mlxsw_reg_ralue_op op)
4102 int err = __mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry, op);
4104 mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, err);
4109 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
4110 struct mlxsw_sp_fib_entry *fib_entry)
4112 return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
4113 MLXSW_REG_RALUE_OP_WRITE_WRITE);
4116 static int mlxsw_sp_fib_entry_del(struct mlxsw_sp *mlxsw_sp,
4117 struct mlxsw_sp_fib_entry *fib_entry)
4119 return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
4120 MLXSW_REG_RALUE_OP_WRITE_DELETE);
4124 mlxsw_sp_fib4_entry_type_set(struct mlxsw_sp *mlxsw_sp,
4125 const struct fib_entry_notifier_info *fen_info,
4126 struct mlxsw_sp_fib_entry *fib_entry)
4128 union mlxsw_sp_l3addr dip = { .addr4 = htonl(fen_info->dst) };
4129 struct net_device *dev = fen_info->fi->fib_dev;
4130 struct mlxsw_sp_ipip_entry *ipip_entry;
4131 struct fib_info *fi = fen_info->fi;
4133 switch (fen_info->type) {
4135 ipip_entry = mlxsw_sp_ipip_entry_find_by_decap(mlxsw_sp, dev,
4136 MLXSW_SP_L3_PROTO_IPV4, dip);
4137 if (ipip_entry && ipip_entry->ol_dev->flags & IFF_UP) {
4138 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP;
4139 return mlxsw_sp_fib_entry_decap_init(mlxsw_sp,
4145 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
4147 case RTN_UNREACHABLE: /* fall through */
4148 case RTN_BLACKHOLE: /* fall through */
4150 /* Packets hitting these routes need to be trapped, but
4151 * can do so with a lower priority than packets directed
4152 * at the host, so use action type local instead of trap.
4154 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
4157 if (mlxsw_sp_fi_is_gateway(mlxsw_sp, fi))
4158 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
4160 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
4167 static struct mlxsw_sp_fib4_entry *
4168 mlxsw_sp_fib4_entry_create(struct mlxsw_sp *mlxsw_sp,
4169 struct mlxsw_sp_fib_node *fib_node,
4170 const struct fib_entry_notifier_info *fen_info)
4172 struct mlxsw_sp_fib4_entry *fib4_entry;
4173 struct mlxsw_sp_fib_entry *fib_entry;
4176 fib4_entry = kzalloc(sizeof(*fib4_entry), GFP_KERNEL);
4178 return ERR_PTR(-ENOMEM);
4179 fib_entry = &fib4_entry->common;
4181 err = mlxsw_sp_fib4_entry_type_set(mlxsw_sp, fen_info, fib_entry);
4183 goto err_fib4_entry_type_set;
4185 err = mlxsw_sp_nexthop4_group_get(mlxsw_sp, fib_entry, fen_info->fi);
4187 goto err_nexthop4_group_get;
4189 fib4_entry->prio = fen_info->fi->fib_priority;
4190 fib4_entry->tb_id = fen_info->tb_id;
4191 fib4_entry->type = fen_info->type;
4192 fib4_entry->tos = fen_info->tos;
4194 fib_entry->fib_node = fib_node;
4198 err_nexthop4_group_get:
4199 err_fib4_entry_type_set:
4201 return ERR_PTR(err);
4204 static void mlxsw_sp_fib4_entry_destroy(struct mlxsw_sp *mlxsw_sp,
4205 struct mlxsw_sp_fib4_entry *fib4_entry)
4207 mlxsw_sp_nexthop4_group_put(mlxsw_sp, &fib4_entry->common);
4211 static struct mlxsw_sp_fib4_entry *
4212 mlxsw_sp_fib4_entry_lookup(struct mlxsw_sp *mlxsw_sp,
4213 const struct fib_entry_notifier_info *fen_info)
4215 struct mlxsw_sp_fib4_entry *fib4_entry;
4216 struct mlxsw_sp_fib_node *fib_node;
4217 struct mlxsw_sp_fib *fib;
4218 struct mlxsw_sp_vr *vr;
4220 vr = mlxsw_sp_vr_find(mlxsw_sp, fen_info->tb_id);
4223 fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV4);
4225 fib_node = mlxsw_sp_fib_node_lookup(fib, &fen_info->dst,
4226 sizeof(fen_info->dst),
4231 list_for_each_entry(fib4_entry, &fib_node->entry_list, common.list) {
4232 if (fib4_entry->tb_id == fen_info->tb_id &&
4233 fib4_entry->tos == fen_info->tos &&
4234 fib4_entry->type == fen_info->type &&
4235 mlxsw_sp_nexthop4_group_fi(fib4_entry->common.nh_group) ==
4244 static const struct rhashtable_params mlxsw_sp_fib_ht_params = {
4245 .key_offset = offsetof(struct mlxsw_sp_fib_node, key),
4246 .head_offset = offsetof(struct mlxsw_sp_fib_node, ht_node),
4247 .key_len = sizeof(struct mlxsw_sp_fib_key),
4248 .automatic_shrinking = true,
4251 static int mlxsw_sp_fib_node_insert(struct mlxsw_sp_fib *fib,
4252 struct mlxsw_sp_fib_node *fib_node)
4254 return rhashtable_insert_fast(&fib->ht, &fib_node->ht_node,
4255 mlxsw_sp_fib_ht_params);
4258 static void mlxsw_sp_fib_node_remove(struct mlxsw_sp_fib *fib,
4259 struct mlxsw_sp_fib_node *fib_node)
4261 rhashtable_remove_fast(&fib->ht, &fib_node->ht_node,
4262 mlxsw_sp_fib_ht_params);
4265 static struct mlxsw_sp_fib_node *
4266 mlxsw_sp_fib_node_lookup(struct mlxsw_sp_fib *fib, const void *addr,
4267 size_t addr_len, unsigned char prefix_len)
4269 struct mlxsw_sp_fib_key key;
4271 memset(&key, 0, sizeof(key));
4272 memcpy(key.addr, addr, addr_len);
4273 key.prefix_len = prefix_len;
4274 return rhashtable_lookup_fast(&fib->ht, &key, mlxsw_sp_fib_ht_params);
4277 static struct mlxsw_sp_fib_node *
4278 mlxsw_sp_fib_node_create(struct mlxsw_sp_fib *fib, const void *addr,
4279 size_t addr_len, unsigned char prefix_len)
4281 struct mlxsw_sp_fib_node *fib_node;
4283 fib_node = kzalloc(sizeof(*fib_node), GFP_KERNEL);
4287 INIT_LIST_HEAD(&fib_node->entry_list);
4288 list_add(&fib_node->list, &fib->node_list);
4289 memcpy(fib_node->key.addr, addr, addr_len);
4290 fib_node->key.prefix_len = prefix_len;
4295 static void mlxsw_sp_fib_node_destroy(struct mlxsw_sp_fib_node *fib_node)
4297 list_del(&fib_node->list);
4298 WARN_ON(!list_empty(&fib_node->entry_list));
4303 mlxsw_sp_fib_node_entry_is_first(const struct mlxsw_sp_fib_node *fib_node,
4304 const struct mlxsw_sp_fib_entry *fib_entry)
4306 return list_first_entry(&fib_node->entry_list,
4307 struct mlxsw_sp_fib_entry, list) == fib_entry;
4310 static int mlxsw_sp_fib_lpm_tree_link(struct mlxsw_sp *mlxsw_sp,
4311 struct mlxsw_sp_fib_node *fib_node)
4313 struct mlxsw_sp_prefix_usage req_prefix_usage;
4314 struct mlxsw_sp_fib *fib = fib_node->fib;
4315 struct mlxsw_sp_lpm_tree *lpm_tree;
4318 lpm_tree = mlxsw_sp->router->lpm.proto_trees[fib->proto];
4319 if (lpm_tree->prefix_ref_count[fib_node->key.prefix_len] != 0)
4322 mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &lpm_tree->prefix_usage);
4323 mlxsw_sp_prefix_usage_set(&req_prefix_usage, fib_node->key.prefix_len);
4324 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
4326 if (IS_ERR(lpm_tree))
4327 return PTR_ERR(lpm_tree);
4329 err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree);
4331 goto err_lpm_tree_replace;
4334 lpm_tree->prefix_ref_count[fib_node->key.prefix_len]++;
4337 err_lpm_tree_replace:
4338 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
4342 static void mlxsw_sp_fib_lpm_tree_unlink(struct mlxsw_sp *mlxsw_sp,
4343 struct mlxsw_sp_fib_node *fib_node)
4345 struct mlxsw_sp_lpm_tree *lpm_tree = fib_node->fib->lpm_tree;
4346 struct mlxsw_sp_prefix_usage req_prefix_usage;
4347 struct mlxsw_sp_fib *fib = fib_node->fib;
4350 if (--lpm_tree->prefix_ref_count[fib_node->key.prefix_len] != 0)
4352 /* Try to construct a new LPM tree from the current prefix usage
4353 * minus the unused one. If we fail, continue using the old one.
4355 mlxsw_sp_prefix_usage_cpy(&req_prefix_usage, &lpm_tree->prefix_usage);
4356 mlxsw_sp_prefix_usage_clear(&req_prefix_usage,
4357 fib_node->key.prefix_len);
4358 lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
4360 if (IS_ERR(lpm_tree))
4363 err = mlxsw_sp_vrs_lpm_tree_replace(mlxsw_sp, fib, lpm_tree);
4365 goto err_lpm_tree_replace;
4369 err_lpm_tree_replace:
4370 mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
4373 static int mlxsw_sp_fib_node_init(struct mlxsw_sp *mlxsw_sp,
4374 struct mlxsw_sp_fib_node *fib_node,
4375 struct mlxsw_sp_fib *fib)
4379 err = mlxsw_sp_fib_node_insert(fib, fib_node);
4382 fib_node->fib = fib;
4384 err = mlxsw_sp_fib_lpm_tree_link(mlxsw_sp, fib_node);
4386 goto err_fib_lpm_tree_link;
4390 err_fib_lpm_tree_link:
4391 fib_node->fib = NULL;
4392 mlxsw_sp_fib_node_remove(fib, fib_node);
4396 static void mlxsw_sp_fib_node_fini(struct mlxsw_sp *mlxsw_sp,
4397 struct mlxsw_sp_fib_node *fib_node)
4399 struct mlxsw_sp_fib *fib = fib_node->fib;
4401 mlxsw_sp_fib_lpm_tree_unlink(mlxsw_sp, fib_node);
4402 fib_node->fib = NULL;
4403 mlxsw_sp_fib_node_remove(fib, fib_node);
4406 static struct mlxsw_sp_fib_node *
4407 mlxsw_sp_fib_node_get(struct mlxsw_sp *mlxsw_sp, u32 tb_id, const void *addr,
4408 size_t addr_len, unsigned char prefix_len,
4409 enum mlxsw_sp_l3proto proto)
4411 struct mlxsw_sp_fib_node *fib_node;
4412 struct mlxsw_sp_fib *fib;
4413 struct mlxsw_sp_vr *vr;
4416 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id, NULL);
4418 return ERR_CAST(vr);
4419 fib = mlxsw_sp_vr_fib(vr, proto);
4421 fib_node = mlxsw_sp_fib_node_lookup(fib, addr, addr_len, prefix_len);
4425 fib_node = mlxsw_sp_fib_node_create(fib, addr, addr_len, prefix_len);
4428 goto err_fib_node_create;
4431 err = mlxsw_sp_fib_node_init(mlxsw_sp, fib_node, fib);
4433 goto err_fib_node_init;
4438 mlxsw_sp_fib_node_destroy(fib_node);
4439 err_fib_node_create:
4440 mlxsw_sp_vr_put(mlxsw_sp, vr);
4441 return ERR_PTR(err);
4444 static void mlxsw_sp_fib_node_put(struct mlxsw_sp *mlxsw_sp,
4445 struct mlxsw_sp_fib_node *fib_node)
4447 struct mlxsw_sp_vr *vr = fib_node->fib->vr;
4449 if (!list_empty(&fib_node->entry_list))
4451 mlxsw_sp_fib_node_fini(mlxsw_sp, fib_node);
4452 mlxsw_sp_fib_node_destroy(fib_node);
4453 mlxsw_sp_vr_put(mlxsw_sp, vr);
4456 static struct mlxsw_sp_fib4_entry *
4457 mlxsw_sp_fib4_node_entry_find(const struct mlxsw_sp_fib_node *fib_node,
4458 const struct mlxsw_sp_fib4_entry *new4_entry)
4460 struct mlxsw_sp_fib4_entry *fib4_entry;
4462 list_for_each_entry(fib4_entry, &fib_node->entry_list, common.list) {
4463 if (fib4_entry->tb_id > new4_entry->tb_id)
4465 if (fib4_entry->tb_id != new4_entry->tb_id)
4467 if (fib4_entry->tos > new4_entry->tos)
4469 if (fib4_entry->prio >= new4_entry->prio ||
4470 fib4_entry->tos < new4_entry->tos)
4478 mlxsw_sp_fib4_node_list_append(struct mlxsw_sp_fib4_entry *fib4_entry,
4479 struct mlxsw_sp_fib4_entry *new4_entry)
4481 struct mlxsw_sp_fib_node *fib_node;
4483 if (WARN_ON(!fib4_entry))
4486 fib_node = fib4_entry->common.fib_node;
4487 list_for_each_entry_from(fib4_entry, &fib_node->entry_list,
4489 if (fib4_entry->tb_id != new4_entry->tb_id ||
4490 fib4_entry->tos != new4_entry->tos ||
4491 fib4_entry->prio != new4_entry->prio)
4495 list_add_tail(&new4_entry->common.list, &fib4_entry->common.list);
4500 mlxsw_sp_fib4_node_list_insert(struct mlxsw_sp_fib4_entry *new4_entry,
4501 bool replace, bool append)
4503 struct mlxsw_sp_fib_node *fib_node = new4_entry->common.fib_node;
4504 struct mlxsw_sp_fib4_entry *fib4_entry;
4506 fib4_entry = mlxsw_sp_fib4_node_entry_find(fib_node, new4_entry);
4509 return mlxsw_sp_fib4_node_list_append(fib4_entry, new4_entry);
4510 if (replace && WARN_ON(!fib4_entry))
4513 /* Insert new entry before replaced one, so that we can later
4514 * remove the second.
4517 list_add_tail(&new4_entry->common.list,
4518 &fib4_entry->common.list);
4520 struct mlxsw_sp_fib4_entry *last;
4522 list_for_each_entry(last, &fib_node->entry_list, common.list) {
4523 if (new4_entry->tb_id > last->tb_id)
4529 list_add(&new4_entry->common.list,
4530 &fib4_entry->common.list);
4532 list_add(&new4_entry->common.list,
4533 &fib_node->entry_list);
4540 mlxsw_sp_fib4_node_list_remove(struct mlxsw_sp_fib4_entry *fib4_entry)
4542 list_del(&fib4_entry->common.list);
4545 static int mlxsw_sp_fib_node_entry_add(struct mlxsw_sp *mlxsw_sp,
4546 struct mlxsw_sp_fib_entry *fib_entry)
4548 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
4550 if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry))
4553 /* To prevent packet loss, overwrite the previously offloaded
4556 if (!list_is_singular(&fib_node->entry_list)) {
4557 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_DELETE;
4558 struct mlxsw_sp_fib_entry *n = list_next_entry(fib_entry, list);
4560 mlxsw_sp_fib_entry_offload_refresh(n, op, 0);
4563 return mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
4566 static void mlxsw_sp_fib_node_entry_del(struct mlxsw_sp *mlxsw_sp,
4567 struct mlxsw_sp_fib_entry *fib_entry)
4569 struct mlxsw_sp_fib_node *fib_node = fib_entry->fib_node;
4571 if (!mlxsw_sp_fib_node_entry_is_first(fib_node, fib_entry))
4574 /* Promote the next entry by overwriting the deleted entry */
4575 if (!list_is_singular(&fib_node->entry_list)) {
4576 struct mlxsw_sp_fib_entry *n = list_next_entry(fib_entry, list);
4577 enum mlxsw_reg_ralue_op op = MLXSW_REG_RALUE_OP_WRITE_DELETE;
4579 mlxsw_sp_fib_entry_update(mlxsw_sp, n);
4580 mlxsw_sp_fib_entry_offload_refresh(fib_entry, op, 0);
4584 mlxsw_sp_fib_entry_del(mlxsw_sp, fib_entry);
4587 static int mlxsw_sp_fib4_node_entry_link(struct mlxsw_sp *mlxsw_sp,
4588 struct mlxsw_sp_fib4_entry *fib4_entry,
4589 bool replace, bool append)
4593 err = mlxsw_sp_fib4_node_list_insert(fib4_entry, replace, append);
4597 err = mlxsw_sp_fib_node_entry_add(mlxsw_sp, &fib4_entry->common);
4599 goto err_fib_node_entry_add;
4603 err_fib_node_entry_add:
4604 mlxsw_sp_fib4_node_list_remove(fib4_entry);
4609 mlxsw_sp_fib4_node_entry_unlink(struct mlxsw_sp *mlxsw_sp,
4610 struct mlxsw_sp_fib4_entry *fib4_entry)
4612 mlxsw_sp_fib_node_entry_del(mlxsw_sp, &fib4_entry->common);
4613 mlxsw_sp_fib4_node_list_remove(fib4_entry);
4615 if (fib4_entry->common.type == MLXSW_SP_FIB_ENTRY_TYPE_IPIP_DECAP)
4616 mlxsw_sp_fib_entry_decap_fini(mlxsw_sp, &fib4_entry->common);
4619 static void mlxsw_sp_fib4_entry_replace(struct mlxsw_sp *mlxsw_sp,
4620 struct mlxsw_sp_fib4_entry *fib4_entry,
4623 struct mlxsw_sp_fib_node *fib_node = fib4_entry->common.fib_node;
4624 struct mlxsw_sp_fib4_entry *replaced;
4629 /* We inserted the new entry before replaced one */
4630 replaced = list_next_entry(fib4_entry, common.list);
4632 mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, replaced);
4633 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, replaced);
4634 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
4638 mlxsw_sp_router_fib4_add(struct mlxsw_sp *mlxsw_sp,
4639 const struct fib_entry_notifier_info *fen_info,
4640 bool replace, bool append)
4642 struct mlxsw_sp_fib4_entry *fib4_entry;
4643 struct mlxsw_sp_fib_node *fib_node;
4646 if (mlxsw_sp->router->aborted)
4649 fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, fen_info->tb_id,
4650 &fen_info->dst, sizeof(fen_info->dst),
4652 MLXSW_SP_L3_PROTO_IPV4);
4653 if (IS_ERR(fib_node)) {
4654 dev_warn(mlxsw_sp->bus_info->dev, "Failed to get FIB node\n");
4655 return PTR_ERR(fib_node);
4658 fib4_entry = mlxsw_sp_fib4_entry_create(mlxsw_sp, fib_node, fen_info);
4659 if (IS_ERR(fib4_entry)) {
4660 dev_warn(mlxsw_sp->bus_info->dev, "Failed to create FIB entry\n");
4661 err = PTR_ERR(fib4_entry);
4662 goto err_fib4_entry_create;
4665 err = mlxsw_sp_fib4_node_entry_link(mlxsw_sp, fib4_entry, replace,
4668 dev_warn(mlxsw_sp->bus_info->dev, "Failed to link FIB entry to node\n");
4669 goto err_fib4_node_entry_link;
4672 mlxsw_sp_fib4_entry_replace(mlxsw_sp, fib4_entry, replace);
4676 err_fib4_node_entry_link:
4677 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
4678 err_fib4_entry_create:
4679 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
4683 static void mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp,
4684 struct fib_entry_notifier_info *fen_info)
4686 struct mlxsw_sp_fib4_entry *fib4_entry;
4687 struct mlxsw_sp_fib_node *fib_node;
4689 if (mlxsw_sp->router->aborted)
4692 fib4_entry = mlxsw_sp_fib4_entry_lookup(mlxsw_sp, fen_info);
4693 if (WARN_ON(!fib4_entry))
4695 fib_node = fib4_entry->common.fib_node;
4697 mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib4_entry);
4698 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
4699 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
4702 static bool mlxsw_sp_fib6_rt_should_ignore(const struct fib6_info *rt)
4704 /* Packets with link-local destination IP arriving to the router
4705 * are trapped to the CPU, so no need to program specific routes
4708 if (ipv6_addr_type(&rt->fib6_dst.addr) & IPV6_ADDR_LINKLOCAL)
4711 /* Multicast routes aren't supported, so ignore them. Neighbour
4712 * Discovery packets are specifically trapped.
4714 if (ipv6_addr_type(&rt->fib6_dst.addr) & IPV6_ADDR_MULTICAST)
4717 /* Cloned routes are irrelevant in the forwarding path. */
4718 if (rt->fib6_flags & RTF_CACHE)
4724 static struct mlxsw_sp_rt6 *mlxsw_sp_rt6_create(struct fib6_info *rt)
4726 struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
4728 mlxsw_sp_rt6 = kzalloc(sizeof(*mlxsw_sp_rt6), GFP_KERNEL);
4730 return ERR_PTR(-ENOMEM);
4732 /* In case of route replace, replaced route is deleted with
4733 * no notification. Take reference to prevent accessing freed
4736 mlxsw_sp_rt6->rt = rt;
4739 return mlxsw_sp_rt6;
4742 #if IS_ENABLED(CONFIG_IPV6)
4743 static void mlxsw_sp_rt6_release(struct fib6_info *rt)
4745 fib6_info_release(rt);
4748 static void mlxsw_sp_rt6_release(struct fib6_info *rt)
4753 static void mlxsw_sp_rt6_destroy(struct mlxsw_sp_rt6 *mlxsw_sp_rt6)
4755 mlxsw_sp_rt6_release(mlxsw_sp_rt6->rt);
4756 kfree(mlxsw_sp_rt6);
4759 static struct fib6_info *
4760 mlxsw_sp_fib6_entry_rt(const struct mlxsw_sp_fib6_entry *fib6_entry)
4762 return list_first_entry(&fib6_entry->rt6_list, struct mlxsw_sp_rt6,
4766 static struct mlxsw_sp_fib6_entry *
4767 mlxsw_sp_fib6_node_mp_entry_find(const struct mlxsw_sp_fib_node *fib_node,
4768 const struct fib6_info *nrt, bool append)
4770 struct mlxsw_sp_fib6_entry *fib6_entry;
4775 list_for_each_entry(fib6_entry, &fib_node->entry_list, common.list) {
4776 struct fib6_info *rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
4778 /* RT6_TABLE_LOCAL and RT6_TABLE_MAIN share the same
4781 if (rt->fib6_table->tb6_id > nrt->fib6_table->tb6_id)
4783 if (rt->fib6_table->tb6_id != nrt->fib6_table->tb6_id)
4785 if (rt->fib6_metric < nrt->fib6_metric)
4787 if (rt->fib6_metric == nrt->fib6_metric)
4789 if (rt->fib6_metric > nrt->fib6_metric)
4796 static struct mlxsw_sp_rt6 *
4797 mlxsw_sp_fib6_entry_rt_find(const struct mlxsw_sp_fib6_entry *fib6_entry,
4798 const struct fib6_info *rt)
4800 struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
4802 list_for_each_entry(mlxsw_sp_rt6, &fib6_entry->rt6_list, list) {
4803 if (mlxsw_sp_rt6->rt == rt)
4804 return mlxsw_sp_rt6;
4810 static bool mlxsw_sp_nexthop6_ipip_type(const struct mlxsw_sp *mlxsw_sp,
4811 const struct fib6_info *rt,
4812 enum mlxsw_sp_ipip_type *ret)
4814 return rt->fib6_nh.nh_dev &&
4815 mlxsw_sp_netdev_ipip_type(mlxsw_sp, rt->fib6_nh.nh_dev, ret);
4818 static int mlxsw_sp_nexthop6_type_init(struct mlxsw_sp *mlxsw_sp,
4819 struct mlxsw_sp_nexthop_group *nh_grp,
4820 struct mlxsw_sp_nexthop *nh,
4821 const struct fib6_info *rt)
4823 const struct mlxsw_sp_ipip_ops *ipip_ops;
4824 struct mlxsw_sp_ipip_entry *ipip_entry;
4825 struct net_device *dev = rt->fib6_nh.nh_dev;
4826 struct mlxsw_sp_rif *rif;
4829 ipip_entry = mlxsw_sp_ipip_entry_find_by_ol_dev(mlxsw_sp, dev);
4831 ipip_ops = mlxsw_sp->router->ipip_ops_arr[ipip_entry->ipipt];
4832 if (ipip_ops->can_offload(mlxsw_sp, dev,
4833 MLXSW_SP_L3_PROTO_IPV6)) {
4834 nh->type = MLXSW_SP_NEXTHOP_TYPE_IPIP;
4835 mlxsw_sp_nexthop_ipip_init(mlxsw_sp, nh, ipip_entry);
4840 nh->type = MLXSW_SP_NEXTHOP_TYPE_ETH;
4841 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
4844 mlxsw_sp_nexthop_rif_init(nh, rif);
4846 err = mlxsw_sp_nexthop_neigh_init(mlxsw_sp, nh);
4848 goto err_nexthop_neigh_init;
4852 err_nexthop_neigh_init:
4853 mlxsw_sp_nexthop_rif_fini(nh);
4857 static void mlxsw_sp_nexthop6_type_fini(struct mlxsw_sp *mlxsw_sp,
4858 struct mlxsw_sp_nexthop *nh)
4860 mlxsw_sp_nexthop_type_fini(mlxsw_sp, nh);
4863 static int mlxsw_sp_nexthop6_init(struct mlxsw_sp *mlxsw_sp,
4864 struct mlxsw_sp_nexthop_group *nh_grp,
4865 struct mlxsw_sp_nexthop *nh,
4866 const struct fib6_info *rt)
4868 struct net_device *dev = rt->fib6_nh.nh_dev;
4870 nh->nh_grp = nh_grp;
4871 nh->nh_weight = rt->fib6_nh.nh_weight;
4872 memcpy(&nh->gw_addr, &rt->fib6_nh.nh_gw, sizeof(nh->gw_addr));
4873 mlxsw_sp_nexthop_counter_alloc(mlxsw_sp, nh);
4875 list_add_tail(&nh->router_list_node, &mlxsw_sp->router->nexthop_list);
4879 nh->ifindex = dev->ifindex;
4881 return mlxsw_sp_nexthop6_type_init(mlxsw_sp, nh_grp, nh, rt);
4884 static void mlxsw_sp_nexthop6_fini(struct mlxsw_sp *mlxsw_sp,
4885 struct mlxsw_sp_nexthop *nh)
4887 mlxsw_sp_nexthop6_type_fini(mlxsw_sp, nh);
4888 list_del(&nh->router_list_node);
4889 mlxsw_sp_nexthop_counter_free(mlxsw_sp, nh);
4892 static bool mlxsw_sp_rt6_is_gateway(const struct mlxsw_sp *mlxsw_sp,
4893 const struct fib6_info *rt)
4895 return rt->fib6_flags & RTF_GATEWAY ||
4896 mlxsw_sp_nexthop6_ipip_type(mlxsw_sp, rt, NULL);
4899 static struct mlxsw_sp_nexthop_group *
4900 mlxsw_sp_nexthop6_group_create(struct mlxsw_sp *mlxsw_sp,
4901 struct mlxsw_sp_fib6_entry *fib6_entry)
4903 struct mlxsw_sp_nexthop_group *nh_grp;
4904 struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
4905 struct mlxsw_sp_nexthop *nh;
4910 alloc_size = sizeof(*nh_grp) +
4911 fib6_entry->nrt6 * sizeof(struct mlxsw_sp_nexthop);
4912 nh_grp = kzalloc(alloc_size, GFP_KERNEL);
4914 return ERR_PTR(-ENOMEM);
4915 INIT_LIST_HEAD(&nh_grp->fib_list);
4916 #if IS_ENABLED(CONFIG_IPV6)
4917 nh_grp->neigh_tbl = &nd_tbl;
4919 mlxsw_sp_rt6 = list_first_entry(&fib6_entry->rt6_list,
4920 struct mlxsw_sp_rt6, list);
4921 nh_grp->gateway = mlxsw_sp_rt6_is_gateway(mlxsw_sp, mlxsw_sp_rt6->rt);
4922 nh_grp->count = fib6_entry->nrt6;
4923 for (i = 0; i < nh_grp->count; i++) {
4924 struct fib6_info *rt = mlxsw_sp_rt6->rt;
4926 nh = &nh_grp->nexthops[i];
4927 err = mlxsw_sp_nexthop6_init(mlxsw_sp, nh_grp, nh, rt);
4929 goto err_nexthop6_init;
4930 mlxsw_sp_rt6 = list_next_entry(mlxsw_sp_rt6, list);
4933 err = mlxsw_sp_nexthop_group_insert(mlxsw_sp, nh_grp);
4935 goto err_nexthop_group_insert;
4937 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
4940 err_nexthop_group_insert:
4942 for (i--; i >= 0; i--) {
4943 nh = &nh_grp->nexthops[i];
4944 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh);
4947 return ERR_PTR(err);
4951 mlxsw_sp_nexthop6_group_destroy(struct mlxsw_sp *mlxsw_sp,
4952 struct mlxsw_sp_nexthop_group *nh_grp)
4954 struct mlxsw_sp_nexthop *nh;
4955 int i = nh_grp->count;
4957 mlxsw_sp_nexthop_group_remove(mlxsw_sp, nh_grp);
4958 for (i--; i >= 0; i--) {
4959 nh = &nh_grp->nexthops[i];
4960 mlxsw_sp_nexthop6_fini(mlxsw_sp, nh);
4962 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
4963 WARN_ON(nh_grp->adj_index_valid);
4967 static int mlxsw_sp_nexthop6_group_get(struct mlxsw_sp *mlxsw_sp,
4968 struct mlxsw_sp_fib6_entry *fib6_entry)
4970 struct mlxsw_sp_nexthop_group *nh_grp;
4972 nh_grp = mlxsw_sp_nexthop6_group_lookup(mlxsw_sp, fib6_entry);
4974 nh_grp = mlxsw_sp_nexthop6_group_create(mlxsw_sp, fib6_entry);
4976 return PTR_ERR(nh_grp);
4979 list_add_tail(&fib6_entry->common.nexthop_group_node,
4981 fib6_entry->common.nh_group = nh_grp;
4986 static void mlxsw_sp_nexthop6_group_put(struct mlxsw_sp *mlxsw_sp,
4987 struct mlxsw_sp_fib_entry *fib_entry)
4989 struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
4991 list_del(&fib_entry->nexthop_group_node);
4992 if (!list_empty(&nh_grp->fib_list))
4994 mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, nh_grp);
4998 mlxsw_sp_nexthop6_group_update(struct mlxsw_sp *mlxsw_sp,
4999 struct mlxsw_sp_fib6_entry *fib6_entry)
5001 struct mlxsw_sp_nexthop_group *old_nh_grp = fib6_entry->common.nh_group;
5004 fib6_entry->common.nh_group = NULL;
5005 list_del(&fib6_entry->common.nexthop_group_node);
5007 err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry);
5009 goto err_nexthop6_group_get;
5011 /* In case this entry is offloaded, then the adjacency index
5012 * currently associated with it in the device's table is that
5013 * of the old group. Start using the new one instead.
5015 err = mlxsw_sp_fib_node_entry_add(mlxsw_sp, &fib6_entry->common);
5017 goto err_fib_node_entry_add;
5019 if (list_empty(&old_nh_grp->fib_list))
5020 mlxsw_sp_nexthop6_group_destroy(mlxsw_sp, old_nh_grp);
5024 err_fib_node_entry_add:
5025 mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common);
5026 err_nexthop6_group_get:
5027 list_add_tail(&fib6_entry->common.nexthop_group_node,
5028 &old_nh_grp->fib_list);
5029 fib6_entry->common.nh_group = old_nh_grp;
5034 mlxsw_sp_fib6_entry_nexthop_add(struct mlxsw_sp *mlxsw_sp,
5035 struct mlxsw_sp_fib6_entry *fib6_entry,
5036 struct fib6_info *rt)
5038 struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
5041 mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt);
5042 if (IS_ERR(mlxsw_sp_rt6))
5043 return PTR_ERR(mlxsw_sp_rt6);
5045 list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list);
5048 err = mlxsw_sp_nexthop6_group_update(mlxsw_sp, fib6_entry);
5050 goto err_nexthop6_group_update;
5054 err_nexthop6_group_update:
5056 list_del(&mlxsw_sp_rt6->list);
5057 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
5062 mlxsw_sp_fib6_entry_nexthop_del(struct mlxsw_sp *mlxsw_sp,
5063 struct mlxsw_sp_fib6_entry *fib6_entry,
5064 struct fib6_info *rt)
5066 struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
5068 mlxsw_sp_rt6 = mlxsw_sp_fib6_entry_rt_find(fib6_entry, rt);
5069 if (WARN_ON(!mlxsw_sp_rt6))
5073 list_del(&mlxsw_sp_rt6->list);
5074 mlxsw_sp_nexthop6_group_update(mlxsw_sp, fib6_entry);
5075 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
5078 static void mlxsw_sp_fib6_entry_type_set(struct mlxsw_sp *mlxsw_sp,
5079 struct mlxsw_sp_fib_entry *fib_entry,
5080 const struct fib6_info *rt)
5082 /* Packets hitting RTF_REJECT routes need to be discarded by the
5083 * stack. We can rely on their destination device not having a
5084 * RIF (it's the loopback device) and can thus use action type
5085 * local, which will cause them to be trapped with a lower
5086 * priority than packets that need to be locally received.
5088 if (rt->fib6_flags & (RTF_LOCAL | RTF_ANYCAST))
5089 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
5090 else if (rt->fib6_flags & RTF_REJECT)
5091 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
5092 else if (mlxsw_sp_rt6_is_gateway(mlxsw_sp, rt))
5093 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
5095 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
5099 mlxsw_sp_fib6_entry_rt_destroy_all(struct mlxsw_sp_fib6_entry *fib6_entry)
5101 struct mlxsw_sp_rt6 *mlxsw_sp_rt6, *tmp;
5103 list_for_each_entry_safe(mlxsw_sp_rt6, tmp, &fib6_entry->rt6_list,
5106 list_del(&mlxsw_sp_rt6->list);
5107 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
5111 static struct mlxsw_sp_fib6_entry *
5112 mlxsw_sp_fib6_entry_create(struct mlxsw_sp *mlxsw_sp,
5113 struct mlxsw_sp_fib_node *fib_node,
5114 struct fib6_info *rt)
5116 struct mlxsw_sp_fib6_entry *fib6_entry;
5117 struct mlxsw_sp_fib_entry *fib_entry;
5118 struct mlxsw_sp_rt6 *mlxsw_sp_rt6;
5121 fib6_entry = kzalloc(sizeof(*fib6_entry), GFP_KERNEL);
5123 return ERR_PTR(-ENOMEM);
5124 fib_entry = &fib6_entry->common;
5126 mlxsw_sp_rt6 = mlxsw_sp_rt6_create(rt);
5127 if (IS_ERR(mlxsw_sp_rt6)) {
5128 err = PTR_ERR(mlxsw_sp_rt6);
5129 goto err_rt6_create;
5132 mlxsw_sp_fib6_entry_type_set(mlxsw_sp, fib_entry, mlxsw_sp_rt6->rt);
5134 INIT_LIST_HEAD(&fib6_entry->rt6_list);
5135 list_add_tail(&mlxsw_sp_rt6->list, &fib6_entry->rt6_list);
5136 fib6_entry->nrt6 = 1;
5137 err = mlxsw_sp_nexthop6_group_get(mlxsw_sp, fib6_entry);
5139 goto err_nexthop6_group_get;
5141 fib_entry->fib_node = fib_node;
5145 err_nexthop6_group_get:
5146 list_del(&mlxsw_sp_rt6->list);
5147 mlxsw_sp_rt6_destroy(mlxsw_sp_rt6);
5150 return ERR_PTR(err);
5153 static void mlxsw_sp_fib6_entry_destroy(struct mlxsw_sp *mlxsw_sp,
5154 struct mlxsw_sp_fib6_entry *fib6_entry)
5156 mlxsw_sp_nexthop6_group_put(mlxsw_sp, &fib6_entry->common);
5157 mlxsw_sp_fib6_entry_rt_destroy_all(fib6_entry);
5158 WARN_ON(fib6_entry->nrt6);
5162 static struct mlxsw_sp_fib6_entry *
5163 mlxsw_sp_fib6_node_entry_find(const struct mlxsw_sp_fib_node *fib_node,
5164 const struct fib6_info *nrt, bool replace)
5166 struct mlxsw_sp_fib6_entry *fib6_entry;
5168 list_for_each_entry(fib6_entry, &fib_node->entry_list, common.list) {
5169 struct fib6_info *rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
5171 if (rt->fib6_table->tb6_id > nrt->fib6_table->tb6_id)
5173 if (rt->fib6_table->tb6_id != nrt->fib6_table->tb6_id)
5175 if (replace && rt->fib6_metric == nrt->fib6_metric)
5177 if (rt->fib6_metric > nrt->fib6_metric)
5185 mlxsw_sp_fib6_node_list_insert(struct mlxsw_sp_fib6_entry *new6_entry,
5188 struct mlxsw_sp_fib_node *fib_node = new6_entry->common.fib_node;
5189 struct fib6_info *nrt = mlxsw_sp_fib6_entry_rt(new6_entry);
5190 struct mlxsw_sp_fib6_entry *fib6_entry;
5192 fib6_entry = mlxsw_sp_fib6_node_entry_find(fib_node, nrt, replace);
5194 if (replace && WARN_ON(!fib6_entry))
5198 list_add_tail(&new6_entry->common.list,
5199 &fib6_entry->common.list);
5201 struct mlxsw_sp_fib6_entry *last;
5203 list_for_each_entry(last, &fib_node->entry_list, common.list) {
5204 struct fib6_info *rt = mlxsw_sp_fib6_entry_rt(last);
5206 if (nrt->fib6_table->tb6_id > rt->fib6_table->tb6_id)
5212 list_add(&new6_entry->common.list,
5213 &fib6_entry->common.list);
5215 list_add(&new6_entry->common.list,
5216 &fib_node->entry_list);
5223 mlxsw_sp_fib6_node_list_remove(struct mlxsw_sp_fib6_entry *fib6_entry)
5225 list_del(&fib6_entry->common.list);
5228 static int mlxsw_sp_fib6_node_entry_link(struct mlxsw_sp *mlxsw_sp,
5229 struct mlxsw_sp_fib6_entry *fib6_entry,
5234 err = mlxsw_sp_fib6_node_list_insert(fib6_entry, replace);
5238 err = mlxsw_sp_fib_node_entry_add(mlxsw_sp, &fib6_entry->common);
5240 goto err_fib_node_entry_add;
5244 err_fib_node_entry_add:
5245 mlxsw_sp_fib6_node_list_remove(fib6_entry);
5250 mlxsw_sp_fib6_node_entry_unlink(struct mlxsw_sp *mlxsw_sp,
5251 struct mlxsw_sp_fib6_entry *fib6_entry)
5253 mlxsw_sp_fib_node_entry_del(mlxsw_sp, &fib6_entry->common);
5254 mlxsw_sp_fib6_node_list_remove(fib6_entry);
5257 static struct mlxsw_sp_fib6_entry *
5258 mlxsw_sp_fib6_entry_lookup(struct mlxsw_sp *mlxsw_sp,
5259 const struct fib6_info *rt)
5261 struct mlxsw_sp_fib6_entry *fib6_entry;
5262 struct mlxsw_sp_fib_node *fib_node;
5263 struct mlxsw_sp_fib *fib;
5264 struct mlxsw_sp_vr *vr;
5266 vr = mlxsw_sp_vr_find(mlxsw_sp, rt->fib6_table->tb6_id);
5269 fib = mlxsw_sp_vr_fib(vr, MLXSW_SP_L3_PROTO_IPV6);
5271 fib_node = mlxsw_sp_fib_node_lookup(fib, &rt->fib6_dst.addr,
5272 sizeof(rt->fib6_dst.addr),
5277 list_for_each_entry(fib6_entry, &fib_node->entry_list, common.list) {
5278 struct fib6_info *iter_rt = mlxsw_sp_fib6_entry_rt(fib6_entry);
5280 if (rt->fib6_table->tb6_id == iter_rt->fib6_table->tb6_id &&
5281 rt->fib6_metric == iter_rt->fib6_metric &&
5282 mlxsw_sp_fib6_entry_rt_find(fib6_entry, rt))
5289 static void mlxsw_sp_fib6_entry_replace(struct mlxsw_sp *mlxsw_sp,
5290 struct mlxsw_sp_fib6_entry *fib6_entry,
5293 struct mlxsw_sp_fib_node *fib_node = fib6_entry->common.fib_node;
5294 struct mlxsw_sp_fib6_entry *replaced;
5299 replaced = list_next_entry(fib6_entry, common.list);
5301 mlxsw_sp_fib6_node_entry_unlink(mlxsw_sp, replaced);
5302 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, replaced);
5303 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
5306 static int mlxsw_sp_router_fib6_add(struct mlxsw_sp *mlxsw_sp,
5307 struct fib6_info *rt, bool replace,
5310 struct mlxsw_sp_fib6_entry *fib6_entry;
5311 struct mlxsw_sp_fib_node *fib_node;
5314 if (mlxsw_sp->router->aborted)
5317 if (rt->fib6_src.plen)
5320 if (mlxsw_sp_fib6_rt_should_ignore(rt))
5323 fib_node = mlxsw_sp_fib_node_get(mlxsw_sp, rt->fib6_table->tb6_id,
5325 sizeof(rt->fib6_dst.addr),
5327 MLXSW_SP_L3_PROTO_IPV6);
5328 if (IS_ERR(fib_node))
5329 return PTR_ERR(fib_node);
5331 /* Before creating a new entry, try to append route to an existing
5334 fib6_entry = mlxsw_sp_fib6_node_mp_entry_find(fib_node, rt, append);
5336 err = mlxsw_sp_fib6_entry_nexthop_add(mlxsw_sp, fib6_entry, rt);
5338 goto err_fib6_entry_nexthop_add;
5342 /* We received an append event, yet did not find any route to
5345 if (WARN_ON(append)) {
5347 goto err_fib6_entry_append;
5350 fib6_entry = mlxsw_sp_fib6_entry_create(mlxsw_sp, fib_node, rt);
5351 if (IS_ERR(fib6_entry)) {
5352 err = PTR_ERR(fib6_entry);
5353 goto err_fib6_entry_create;
5356 err = mlxsw_sp_fib6_node_entry_link(mlxsw_sp, fib6_entry, replace);
5358 goto err_fib6_node_entry_link;
5360 mlxsw_sp_fib6_entry_replace(mlxsw_sp, fib6_entry, replace);
5364 err_fib6_node_entry_link:
5365 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
5366 err_fib6_entry_create:
5367 err_fib6_entry_append:
5368 err_fib6_entry_nexthop_add:
5369 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
5373 static void mlxsw_sp_router_fib6_del(struct mlxsw_sp *mlxsw_sp,
5374 struct fib6_info *rt)
5376 struct mlxsw_sp_fib6_entry *fib6_entry;
5377 struct mlxsw_sp_fib_node *fib_node;
5379 if (mlxsw_sp->router->aborted)
5382 if (mlxsw_sp_fib6_rt_should_ignore(rt))
5385 fib6_entry = mlxsw_sp_fib6_entry_lookup(mlxsw_sp, rt);
5386 if (WARN_ON(!fib6_entry))
5389 /* If route is part of a multipath entry, but not the last one
5390 * removed, then only reduce its nexthop group.
5392 if (!list_is_singular(&fib6_entry->rt6_list)) {
5393 mlxsw_sp_fib6_entry_nexthop_del(mlxsw_sp, fib6_entry, rt);
5397 fib_node = fib6_entry->common.fib_node;
5399 mlxsw_sp_fib6_node_entry_unlink(mlxsw_sp, fib6_entry);
5400 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
5401 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
5404 static int __mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp,
5405 enum mlxsw_reg_ralxx_protocol proto,
5408 char ralta_pl[MLXSW_REG_RALTA_LEN];
5409 char ralst_pl[MLXSW_REG_RALST_LEN];
5412 mlxsw_reg_ralta_pack(ralta_pl, true, proto, tree_id);
5413 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
5417 mlxsw_reg_ralst_pack(ralst_pl, 0xff, tree_id);
5418 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
5422 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
5423 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
5424 char raltb_pl[MLXSW_REG_RALTB_LEN];
5425 char ralue_pl[MLXSW_REG_RALUE_LEN];
5427 mlxsw_reg_raltb_pack(raltb_pl, vr->id, proto, tree_id);
5428 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb),
5433 mlxsw_reg_ralue_pack(ralue_pl, proto,
5434 MLXSW_REG_RALUE_OP_WRITE_WRITE, vr->id, 0);
5435 mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
5436 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue),
5445 static struct mlxsw_sp_mr_table *
5446 mlxsw_sp_router_fibmr_family_to_table(struct mlxsw_sp_vr *vr, int family)
5448 if (family == RTNL_FAMILY_IPMR)
5449 return vr->mr_table[MLXSW_SP_L3_PROTO_IPV4];
5451 return vr->mr_table[MLXSW_SP_L3_PROTO_IPV6];
5454 static int mlxsw_sp_router_fibmr_add(struct mlxsw_sp *mlxsw_sp,
5455 struct mfc_entry_notifier_info *men_info,
5458 struct mlxsw_sp_mr_table *mrt;
5459 struct mlxsw_sp_vr *vr;
5461 if (mlxsw_sp->router->aborted)
5464 vr = mlxsw_sp_vr_get(mlxsw_sp, men_info->tb_id, NULL);
5468 mrt = mlxsw_sp_router_fibmr_family_to_table(vr, men_info->info.family);
5469 return mlxsw_sp_mr_route_add(mrt, men_info->mfc, replace);
5472 static void mlxsw_sp_router_fibmr_del(struct mlxsw_sp *mlxsw_sp,
5473 struct mfc_entry_notifier_info *men_info)
5475 struct mlxsw_sp_mr_table *mrt;
5476 struct mlxsw_sp_vr *vr;
5478 if (mlxsw_sp->router->aborted)
5481 vr = mlxsw_sp_vr_find(mlxsw_sp, men_info->tb_id);
5485 mrt = mlxsw_sp_router_fibmr_family_to_table(vr, men_info->info.family);
5486 mlxsw_sp_mr_route_del(mrt, men_info->mfc);
5487 mlxsw_sp_vr_put(mlxsw_sp, vr);
5491 mlxsw_sp_router_fibmr_vif_add(struct mlxsw_sp *mlxsw_sp,
5492 struct vif_entry_notifier_info *ven_info)
5494 struct mlxsw_sp_mr_table *mrt;
5495 struct mlxsw_sp_rif *rif;
5496 struct mlxsw_sp_vr *vr;
5498 if (mlxsw_sp->router->aborted)
5501 vr = mlxsw_sp_vr_get(mlxsw_sp, ven_info->tb_id, NULL);
5505 mrt = mlxsw_sp_router_fibmr_family_to_table(vr, ven_info->info.family);
5506 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, ven_info->dev);
5507 return mlxsw_sp_mr_vif_add(mrt, ven_info->dev,
5508 ven_info->vif_index,
5509 ven_info->vif_flags, rif);
5513 mlxsw_sp_router_fibmr_vif_del(struct mlxsw_sp *mlxsw_sp,
5514 struct vif_entry_notifier_info *ven_info)
5516 struct mlxsw_sp_mr_table *mrt;
5517 struct mlxsw_sp_vr *vr;
5519 if (mlxsw_sp->router->aborted)
5522 vr = mlxsw_sp_vr_find(mlxsw_sp, ven_info->tb_id);
5526 mrt = mlxsw_sp_router_fibmr_family_to_table(vr, ven_info->info.family);
5527 mlxsw_sp_mr_vif_del(mrt, ven_info->vif_index);
5528 mlxsw_sp_vr_put(mlxsw_sp, vr);
5531 static int mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp)
5533 enum mlxsw_reg_ralxx_protocol proto = MLXSW_REG_RALXX_PROTOCOL_IPV4;
5536 err = __mlxsw_sp_router_set_abort_trap(mlxsw_sp, proto,
5537 MLXSW_SP_LPM_TREE_MIN);
5541 /* The multicast router code does not need an abort trap as by default,
5542 * packets that don't match any routes are trapped to the CPU.
5545 proto = MLXSW_REG_RALXX_PROTOCOL_IPV6;
5546 return __mlxsw_sp_router_set_abort_trap(mlxsw_sp, proto,
5547 MLXSW_SP_LPM_TREE_MIN + 1);
5550 static void mlxsw_sp_fib4_node_flush(struct mlxsw_sp *mlxsw_sp,
5551 struct mlxsw_sp_fib_node *fib_node)
5553 struct mlxsw_sp_fib4_entry *fib4_entry, *tmp;
5555 list_for_each_entry_safe(fib4_entry, tmp, &fib_node->entry_list,
5557 bool do_break = &tmp->common.list == &fib_node->entry_list;
5559 mlxsw_sp_fib4_node_entry_unlink(mlxsw_sp, fib4_entry);
5560 mlxsw_sp_fib4_entry_destroy(mlxsw_sp, fib4_entry);
5561 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
5562 /* Break when entry list is empty and node was freed.
5563 * Otherwise, we'll access freed memory in the next
5571 static void mlxsw_sp_fib6_node_flush(struct mlxsw_sp *mlxsw_sp,
5572 struct mlxsw_sp_fib_node *fib_node)
5574 struct mlxsw_sp_fib6_entry *fib6_entry, *tmp;
5576 list_for_each_entry_safe(fib6_entry, tmp, &fib_node->entry_list,
5578 bool do_break = &tmp->common.list == &fib_node->entry_list;
5580 mlxsw_sp_fib6_node_entry_unlink(mlxsw_sp, fib6_entry);
5581 mlxsw_sp_fib6_entry_destroy(mlxsw_sp, fib6_entry);
5582 mlxsw_sp_fib_node_put(mlxsw_sp, fib_node);
5588 static void mlxsw_sp_fib_node_flush(struct mlxsw_sp *mlxsw_sp,
5589 struct mlxsw_sp_fib_node *fib_node)
5591 switch (fib_node->fib->proto) {
5592 case MLXSW_SP_L3_PROTO_IPV4:
5593 mlxsw_sp_fib4_node_flush(mlxsw_sp, fib_node);
5595 case MLXSW_SP_L3_PROTO_IPV6:
5596 mlxsw_sp_fib6_node_flush(mlxsw_sp, fib_node);
5601 static void mlxsw_sp_vr_fib_flush(struct mlxsw_sp *mlxsw_sp,
5602 struct mlxsw_sp_vr *vr,
5603 enum mlxsw_sp_l3proto proto)
5605 struct mlxsw_sp_fib *fib = mlxsw_sp_vr_fib(vr, proto);
5606 struct mlxsw_sp_fib_node *fib_node, *tmp;
5608 list_for_each_entry_safe(fib_node, tmp, &fib->node_list, list) {
5609 bool do_break = &tmp->list == &fib->node_list;
5611 mlxsw_sp_fib_node_flush(mlxsw_sp, fib_node);
5617 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp)
5621 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_VRS); i++) {
5622 struct mlxsw_sp_vr *vr = &mlxsw_sp->router->vrs[i];
5624 if (!mlxsw_sp_vr_is_used(vr))
5627 for (j = 0; j < MLXSW_SP_L3_PROTO_MAX; j++)
5628 mlxsw_sp_mr_table_flush(vr->mr_table[j]);
5629 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV4);
5631 /* If virtual router was only used for IPv4, then it's no
5634 if (!mlxsw_sp_vr_is_used(vr))
5636 mlxsw_sp_vr_fib_flush(mlxsw_sp, vr, MLXSW_SP_L3_PROTO_IPV6);
5640 static void mlxsw_sp_router_fib_abort(struct mlxsw_sp *mlxsw_sp)
5644 if (mlxsw_sp->router->aborted)
5646 dev_warn(mlxsw_sp->bus_info->dev, "FIB abort triggered. Note that FIB entries are no longer being offloaded to this device.\n");
5647 mlxsw_sp_router_fib_flush(mlxsw_sp);
5648 mlxsw_sp->router->aborted = true;
5649 err = mlxsw_sp_router_set_abort_trap(mlxsw_sp);
5651 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set abort trap.\n");
5654 struct mlxsw_sp_fib_event_work {
5655 struct work_struct work;
5657 struct fib6_entry_notifier_info fen6_info;
5658 struct fib_entry_notifier_info fen_info;
5659 struct fib_rule_notifier_info fr_info;
5660 struct fib_nh_notifier_info fnh_info;
5661 struct mfc_entry_notifier_info men_info;
5662 struct vif_entry_notifier_info ven_info;
5664 struct mlxsw_sp *mlxsw_sp;
5665 unsigned long event;
5668 static void mlxsw_sp_router_fib4_event_work(struct work_struct *work)
5670 struct mlxsw_sp_fib_event_work *fib_work =
5671 container_of(work, struct mlxsw_sp_fib_event_work, work);
5672 struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp;
5673 bool replace, append;
5676 /* Protect internal structures from changes */
5678 mlxsw_sp_span_respin(mlxsw_sp);
5680 switch (fib_work->event) {
5681 case FIB_EVENT_ENTRY_REPLACE: /* fall through */
5682 case FIB_EVENT_ENTRY_APPEND: /* fall through */
5683 case FIB_EVENT_ENTRY_ADD:
5684 replace = fib_work->event == FIB_EVENT_ENTRY_REPLACE;
5685 append = fib_work->event == FIB_EVENT_ENTRY_APPEND;
5686 err = mlxsw_sp_router_fib4_add(mlxsw_sp, &fib_work->fen_info,
5689 mlxsw_sp_router_fib_abort(mlxsw_sp);
5690 fib_info_put(fib_work->fen_info.fi);
5692 case FIB_EVENT_ENTRY_DEL:
5693 mlxsw_sp_router_fib4_del(mlxsw_sp, &fib_work->fen_info);
5694 fib_info_put(fib_work->fen_info.fi);
5696 case FIB_EVENT_RULE_ADD:
5697 /* if we get here, a rule was added that we do not support.
5698 * just do the fib_abort
5700 mlxsw_sp_router_fib_abort(mlxsw_sp);
5702 case FIB_EVENT_NH_ADD: /* fall through */
5703 case FIB_EVENT_NH_DEL:
5704 mlxsw_sp_nexthop4_event(mlxsw_sp, fib_work->event,
5705 fib_work->fnh_info.fib_nh);
5706 fib_info_put(fib_work->fnh_info.fib_nh->nh_parent);
5713 static void mlxsw_sp_router_fib6_event_work(struct work_struct *work)
5715 struct mlxsw_sp_fib_event_work *fib_work =
5716 container_of(work, struct mlxsw_sp_fib_event_work, work);
5717 struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp;
5718 bool replace, append;
5722 mlxsw_sp_span_respin(mlxsw_sp);
5724 switch (fib_work->event) {
5725 case FIB_EVENT_ENTRY_REPLACE: /* fall through */
5726 case FIB_EVENT_ENTRY_APPEND: /* fall through */
5727 case FIB_EVENT_ENTRY_ADD:
5728 replace = fib_work->event == FIB_EVENT_ENTRY_REPLACE;
5729 append = fib_work->event == FIB_EVENT_ENTRY_APPEND;
5730 err = mlxsw_sp_router_fib6_add(mlxsw_sp,
5731 fib_work->fen6_info.rt, replace,
5734 mlxsw_sp_router_fib_abort(mlxsw_sp);
5735 mlxsw_sp_rt6_release(fib_work->fen6_info.rt);
5737 case FIB_EVENT_ENTRY_DEL:
5738 mlxsw_sp_router_fib6_del(mlxsw_sp, fib_work->fen6_info.rt);
5739 mlxsw_sp_rt6_release(fib_work->fen6_info.rt);
5741 case FIB_EVENT_RULE_ADD:
5742 /* if we get here, a rule was added that we do not support.
5743 * just do the fib_abort
5745 mlxsw_sp_router_fib_abort(mlxsw_sp);
5752 static void mlxsw_sp_router_fibmr_event_work(struct work_struct *work)
5754 struct mlxsw_sp_fib_event_work *fib_work =
5755 container_of(work, struct mlxsw_sp_fib_event_work, work);
5756 struct mlxsw_sp *mlxsw_sp = fib_work->mlxsw_sp;
5761 switch (fib_work->event) {
5762 case FIB_EVENT_ENTRY_REPLACE: /* fall through */
5763 case FIB_EVENT_ENTRY_ADD:
5764 replace = fib_work->event == FIB_EVENT_ENTRY_REPLACE;
5766 err = mlxsw_sp_router_fibmr_add(mlxsw_sp, &fib_work->men_info,
5769 mlxsw_sp_router_fib_abort(mlxsw_sp);
5770 mr_cache_put(fib_work->men_info.mfc);
5772 case FIB_EVENT_ENTRY_DEL:
5773 mlxsw_sp_router_fibmr_del(mlxsw_sp, &fib_work->men_info);
5774 mr_cache_put(fib_work->men_info.mfc);
5776 case FIB_EVENT_VIF_ADD:
5777 err = mlxsw_sp_router_fibmr_vif_add(mlxsw_sp,
5778 &fib_work->ven_info);
5780 mlxsw_sp_router_fib_abort(mlxsw_sp);
5781 dev_put(fib_work->ven_info.dev);
5783 case FIB_EVENT_VIF_DEL:
5784 mlxsw_sp_router_fibmr_vif_del(mlxsw_sp,
5785 &fib_work->ven_info);
5786 dev_put(fib_work->ven_info.dev);
5788 case FIB_EVENT_RULE_ADD:
5789 /* if we get here, a rule was added that we do not support.
5790 * just do the fib_abort
5792 mlxsw_sp_router_fib_abort(mlxsw_sp);
5799 static void mlxsw_sp_router_fib4_event(struct mlxsw_sp_fib_event_work *fib_work,
5800 struct fib_notifier_info *info)
5802 struct fib_entry_notifier_info *fen_info;
5803 struct fib_nh_notifier_info *fnh_info;
5805 switch (fib_work->event) {
5806 case FIB_EVENT_ENTRY_REPLACE: /* fall through */
5807 case FIB_EVENT_ENTRY_APPEND: /* fall through */
5808 case FIB_EVENT_ENTRY_ADD: /* fall through */
5809 case FIB_EVENT_ENTRY_DEL:
5810 fen_info = container_of(info, struct fib_entry_notifier_info,
5812 fib_work->fen_info = *fen_info;
5813 /* Take reference on fib_info to prevent it from being
5814 * freed while work is queued. Release it afterwards.
5816 fib_info_hold(fib_work->fen_info.fi);
5818 case FIB_EVENT_NH_ADD: /* fall through */
5819 case FIB_EVENT_NH_DEL:
5820 fnh_info = container_of(info, struct fib_nh_notifier_info,
5822 fib_work->fnh_info = *fnh_info;
5823 fib_info_hold(fib_work->fnh_info.fib_nh->nh_parent);
5828 static void mlxsw_sp_router_fib6_event(struct mlxsw_sp_fib_event_work *fib_work,
5829 struct fib_notifier_info *info)
5831 struct fib6_entry_notifier_info *fen6_info;
5833 switch (fib_work->event) {
5834 case FIB_EVENT_ENTRY_REPLACE: /* fall through */
5835 case FIB_EVENT_ENTRY_APPEND: /* fall through */
5836 case FIB_EVENT_ENTRY_ADD: /* fall through */
5837 case FIB_EVENT_ENTRY_DEL:
5838 fen6_info = container_of(info, struct fib6_entry_notifier_info,
5840 fib_work->fen6_info = *fen6_info;
5841 fib6_info_hold(fib_work->fen6_info.rt);
5847 mlxsw_sp_router_fibmr_event(struct mlxsw_sp_fib_event_work *fib_work,
5848 struct fib_notifier_info *info)
5850 switch (fib_work->event) {
5851 case FIB_EVENT_ENTRY_REPLACE: /* fall through */
5852 case FIB_EVENT_ENTRY_ADD: /* fall through */
5853 case FIB_EVENT_ENTRY_DEL:
5854 memcpy(&fib_work->men_info, info, sizeof(fib_work->men_info));
5855 mr_cache_hold(fib_work->men_info.mfc);
5857 case FIB_EVENT_VIF_ADD: /* fall through */
5858 case FIB_EVENT_VIF_DEL:
5859 memcpy(&fib_work->ven_info, info, sizeof(fib_work->ven_info));
5860 dev_hold(fib_work->ven_info.dev);
5865 static int mlxsw_sp_router_fib_rule_event(unsigned long event,
5866 struct fib_notifier_info *info,
5867 struct mlxsw_sp *mlxsw_sp)
5869 struct netlink_ext_ack *extack = info->extack;
5870 struct fib_rule_notifier_info *fr_info;
5871 struct fib_rule *rule;
5874 /* nothing to do at the moment */
5875 if (event == FIB_EVENT_RULE_DEL)
5878 if (mlxsw_sp->router->aborted)
5881 fr_info = container_of(info, struct fib_rule_notifier_info, info);
5882 rule = fr_info->rule;
5884 switch (info->family) {
5886 if (!fib4_rule_default(rule) && !rule->l3mdev)
5890 if (!fib6_rule_default(rule) && !rule->l3mdev)
5893 case RTNL_FAMILY_IPMR:
5894 if (!ipmr_rule_default(rule) && !rule->l3mdev)
5897 case RTNL_FAMILY_IP6MR:
5898 if (!ip6mr_rule_default(rule) && !rule->l3mdev)
5904 NL_SET_ERR_MSG_MOD(extack, "FIB rules not supported");
5909 /* Called with rcu_read_lock() */
5910 static int mlxsw_sp_router_fib_event(struct notifier_block *nb,
5911 unsigned long event, void *ptr)
5913 struct mlxsw_sp_fib_event_work *fib_work;
5914 struct fib_notifier_info *info = ptr;
5915 struct mlxsw_sp_router *router;
5918 if (!net_eq(info->net, &init_net) ||
5919 (info->family != AF_INET && info->family != AF_INET6 &&
5920 info->family != RTNL_FAMILY_IPMR &&
5921 info->family != RTNL_FAMILY_IP6MR))
5924 router = container_of(nb, struct mlxsw_sp_router, fib_nb);
5927 case FIB_EVENT_RULE_ADD: /* fall through */
5928 case FIB_EVENT_RULE_DEL:
5929 err = mlxsw_sp_router_fib_rule_event(event, info,
5931 if (!err || info->extack)
5932 return notifier_from_errno(err);
5934 case FIB_EVENT_ENTRY_ADD:
5935 if (router->aborted) {
5936 NL_SET_ERR_MSG_MOD(info->extack, "FIB offload was aborted. Not configuring route");
5937 return notifier_from_errno(-EINVAL);
5942 fib_work = kzalloc(sizeof(*fib_work), GFP_ATOMIC);
5943 if (WARN_ON(!fib_work))
5946 fib_work->mlxsw_sp = router->mlxsw_sp;
5947 fib_work->event = event;
5949 switch (info->family) {
5951 INIT_WORK(&fib_work->work, mlxsw_sp_router_fib4_event_work);
5952 mlxsw_sp_router_fib4_event(fib_work, info);
5955 INIT_WORK(&fib_work->work, mlxsw_sp_router_fib6_event_work);
5956 mlxsw_sp_router_fib6_event(fib_work, info);
5958 case RTNL_FAMILY_IP6MR:
5959 case RTNL_FAMILY_IPMR:
5960 INIT_WORK(&fib_work->work, mlxsw_sp_router_fibmr_event_work);
5961 mlxsw_sp_router_fibmr_event(fib_work, info);
5965 mlxsw_core_schedule_work(&fib_work->work);
5970 static struct mlxsw_sp_rif *
5971 mlxsw_sp_rif_find_by_dev(const struct mlxsw_sp *mlxsw_sp,
5972 const struct net_device *dev)
5976 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
5977 if (mlxsw_sp->router->rifs[i] &&
5978 mlxsw_sp->router->rifs[i]->dev == dev)
5979 return mlxsw_sp->router->rifs[i];
5984 static int mlxsw_sp_router_rif_disable(struct mlxsw_sp *mlxsw_sp, u16 rif)
5986 char ritr_pl[MLXSW_REG_RITR_LEN];
5989 mlxsw_reg_ritr_rif_pack(ritr_pl, rif);
5990 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
5991 if (WARN_ON_ONCE(err))
5994 mlxsw_reg_ritr_enable_set(ritr_pl, false);
5995 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
5998 static void mlxsw_sp_router_rif_gone_sync(struct mlxsw_sp *mlxsw_sp,
5999 struct mlxsw_sp_rif *rif)
6001 mlxsw_sp_router_rif_disable(mlxsw_sp, rif->rif_index);
6002 mlxsw_sp_nexthop_rif_gone_sync(mlxsw_sp, rif);
6003 mlxsw_sp_neigh_rif_gone_sync(mlxsw_sp, rif);
6007 mlxsw_sp_rif_should_config(struct mlxsw_sp_rif *rif, struct net_device *dev,
6008 unsigned long event)
6010 struct inet6_dev *inet6_dev;
6011 bool addr_list_empty = true;
6012 struct in_device *idev;
6018 idev = __in_dev_get_rtnl(dev);
6019 if (idev && idev->ifa_list)
6020 addr_list_empty = false;
6022 inet6_dev = __in6_dev_get(dev);
6023 if (addr_list_empty && inet6_dev &&
6024 !list_empty(&inet6_dev->addr_list))
6025 addr_list_empty = false;
6027 if (rif && addr_list_empty &&
6028 !netif_is_l3_slave(rif->dev))
6030 /* It is possible we already removed the RIF ourselves
6031 * if it was assigned to a netdev that is now a bridge
6040 static enum mlxsw_sp_rif_type
6041 mlxsw_sp_dev_rif_type(const struct mlxsw_sp *mlxsw_sp,
6042 const struct net_device *dev)
6044 enum mlxsw_sp_fid_type type;
6046 if (mlxsw_sp_netdev_ipip_type(mlxsw_sp, dev, NULL))
6047 return MLXSW_SP_RIF_TYPE_IPIP_LB;
6049 /* Otherwise RIF type is derived from the type of the underlying FID. */
6050 if (is_vlan_dev(dev) && netif_is_bridge_master(vlan_dev_real_dev(dev)))
6051 type = MLXSW_SP_FID_TYPE_8021Q;
6052 else if (netif_is_bridge_master(dev) && br_vlan_enabled(dev))
6053 type = MLXSW_SP_FID_TYPE_8021Q;
6054 else if (netif_is_bridge_master(dev))
6055 type = MLXSW_SP_FID_TYPE_8021D;
6057 type = MLXSW_SP_FID_TYPE_RFID;
6059 return mlxsw_sp_fid_type_rif_type(mlxsw_sp, type);
6062 static int mlxsw_sp_rif_index_alloc(struct mlxsw_sp *mlxsw_sp, u16 *p_rif_index)
6066 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++) {
6067 if (!mlxsw_sp->router->rifs[i]) {
6076 static struct mlxsw_sp_rif *mlxsw_sp_rif_alloc(size_t rif_size, u16 rif_index,
6078 struct net_device *l3_dev)
6080 struct mlxsw_sp_rif *rif;
6082 rif = kzalloc(rif_size, GFP_KERNEL);
6086 INIT_LIST_HEAD(&rif->nexthop_list);
6087 INIT_LIST_HEAD(&rif->neigh_list);
6088 ether_addr_copy(rif->addr, l3_dev->dev_addr);
6089 rif->mtu = l3_dev->mtu;
6092 rif->rif_index = rif_index;
6097 struct mlxsw_sp_rif *mlxsw_sp_rif_by_index(const struct mlxsw_sp *mlxsw_sp,
6100 return mlxsw_sp->router->rifs[rif_index];
6103 u16 mlxsw_sp_rif_index(const struct mlxsw_sp_rif *rif)
6105 return rif->rif_index;
6108 u16 mlxsw_sp_ipip_lb_rif_index(const struct mlxsw_sp_rif_ipip_lb *lb_rif)
6110 return lb_rif->common.rif_index;
6113 u16 mlxsw_sp_ipip_lb_ul_vr_id(const struct mlxsw_sp_rif_ipip_lb *lb_rif)
6115 return lb_rif->ul_vr_id;
6118 int mlxsw_sp_rif_dev_ifindex(const struct mlxsw_sp_rif *rif)
6120 return rif->dev->ifindex;
6123 const struct net_device *mlxsw_sp_rif_dev(const struct mlxsw_sp_rif *rif)
6128 static struct mlxsw_sp_rif *
6129 mlxsw_sp_rif_create(struct mlxsw_sp *mlxsw_sp,
6130 const struct mlxsw_sp_rif_params *params,
6131 struct netlink_ext_ack *extack)
6133 u32 tb_id = l3mdev_fib_table(params->dev);
6134 const struct mlxsw_sp_rif_ops *ops;
6135 struct mlxsw_sp_fid *fid = NULL;
6136 enum mlxsw_sp_rif_type type;
6137 struct mlxsw_sp_rif *rif;
6138 struct mlxsw_sp_vr *vr;
6142 type = mlxsw_sp_dev_rif_type(mlxsw_sp, params->dev);
6143 ops = mlxsw_sp->router->rif_ops_arr[type];
6145 vr = mlxsw_sp_vr_get(mlxsw_sp, tb_id ? : RT_TABLE_MAIN, extack);
6147 return ERR_CAST(vr);
6150 err = mlxsw_sp_rif_index_alloc(mlxsw_sp, &rif_index);
6152 NL_SET_ERR_MSG_MOD(extack, "Exceeded number of supported router interfaces");
6153 goto err_rif_index_alloc;
6156 rif = mlxsw_sp_rif_alloc(ops->rif_size, rif_index, vr->id, params->dev);
6161 rif->mlxsw_sp = mlxsw_sp;
6165 fid = ops->fid_get(rif);
6174 ops->setup(rif, params);
6176 err = ops->configure(rif);
6180 for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++) {
6181 err = mlxsw_sp_mr_rif_add(vr->mr_table[i], rif);
6183 goto err_mr_rif_add;
6186 mlxsw_sp_rif_counters_alloc(rif);
6187 mlxsw_sp->router->rifs[rif_index] = rif;
6192 for (i--; i >= 0; i--)
6193 mlxsw_sp_mr_rif_del(vr->mr_table[i], rif);
6194 ops->deconfigure(rif);
6197 mlxsw_sp_fid_put(fid);
6201 err_rif_index_alloc:
6203 mlxsw_sp_vr_put(mlxsw_sp, vr);
6204 return ERR_PTR(err);
6207 void mlxsw_sp_rif_destroy(struct mlxsw_sp_rif *rif)
6209 const struct mlxsw_sp_rif_ops *ops = rif->ops;
6210 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
6211 struct mlxsw_sp_fid *fid = rif->fid;
6212 struct mlxsw_sp_vr *vr;
6215 mlxsw_sp_router_rif_gone_sync(mlxsw_sp, rif);
6216 vr = &mlxsw_sp->router->vrs[rif->vr_id];
6218 mlxsw_sp->router->rifs[rif->rif_index] = NULL;
6219 mlxsw_sp_rif_counters_free(rif);
6220 for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++)
6221 mlxsw_sp_mr_rif_del(vr->mr_table[i], rif);
6222 ops->deconfigure(rif);
6224 /* Loopback RIFs are not associated with a FID. */
6225 mlxsw_sp_fid_put(fid);
6228 mlxsw_sp_vr_put(mlxsw_sp, vr);
6232 mlxsw_sp_rif_subport_params_init(struct mlxsw_sp_rif_params *params,
6233 struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
6235 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
6237 params->vid = mlxsw_sp_port_vlan->vid;
6238 params->lag = mlxsw_sp_port->lagged;
6240 params->lag_id = mlxsw_sp_port->lag_id;
6242 params->system_port = mlxsw_sp_port->local_port;
6246 mlxsw_sp_port_vlan_router_join(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan,
6247 struct net_device *l3_dev,
6248 struct netlink_ext_ack *extack)
6250 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
6251 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
6252 u16 vid = mlxsw_sp_port_vlan->vid;
6253 struct mlxsw_sp_rif *rif;
6254 struct mlxsw_sp_fid *fid;
6257 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
6259 struct mlxsw_sp_rif_params params = {
6263 mlxsw_sp_rif_subport_params_init(¶ms, mlxsw_sp_port_vlan);
6264 rif = mlxsw_sp_rif_create(mlxsw_sp, ¶ms, extack);
6266 return PTR_ERR(rif);
6269 /* FID was already created, just take a reference */
6270 fid = rif->ops->fid_get(rif);
6271 err = mlxsw_sp_fid_port_vid_map(fid, mlxsw_sp_port, vid);
6273 goto err_fid_port_vid_map;
6275 err = mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, false);
6277 goto err_port_vid_learning_set;
6279 err = mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid,
6280 BR_STATE_FORWARDING);
6282 goto err_port_vid_stp_set;
6284 mlxsw_sp_port_vlan->fid = fid;
6288 err_port_vid_stp_set:
6289 mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true);
6290 err_port_vid_learning_set:
6291 mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid);
6292 err_fid_port_vid_map:
6293 mlxsw_sp_fid_put(fid);
6298 mlxsw_sp_port_vlan_router_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan)
6300 struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port;
6301 struct mlxsw_sp_fid *fid = mlxsw_sp_port_vlan->fid;
6302 u16 vid = mlxsw_sp_port_vlan->vid;
6304 if (WARN_ON(mlxsw_sp_fid_type(fid) != MLXSW_SP_FID_TYPE_RFID))
6307 mlxsw_sp_port_vlan->fid = NULL;
6308 mlxsw_sp_port_vid_stp_set(mlxsw_sp_port, vid, BR_STATE_BLOCKING);
6309 mlxsw_sp_port_vid_learning_set(mlxsw_sp_port, vid, true);
6310 mlxsw_sp_fid_port_vid_unmap(fid, mlxsw_sp_port, vid);
6311 /* If router port holds the last reference on the rFID, then the
6312 * associated Sub-port RIF will be destroyed.
6314 mlxsw_sp_fid_put(fid);
6317 static int mlxsw_sp_inetaddr_port_vlan_event(struct net_device *l3_dev,
6318 struct net_device *port_dev,
6319 unsigned long event, u16 vid,
6320 struct netlink_ext_ack *extack)
6322 struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(port_dev);
6323 struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan;
6325 mlxsw_sp_port_vlan = mlxsw_sp_port_vlan_find_by_vid(mlxsw_sp_port, vid);
6326 if (WARN_ON(!mlxsw_sp_port_vlan))
6331 return mlxsw_sp_port_vlan_router_join(mlxsw_sp_port_vlan,
6334 mlxsw_sp_port_vlan_router_leave(mlxsw_sp_port_vlan);
6341 static int mlxsw_sp_inetaddr_port_event(struct net_device *port_dev,
6342 unsigned long event,
6343 struct netlink_ext_ack *extack)
6345 if (netif_is_bridge_port(port_dev) ||
6346 netif_is_lag_port(port_dev) ||
6347 netif_is_ovs_port(port_dev))
6350 return mlxsw_sp_inetaddr_port_vlan_event(port_dev, port_dev, event, 1,
6354 static int __mlxsw_sp_inetaddr_lag_event(struct net_device *l3_dev,
6355 struct net_device *lag_dev,
6356 unsigned long event, u16 vid,
6357 struct netlink_ext_ack *extack)
6359 struct net_device *port_dev;
6360 struct list_head *iter;
6363 netdev_for_each_lower_dev(lag_dev, port_dev, iter) {
6364 if (mlxsw_sp_port_dev_check(port_dev)) {
6365 err = mlxsw_sp_inetaddr_port_vlan_event(l3_dev,
6377 static int mlxsw_sp_inetaddr_lag_event(struct net_device *lag_dev,
6378 unsigned long event,
6379 struct netlink_ext_ack *extack)
6381 if (netif_is_bridge_port(lag_dev))
6384 return __mlxsw_sp_inetaddr_lag_event(lag_dev, lag_dev, event, 1,
6388 static int mlxsw_sp_inetaddr_bridge_event(struct net_device *l3_dev,
6389 unsigned long event,
6390 struct netlink_ext_ack *extack)
6392 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev);
6393 struct mlxsw_sp_rif_params params = {
6396 struct mlxsw_sp_rif *rif;
6400 rif = mlxsw_sp_rif_create(mlxsw_sp, ¶ms, extack);
6402 return PTR_ERR(rif);
6405 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
6406 mlxsw_sp_rif_destroy(rif);
6413 static int mlxsw_sp_inetaddr_vlan_event(struct net_device *vlan_dev,
6414 unsigned long event,
6415 struct netlink_ext_ack *extack)
6417 struct net_device *real_dev = vlan_dev_real_dev(vlan_dev);
6418 u16 vid = vlan_dev_vlan_id(vlan_dev);
6420 if (netif_is_bridge_port(vlan_dev))
6423 if (mlxsw_sp_port_dev_check(real_dev))
6424 return mlxsw_sp_inetaddr_port_vlan_event(vlan_dev, real_dev,
6425 event, vid, extack);
6426 else if (netif_is_lag_master(real_dev))
6427 return __mlxsw_sp_inetaddr_lag_event(vlan_dev, real_dev, event,
6429 else if (netif_is_bridge_master(real_dev) && br_vlan_enabled(real_dev))
6430 return mlxsw_sp_inetaddr_bridge_event(vlan_dev, event, extack);
6435 static int __mlxsw_sp_inetaddr_event(struct net_device *dev,
6436 unsigned long event,
6437 struct netlink_ext_ack *extack)
6439 if (mlxsw_sp_port_dev_check(dev))
6440 return mlxsw_sp_inetaddr_port_event(dev, event, extack);
6441 else if (netif_is_lag_master(dev))
6442 return mlxsw_sp_inetaddr_lag_event(dev, event, extack);
6443 else if (netif_is_bridge_master(dev))
6444 return mlxsw_sp_inetaddr_bridge_event(dev, event, extack);
6445 else if (is_vlan_dev(dev))
6446 return mlxsw_sp_inetaddr_vlan_event(dev, event, extack);
6451 int mlxsw_sp_inetaddr_event(struct notifier_block *unused,
6452 unsigned long event, void *ptr)
6454 struct in_ifaddr *ifa = (struct in_ifaddr *) ptr;
6455 struct net_device *dev = ifa->ifa_dev->dev;
6456 struct mlxsw_sp *mlxsw_sp;
6457 struct mlxsw_sp_rif *rif;
6460 /* NETDEV_UP event is handled by mlxsw_sp_inetaddr_valid_event */
6461 if (event == NETDEV_UP)
6464 mlxsw_sp = mlxsw_sp_lower_get(dev);
6468 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
6469 if (!mlxsw_sp_rif_should_config(rif, dev, event))
6472 err = __mlxsw_sp_inetaddr_event(dev, event, NULL);
6474 return notifier_from_errno(err);
6477 int mlxsw_sp_inetaddr_valid_event(struct notifier_block *unused,
6478 unsigned long event, void *ptr)
6480 struct in_validator_info *ivi = (struct in_validator_info *) ptr;
6481 struct net_device *dev = ivi->ivi_dev->dev;
6482 struct mlxsw_sp *mlxsw_sp;
6483 struct mlxsw_sp_rif *rif;
6486 mlxsw_sp = mlxsw_sp_lower_get(dev);
6490 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
6491 if (!mlxsw_sp_rif_should_config(rif, dev, event))
6494 err = __mlxsw_sp_inetaddr_event(dev, event, ivi->extack);
6496 return notifier_from_errno(err);
6499 struct mlxsw_sp_inet6addr_event_work {
6500 struct work_struct work;
6501 struct net_device *dev;
6502 unsigned long event;
6505 static void mlxsw_sp_inet6addr_event_work(struct work_struct *work)
6507 struct mlxsw_sp_inet6addr_event_work *inet6addr_work =
6508 container_of(work, struct mlxsw_sp_inet6addr_event_work, work);
6509 struct net_device *dev = inet6addr_work->dev;
6510 unsigned long event = inet6addr_work->event;
6511 struct mlxsw_sp *mlxsw_sp;
6512 struct mlxsw_sp_rif *rif;
6515 mlxsw_sp = mlxsw_sp_lower_get(dev);
6519 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
6520 if (!mlxsw_sp_rif_should_config(rif, dev, event))
6523 __mlxsw_sp_inetaddr_event(dev, event, NULL);
6527 kfree(inet6addr_work);
6530 /* Called with rcu_read_lock() */
6531 int mlxsw_sp_inet6addr_event(struct notifier_block *unused,
6532 unsigned long event, void *ptr)
6534 struct inet6_ifaddr *if6 = (struct inet6_ifaddr *) ptr;
6535 struct mlxsw_sp_inet6addr_event_work *inet6addr_work;
6536 struct net_device *dev = if6->idev->dev;
6538 /* NETDEV_UP event is handled by mlxsw_sp_inet6addr_valid_event */
6539 if (event == NETDEV_UP)
6542 if (!mlxsw_sp_port_dev_lower_find_rcu(dev))
6545 inet6addr_work = kzalloc(sizeof(*inet6addr_work), GFP_ATOMIC);
6546 if (!inet6addr_work)
6549 INIT_WORK(&inet6addr_work->work, mlxsw_sp_inet6addr_event_work);
6550 inet6addr_work->dev = dev;
6551 inet6addr_work->event = event;
6553 mlxsw_core_schedule_work(&inet6addr_work->work);
6558 int mlxsw_sp_inet6addr_valid_event(struct notifier_block *unused,
6559 unsigned long event, void *ptr)
6561 struct in6_validator_info *i6vi = (struct in6_validator_info *) ptr;
6562 struct net_device *dev = i6vi->i6vi_dev->dev;
6563 struct mlxsw_sp *mlxsw_sp;
6564 struct mlxsw_sp_rif *rif;
6567 mlxsw_sp = mlxsw_sp_lower_get(dev);
6571 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
6572 if (!mlxsw_sp_rif_should_config(rif, dev, event))
6575 err = __mlxsw_sp_inetaddr_event(dev, event, i6vi->extack);
6577 return notifier_from_errno(err);
6580 static int mlxsw_sp_rif_edit(struct mlxsw_sp *mlxsw_sp, u16 rif_index,
6581 const char *mac, int mtu)
6583 char ritr_pl[MLXSW_REG_RITR_LEN];
6586 mlxsw_reg_ritr_rif_pack(ritr_pl, rif_index);
6587 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
6591 mlxsw_reg_ritr_mtu_set(ritr_pl, mtu);
6592 mlxsw_reg_ritr_if_mac_memcpy_to(ritr_pl, mac);
6593 mlxsw_reg_ritr_op_set(ritr_pl, MLXSW_REG_RITR_RIF_CREATE);
6594 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
6597 int mlxsw_sp_netdevice_router_port_event(struct net_device *dev)
6599 struct mlxsw_sp *mlxsw_sp;
6600 struct mlxsw_sp_rif *rif;
6604 mlxsw_sp = mlxsw_sp_lower_get(dev);
6608 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
6611 fid_index = mlxsw_sp_fid_index(rif->fid);
6613 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, false);
6617 err = mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, dev->dev_addr,
6622 err = mlxsw_sp_rif_fdb_op(mlxsw_sp, dev->dev_addr, fid_index, true);
6624 goto err_rif_fdb_op;
6626 if (rif->mtu != dev->mtu) {
6627 struct mlxsw_sp_vr *vr;
6630 /* The RIF is relevant only to its mr_table instance, as unlike
6631 * unicast routing, in multicast routing a RIF cannot be shared
6632 * between several multicast routing tables.
6634 vr = &mlxsw_sp->router->vrs[rif->vr_id];
6635 for (i = 0; i < MLXSW_SP_L3_PROTO_MAX; i++)
6636 mlxsw_sp_mr_rif_mtu_update(vr->mr_table[i],
6640 ether_addr_copy(rif->addr, dev->dev_addr);
6641 rif->mtu = dev->mtu;
6643 netdev_dbg(dev, "Updated RIF=%d\n", rif->rif_index);
6648 mlxsw_sp_rif_edit(mlxsw_sp, rif->rif_index, rif->addr, rif->mtu);
6650 mlxsw_sp_rif_fdb_op(mlxsw_sp, rif->addr, fid_index, true);
6654 static int mlxsw_sp_port_vrf_join(struct mlxsw_sp *mlxsw_sp,
6655 struct net_device *l3_dev,
6656 struct netlink_ext_ack *extack)
6658 struct mlxsw_sp_rif *rif;
6660 /* If netdev is already associated with a RIF, then we need to
6661 * destroy it and create a new one with the new virtual router ID.
6663 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
6665 __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_DOWN, extack);
6667 return __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_UP, extack);
6670 static void mlxsw_sp_port_vrf_leave(struct mlxsw_sp *mlxsw_sp,
6671 struct net_device *l3_dev)
6673 struct mlxsw_sp_rif *rif;
6675 rif = mlxsw_sp_rif_find_by_dev(mlxsw_sp, l3_dev);
6678 __mlxsw_sp_inetaddr_event(l3_dev, NETDEV_DOWN, NULL);
6681 int mlxsw_sp_netdevice_vrf_event(struct net_device *l3_dev, unsigned long event,
6682 struct netdev_notifier_changeupper_info *info)
6684 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_lower_get(l3_dev);
6691 case NETDEV_PRECHANGEUPPER:
6693 case NETDEV_CHANGEUPPER:
6694 if (info->linking) {
6695 struct netlink_ext_ack *extack;
6697 extack = netdev_notifier_info_to_extack(&info->info);
6698 err = mlxsw_sp_port_vrf_join(mlxsw_sp, l3_dev, extack);
6700 mlxsw_sp_port_vrf_leave(mlxsw_sp, l3_dev);
6708 static struct mlxsw_sp_rif_subport *
6709 mlxsw_sp_rif_subport_rif(const struct mlxsw_sp_rif *rif)
6711 return container_of(rif, struct mlxsw_sp_rif_subport, common);
6714 static void mlxsw_sp_rif_subport_setup(struct mlxsw_sp_rif *rif,
6715 const struct mlxsw_sp_rif_params *params)
6717 struct mlxsw_sp_rif_subport *rif_subport;
6719 rif_subport = mlxsw_sp_rif_subport_rif(rif);
6720 rif_subport->vid = params->vid;
6721 rif_subport->lag = params->lag;
6723 rif_subport->lag_id = params->lag_id;
6725 rif_subport->system_port = params->system_port;
6728 static int mlxsw_sp_rif_subport_op(struct mlxsw_sp_rif *rif, bool enable)
6730 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
6731 struct mlxsw_sp_rif_subport *rif_subport;
6732 char ritr_pl[MLXSW_REG_RITR_LEN];
6734 rif_subport = mlxsw_sp_rif_subport_rif(rif);
6735 mlxsw_reg_ritr_pack(ritr_pl, enable, MLXSW_REG_RITR_SP_IF,
6736 rif->rif_index, rif->vr_id, rif->dev->mtu);
6737 mlxsw_reg_ritr_mac_pack(ritr_pl, rif->dev->dev_addr);
6738 mlxsw_reg_ritr_sp_if_pack(ritr_pl, rif_subport->lag,
6739 rif_subport->lag ? rif_subport->lag_id :
6740 rif_subport->system_port,
6743 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
6746 static int mlxsw_sp_rif_subport_configure(struct mlxsw_sp_rif *rif)
6750 err = mlxsw_sp_rif_subport_op(rif, true);
6754 err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
6755 mlxsw_sp_fid_index(rif->fid), true);
6757 goto err_rif_fdb_op;
6759 mlxsw_sp_fid_rif_set(rif->fid, rif);
6763 mlxsw_sp_rif_subport_op(rif, false);
6767 static void mlxsw_sp_rif_subport_deconfigure(struct mlxsw_sp_rif *rif)
6769 struct mlxsw_sp_fid *fid = rif->fid;
6771 mlxsw_sp_fid_rif_set(fid, NULL);
6772 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
6773 mlxsw_sp_fid_index(fid), false);
6774 mlxsw_sp_rif_subport_op(rif, false);
6777 static struct mlxsw_sp_fid *
6778 mlxsw_sp_rif_subport_fid_get(struct mlxsw_sp_rif *rif)
6780 return mlxsw_sp_fid_rfid_get(rif->mlxsw_sp, rif->rif_index);
6783 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_subport_ops = {
6784 .type = MLXSW_SP_RIF_TYPE_SUBPORT,
6785 .rif_size = sizeof(struct mlxsw_sp_rif_subport),
6786 .setup = mlxsw_sp_rif_subport_setup,
6787 .configure = mlxsw_sp_rif_subport_configure,
6788 .deconfigure = mlxsw_sp_rif_subport_deconfigure,
6789 .fid_get = mlxsw_sp_rif_subport_fid_get,
6792 static int mlxsw_sp_rif_vlan_fid_op(struct mlxsw_sp_rif *rif,
6793 enum mlxsw_reg_ritr_if_type type,
6794 u16 vid_fid, bool enable)
6796 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
6797 char ritr_pl[MLXSW_REG_RITR_LEN];
6799 mlxsw_reg_ritr_pack(ritr_pl, enable, type, rif->rif_index, rif->vr_id,
6801 mlxsw_reg_ritr_mac_pack(ritr_pl, rif->dev->dev_addr);
6802 mlxsw_reg_ritr_fid_set(ritr_pl, type, vid_fid);
6804 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl);
6807 u8 mlxsw_sp_router_port(const struct mlxsw_sp *mlxsw_sp)
6809 return mlxsw_core_max_ports(mlxsw_sp->core) + 1;
6812 static int mlxsw_sp_rif_vlan_configure(struct mlxsw_sp_rif *rif)
6814 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
6815 u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid);
6818 err = mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, true);
6822 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
6823 mlxsw_sp_router_port(mlxsw_sp), true);
6825 goto err_fid_mc_flood_set;
6827 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
6828 mlxsw_sp_router_port(mlxsw_sp), true);
6830 goto err_fid_bc_flood_set;
6832 err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
6833 mlxsw_sp_fid_index(rif->fid), true);
6835 goto err_rif_fdb_op;
6837 mlxsw_sp_fid_rif_set(rif->fid, rif);
6841 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
6842 mlxsw_sp_router_port(mlxsw_sp), false);
6843 err_fid_bc_flood_set:
6844 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
6845 mlxsw_sp_router_port(mlxsw_sp), false);
6846 err_fid_mc_flood_set:
6847 mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, false);
6851 static void mlxsw_sp_rif_vlan_deconfigure(struct mlxsw_sp_rif *rif)
6853 u16 vid = mlxsw_sp_fid_8021q_vid(rif->fid);
6854 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
6855 struct mlxsw_sp_fid *fid = rif->fid;
6857 mlxsw_sp_fid_rif_set(fid, NULL);
6858 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
6859 mlxsw_sp_fid_index(fid), false);
6860 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
6861 mlxsw_sp_router_port(mlxsw_sp), false);
6862 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
6863 mlxsw_sp_router_port(mlxsw_sp), false);
6864 mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_VLAN_IF, vid, false);
6867 static struct mlxsw_sp_fid *
6868 mlxsw_sp_rif_vlan_fid_get(struct mlxsw_sp_rif *rif)
6870 u16 vid = is_vlan_dev(rif->dev) ? vlan_dev_vlan_id(rif->dev) : 1;
6872 return mlxsw_sp_fid_8021q_get(rif->mlxsw_sp, vid);
6875 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_vlan_ops = {
6876 .type = MLXSW_SP_RIF_TYPE_VLAN,
6877 .rif_size = sizeof(struct mlxsw_sp_rif),
6878 .configure = mlxsw_sp_rif_vlan_configure,
6879 .deconfigure = mlxsw_sp_rif_vlan_deconfigure,
6880 .fid_get = mlxsw_sp_rif_vlan_fid_get,
6883 static int mlxsw_sp_rif_fid_configure(struct mlxsw_sp_rif *rif)
6885 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
6886 u16 fid_index = mlxsw_sp_fid_index(rif->fid);
6889 err = mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index,
6894 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
6895 mlxsw_sp_router_port(mlxsw_sp), true);
6897 goto err_fid_mc_flood_set;
6899 err = mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
6900 mlxsw_sp_router_port(mlxsw_sp), true);
6902 goto err_fid_bc_flood_set;
6904 err = mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
6905 mlxsw_sp_fid_index(rif->fid), true);
6907 goto err_rif_fdb_op;
6909 mlxsw_sp_fid_rif_set(rif->fid, rif);
6913 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
6914 mlxsw_sp_router_port(mlxsw_sp), false);
6915 err_fid_bc_flood_set:
6916 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
6917 mlxsw_sp_router_port(mlxsw_sp), false);
6918 err_fid_mc_flood_set:
6919 mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false);
6923 static void mlxsw_sp_rif_fid_deconfigure(struct mlxsw_sp_rif *rif)
6925 u16 fid_index = mlxsw_sp_fid_index(rif->fid);
6926 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
6927 struct mlxsw_sp_fid *fid = rif->fid;
6929 mlxsw_sp_fid_rif_set(fid, NULL);
6930 mlxsw_sp_rif_fdb_op(rif->mlxsw_sp, rif->dev->dev_addr,
6931 mlxsw_sp_fid_index(fid), false);
6932 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_BC,
6933 mlxsw_sp_router_port(mlxsw_sp), false);
6934 mlxsw_sp_fid_flood_set(rif->fid, MLXSW_SP_FLOOD_TYPE_MC,
6935 mlxsw_sp_router_port(mlxsw_sp), false);
6936 mlxsw_sp_rif_vlan_fid_op(rif, MLXSW_REG_RITR_FID_IF, fid_index, false);
6939 static struct mlxsw_sp_fid *
6940 mlxsw_sp_rif_fid_fid_get(struct mlxsw_sp_rif *rif)
6942 return mlxsw_sp_fid_8021d_get(rif->mlxsw_sp, rif->dev->ifindex);
6945 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_fid_ops = {
6946 .type = MLXSW_SP_RIF_TYPE_FID,
6947 .rif_size = sizeof(struct mlxsw_sp_rif),
6948 .configure = mlxsw_sp_rif_fid_configure,
6949 .deconfigure = mlxsw_sp_rif_fid_deconfigure,
6950 .fid_get = mlxsw_sp_rif_fid_fid_get,
6953 static struct mlxsw_sp_rif_ipip_lb *
6954 mlxsw_sp_rif_ipip_lb_rif(struct mlxsw_sp_rif *rif)
6956 return container_of(rif, struct mlxsw_sp_rif_ipip_lb, common);
6960 mlxsw_sp_rif_ipip_lb_setup(struct mlxsw_sp_rif *rif,
6961 const struct mlxsw_sp_rif_params *params)
6963 struct mlxsw_sp_rif_params_ipip_lb *params_lb;
6964 struct mlxsw_sp_rif_ipip_lb *rif_lb;
6966 params_lb = container_of(params, struct mlxsw_sp_rif_params_ipip_lb,
6968 rif_lb = mlxsw_sp_rif_ipip_lb_rif(rif);
6969 rif_lb->lb_config = params_lb->lb_config;
6973 mlxsw_sp_rif_ipip_lb_configure(struct mlxsw_sp_rif *rif)
6975 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
6976 u32 ul_tb_id = mlxsw_sp_ipip_dev_ul_tb_id(rif->dev);
6977 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
6978 struct mlxsw_sp_vr *ul_vr;
6981 ul_vr = mlxsw_sp_vr_get(mlxsw_sp, ul_tb_id, NULL);
6983 return PTR_ERR(ul_vr);
6985 err = mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr, true);
6987 goto err_loopback_op;
6989 lb_rif->ul_vr_id = ul_vr->id;
6994 mlxsw_sp_vr_put(mlxsw_sp, ul_vr);
6998 static void mlxsw_sp_rif_ipip_lb_deconfigure(struct mlxsw_sp_rif *rif)
7000 struct mlxsw_sp_rif_ipip_lb *lb_rif = mlxsw_sp_rif_ipip_lb_rif(rif);
7001 struct mlxsw_sp *mlxsw_sp = rif->mlxsw_sp;
7002 struct mlxsw_sp_vr *ul_vr;
7004 ul_vr = &mlxsw_sp->router->vrs[lb_rif->ul_vr_id];
7005 mlxsw_sp_rif_ipip_lb_op(lb_rif, ul_vr, false);
7008 mlxsw_sp_vr_put(mlxsw_sp, ul_vr);
7011 static const struct mlxsw_sp_rif_ops mlxsw_sp_rif_ipip_lb_ops = {
7012 .type = MLXSW_SP_RIF_TYPE_IPIP_LB,
7013 .rif_size = sizeof(struct mlxsw_sp_rif_ipip_lb),
7014 .setup = mlxsw_sp_rif_ipip_lb_setup,
7015 .configure = mlxsw_sp_rif_ipip_lb_configure,
7016 .deconfigure = mlxsw_sp_rif_ipip_lb_deconfigure,
7019 static const struct mlxsw_sp_rif_ops *mlxsw_sp_rif_ops_arr[] = {
7020 [MLXSW_SP_RIF_TYPE_SUBPORT] = &mlxsw_sp_rif_subport_ops,
7021 [MLXSW_SP_RIF_TYPE_VLAN] = &mlxsw_sp_rif_vlan_ops,
7022 [MLXSW_SP_RIF_TYPE_FID] = &mlxsw_sp_rif_fid_ops,
7023 [MLXSW_SP_RIF_TYPE_IPIP_LB] = &mlxsw_sp_rif_ipip_lb_ops,
7026 static int mlxsw_sp_rifs_init(struct mlxsw_sp *mlxsw_sp)
7028 u64 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
7030 mlxsw_sp->router->rifs = kcalloc(max_rifs,
7031 sizeof(struct mlxsw_sp_rif *),
7033 if (!mlxsw_sp->router->rifs)
7036 mlxsw_sp->router->rif_ops_arr = mlxsw_sp_rif_ops_arr;
7041 static void mlxsw_sp_rifs_fini(struct mlxsw_sp *mlxsw_sp)
7045 for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS); i++)
7046 WARN_ON_ONCE(mlxsw_sp->router->rifs[i]);
7048 kfree(mlxsw_sp->router->rifs);
7052 mlxsw_sp_ipip_config_tigcr(struct mlxsw_sp *mlxsw_sp)
7054 char tigcr_pl[MLXSW_REG_TIGCR_LEN];
7056 mlxsw_reg_tigcr_pack(tigcr_pl, true, 0);
7057 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(tigcr), tigcr_pl);
7060 static int mlxsw_sp_ipips_init(struct mlxsw_sp *mlxsw_sp)
7062 mlxsw_sp->router->ipip_ops_arr = mlxsw_sp_ipip_ops_arr;
7063 INIT_LIST_HEAD(&mlxsw_sp->router->ipip_list);
7064 return mlxsw_sp_ipip_config_tigcr(mlxsw_sp);
7067 static void mlxsw_sp_ipips_fini(struct mlxsw_sp *mlxsw_sp)
7069 WARN_ON(!list_empty(&mlxsw_sp->router->ipip_list));
7072 static void mlxsw_sp_router_fib_dump_flush(struct notifier_block *nb)
7074 struct mlxsw_sp_router *router;
7076 /* Flush pending FIB notifications and then flush the device's
7077 * table before requesting another dump. The FIB notification
7078 * block is unregistered, so no need to take RTNL.
7080 mlxsw_core_flush_owq();
7081 router = container_of(nb, struct mlxsw_sp_router, fib_nb);
7082 mlxsw_sp_router_fib_flush(router->mlxsw_sp);
7085 #ifdef CONFIG_IP_ROUTE_MULTIPATH
7086 static void mlxsw_sp_mp_hash_header_set(char *recr2_pl, int header)
7088 mlxsw_reg_recr2_outer_header_enables_set(recr2_pl, header, true);
7091 static void mlxsw_sp_mp_hash_field_set(char *recr2_pl, int field)
7093 mlxsw_reg_recr2_outer_header_fields_enable_set(recr2_pl, field, true);
7096 static void mlxsw_sp_mp4_hash_init(char *recr2_pl)
7098 bool only_l3 = !init_net.ipv4.sysctl_fib_multipath_hash_policy;
7100 mlxsw_sp_mp_hash_header_set(recr2_pl,
7101 MLXSW_REG_RECR2_IPV4_EN_NOT_TCP_NOT_UDP);
7102 mlxsw_sp_mp_hash_header_set(recr2_pl, MLXSW_REG_RECR2_IPV4_EN_TCP_UDP);
7103 mlxsw_reg_recr2_ipv4_sip_enable(recr2_pl);
7104 mlxsw_reg_recr2_ipv4_dip_enable(recr2_pl);
7107 mlxsw_sp_mp_hash_header_set(recr2_pl, MLXSW_REG_RECR2_TCP_UDP_EN_IPV4);
7108 mlxsw_sp_mp_hash_field_set(recr2_pl, MLXSW_REG_RECR2_IPV4_PROTOCOL);
7109 mlxsw_sp_mp_hash_field_set(recr2_pl, MLXSW_REG_RECR2_TCP_UDP_SPORT);
7110 mlxsw_sp_mp_hash_field_set(recr2_pl, MLXSW_REG_RECR2_TCP_UDP_DPORT);
7113 static void mlxsw_sp_mp6_hash_init(char *recr2_pl)
7115 bool only_l3 = !ip6_multipath_hash_policy(&init_net);
7117 mlxsw_sp_mp_hash_header_set(recr2_pl,
7118 MLXSW_REG_RECR2_IPV6_EN_NOT_TCP_NOT_UDP);
7119 mlxsw_sp_mp_hash_header_set(recr2_pl, MLXSW_REG_RECR2_IPV6_EN_TCP_UDP);
7120 mlxsw_reg_recr2_ipv6_sip_enable(recr2_pl);
7121 mlxsw_reg_recr2_ipv6_dip_enable(recr2_pl);
7122 mlxsw_sp_mp_hash_field_set(recr2_pl, MLXSW_REG_RECR2_IPV6_NEXT_HEADER);
7124 mlxsw_sp_mp_hash_field_set(recr2_pl,
7125 MLXSW_REG_RECR2_IPV6_FLOW_LABEL);
7127 mlxsw_sp_mp_hash_header_set(recr2_pl,
7128 MLXSW_REG_RECR2_TCP_UDP_EN_IPV6);
7129 mlxsw_sp_mp_hash_field_set(recr2_pl,
7130 MLXSW_REG_RECR2_TCP_UDP_SPORT);
7131 mlxsw_sp_mp_hash_field_set(recr2_pl,
7132 MLXSW_REG_RECR2_TCP_UDP_DPORT);
7136 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp)
7138 char recr2_pl[MLXSW_REG_RECR2_LEN];
7141 get_random_bytes(&seed, sizeof(seed));
7142 mlxsw_reg_recr2_pack(recr2_pl, seed);
7143 mlxsw_sp_mp4_hash_init(recr2_pl);
7144 mlxsw_sp_mp6_hash_init(recr2_pl);
7146 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(recr2), recr2_pl);
7149 static int mlxsw_sp_mp_hash_init(struct mlxsw_sp *mlxsw_sp)
7155 static int mlxsw_sp_dscp_init(struct mlxsw_sp *mlxsw_sp)
7157 char rdpm_pl[MLXSW_REG_RDPM_LEN];
7160 MLXSW_REG_ZERO(rdpm, rdpm_pl);
7162 /* HW is determining switch priority based on DSCP-bits, but the
7163 * kernel is still doing that based on the ToS. Since there's a
7164 * mismatch in bits we need to make sure to translate the right
7165 * value ToS would observe, skipping the 2 least-significant ECN bits.
7167 for (i = 0; i < MLXSW_REG_RDPM_DSCP_ENTRY_REC_MAX_COUNT; i++)
7168 mlxsw_reg_rdpm_pack(rdpm_pl, i, rt_tos2priority(i << 2));
7170 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rdpm), rdpm_pl);
7173 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
7175 char rgcr_pl[MLXSW_REG_RGCR_LEN];
7179 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_RIFS))
7181 max_rifs = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_RIFS);
7183 mlxsw_reg_rgcr_pack(rgcr_pl, true, true);
7184 mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, max_rifs);
7185 mlxsw_reg_rgcr_usp_set(rgcr_pl, true);
7186 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
7192 static void __mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
7194 char rgcr_pl[MLXSW_REG_RGCR_LEN];
7196 mlxsw_reg_rgcr_pack(rgcr_pl, false, false);
7197 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
7200 int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
7202 struct mlxsw_sp_router *router;
7205 router = kzalloc(sizeof(*mlxsw_sp->router), GFP_KERNEL);
7208 mlxsw_sp->router = router;
7209 router->mlxsw_sp = mlxsw_sp;
7211 INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_neighs_list);
7212 err = __mlxsw_sp_router_init(mlxsw_sp);
7214 goto err_router_init;
7216 err = mlxsw_sp_rifs_init(mlxsw_sp);
7220 err = mlxsw_sp_ipips_init(mlxsw_sp);
7222 goto err_ipips_init;
7224 err = rhashtable_init(&mlxsw_sp->router->nexthop_ht,
7225 &mlxsw_sp_nexthop_ht_params);
7227 goto err_nexthop_ht_init;
7229 err = rhashtable_init(&mlxsw_sp->router->nexthop_group_ht,
7230 &mlxsw_sp_nexthop_group_ht_params);
7232 goto err_nexthop_group_ht_init;
7234 INIT_LIST_HEAD(&mlxsw_sp->router->nexthop_list);
7235 err = mlxsw_sp_lpm_init(mlxsw_sp);
7239 err = mlxsw_sp_mr_init(mlxsw_sp, &mlxsw_sp_mr_tcam_ops);
7243 err = mlxsw_sp_vrs_init(mlxsw_sp);
7247 err = mlxsw_sp_neigh_init(mlxsw_sp);
7249 goto err_neigh_init;
7251 mlxsw_sp->router->netevent_nb.notifier_call =
7252 mlxsw_sp_router_netevent_event;
7253 err = register_netevent_notifier(&mlxsw_sp->router->netevent_nb);
7255 goto err_register_netevent_notifier;
7257 err = mlxsw_sp_mp_hash_init(mlxsw_sp);
7259 goto err_mp_hash_init;
7261 err = mlxsw_sp_dscp_init(mlxsw_sp);
7265 mlxsw_sp->router->fib_nb.notifier_call = mlxsw_sp_router_fib_event;
7266 err = register_fib_notifier(&mlxsw_sp->router->fib_nb,
7267 mlxsw_sp_router_fib_dump_flush);
7269 goto err_register_fib_notifier;
7273 err_register_fib_notifier:
7276 unregister_netevent_notifier(&mlxsw_sp->router->netevent_nb);
7277 err_register_netevent_notifier:
7278 mlxsw_sp_neigh_fini(mlxsw_sp);
7280 mlxsw_sp_vrs_fini(mlxsw_sp);
7282 mlxsw_sp_mr_fini(mlxsw_sp);
7284 mlxsw_sp_lpm_fini(mlxsw_sp);
7286 rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht);
7287 err_nexthop_group_ht_init:
7288 rhashtable_destroy(&mlxsw_sp->router->nexthop_ht);
7289 err_nexthop_ht_init:
7290 mlxsw_sp_ipips_fini(mlxsw_sp);
7292 mlxsw_sp_rifs_fini(mlxsw_sp);
7294 __mlxsw_sp_router_fini(mlxsw_sp);
7296 kfree(mlxsw_sp->router);
7300 void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
7302 unregister_fib_notifier(&mlxsw_sp->router->fib_nb);
7303 unregister_netevent_notifier(&mlxsw_sp->router->netevent_nb);
7304 mlxsw_sp_neigh_fini(mlxsw_sp);
7305 mlxsw_sp_vrs_fini(mlxsw_sp);
7306 mlxsw_sp_mr_fini(mlxsw_sp);
7307 mlxsw_sp_lpm_fini(mlxsw_sp);
7308 rhashtable_destroy(&mlxsw_sp->router->nexthop_group_ht);
7309 rhashtable_destroy(&mlxsw_sp->router->nexthop_ht);
7310 mlxsw_sp_ipips_fini(mlxsw_sp);
7311 mlxsw_sp_rifs_fini(mlxsw_sp);
7312 __mlxsw_sp_router_fini(mlxsw_sp);
7313 kfree(mlxsw_sp->router);