drivers/net/ethernet/mellanox/mlx5/core/steering/dr_matcher.c

   1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
   2 /* Copyright (c) 2019 Mellanox Technologies. */
   3
   4 #include "dr_types.h"
   5
   6 static bool dr_mask_is_smac_set(struct mlx5dr_match_spec *spec)
   7 {
   8         return (spec->smac_47_16 || spec->smac_15_0);
   9 }
  10
  11 static bool dr_mask_is_dmac_set(struct mlx5dr_match_spec *spec)
  12 {
  13         return (spec->dmac_47_16 || spec->dmac_15_0);
  14 }
  15
  16 static bool dr_mask_is_src_addr_set(struct mlx5dr_match_spec *spec)
  17 {
  18         return (spec->src_ip_127_96 || spec->src_ip_95_64 ||
  19                 spec->src_ip_63_32 || spec->src_ip_31_0);
  20 }
  21
  22 static bool dr_mask_is_dst_addr_set(struct mlx5dr_match_spec *spec)
  23 {
  24         return (spec->dst_ip_127_96 || spec->dst_ip_95_64 ||
  25                 spec->dst_ip_63_32 || spec->dst_ip_31_0);
  26 }
  27
  28 static bool dr_mask_is_l3_base_set(struct mlx5dr_match_spec *spec)
  29 {
  30         return (spec->ip_protocol || spec->frag || spec->tcp_flags ||
  31                 spec->ip_ecn || spec->ip_dscp);
  32 }
  33
  34 static bool dr_mask_is_tcp_udp_base_set(struct mlx5dr_match_spec *spec)
  35 {
  36         return (spec->tcp_sport || spec->tcp_dport ||
  37                 spec->udp_sport || spec->udp_dport);
  38 }
  39
  40 static bool dr_mask_is_ipv4_set(struct mlx5dr_match_spec *spec)
  41 {
  42         return (spec->dst_ip_31_0 || spec->src_ip_31_0);
  43 }
  44
  45 static bool dr_mask_is_ipv4_5_tuple_set(struct mlx5dr_match_spec *spec)
  46 {
  47         return (dr_mask_is_l3_base_set(spec) ||
  48                 dr_mask_is_tcp_udp_base_set(spec) ||
  49                 dr_mask_is_ipv4_set(spec));
  50 }
  51
  52 static bool dr_mask_is_eth_l2_tnl_set(struct mlx5dr_match_misc *misc)
  53 {
  54         return misc->vxlan_vni;
  55 }
  56
  57 static bool dr_mask_is_ttl_set(struct mlx5dr_match_spec *spec)
  58 {
  59         return spec->ttl_hoplimit;
  60 }
  61
  62 #define DR_MASK_IS_L2_DST(_spec, _misc, _inner_outer) (_spec.first_vid || \
  63         (_spec).first_cfi || (_spec).first_prio || (_spec).cvlan_tag || \
  64         (_spec).svlan_tag || (_spec).dmac_47_16 || (_spec).dmac_15_0 || \
  65         (_spec).ethertype || (_spec).ip_version || \
  66         (_misc)._inner_outer##_second_vid || \
  67         (_misc)._inner_outer##_second_cfi || \
  68         (_misc)._inner_outer##_second_prio || \
  69         (_misc)._inner_outer##_second_cvlan_tag || \
  70         (_misc)._inner_outer##_second_svlan_tag)
  71
  72 #define DR_MASK_IS_ETH_L4_SET(_spec, _misc, _inner_outer) ( \
  73         dr_mask_is_l3_base_set(&(_spec)) || \
  74         dr_mask_is_tcp_udp_base_set(&(_spec)) || \
  75         dr_mask_is_ttl_set(&(_spec)) || \
  76         (_misc)._inner_outer##_ipv6_flow_label)
  77
  78 #define DR_MASK_IS_ETH_L4_MISC_SET(_misc3, _inner_outer) ( \
  79         (_misc3)._inner_outer##_tcp_seq_num || \
  80         (_misc3)._inner_outer##_tcp_ack_num)
  81
  82 #define DR_MASK_IS_FIRST_MPLS_SET(_misc2, _inner_outer) ( \
  83         (_misc2)._inner_outer##_first_mpls_label || \
  84         (_misc2)._inner_outer##_first_mpls_exp || \
  85         (_misc2)._inner_outer##_first_mpls_s_bos || \
  86         (_misc2)._inner_outer##_first_mpls_ttl)
  87
  88 static bool dr_mask_is_gre_set(struct mlx5dr_match_misc *misc)
  89 {
  90         return (misc->gre_key_h || misc->gre_key_l ||
  91                 misc->gre_protocol || misc->gre_c_present ||
  92                 misc->gre_k_present || misc->gre_s_present);
  93 }
  94
  95 #define DR_MASK_IS_OUTER_MPLS_OVER_GRE_UDP_SET(_misc2, gre_udp) ( \
  96         (_misc2).outer_first_mpls_over_##gre_udp##_label || \
  97         (_misc2).outer_first_mpls_over_##gre_udp##_exp || \
  98         (_misc2).outer_first_mpls_over_##gre_udp##_s_bos || \
  99         (_misc2).outer_first_mpls_over_##gre_udp##_ttl)
 100
 101 #define DR_MASK_IS_FLEX_PARSER_0_SET(_misc2) ( \
 102         DR_MASK_IS_OUTER_MPLS_OVER_GRE_UDP_SET((_misc2), gre) || \
 103         DR_MASK_IS_OUTER_MPLS_OVER_GRE_UDP_SET((_misc2), udp))
 104
 105 static bool dr_mask_is_flex_parser_tnl_set(struct mlx5dr_match_misc3 *misc3)
 106 {
 107         return (misc3->outer_vxlan_gpe_vni ||
 108                 misc3->outer_vxlan_gpe_next_protocol ||
 109                 misc3->outer_vxlan_gpe_flags);
 110 }
 111
 112 static bool dr_mask_is_flex_parser_icmpv6_set(struct mlx5dr_match_misc3 *misc3)
 113 {
 114         return (misc3->icmpv6_type || misc3->icmpv6_code ||
 115                 misc3->icmpv6_header_data);
 116 }
 117
 118 static bool dr_mask_is_wqe_metadata_set(struct mlx5dr_match_misc2 *misc2)
 119 {
 120         return misc2->metadata_reg_a;
 121 }
 122
 123 static bool dr_mask_is_reg_c_0_3_set(struct mlx5dr_match_misc2 *misc2)
 124 {
 125         return (misc2->metadata_reg_c_0 || misc2->metadata_reg_c_1 ||
 126                 misc2->metadata_reg_c_2 || misc2->metadata_reg_c_3);
 127 }
 128
 129 static bool dr_mask_is_reg_c_4_7_set(struct mlx5dr_match_misc2 *misc2)
 130 {
 131         return (misc2->metadata_reg_c_4 || misc2->metadata_reg_c_5 ||
 132                 misc2->metadata_reg_c_6 || misc2->metadata_reg_c_7);
 133 }
 134
 135 static bool dr_mask_is_gvmi_or_qpn_set(struct mlx5dr_match_misc *misc)
 136 {
 137         return (misc->source_sqn || misc->source_port);
 138 }
 139
 140 static bool
 141 dr_matcher_supp_flex_parser_vxlan_gpe(struct mlx5dr_domain *dmn)
 142 {
 143         return dmn->info.caps.flex_protocols &
 144                MLX5_FLEX_PARSER_VXLAN_GPE_ENABLED;
 145 }
 146
 147 int mlx5dr_matcher_select_builders(struct mlx5dr_matcher *matcher,
 148                                    struct mlx5dr_matcher_rx_tx *nic_matcher,
 149                                    enum mlx5dr_ipv outer_ipv,
 150                                    enum mlx5dr_ipv inner_ipv)
 151 {
 152         nic_matcher->ste_builder =
 153                 nic_matcher->ste_builder_arr[outer_ipv][inner_ipv];
 154         nic_matcher->num_of_builders =
 155                 nic_matcher->num_of_builders_arr[outer_ipv][inner_ipv];
 156
 157         if (!nic_matcher->num_of_builders) {
 158                 mlx5dr_dbg(matcher->tbl->dmn,
 159                            "Rule not supported on this matcher due to IP related fields\n");
 160                 return -EINVAL;
 161         }
 162
 163         return 0;
 164 }
 165
 166 static int dr_matcher_set_ste_builders(struct mlx5dr_matcher *matcher,
 167                                        struct mlx5dr_matcher_rx_tx *nic_matcher,
 168                                        enum mlx5dr_ipv outer_ipv,
 169                                        enum mlx5dr_ipv inner_ipv)
 170 {
 171         struct mlx5dr_domain_rx_tx *nic_dmn = nic_matcher->nic_tbl->nic_dmn;
 172         struct mlx5dr_domain *dmn = matcher->tbl->dmn;
 173         struct mlx5dr_match_param mask = {};
 174         struct mlx5dr_match_misc3 *misc3;
 175         struct mlx5dr_ste_build *sb;
 176         bool inner, rx;
 177         int idx = 0;
 178         int ret, i;
 179
 180         sb = nic_matcher->ste_builder_arr[outer_ipv][inner_ipv];
 181         rx = nic_dmn->ste_type == MLX5DR_STE_TYPE_RX;
 182
 183         /* Create a temporary mask to track and clear used mask fields */
 184         if (matcher->match_criteria & DR_MATCHER_CRITERIA_OUTER)
 185                 mask.outer = matcher->mask.outer;
 186
 187         if (matcher->match_criteria & DR_MATCHER_CRITERIA_MISC)
 188                 mask.misc = matcher->mask.misc;
 189
 190         if (matcher->match_criteria & DR_MATCHER_CRITERIA_INNER)
 191                 mask.inner = matcher->mask.inner;
 192
 193         if (matcher->match_criteria & DR_MATCHER_CRITERIA_MISC2)
 194                 mask.misc2 = matcher->mask.misc2;
 195
 196         if (matcher->match_criteria & DR_MATCHER_CRITERIA_MISC3)
 197                 mask.misc3 = matcher->mask.misc3;
 198
 199         ret = mlx5dr_ste_build_pre_check(dmn, matcher->match_criteria,
 200                                          &matcher->mask, NULL);
 201         if (ret)
 202                 return ret;
 203
 204         /* Outer */
 205         if (matcher->match_criteria & (DR_MATCHER_CRITERIA_OUTER |
 206                                        DR_MATCHER_CRITERIA_MISC |
 207                                        DR_MATCHER_CRITERIA_MISC2 |
 208                                        DR_MATCHER_CRITERIA_MISC3)) {
 209                 inner = false;
 210
 211                 if (dr_mask_is_wqe_metadata_set(&mask.misc2))
 212                         mlx5dr_ste_build_general_purpose(&sb[idx++], &mask, inner, rx);
 213
 214                 if (dr_mask_is_reg_c_0_3_set(&mask.misc2))
 215                         mlx5dr_ste_build_register_0(&sb[idx++], &mask, inner, rx);
 216
 217                 if (dr_mask_is_reg_c_4_7_set(&mask.misc2))
 218                         mlx5dr_ste_build_register_1(&sb[idx++], &mask, inner, rx);
 219
 220                 if (dr_mask_is_gvmi_or_qpn_set(&mask.misc) &&
 221                     (dmn->type == MLX5DR_DOMAIN_TYPE_FDB ||
 222                      dmn->type == MLX5DR_DOMAIN_TYPE_NIC_RX)) {
 223                         ret = mlx5dr_ste_build_src_gvmi_qpn(&sb[idx++], &mask,
 224                                                             dmn, inner, rx);
 225                         if (ret)
 226                                 return ret;
 227                 }
 228
 229                 if (dr_mask_is_smac_set(&mask.outer) &&
 230                     dr_mask_is_dmac_set(&mask.outer)) {
 231                         ret = mlx5dr_ste_build_eth_l2_src_des(&sb[idx++], &mask,
 232                                                               inner, rx);
 233                         if (ret)
 234                                 return ret;
 235                 }
 236
 237                 if (dr_mask_is_smac_set(&mask.outer))
 238                         mlx5dr_ste_build_eth_l2_src(&sb[idx++], &mask, inner, rx);
 239
 240                 if (DR_MASK_IS_L2_DST(mask.outer, mask.misc, outer))
 241                         mlx5dr_ste_build_eth_l2_dst(&sb[idx++], &mask, inner, rx);
 242
 243                 if (outer_ipv == DR_RULE_IPV6) {
 244                         if (dr_mask_is_dst_addr_set(&mask.outer))
 245                                 mlx5dr_ste_build_eth_l3_ipv6_dst(&sb[idx++], &mask,
 246                                                                  inner, rx);
 247
 248                         if (dr_mask_is_src_addr_set(&mask.outer))
 249                                 mlx5dr_ste_build_eth_l3_ipv6_src(&sb[idx++], &mask,
 250                                                                  inner, rx);
 251
 252                         if (DR_MASK_IS_ETH_L4_SET(mask.outer, mask.misc, outer))
 253                                 mlx5dr_ste_build_ipv6_l3_l4(&sb[idx++], &mask,
 254                                                             inner, rx);
 255                 } else {
 256                         if (dr_mask_is_ipv4_5_tuple_set(&mask.outer))
 257                                 mlx5dr_ste_build_eth_l3_ipv4_5_tuple(&sb[idx++], &mask,
 258                                                                      inner, rx);
 259
 260                         if (dr_mask_is_ttl_set(&mask.outer))
 261                                 mlx5dr_ste_build_eth_l3_ipv4_misc(&sb[idx++], &mask,
 262                                                                   inner, rx);
 263                 }
 264
 265                 if (dr_mask_is_flex_parser_tnl_set(&mask.misc3) &&
 266                     dr_matcher_supp_flex_parser_vxlan_gpe(dmn))
 267                         mlx5dr_ste_build_flex_parser_tnl(&sb[idx++], &mask,
 268                                                          inner, rx);
 269
 270                 if (DR_MASK_IS_ETH_L4_MISC_SET(mask.misc3, outer))
 271                         mlx5dr_ste_build_eth_l4_misc(&sb[idx++], &mask, inner, rx);
 272
 273                 if (DR_MASK_IS_FIRST_MPLS_SET(mask.misc2, outer))
 274                         mlx5dr_ste_build_mpls(&sb[idx++], &mask, inner, rx);
 275
 276                 if (DR_MASK_IS_FLEX_PARSER_0_SET(mask.misc2))
 277                         mlx5dr_ste_build_flex_parser_0(&sb[idx++], &mask,
 278                                                        inner, rx);
 279
 280                 misc3 = &mask.misc3;
 281                 if ((DR_MASK_IS_FLEX_PARSER_ICMPV4_SET(misc3) &&
 282                      mlx5dr_matcher_supp_flex_parser_icmp_v4(&dmn->info.caps)) ||
 283                     (dr_mask_is_flex_parser_icmpv6_set(&mask.misc3) &&
 284                      mlx5dr_matcher_supp_flex_parser_icmp_v6(&dmn->info.caps))) {
 285                         ret = mlx5dr_ste_build_flex_parser_1(&sb[idx++],
 286                                                              &mask, &dmn->info.caps,
 287                                                              inner, rx);
 288                         if (ret)
 289                                 return ret;
 290                 }
 291                 if (dr_mask_is_gre_set(&mask.misc))
 292                         mlx5dr_ste_build_gre(&sb[idx++], &mask, inner, rx);
 293         }
 294
 295         /* Inner */
 296         if (matcher->match_criteria & (DR_MATCHER_CRITERIA_INNER |
 297                                        DR_MATCHER_CRITERIA_MISC |
 298                                        DR_MATCHER_CRITERIA_MISC2 |
 299                                        DR_MATCHER_CRITERIA_MISC3)) {
 300                 inner = true;
 301
 302                 if (dr_mask_is_eth_l2_tnl_set(&mask.misc))
 303                         mlx5dr_ste_build_eth_l2_tnl(&sb[idx++], &mask, inner, rx);
 304
 305                 if (dr_mask_is_smac_set(&mask.inner) &&
 306                     dr_mask_is_dmac_set(&mask.inner)) {
 307                         ret = mlx5dr_ste_build_eth_l2_src_des(&sb[idx++],
 308                                                               &mask, inner, rx);
 309                         if (ret)
 310                                 return ret;
 311                 }
 312
 313                 if (dr_mask_is_smac_set(&mask.inner))
 314                         mlx5dr_ste_build_eth_l2_src(&sb[idx++], &mask, inner, rx);
 315
 316                 if (DR_MASK_IS_L2_DST(mask.inner, mask.misc, inner))
 317                         mlx5dr_ste_build_eth_l2_dst(&sb[idx++], &mask, inner, rx);
 318
 319                 if (inner_ipv == DR_RULE_IPV6) {
 320                         if (dr_mask_is_dst_addr_set(&mask.inner))
 321                                 mlx5dr_ste_build_eth_l3_ipv6_dst(&sb[idx++], &mask,
 322                                                                  inner, rx);
 323
 324                         if (dr_mask_is_src_addr_set(&mask.inner))
 325                                 mlx5dr_ste_build_eth_l3_ipv6_src(&sb[idx++], &mask,
 326                                                                  inner, rx);
 327
 328                         if (DR_MASK_IS_ETH_L4_SET(mask.inner, mask.misc, inner))
 329                                 mlx5dr_ste_build_ipv6_l3_l4(&sb[idx++], &mask,
 330                                                             inner, rx);
 331                 } else {
 332                         if (dr_mask_is_ipv4_5_tuple_set(&mask.inner))
 333                                 mlx5dr_ste_build_eth_l3_ipv4_5_tuple(&sb[idx++], &mask,
 334                                                                      inner, rx);
 335
 336                         if (dr_mask_is_ttl_set(&mask.inner))
 337                                 mlx5dr_ste_build_eth_l3_ipv4_misc(&sb[idx++], &mask,
 338                                                                   inner, rx);
 339                 }
 340
 341                 if (DR_MASK_IS_ETH_L4_MISC_SET(mask.misc3, inner))
 342                         mlx5dr_ste_build_eth_l4_misc(&sb[idx++], &mask, inner, rx);
 343
 344                 if (DR_MASK_IS_FIRST_MPLS_SET(mask.misc2, inner))
 345                         mlx5dr_ste_build_mpls(&sb[idx++], &mask, inner, rx);
 346
 347                 if (DR_MASK_IS_FLEX_PARSER_0_SET(mask.misc2))
 348                         mlx5dr_ste_build_flex_parser_0(&sb[idx++], &mask, inner, rx);
 349         }
 350         /* Empty matcher, takes all */
 351         if (matcher->match_criteria == DR_MATCHER_CRITERIA_EMPTY)
 352                 mlx5dr_ste_build_empty_always_hit(&sb[idx++], rx);
 353
 354         if (idx == 0) {
 355                 mlx5dr_dbg(dmn, "Cannot generate any valid rules from mask\n");
 356                 return -EINVAL;
 357         }
 358
 359         /* Check that all mask fields were consumed */
 360         for (i = 0; i < sizeof(struct mlx5dr_match_param); i++) {
 361                 if (((u8 *)&mask)[i] != 0) {
 362                         mlx5dr_info(dmn, "Mask contains unsupported parameters\n");
 363                         return -EOPNOTSUPP;
 364                 }
 365         }
 366
 367         nic_matcher->ste_builder = sb;
 368         nic_matcher->num_of_builders_arr[outer_ipv][inner_ipv] = idx;
 369
 370         return 0;
 371 }
 372
 373 static int dr_matcher_connect(struct mlx5dr_domain *dmn,
 374                               struct mlx5dr_matcher_rx_tx *curr_nic_matcher,
 375                               struct mlx5dr_matcher_rx_tx *next_nic_matcher,
 376                               struct mlx5dr_matcher_rx_tx *prev_nic_matcher)
 377 {
 378         struct mlx5dr_table_rx_tx *nic_tbl = curr_nic_matcher->nic_tbl;
 379         struct mlx5dr_domain_rx_tx *nic_dmn = nic_tbl->nic_dmn;
 380         struct mlx5dr_htbl_connect_info info;
 381         struct mlx5dr_ste_htbl *prev_htbl;
 382         int ret;
 383
 384         /* Connect end anchor hash table to next_htbl or to the default address */
 385         if (next_nic_matcher) {
 386                 info.type = CONNECT_HIT;
 387                 info.hit_next_htbl = next_nic_matcher->s_htbl;
 388         } else {
 389                 info.type = CONNECT_MISS;
 390                 info.miss_icm_addr = nic_tbl->default_icm_addr;
 391         }
 392         ret = mlx5dr_ste_htbl_init_and_postsend(dmn, nic_dmn,
 393                                                 curr_nic_matcher->e_anchor,
 394                                                 &info, info.type == CONNECT_HIT);
 395         if (ret)
 396                 return ret;
 397
 398         /* Connect start hash table to end anchor */
 399         info.type = CONNECT_MISS;
 400         info.miss_icm_addr = curr_nic_matcher->e_anchor->chunk->icm_addr;
 401         ret = mlx5dr_ste_htbl_init_and_postsend(dmn, nic_dmn,
 402                                                 curr_nic_matcher->s_htbl,
 403                                                 &info, false);
 404         if (ret)
 405                 return ret;
 406
 407         /* Connect previous hash table to matcher start hash table */
 408         if (prev_nic_matcher)
 409                 prev_htbl = prev_nic_matcher->e_anchor;
 410         else
 411                 prev_htbl = nic_tbl->s_anchor;
 412
 413         info.type = CONNECT_HIT;
 414         info.hit_next_htbl = curr_nic_matcher->s_htbl;
 415         ret = mlx5dr_ste_htbl_init_and_postsend(dmn, nic_dmn, prev_htbl,
 416                                                 &info, true);
 417         if (ret)
 418                 return ret;
 419
 420         /* Update the pointing ste and next hash table */
 421         curr_nic_matcher->s_htbl->pointing_ste = prev_htbl->ste_arr;
 422         prev_htbl->ste_arr[0].next_htbl = curr_nic_matcher->s_htbl;
 423
 424         if (next_nic_matcher) {
 425                 next_nic_matcher->s_htbl->pointing_ste = curr_nic_matcher->e_anchor->ste_arr;
 426                 curr_nic_matcher->e_anchor->ste_arr[0].next_htbl = next_nic_matcher->s_htbl;
 427         }
 428
 429         return 0;
 430 }
 431
 432 static int dr_matcher_add_to_tbl(struct mlx5dr_matcher *matcher)
 433 {
 434         struct mlx5dr_matcher *next_matcher, *prev_matcher, *tmp_matcher;
 435         struct mlx5dr_table *tbl = matcher->tbl;
 436         struct mlx5dr_domain *dmn = tbl->dmn;
 437         bool first = true;
 438         int ret;
 439
 440         next_matcher = NULL;
 441         if (!list_empty(&tbl->matcher_list))
 442                 list_for_each_entry(tmp_matcher, &tbl->matcher_list, matcher_list) {
 443                         if (tmp_matcher->prio >= matcher->prio) {
 444                                 next_matcher = tmp_matcher;
 445                                 break;
 446                         }
 447                         first = false;
 448                 }
 449
 450         prev_matcher = NULL;
 451         if (next_matcher && !first)
 452                 prev_matcher = list_prev_entry(next_matcher, matcher_list);
 453         else if (!first)
 454                 prev_matcher = list_last_entry(&tbl->matcher_list,
 455                                                struct mlx5dr_matcher,
 456                                                matcher_list);
 457
 458         if (dmn->type == MLX5DR_DOMAIN_TYPE_FDB ||
 459             dmn->type == MLX5DR_DOMAIN_TYPE_NIC_RX) {
 460                 ret = dr_matcher_connect(dmn, &matcher->rx,
 461                                          next_matcher ? &next_matcher->rx : NULL,
 462                                          prev_matcher ? &prev_matcher->rx : NULL);
 463                 if (ret)
 464                         return ret;
 465         }
 466
 467         if (dmn->type == MLX5DR_DOMAIN_TYPE_FDB ||
 468             dmn->type == MLX5DR_DOMAIN_TYPE_NIC_TX) {
 469                 ret = dr_matcher_connect(dmn, &matcher->tx,
 470                                          next_matcher ? &next_matcher->tx : NULL,
 471                                          prev_matcher ? &prev_matcher->tx : NULL);
 472                 if (ret)
 473                         return ret;
 474         }
 475
 476         if (prev_matcher)
 477                 list_add(&matcher->matcher_list, &prev_matcher->matcher_list);
 478         else if (next_matcher)
 479                 list_add_tail(&matcher->matcher_list,
 480                               &next_matcher->matcher_list);
 481         else
 482                 list_add(&matcher->matcher_list, &tbl->matcher_list);
 483
 484         return 0;
 485 }
 486
 487 static void dr_matcher_uninit_nic(struct mlx5dr_matcher_rx_tx *nic_matcher)
 488 {
 489         mlx5dr_htbl_put(nic_matcher->s_htbl);
 490         mlx5dr_htbl_put(nic_matcher->e_anchor);
 491 }
 492
 493 static void dr_matcher_uninit_fdb(struct mlx5dr_matcher *matcher)
 494 {
 495         dr_matcher_uninit_nic(&matcher->rx);
 496         dr_matcher_uninit_nic(&matcher->tx);
 497 }
 498
 499 static void dr_matcher_uninit(struct mlx5dr_matcher *matcher)
 500 {
 501         struct mlx5dr_domain *dmn = matcher->tbl->dmn;
 502
 503         switch (dmn->type) {
 504         case MLX5DR_DOMAIN_TYPE_NIC_RX:
 505                 dr_matcher_uninit_nic(&matcher->rx);
 506                 break;
 507         case MLX5DR_DOMAIN_TYPE_NIC_TX:
 508                 dr_matcher_uninit_nic(&matcher->tx);
 509                 break;
 510         case MLX5DR_DOMAIN_TYPE_FDB:
 511                 dr_matcher_uninit_fdb(matcher);
 512                 break;
 513         default:
 514                 WARN_ON(true);
 515                 break;
 516         }
 517 }
 518
 519 static int dr_matcher_set_all_ste_builders(struct mlx5dr_matcher *matcher,
 520                                            struct mlx5dr_matcher_rx_tx *nic_matcher)
 521 {
 522         struct mlx5dr_domain *dmn = matcher->tbl->dmn;
 523
 524         dr_matcher_set_ste_builders(matcher, nic_matcher, DR_RULE_IPV4, DR_RULE_IPV4);
 525         dr_matcher_set_ste_builders(matcher, nic_matcher, DR_RULE_IPV4, DR_RULE_IPV6);
 526         dr_matcher_set_ste_builders(matcher, nic_matcher, DR_RULE_IPV6, DR_RULE_IPV4);
 527         dr_matcher_set_ste_builders(matcher, nic_matcher, DR_RULE_IPV6, DR_RULE_IPV6);
 528
 529         if (!nic_matcher->ste_builder) {
 530                 mlx5dr_dbg(dmn, "Cannot generate IPv4 or IPv6 rules with given mask\n");
 531                 return -EINVAL;
 532         }
 533
 534         return 0;
 535 }
 536
 537 static int dr_matcher_init_nic(struct mlx5dr_matcher *matcher,
 538                                struct mlx5dr_matcher_rx_tx *nic_matcher)
 539 {
 540         struct mlx5dr_domain *dmn = matcher->tbl->dmn;
 541         int ret;
 542
 543         ret = dr_matcher_set_all_ste_builders(matcher, nic_matcher);
 544         if (ret)
 545                 return ret;
 546
 547         nic_matcher->e_anchor = mlx5dr_ste_htbl_alloc(dmn->ste_icm_pool,
 548                                                       DR_CHUNK_SIZE_1,
 549                                                       MLX5DR_STE_LU_TYPE_DONT_CARE,
 550                                                       0);
 551         if (!nic_matcher->e_anchor)
 552                 return -ENOMEM;
 553
 554         nic_matcher->s_htbl = mlx5dr_ste_htbl_alloc(dmn->ste_icm_pool,
 555                                                     DR_CHUNK_SIZE_1,
 556                                                     nic_matcher->ste_builder[0].lu_type,
 557                                                     nic_matcher->ste_builder[0].byte_mask);
 558         if (!nic_matcher->s_htbl) {
 559                 ret = -ENOMEM;
 560                 goto free_e_htbl;
 561         }
 562
 563         /* make sure the tables exist while empty */
 564         mlx5dr_htbl_get(nic_matcher->s_htbl);
 565         mlx5dr_htbl_get(nic_matcher->e_anchor);
 566
 567         return 0;
 568
 569 free_e_htbl:
 570         mlx5dr_ste_htbl_free(nic_matcher->e_anchor);
 571         return ret;
 572 }
 573
 574 static int dr_matcher_init_fdb(struct mlx5dr_matcher *matcher)
 575 {
 576         int ret;
 577
 578         ret = dr_matcher_init_nic(matcher, &matcher->rx);
 579         if (ret)
 580                 return ret;
 581
 582         ret = dr_matcher_init_nic(matcher, &matcher->tx);
 583         if (ret)
 584                 goto uninit_nic_rx;
 585
 586         return 0;
 587
 588 uninit_nic_rx:
 589         dr_matcher_uninit_nic(&matcher->rx);
 590         return ret;
 591 }
 592
 593 static int dr_matcher_init(struct mlx5dr_matcher *matcher,
 594                            struct mlx5dr_match_parameters *mask)
 595 {
 596         struct mlx5dr_table *tbl = matcher->tbl;
 597         struct mlx5dr_domain *dmn = tbl->dmn;
 598         int ret;
 599
 600         if (matcher->match_criteria >= DR_MATCHER_CRITERIA_MAX) {
 601                 mlx5dr_info(dmn, "Invalid match criteria attribute\n");
 602                 return -EINVAL;
 603         }
 604
 605         if (mask) {
 606                 if (mask->match_sz > sizeof(struct mlx5dr_match_param)) {
 607                         mlx5dr_info(dmn, "Invalid match size attribute\n");
 608                         return -EINVAL;
 609                 }
 610                 mlx5dr_ste_copy_param(matcher->match_criteria,
 611                                       &matcher->mask, mask);
 612         }
 613
 614         switch (dmn->type) {
 615         case MLX5DR_DOMAIN_TYPE_NIC_RX:
 616                 matcher->rx.nic_tbl = &tbl->rx;
 617                 ret = dr_matcher_init_nic(matcher, &matcher->rx);
 618                 break;
 619         case MLX5DR_DOMAIN_TYPE_NIC_TX:
 620                 matcher->tx.nic_tbl = &tbl->tx;
 621                 ret = dr_matcher_init_nic(matcher, &matcher->tx);
 622                 break;
 623         case MLX5DR_DOMAIN_TYPE_FDB:
 624                 matcher->rx.nic_tbl = &tbl->rx;
 625                 matcher->tx.nic_tbl = &tbl->tx;
 626                 ret = dr_matcher_init_fdb(matcher);
 627                 break;
 628         default:
 629                 WARN_ON(true);
 630                 return -EINVAL;
 631         }
 632
 633         return ret;
 634 }
 635
 636 struct mlx5dr_matcher *
 637 mlx5dr_matcher_create(struct mlx5dr_table *tbl,
 638                       u16 priority,
 639                       u8 match_criteria_enable,
 640                       struct mlx5dr_match_parameters *mask)
 641 {
 642         struct mlx5dr_matcher *matcher;
 643         int ret;
 644
 645         refcount_inc(&tbl->refcount);
 646
 647         matcher = kzalloc(sizeof(*matcher), GFP_KERNEL);
 648         if (!matcher)
 649                 goto dec_ref;
 650
 651         matcher->tbl = tbl;
 652         matcher->prio = priority;
 653         matcher->match_criteria = match_criteria_enable;
 654         refcount_set(&matcher->refcount, 1);
 655         INIT_LIST_HEAD(&matcher->matcher_list);
 656
 657         mutex_lock(&tbl->dmn->mutex);
 658
 659         ret = dr_matcher_init(matcher, mask);
 660         if (ret)
 661                 goto free_matcher;
 662
 663         ret = dr_matcher_add_to_tbl(matcher);
 664         if (ret)
 665                 goto matcher_uninit;
 666
 667         mutex_unlock(&tbl->dmn->mutex);
 668
 669         return matcher;
 670
 671 matcher_uninit:
 672         dr_matcher_uninit(matcher);
 673 free_matcher:
 674         mutex_unlock(&tbl->dmn->mutex);
 675         kfree(matcher);
 676 dec_ref:
 677         refcount_dec(&tbl->refcount);
 678         return NULL;
 679 }
 680
 681 static int dr_matcher_disconnect(struct mlx5dr_domain *dmn,
 682                                  struct mlx5dr_table_rx_tx *nic_tbl,
 683                                  struct mlx5dr_matcher_rx_tx *next_nic_matcher,
 684                                  struct mlx5dr_matcher_rx_tx *prev_nic_matcher)
 685 {
 686         struct mlx5dr_domain_rx_tx *nic_dmn = nic_tbl->nic_dmn;
 687         struct mlx5dr_htbl_connect_info info;
 688         struct mlx5dr_ste_htbl *prev_anchor;
 689
 690         if (prev_nic_matcher)
 691                 prev_anchor = prev_nic_matcher->e_anchor;
 692         else
 693                 prev_anchor = nic_tbl->s_anchor;
 694
 695         /* Connect previous anchor hash table to next matcher or to the default address */
 696         if (next_nic_matcher) {
 697                 info.type = CONNECT_HIT;
 698                 info.hit_next_htbl = next_nic_matcher->s_htbl;
 699                 next_nic_matcher->s_htbl->pointing_ste = prev_anchor->ste_arr;
 700                 prev_anchor->ste_arr[0].next_htbl = next_nic_matcher->s_htbl;
 701         } else {
 702                 info.type = CONNECT_MISS;
 703                 info.miss_icm_addr = nic_tbl->default_icm_addr;
 704                 prev_anchor->ste_arr[0].next_htbl = NULL;
 705         }
 706
 707         return mlx5dr_ste_htbl_init_and_postsend(dmn, nic_dmn, prev_anchor,
 708                                                  &info, true);
 709 }
 710
 711 static int dr_matcher_remove_from_tbl(struct mlx5dr_matcher *matcher)
 712 {
 713         struct mlx5dr_matcher *prev_matcher, *next_matcher;
 714         struct mlx5dr_table *tbl = matcher->tbl;
 715         struct mlx5dr_domain *dmn = tbl->dmn;
 716         int ret = 0;
 717
 718         if (list_is_last(&matcher->matcher_list, &tbl->matcher_list))
 719                 next_matcher = NULL;
 720         else
 721                 next_matcher = list_next_entry(matcher, matcher_list);
 722
 723         if (matcher->matcher_list.prev == &tbl->matcher_list)
 724                 prev_matcher = NULL;
 725         else
 726                 prev_matcher = list_prev_entry(matcher, matcher_list);
 727
 728         if (dmn->type == MLX5DR_DOMAIN_TYPE_FDB ||
 729             dmn->type == MLX5DR_DOMAIN_TYPE_NIC_RX) {
 730                 ret = dr_matcher_disconnect(dmn, &tbl->rx,
 731                                             next_matcher ? &next_matcher->rx : NULL,
 732                                             prev_matcher ? &prev_matcher->rx : NULL);
 733                 if (ret)
 734                         return ret;
 735         }
 736
 737         if (dmn->type == MLX5DR_DOMAIN_TYPE_FDB ||
 738             dmn->type == MLX5DR_DOMAIN_TYPE_NIC_TX) {
 739                 ret = dr_matcher_disconnect(dmn, &tbl->tx,
 740                                             next_matcher ? &next_matcher->tx : NULL,
 741                                             prev_matcher ? &prev_matcher->tx : NULL);
 742                 if (ret)
 743                         return ret;
 744         }
 745
 746         list_del(&matcher->matcher_list);
 747
 748         return 0;
 749 }
 750
 751 int mlx5dr_matcher_destroy(struct mlx5dr_matcher *matcher)
 752 {
 753         struct mlx5dr_table *tbl = matcher->tbl;
 754
 755         if (refcount_read(&matcher->refcount) > 1)
 756                 return -EBUSY;
 757
 758         mutex_lock(&tbl->dmn->mutex);
 759
 760         dr_matcher_remove_from_tbl(matcher);
 761         dr_matcher_uninit(matcher);
 762         refcount_dec(&matcher->tbl->refcount);
 763
 764         mutex_unlock(&tbl->dmn->mutex);
 765         kfree(matcher);
 766
 767         return 0;
 768 }