net/sched/cls_flower.c

   1 /*
   2  * net/sched/cls_flower.c               Flower classifier
   3  *
   4  * Copyright (c) 2015 Jiri Pirko <jiri@resnulli.us>
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation; either version 2 of the License, or
   9  * (at your option) any later version.
  10  */
  11
  12 #include <linux/kernel.h>
  13 #include <linux/init.h>
  14 #include <linux/module.h>
  15 #include <linux/rhashtable.h>
  16 #include <linux/workqueue.h>
  17 #include <linux/refcount.h>
  18
  19 #include <linux/if_ether.h>
  20 #include <linux/in6.h>
  21 #include <linux/ip.h>
  22 #include <linux/mpls.h>
  23
  24 #include <net/sch_generic.h>
  25 #include <net/pkt_cls.h>
  26 #include <net/ip.h>
  27 #include <net/flow_dissector.h>
  28 #include <net/geneve.h>
  29
  30 #include <net/dst.h>
  31 #include <net/dst_metadata.h>
  32
  33 struct fl_flow_key {
  34         int     indev_ifindex;
  35         struct flow_dissector_key_control control;
  36         struct flow_dissector_key_control enc_control;
  37         struct flow_dissector_key_basic basic;
  38         struct flow_dissector_key_eth_addrs eth;
  39         struct flow_dissector_key_vlan vlan;
  40         struct flow_dissector_key_vlan cvlan;
  41         union {
  42                 struct flow_dissector_key_ipv4_addrs ipv4;
  43                 struct flow_dissector_key_ipv6_addrs ipv6;
  44         };
  45         struct flow_dissector_key_ports tp;
  46         struct flow_dissector_key_icmp icmp;
  47         struct flow_dissector_key_arp arp;
  48         struct flow_dissector_key_keyid enc_key_id;
  49         union {
  50                 struct flow_dissector_key_ipv4_addrs enc_ipv4;
  51                 struct flow_dissector_key_ipv6_addrs enc_ipv6;
  52         };
  53         struct flow_dissector_key_ports enc_tp;
  54         struct flow_dissector_key_mpls mpls;
  55         struct flow_dissector_key_tcp tcp;
  56         struct flow_dissector_key_ip ip;
  57         struct flow_dissector_key_ip enc_ip;
  58         struct flow_dissector_key_enc_opts enc_opts;
  59         struct flow_dissector_key_ports tp_min;
  60         struct flow_dissector_key_ports tp_max;
  61 } __aligned(BITS_PER_LONG / 8); /* Ensure that we can do comparisons as longs. */
  62
  63 struct fl_flow_mask_range {
  64         unsigned short int start;
  65         unsigned short int end;
  66 };
  67
  68 struct fl_flow_mask {
  69         struct fl_flow_key key;
  70         struct fl_flow_mask_range range;
  71         u32 flags;
  72         struct rhash_head ht_node;
  73         struct rhashtable ht;
  74         struct rhashtable_params filter_ht_params;
  75         struct flow_dissector dissector;
  76         struct list_head filters;
  77         struct rcu_work rwork;
  78         struct list_head list;
  79 };
  80
  81 struct fl_flow_tmplt {
  82         struct fl_flow_key dummy_key;
  83         struct fl_flow_key mask;
  84         struct flow_dissector dissector;
  85         struct tcf_chain *chain;
  86 };
  87
  88 struct cls_fl_head {
  89         struct rhashtable ht;
  90         struct list_head masks;
  91         struct rcu_work rwork;
  92         struct idr handle_idr;
  93 };
  94
  95 struct cls_fl_filter {
  96         struct fl_flow_mask *mask;
  97         struct rhash_head ht_node;
  98         struct fl_flow_key mkey;
  99         struct tcf_exts exts;
 100         struct tcf_result res;
 101         struct fl_flow_key key;
 102         struct list_head list;
 103         u32 handle;
 104         u32 flags;
 105         u32 in_hw_count;
 106         struct rcu_work rwork;
 107         struct net_device *hw_dev;
 108         /* Flower classifier is unlocked, which means that its reference counter
 109          * can be changed concurrently without any kind of external
 110          * synchronization. Use atomic reference counter to be concurrency-safe.
 111          */
 112         refcount_t refcnt;
 113 };
 114
 115 static const struct rhashtable_params mask_ht_params = {
 116         .key_offset = offsetof(struct fl_flow_mask, key),
 117         .key_len = sizeof(struct fl_flow_key),
 118         .head_offset = offsetof(struct fl_flow_mask, ht_node),
 119         .automatic_shrinking = true,
 120 };
 121
 122 static unsigned short int fl_mask_range(const struct fl_flow_mask *mask)
 123 {
 124         return mask->range.end - mask->range.start;
 125 }
 126
 127 static void fl_mask_update_range(struct fl_flow_mask *mask)
 128 {
 129         const u8 *bytes = (const u8 *) &mask->key;
 130         size_t size = sizeof(mask->key);
 131         size_t i, first = 0, last;
 132
 133         for (i = 0; i < size; i++) {
 134                 if (bytes[i]) {
 135                         first = i;
 136                         break;
 137                 }
 138         }
 139         last = first;
 140         for (i = size - 1; i != first; i--) {
 141                 if (bytes[i]) {
 142                         last = i;
 143                         break;
 144                 }
 145         }
 146         mask->range.start = rounddown(first, sizeof(long));
 147         mask->range.end = roundup(last + 1, sizeof(long));
 148 }
 149
 150 static void *fl_key_get_start(struct fl_flow_key *key,
 151                               const struct fl_flow_mask *mask)
 152 {
 153         return (u8 *) key + mask->range.start;
 154 }
 155
 156 static void fl_set_masked_key(struct fl_flow_key *mkey, struct fl_flow_key *key,
 157                               struct fl_flow_mask *mask)
 158 {
 159         const long *lkey = fl_key_get_start(key, mask);
 160         const long *lmask = fl_key_get_start(&mask->key, mask);
 161         long *lmkey = fl_key_get_start(mkey, mask);
 162         int i;
 163
 164         for (i = 0; i < fl_mask_range(mask); i += sizeof(long))
 165                 *lmkey++ = *lkey++ & *lmask++;
 166 }
 167
 168 static bool fl_mask_fits_tmplt(struct fl_flow_tmplt *tmplt,
 169                                struct fl_flow_mask *mask)
 170 {
 171         const long *lmask = fl_key_get_start(&mask->key, mask);
 172         const long *ltmplt;
 173         int i;
 174
 175         if (!tmplt)
 176                 return true;
 177         ltmplt = fl_key_get_start(&tmplt->mask, mask);
 178         for (i = 0; i < fl_mask_range(mask); i += sizeof(long)) {
 179                 if (~*ltmplt++ & *lmask++)
 180                         return false;
 181         }
 182         return true;
 183 }
 184
 185 static void fl_clear_masked_range(struct fl_flow_key *key,
 186                                   struct fl_flow_mask *mask)
 187 {
 188         memset(fl_key_get_start(key, mask), 0, fl_mask_range(mask));
 189 }
 190
 191 static bool fl_range_port_dst_cmp(struct cls_fl_filter *filter,
 192                                   struct fl_flow_key *key,
 193                                   struct fl_flow_key *mkey)
 194 {
 195         __be16 min_mask, max_mask, min_val, max_val;
 196
 197         min_mask = htons(filter->mask->key.tp_min.dst);
 198         max_mask = htons(filter->mask->key.tp_max.dst);
 199         min_val = htons(filter->key.tp_min.dst);
 200         max_val = htons(filter->key.tp_max.dst);
 201
 202         if (min_mask && max_mask) {
 203                 if (htons(key->tp.dst) < min_val ||
 204                     htons(key->tp.dst) > max_val)
 205                         return false;
 206
 207                 /* skb does not have min and max values */
 208                 mkey->tp_min.dst = filter->mkey.tp_min.dst;
 209                 mkey->tp_max.dst = filter->mkey.tp_max.dst;
 210         }
 211         return true;
 212 }
 213
 214 static bool fl_range_port_src_cmp(struct cls_fl_filter *filter,
 215                                   struct fl_flow_key *key,
 216                                   struct fl_flow_key *mkey)
 217 {
 218         __be16 min_mask, max_mask, min_val, max_val;
 219
 220         min_mask = htons(filter->mask->key.tp_min.src);
 221         max_mask = htons(filter->mask->key.tp_max.src);
 222         min_val = htons(filter->key.tp_min.src);
 223         max_val = htons(filter->key.tp_max.src);
 224
 225         if (min_mask && max_mask) {
 226                 if (htons(key->tp.src) < min_val ||
 227                     htons(key->tp.src) > max_val)
 228                         return false;
 229
 230                 /* skb does not have min and max values */
 231                 mkey->tp_min.src = filter->mkey.tp_min.src;
 232                 mkey->tp_max.src = filter->mkey.tp_max.src;
 233         }
 234         return true;
 235 }
 236
 237 static struct cls_fl_filter *__fl_lookup(struct fl_flow_mask *mask,
 238                                          struct fl_flow_key *mkey)
 239 {
 240         return rhashtable_lookup_fast(&mask->ht, fl_key_get_start(mkey, mask),
 241                                       mask->filter_ht_params);
 242 }
 243
 244 static struct cls_fl_filter *fl_lookup_range(struct fl_flow_mask *mask,
 245                                              struct fl_flow_key *mkey,
 246                                              struct fl_flow_key *key)
 247 {
 248         struct cls_fl_filter *filter, *f;
 249
 250         list_for_each_entry_rcu(filter, &mask->filters, list) {
 251                 if (!fl_range_port_dst_cmp(filter, key, mkey))
 252                         continue;
 253
 254                 if (!fl_range_port_src_cmp(filter, key, mkey))
 255                         continue;
 256
 257                 f = __fl_lookup(mask, mkey);
 258                 if (f)
 259                         return f;
 260         }
 261         return NULL;
 262 }
 263
 264 static struct cls_fl_filter *fl_lookup(struct fl_flow_mask *mask,
 265                                        struct fl_flow_key *mkey,
 266                                        struct fl_flow_key *key)
 267 {
 268         if ((mask->flags & TCA_FLOWER_MASK_FLAGS_RANGE))
 269                 return fl_lookup_range(mask, mkey, key);
 270
 271         return __fl_lookup(mask, mkey);
 272 }
 273
 274 static int fl_classify(struct sk_buff *skb, const struct tcf_proto *tp,
 275                        struct tcf_result *res)
 276 {
 277         struct cls_fl_head *head = rcu_dereference_bh(tp->root);
 278         struct cls_fl_filter *f;
 279         struct fl_flow_mask *mask;
 280         struct fl_flow_key skb_key;
 281         struct fl_flow_key skb_mkey;
 282
 283         list_for_each_entry_rcu(mask, &head->masks, list) {
 284                 fl_clear_masked_range(&skb_key, mask);
 285
 286                 skb_key.indev_ifindex = skb->skb_iif;
 287                 /* skb_flow_dissect() does not set n_proto in case an unknown
 288                  * protocol, so do it rather here.
 289                  */
 290                 skb_key.basic.n_proto = skb->protocol;
 291                 skb_flow_dissect_tunnel_info(skb, &mask->dissector, &skb_key);
 292                 skb_flow_dissect(skb, &mask->dissector, &skb_key, 0);
 293
 294                 fl_set_masked_key(&skb_mkey, &skb_key, mask);
 295
 296                 f = fl_lookup(mask, &skb_mkey, &skb_key);
 297                 if (f && !tc_skip_sw(f->flags)) {
 298                         *res = f->res;
 299                         return tcf_exts_exec(skb, &f->exts, res);
 300                 }
 301         }
 302         return -1;
 303 }
 304
 305 static int fl_init(struct tcf_proto *tp)
 306 {
 307         struct cls_fl_head *head;
 308
 309         head = kzalloc(sizeof(*head), GFP_KERNEL);
 310         if (!head)
 311                 return -ENOBUFS;
 312
 313         INIT_LIST_HEAD_RCU(&head->masks);
 314         rcu_assign_pointer(tp->root, head);
 315         idr_init(&head->handle_idr);
 316
 317         return rhashtable_init(&head->ht, &mask_ht_params);
 318 }
 319
 320 static void fl_mask_free(struct fl_flow_mask *mask)
 321 {
 322         rhashtable_destroy(&mask->ht);
 323         kfree(mask);
 324 }
 325
 326 static void fl_mask_free_work(struct work_struct *work)
 327 {
 328         struct fl_flow_mask *mask = container_of(to_rcu_work(work),
 329                                                  struct fl_flow_mask, rwork);
 330
 331         fl_mask_free(mask);
 332 }
 333
 334 static bool fl_mask_put(struct cls_fl_head *head, struct fl_flow_mask *mask,
 335                         bool async)
 336 {
 337         if (!list_empty(&mask->filters))
 338                 return false;
 339
 340         rhashtable_remove_fast(&head->ht, &mask->ht_node, mask_ht_params);
 341         list_del_rcu(&mask->list);
 342         if (async)
 343                 tcf_queue_work(&mask->rwork, fl_mask_free_work);
 344         else
 345                 fl_mask_free(mask);
 346
 347         return true;
 348 }
 349
 350 static void __fl_destroy_filter(struct cls_fl_filter *f)
 351 {
 352         tcf_exts_destroy(&f->exts);
 353         tcf_exts_put_net(&f->exts);
 354         kfree(f);
 355 }
 356
 357 static void fl_destroy_filter_work(struct work_struct *work)
 358 {
 359         struct cls_fl_filter *f = container_of(to_rcu_work(work),
 360                                         struct cls_fl_filter, rwork);
 361
 362         rtnl_lock();
 363         __fl_destroy_filter(f);
 364         rtnl_unlock();
 365 }
 366
 367 static void fl_hw_destroy_filter(struct tcf_proto *tp, struct cls_fl_filter *f,
 368                                  struct netlink_ext_ack *extack)
 369 {
 370         struct tc_cls_flower_offload cls_flower = {};
 371         struct tcf_block *block = tp->chain->block;
 372
 373         tc_cls_common_offload_init(&cls_flower.common, tp, f->flags, extack);
 374         cls_flower.command = TC_CLSFLOWER_DESTROY;
 375         cls_flower.cookie = (unsigned long) f;
 376
 377         tc_setup_cb_call(block, TC_SETUP_CLSFLOWER, &cls_flower, false);
 378         tcf_block_offload_dec(block, &f->flags);
 379 }
 380
 381 static int fl_hw_replace_filter(struct tcf_proto *tp,
 382                                 struct cls_fl_filter *f,
 383                                 struct netlink_ext_ack *extack)
 384 {
 385         struct tc_cls_flower_offload cls_flower = {};
 386         struct tcf_block *block = tp->chain->block;
 387         bool skip_sw = tc_skip_sw(f->flags);
 388         int err;
 389
 390         cls_flower.rule = flow_rule_alloc(tcf_exts_num_actions(&f->exts));
 391         if (!cls_flower.rule)
 392                 return -ENOMEM;
 393
 394         tc_cls_common_offload_init(&cls_flower.common, tp, f->flags, extack);
 395         cls_flower.command = TC_CLSFLOWER_REPLACE;
 396         cls_flower.cookie = (unsigned long) f;
 397         cls_flower.rule->match.dissector = &f->mask->dissector;
 398         cls_flower.rule->match.mask = &f->mask->key;
 399         cls_flower.rule->match.key = &f->mkey;
 400         cls_flower.classid = f->res.classid;
 401
 402         err = tc_setup_flow_action(&cls_flower.rule->action, &f->exts);
 403         if (err) {
 404                 kfree(cls_flower.rule);
 405                 if (skip_sw) {
 406                         NL_SET_ERR_MSG_MOD(extack, "Failed to setup flow action");
 407                         return err;
 408                 }
 409                 return 0;
 410         }
 411
 412         err = tc_setup_cb_call(block, TC_SETUP_CLSFLOWER, &cls_flower, skip_sw);
 413         kfree(cls_flower.rule);
 414
 415         if (err < 0) {
 416                 fl_hw_destroy_filter(tp, f, NULL);
 417                 return err;
 418         } else if (err > 0) {
 419                 f->in_hw_count = err;
 420                 tcf_block_offload_inc(block, &f->flags);
 421         }
 422
 423         if (skip_sw && !(f->flags & TCA_CLS_FLAGS_IN_HW))
 424                 return -EINVAL;
 425
 426         return 0;
 427 }
 428
 429 static void fl_hw_update_stats(struct tcf_proto *tp, struct cls_fl_filter *f)
 430 {
 431         struct tc_cls_flower_offload cls_flower = {};
 432         struct tcf_block *block = tp->chain->block;
 433
 434         tc_cls_common_offload_init(&cls_flower.common, tp, f->flags, NULL);
 435         cls_flower.command = TC_CLSFLOWER_STATS;
 436         cls_flower.cookie = (unsigned long) f;
 437         cls_flower.classid = f->res.classid;
 438
 439         tc_setup_cb_call(block, TC_SETUP_CLSFLOWER, &cls_flower, false);
 440
 441         tcf_exts_stats_update(&f->exts, cls_flower.stats.bytes,
 442                               cls_flower.stats.pkts,
 443                               cls_flower.stats.lastused);
 444 }
 445
 446 static struct cls_fl_head *fl_head_dereference(struct tcf_proto *tp)
 447 {
 448         /* Flower classifier only changes root pointer during init and destroy.
 449          * Users must obtain reference to tcf_proto instance before calling its
 450          * API, so tp->root pointer is protected from concurrent call to
 451          * fl_destroy() by reference counting.
 452          */
 453         return rcu_dereference_raw(tp->root);
 454 }
 455
 456 static void __fl_put(struct cls_fl_filter *f)
 457 {
 458         if (!refcount_dec_and_test(&f->refcnt))
 459                 return;
 460
 461         if (tcf_exts_get_net(&f->exts))
 462                 tcf_queue_work(&f->rwork, fl_destroy_filter_work);
 463         else
 464                 __fl_destroy_filter(f);
 465 }
 466
 467 static struct cls_fl_filter *__fl_get(struct cls_fl_head *head, u32 handle)
 468 {
 469         struct cls_fl_filter *f;
 470
 471         rcu_read_lock();
 472         f = idr_find(&head->handle_idr, handle);
 473         if (f && !refcount_inc_not_zero(&f->refcnt))
 474                 f = NULL;
 475         rcu_read_unlock();
 476
 477         return f;
 478 }
 479
 480 static struct cls_fl_filter *fl_get_next_filter(struct tcf_proto *tp,
 481                                                 unsigned long *handle)
 482 {
 483         struct cls_fl_head *head = fl_head_dereference(tp);
 484         struct cls_fl_filter *f;
 485
 486         rcu_read_lock();
 487         while ((f = idr_get_next_ul(&head->handle_idr, handle))) {
 488                 /* don't return filters that are being deleted */
 489                 if (refcount_inc_not_zero(&f->refcnt))
 490                         break;
 491                 ++(*handle);
 492         }
 493         rcu_read_unlock();
 494
 495         return f;
 496 }
 497
 498 static bool __fl_delete(struct tcf_proto *tp, struct cls_fl_filter *f,
 499                         struct netlink_ext_ack *extack)
 500 {
 501         struct cls_fl_head *head = fl_head_dereference(tp);
 502         bool async = tcf_exts_get_net(&f->exts);
 503         bool last;
 504
 505         idr_remove(&head->handle_idr, f->handle);
 506         list_del_rcu(&f->list);
 507         last = fl_mask_put(head, f->mask, async);
 508         if (!tc_skip_hw(f->flags))
 509                 fl_hw_destroy_filter(tp, f, extack);
 510         tcf_unbind_filter(tp, &f->res);
 511         __fl_put(f);
 512
 513         return last;
 514 }
 515
 516 static void fl_destroy_sleepable(struct work_struct *work)
 517 {
 518         struct cls_fl_head *head = container_of(to_rcu_work(work),
 519                                                 struct cls_fl_head,
 520                                                 rwork);
 521
 522         rhashtable_destroy(&head->ht);
 523         kfree(head);
 524         module_put(THIS_MODULE);
 525 }
 526
 527 static void fl_destroy(struct tcf_proto *tp, bool rtnl_held,
 528                        struct netlink_ext_ack *extack)
 529 {
 530         struct cls_fl_head *head = fl_head_dereference(tp);
 531         struct fl_flow_mask *mask, *next_mask;
 532         struct cls_fl_filter *f, *next;
 533
 534         list_for_each_entry_safe(mask, next_mask, &head->masks, list) {
 535                 list_for_each_entry_safe(f, next, &mask->filters, list) {
 536                         if (__fl_delete(tp, f, extack))
 537                                 break;
 538                 }
 539         }
 540         idr_destroy(&head->handle_idr);
 541
 542         __module_get(THIS_MODULE);
 543         tcf_queue_work(&head->rwork, fl_destroy_sleepable);
 544 }
 545
 546 static void fl_put(struct tcf_proto *tp, void *arg)
 547 {
 548         struct cls_fl_filter *f = arg;
 549
 550         __fl_put(f);
 551 }
 552
 553 static void *fl_get(struct tcf_proto *tp, u32 handle)
 554 {
 555         struct cls_fl_head *head = fl_head_dereference(tp);
 556
 557         return __fl_get(head, handle);
 558 }
 559
 560 static const struct nla_policy fl_policy[TCA_FLOWER_MAX + 1] = {
 561         [TCA_FLOWER_UNSPEC]             = { .type = NLA_UNSPEC },
 562         [TCA_FLOWER_CLASSID]            = { .type = NLA_U32 },
 563         [TCA_FLOWER_INDEV]              = { .type = NLA_STRING,
 564                                             .len = IFNAMSIZ },
 565         [TCA_FLOWER_KEY_ETH_DST]        = { .len = ETH_ALEN },
 566         [TCA_FLOWER_KEY_ETH_DST_MASK]   = { .len = ETH_ALEN },
 567         [TCA_FLOWER_KEY_ETH_SRC]        = { .len = ETH_ALEN },
 568         [TCA_FLOWER_KEY_ETH_SRC_MASK]   = { .len = ETH_ALEN },
 569         [TCA_FLOWER_KEY_ETH_TYPE]       = { .type = NLA_U16 },
 570         [TCA_FLOWER_KEY_IP_PROTO]       = { .type = NLA_U8 },
 571         [TCA_FLOWER_KEY_IPV4_SRC]       = { .type = NLA_U32 },
 572         [TCA_FLOWER_KEY_IPV4_SRC_MASK]  = { .type = NLA_U32 },
 573         [TCA_FLOWER_KEY_IPV4_DST]       = { .type = NLA_U32 },
 574         [TCA_FLOWER_KEY_IPV4_DST_MASK]  = { .type = NLA_U32 },
 575         [TCA_FLOWER_KEY_IPV6_SRC]       = { .len = sizeof(struct in6_addr) },
 576         [TCA_FLOWER_KEY_IPV6_SRC_MASK]  = { .len = sizeof(struct in6_addr) },
 577         [TCA_FLOWER_KEY_IPV6_DST]       = { .len = sizeof(struct in6_addr) },
 578         [TCA_FLOWER_KEY_IPV6_DST_MASK]  = { .len = sizeof(struct in6_addr) },
 579         [TCA_FLOWER_KEY_TCP_SRC]        = { .type = NLA_U16 },
 580         [TCA_FLOWER_KEY_TCP_DST]        = { .type = NLA_U16 },
 581         [TCA_FLOWER_KEY_UDP_SRC]        = { .type = NLA_U16 },
 582         [TCA_FLOWER_KEY_UDP_DST]        = { .type = NLA_U16 },
 583         [TCA_FLOWER_KEY_VLAN_ID]        = { .type = NLA_U16 },
 584         [TCA_FLOWER_KEY_VLAN_PRIO]      = { .type = NLA_U8 },
 585         [TCA_FLOWER_KEY_VLAN_ETH_TYPE]  = { .type = NLA_U16 },
 586         [TCA_FLOWER_KEY_ENC_KEY_ID]     = { .type = NLA_U32 },
 587         [TCA_FLOWER_KEY_ENC_IPV4_SRC]   = { .type = NLA_U32 },
 588         [TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK] = { .type = NLA_U32 },
 589         [TCA_FLOWER_KEY_ENC_IPV4_DST]   = { .type = NLA_U32 },
 590         [TCA_FLOWER_KEY_ENC_IPV4_DST_MASK] = { .type = NLA_U32 },
 591         [TCA_FLOWER_KEY_ENC_IPV6_SRC]   = { .len = sizeof(struct in6_addr) },
 592         [TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK] = { .len = sizeof(struct in6_addr) },
 593         [TCA_FLOWER_KEY_ENC_IPV6_DST]   = { .len = sizeof(struct in6_addr) },
 594         [TCA_FLOWER_KEY_ENC_IPV6_DST_MASK] = { .len = sizeof(struct in6_addr) },
 595         [TCA_FLOWER_KEY_TCP_SRC_MASK]   = { .type = NLA_U16 },
 596         [TCA_FLOWER_KEY_TCP_DST_MASK]   = { .type = NLA_U16 },
 597         [TCA_FLOWER_KEY_UDP_SRC_MASK]   = { .type = NLA_U16 },
 598         [TCA_FLOWER_KEY_UDP_DST_MASK]   = { .type = NLA_U16 },
 599         [TCA_FLOWER_KEY_SCTP_SRC_MASK]  = { .type = NLA_U16 },
 600         [TCA_FLOWER_KEY_SCTP_DST_MASK]  = { .type = NLA_U16 },
 601         [TCA_FLOWER_KEY_SCTP_SRC]       = { .type = NLA_U16 },
 602         [TCA_FLOWER_KEY_SCTP_DST]       = { .type = NLA_U16 },
 603         [TCA_FLOWER_KEY_ENC_UDP_SRC_PORT]       = { .type = NLA_U16 },
 604         [TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK]  = { .type = NLA_U16 },
 605         [TCA_FLOWER_KEY_ENC_UDP_DST_PORT]       = { .type = NLA_U16 },
 606         [TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK]  = { .type = NLA_U16 },
 607         [TCA_FLOWER_KEY_FLAGS]          = { .type = NLA_U32 },
 608         [TCA_FLOWER_KEY_FLAGS_MASK]     = { .type = NLA_U32 },
 609         [TCA_FLOWER_KEY_ICMPV4_TYPE]    = { .type = NLA_U8 },
 610         [TCA_FLOWER_KEY_ICMPV4_TYPE_MASK] = { .type = NLA_U8 },
 611         [TCA_FLOWER_KEY_ICMPV4_CODE]    = { .type = NLA_U8 },
 612         [TCA_FLOWER_KEY_ICMPV4_CODE_MASK] = { .type = NLA_U8 },
 613         [TCA_FLOWER_KEY_ICMPV6_TYPE]    = { .type = NLA_U8 },
 614         [TCA_FLOWER_KEY_ICMPV6_TYPE_MASK] = { .type = NLA_U8 },
 615         [TCA_FLOWER_KEY_ICMPV6_CODE]    = { .type = NLA_U8 },
 616         [TCA_FLOWER_KEY_ICMPV6_CODE_MASK] = { .type = NLA_U8 },
 617         [TCA_FLOWER_KEY_ARP_SIP]        = { .type = NLA_U32 },
 618         [TCA_FLOWER_KEY_ARP_SIP_MASK]   = { .type = NLA_U32 },
 619         [TCA_FLOWER_KEY_ARP_TIP]        = { .type = NLA_U32 },
 620         [TCA_FLOWER_KEY_ARP_TIP_MASK]   = { .type = NLA_U32 },
 621         [TCA_FLOWER_KEY_ARP_OP]         = { .type = NLA_U8 },
 622         [TCA_FLOWER_KEY_ARP_OP_MASK]    = { .type = NLA_U8 },
 623         [TCA_FLOWER_KEY_ARP_SHA]        = { .len = ETH_ALEN },
 624         [TCA_FLOWER_KEY_ARP_SHA_MASK]   = { .len = ETH_ALEN },
 625         [TCA_FLOWER_KEY_ARP_THA]        = { .len = ETH_ALEN },
 626         [TCA_FLOWER_KEY_ARP_THA_MASK]   = { .len = ETH_ALEN },
 627         [TCA_FLOWER_KEY_MPLS_TTL]       = { .type = NLA_U8 },
 628         [TCA_FLOWER_KEY_MPLS_BOS]       = { .type = NLA_U8 },
 629         [TCA_FLOWER_KEY_MPLS_TC]        = { .type = NLA_U8 },
 630         [TCA_FLOWER_KEY_MPLS_LABEL]     = { .type = NLA_U32 },
 631         [TCA_FLOWER_KEY_TCP_FLAGS]      = { .type = NLA_U16 },
 632         [TCA_FLOWER_KEY_TCP_FLAGS_MASK] = { .type = NLA_U16 },
 633         [TCA_FLOWER_KEY_IP_TOS]         = { .type = NLA_U8 },
 634         [TCA_FLOWER_KEY_IP_TOS_MASK]    = { .type = NLA_U8 },
 635         [TCA_FLOWER_KEY_IP_TTL]         = { .type = NLA_U8 },
 636         [TCA_FLOWER_KEY_IP_TTL_MASK]    = { .type = NLA_U8 },
 637         [TCA_FLOWER_KEY_CVLAN_ID]       = { .type = NLA_U16 },
 638         [TCA_FLOWER_KEY_CVLAN_PRIO]     = { .type = NLA_U8 },
 639         [TCA_FLOWER_KEY_CVLAN_ETH_TYPE] = { .type = NLA_U16 },
 640         [TCA_FLOWER_KEY_ENC_IP_TOS]     = { .type = NLA_U8 },
 641         [TCA_FLOWER_KEY_ENC_IP_TOS_MASK] = { .type = NLA_U8 },
 642         [TCA_FLOWER_KEY_ENC_IP_TTL]      = { .type = NLA_U8 },
 643         [TCA_FLOWER_KEY_ENC_IP_TTL_MASK] = { .type = NLA_U8 },
 644         [TCA_FLOWER_KEY_ENC_OPTS]       = { .type = NLA_NESTED },
 645         [TCA_FLOWER_KEY_ENC_OPTS_MASK]  = { .type = NLA_NESTED },
 646 };
 647
 648 static const struct nla_policy
 649 enc_opts_policy[TCA_FLOWER_KEY_ENC_OPTS_MAX + 1] = {
 650         [TCA_FLOWER_KEY_ENC_OPTS_GENEVE]        = { .type = NLA_NESTED },
 651 };
 652
 653 static const struct nla_policy
 654 geneve_opt_policy[TCA_FLOWER_KEY_ENC_OPT_GENEVE_MAX + 1] = {
 655         [TCA_FLOWER_KEY_ENC_OPT_GENEVE_CLASS]      = { .type = NLA_U16 },
 656         [TCA_FLOWER_KEY_ENC_OPT_GENEVE_TYPE]       = { .type = NLA_U8 },
 657         [TCA_FLOWER_KEY_ENC_OPT_GENEVE_DATA]       = { .type = NLA_BINARY,
 658                                                        .len = 128 },
 659 };
 660
 661 static void fl_set_key_val(struct nlattr **tb,
 662                            void *val, int val_type,
 663                            void *mask, int mask_type, int len)
 664 {
 665         if (!tb[val_type])
 666                 return;
 667         memcpy(val, nla_data(tb[val_type]), len);
 668         if (mask_type == TCA_FLOWER_UNSPEC || !tb[mask_type])
 669                 memset(mask, 0xff, len);
 670         else
 671                 memcpy(mask, nla_data(tb[mask_type]), len);
 672 }
 673
 674 static int fl_set_key_port_range(struct nlattr **tb, struct fl_flow_key *key,
 675                                  struct fl_flow_key *mask)
 676 {
 677         fl_set_key_val(tb, &key->tp_min.dst,
 678                        TCA_FLOWER_KEY_PORT_DST_MIN, &mask->tp_min.dst,
 679                        TCA_FLOWER_UNSPEC, sizeof(key->tp_min.dst));
 680         fl_set_key_val(tb, &key->tp_max.dst,
 681                        TCA_FLOWER_KEY_PORT_DST_MAX, &mask->tp_max.dst,
 682                        TCA_FLOWER_UNSPEC, sizeof(key->tp_max.dst));
 683         fl_set_key_val(tb, &key->tp_min.src,
 684                        TCA_FLOWER_KEY_PORT_SRC_MIN, &mask->tp_min.src,
 685                        TCA_FLOWER_UNSPEC, sizeof(key->tp_min.src));
 686         fl_set_key_val(tb, &key->tp_max.src,
 687                        TCA_FLOWER_KEY_PORT_SRC_MAX, &mask->tp_max.src,
 688                        TCA_FLOWER_UNSPEC, sizeof(key->tp_max.src));
 689
 690         if ((mask->tp_min.dst && mask->tp_max.dst &&
 691              htons(key->tp_max.dst) <= htons(key->tp_min.dst)) ||
 692              (mask->tp_min.src && mask->tp_max.src &&
 693               htons(key->tp_max.src) <= htons(key->tp_min.src)))
 694                 return -EINVAL;
 695
 696         return 0;
 697 }
 698
 699 static int fl_set_key_mpls(struct nlattr **tb,
 700                            struct flow_dissector_key_mpls *key_val,
 701                            struct flow_dissector_key_mpls *key_mask)
 702 {
 703         if (tb[TCA_FLOWER_KEY_MPLS_TTL]) {
 704                 key_val->mpls_ttl = nla_get_u8(tb[TCA_FLOWER_KEY_MPLS_TTL]);
 705                 key_mask->mpls_ttl = MPLS_TTL_MASK;
 706         }
 707         if (tb[TCA_FLOWER_KEY_MPLS_BOS]) {
 708                 u8 bos = nla_get_u8(tb[TCA_FLOWER_KEY_MPLS_BOS]);
 709
 710                 if (bos & ~MPLS_BOS_MASK)
 711                         return -EINVAL;
 712                 key_val->mpls_bos = bos;
 713                 key_mask->mpls_bos = MPLS_BOS_MASK;
 714         }
 715         if (tb[TCA_FLOWER_KEY_MPLS_TC]) {
 716                 u8 tc = nla_get_u8(tb[TCA_FLOWER_KEY_MPLS_TC]);
 717
 718                 if (tc & ~MPLS_TC_MASK)
 719                         return -EINVAL;
 720                 key_val->mpls_tc = tc;
 721                 key_mask->mpls_tc = MPLS_TC_MASK;
 722         }
 723         if (tb[TCA_FLOWER_KEY_MPLS_LABEL]) {
 724                 u32 label = nla_get_u32(tb[TCA_FLOWER_KEY_MPLS_LABEL]);
 725
 726                 if (label & ~MPLS_LABEL_MASK)
 727                         return -EINVAL;
 728                 key_val->mpls_label = label;
 729                 key_mask->mpls_label = MPLS_LABEL_MASK;
 730         }
 731         return 0;
 732 }
 733
 734 static void fl_set_key_vlan(struct nlattr **tb,
 735                             __be16 ethertype,
 736                             int vlan_id_key, int vlan_prio_key,
 737                             struct flow_dissector_key_vlan *key_val,
 738                             struct flow_dissector_key_vlan *key_mask)
 739 {
 740 #define VLAN_PRIORITY_MASK      0x7
 741
 742         if (tb[vlan_id_key]) {
 743                 key_val->vlan_id =
 744                         nla_get_u16(tb[vlan_id_key]) & VLAN_VID_MASK;
 745                 key_mask->vlan_id = VLAN_VID_MASK;
 746         }
 747         if (tb[vlan_prio_key]) {
 748                 key_val->vlan_priority =
 749                         nla_get_u8(tb[vlan_prio_key]) &
 750                         VLAN_PRIORITY_MASK;
 751                 key_mask->vlan_priority = VLAN_PRIORITY_MASK;
 752         }
 753         key_val->vlan_tpid = ethertype;
 754         key_mask->vlan_tpid = cpu_to_be16(~0);
 755 }
 756
 757 static void fl_set_key_flag(u32 flower_key, u32 flower_mask,
 758                             u32 *dissector_key, u32 *dissector_mask,
 759                             u32 flower_flag_bit, u32 dissector_flag_bit)
 760 {
 761         if (flower_mask & flower_flag_bit) {
 762                 *dissector_mask |= dissector_flag_bit;
 763                 if (flower_key & flower_flag_bit)
 764                         *dissector_key |= dissector_flag_bit;
 765         }
 766 }
 767
 768 static int fl_set_key_flags(struct nlattr **tb,
 769                             u32 *flags_key, u32 *flags_mask)
 770 {
 771         u32 key, mask;
 772
 773         /* mask is mandatory for flags */
 774         if (!tb[TCA_FLOWER_KEY_FLAGS_MASK])
 775                 return -EINVAL;
 776
 777         key = be32_to_cpu(nla_get_u32(tb[TCA_FLOWER_KEY_FLAGS]));
 778         mask = be32_to_cpu(nla_get_u32(tb[TCA_FLOWER_KEY_FLAGS_MASK]));
 779
 780         *flags_key  = 0;
 781         *flags_mask = 0;
 782
 783         fl_set_key_flag(key, mask, flags_key, flags_mask,
 784                         TCA_FLOWER_KEY_FLAGS_IS_FRAGMENT, FLOW_DIS_IS_FRAGMENT);
 785         fl_set_key_flag(key, mask, flags_key, flags_mask,
 786                         TCA_FLOWER_KEY_FLAGS_FRAG_IS_FIRST,
 787                         FLOW_DIS_FIRST_FRAG);
 788
 789         return 0;
 790 }
 791
 792 static void fl_set_key_ip(struct nlattr **tb, bool encap,
 793                           struct flow_dissector_key_ip *key,
 794                           struct flow_dissector_key_ip *mask)
 795 {
 796         int tos_key = encap ? TCA_FLOWER_KEY_ENC_IP_TOS : TCA_FLOWER_KEY_IP_TOS;
 797         int ttl_key = encap ? TCA_FLOWER_KEY_ENC_IP_TTL : TCA_FLOWER_KEY_IP_TTL;
 798         int tos_mask = encap ? TCA_FLOWER_KEY_ENC_IP_TOS_MASK : TCA_FLOWER_KEY_IP_TOS_MASK;
 799         int ttl_mask = encap ? TCA_FLOWER_KEY_ENC_IP_TTL_MASK : TCA_FLOWER_KEY_IP_TTL_MASK;
 800
 801         fl_set_key_val(tb, &key->tos, tos_key, &mask->tos, tos_mask, sizeof(key->tos));
 802         fl_set_key_val(tb, &key->ttl, ttl_key, &mask->ttl, ttl_mask, sizeof(key->ttl));
 803 }
 804
 805 static int fl_set_geneve_opt(const struct nlattr *nla, struct fl_flow_key *key,
 806                              int depth, int option_len,
 807                              struct netlink_ext_ack *extack)
 808 {
 809         struct nlattr *tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_MAX + 1];
 810         struct nlattr *class = NULL, *type = NULL, *data = NULL;
 811         struct geneve_opt *opt;
 812         int err, data_len = 0;
 813
 814         if (option_len > sizeof(struct geneve_opt))
 815                 data_len = option_len - sizeof(struct geneve_opt);
 816
 817         opt = (struct geneve_opt *)&key->enc_opts.data[key->enc_opts.len];
 818         memset(opt, 0xff, option_len);
 819         opt->length = data_len / 4;
 820         opt->r1 = 0;
 821         opt->r2 = 0;
 822         opt->r3 = 0;
 823
 824         /* If no mask has been prodived we assume an exact match. */
 825         if (!depth)
 826                 return sizeof(struct geneve_opt) + data_len;
 827
 828         if (nla_type(nla) != TCA_FLOWER_KEY_ENC_OPTS_GENEVE) {
 829                 NL_SET_ERR_MSG(extack, "Non-geneve option type for mask");
 830                 return -EINVAL;
 831         }
 832
 833         err = nla_parse_nested(tb, TCA_FLOWER_KEY_ENC_OPT_GENEVE_MAX,
 834                                nla, geneve_opt_policy, extack);
 835         if (err < 0)
 836                 return err;
 837
 838         /* We are not allowed to omit any of CLASS, TYPE or DATA
 839          * fields from the key.
 840          */
 841         if (!option_len &&
 842             (!tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_CLASS] ||
 843              !tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_TYPE] ||
 844              !tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_DATA])) {
 845                 NL_SET_ERR_MSG(extack, "Missing tunnel key geneve option class, type or data");
 846                 return -EINVAL;
 847         }
 848
 849         /* Omitting any of CLASS, TYPE or DATA fields is allowed
 850          * for the mask.
 851          */
 852         if (tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_DATA]) {
 853                 int new_len = key->enc_opts.len;
 854
 855                 data = tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_DATA];
 856                 data_len = nla_len(data);
 857                 if (data_len < 4) {
 858                         NL_SET_ERR_MSG(extack, "Tunnel key geneve option data is less than 4 bytes long");
 859                         return -ERANGE;
 860                 }
 861                 if (data_len % 4) {
 862                         NL_SET_ERR_MSG(extack, "Tunnel key geneve option data is not a multiple of 4 bytes long");
 863                         return -ERANGE;
 864                 }
 865
 866                 new_len += sizeof(struct geneve_opt) + data_len;
 867                 BUILD_BUG_ON(FLOW_DIS_TUN_OPTS_MAX != IP_TUNNEL_OPTS_MAX);
 868                 if (new_len > FLOW_DIS_TUN_OPTS_MAX) {
 869                         NL_SET_ERR_MSG(extack, "Tunnel options exceeds max size");
 870                         return -ERANGE;
 871                 }
 872                 opt->length = data_len / 4;
 873                 memcpy(opt->opt_data, nla_data(data), data_len);
 874         }
 875
 876         if (tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_CLASS]) {
 877                 class = tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_CLASS];
 878                 opt->opt_class = nla_get_be16(class);
 879         }
 880
 881         if (tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_TYPE]) {
 882                 type = tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_TYPE];
 883                 opt->type = nla_get_u8(type);
 884         }
 885
 886         return sizeof(struct geneve_opt) + data_len;
 887 }
 888
 889 static int fl_set_enc_opt(struct nlattr **tb, struct fl_flow_key *key,
 890                           struct fl_flow_key *mask,
 891                           struct netlink_ext_ack *extack)
 892 {
 893         const struct nlattr *nla_enc_key, *nla_opt_key, *nla_opt_msk = NULL;
 894         int err, option_len, key_depth, msk_depth = 0;
 895
 896         err = nla_validate_nested(tb[TCA_FLOWER_KEY_ENC_OPTS],
 897                                   TCA_FLOWER_KEY_ENC_OPTS_MAX,
 898                                   enc_opts_policy, extack);
 899         if (err)
 900                 return err;
 901
 902         nla_enc_key = nla_data(tb[TCA_FLOWER_KEY_ENC_OPTS]);
 903
 904         if (tb[TCA_FLOWER_KEY_ENC_OPTS_MASK]) {
 905                 err = nla_validate_nested(tb[TCA_FLOWER_KEY_ENC_OPTS_MASK],
 906                                           TCA_FLOWER_KEY_ENC_OPTS_MAX,
 907                                           enc_opts_policy, extack);
 908                 if (err)
 909                         return err;
 910
 911                 nla_opt_msk = nla_data(tb[TCA_FLOWER_KEY_ENC_OPTS_MASK]);
 912                 msk_depth = nla_len(tb[TCA_FLOWER_KEY_ENC_OPTS_MASK]);
 913         }
 914
 915         nla_for_each_attr(nla_opt_key, nla_enc_key,
 916                           nla_len(tb[TCA_FLOWER_KEY_ENC_OPTS]), key_depth) {
 917                 switch (nla_type(nla_opt_key)) {
 918                 case TCA_FLOWER_KEY_ENC_OPTS_GENEVE:
 919                         option_len = 0;
 920                         key->enc_opts.dst_opt_type = TUNNEL_GENEVE_OPT;
 921                         option_len = fl_set_geneve_opt(nla_opt_key, key,
 922                                                        key_depth, option_len,
 923                                                        extack);
 924                         if (option_len < 0)
 925                                 return option_len;
 926
 927                         key->enc_opts.len += option_len;
 928                         /* At the same time we need to parse through the mask
 929                          * in order to verify exact and mask attribute lengths.
 930                          */
 931                         mask->enc_opts.dst_opt_type = TUNNEL_GENEVE_OPT;
 932                         option_len = fl_set_geneve_opt(nla_opt_msk, mask,
 933                                                        msk_depth, option_len,
 934                                                        extack);
 935                         if (option_len < 0)
 936                                 return option_len;
 937
 938                         mask->enc_opts.len += option_len;
 939                         if (key->enc_opts.len != mask->enc_opts.len) {
 940                                 NL_SET_ERR_MSG(extack, "Key and mask miss aligned");
 941                                 return -EINVAL;
 942                         }
 943
 944                         if (msk_depth)
 945                                 nla_opt_msk = nla_next(nla_opt_msk, &msk_depth);
 946                         break;
 947                 default:
 948                         NL_SET_ERR_MSG(extack, "Unknown tunnel option type");
 949                         return -EINVAL;
 950                 }
 951         }
 952
 953         return 0;
 954 }
 955
 956 static int fl_set_key(struct net *net, struct nlattr **tb,
 957                       struct fl_flow_key *key, struct fl_flow_key *mask,
 958                       struct netlink_ext_ack *extack)
 959 {
 960         __be16 ethertype;
 961         int ret = 0;
 962 #ifdef CONFIG_NET_CLS_IND
 963         if (tb[TCA_FLOWER_INDEV]) {
 964                 int err = tcf_change_indev(net, tb[TCA_FLOWER_INDEV], extack);
 965                 if (err < 0)
 966                         return err;
 967                 key->indev_ifindex = err;
 968                 mask->indev_ifindex = 0xffffffff;
 969         }
 970 #endif
 971
 972         fl_set_key_val(tb, key->eth.dst, TCA_FLOWER_KEY_ETH_DST,
 973                        mask->eth.dst, TCA_FLOWER_KEY_ETH_DST_MASK,
 974                        sizeof(key->eth.dst));
 975         fl_set_key_val(tb, key->eth.src, TCA_FLOWER_KEY_ETH_SRC,
 976                        mask->eth.src, TCA_FLOWER_KEY_ETH_SRC_MASK,
 977                        sizeof(key->eth.src));
 978
 979         if (tb[TCA_FLOWER_KEY_ETH_TYPE]) {
 980                 ethertype = nla_get_be16(tb[TCA_FLOWER_KEY_ETH_TYPE]);
 981
 982                 if (eth_type_vlan(ethertype)) {
 983                         fl_set_key_vlan(tb, ethertype, TCA_FLOWER_KEY_VLAN_ID,
 984                                         TCA_FLOWER_KEY_VLAN_PRIO, &key->vlan,
 985                                         &mask->vlan);
 986
 987                         if (tb[TCA_FLOWER_KEY_VLAN_ETH_TYPE]) {
 988                                 ethertype = nla_get_be16(tb[TCA_FLOWER_KEY_VLAN_ETH_TYPE]);
 989                                 if (eth_type_vlan(ethertype)) {
 990                                         fl_set_key_vlan(tb, ethertype,
 991                                                         TCA_FLOWER_KEY_CVLAN_ID,
 992                                                         TCA_FLOWER_KEY_CVLAN_PRIO,
 993                                                         &key->cvlan, &mask->cvlan);
 994                                         fl_set_key_val(tb, &key->basic.n_proto,
 995                                                        TCA_FLOWER_KEY_CVLAN_ETH_TYPE,
 996                                                        &mask->basic.n_proto,
 997                                                        TCA_FLOWER_UNSPEC,
 998                                                        sizeof(key->basic.n_proto));
 999                                 } else {
1000                                         key->basic.n_proto = ethertype;
1001                                         mask->basic.n_proto = cpu_to_be16(~0);
1002                                 }
1003                         }
1004                 } else {
1005                         key->basic.n_proto = ethertype;
1006                         mask->basic.n_proto = cpu_to_be16(~0);
1007                 }
1008         }
1009
1010         if (key->basic.n_proto == htons(ETH_P_IP) ||
1011             key->basic.n_proto == htons(ETH_P_IPV6)) {
1012                 fl_set_key_val(tb, &key->basic.ip_proto, TCA_FLOWER_KEY_IP_PROTO,
1013                                &mask->basic.ip_proto, TCA_FLOWER_UNSPEC,
1014                                sizeof(key->basic.ip_proto));
1015                 fl_set_key_ip(tb, false, &key->ip, &mask->ip);
1016         }
1017
1018         if (tb[TCA_FLOWER_KEY_IPV4_SRC] || tb[TCA_FLOWER_KEY_IPV4_DST]) {
1019                 key->control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
1020                 mask->control.addr_type = ~0;
1021                 fl_set_key_val(tb, &key->ipv4.src, TCA_FLOWER_KEY_IPV4_SRC,
1022                                &mask->ipv4.src, TCA_FLOWER_KEY_IPV4_SRC_MASK,
1023                                sizeof(key->ipv4.src));
1024                 fl_set_key_val(tb, &key->ipv4.dst, TCA_FLOWER_KEY_IPV4_DST,
1025                                &mask->ipv4.dst, TCA_FLOWER_KEY_IPV4_DST_MASK,
1026                                sizeof(key->ipv4.dst));
1027         } else if (tb[TCA_FLOWER_KEY_IPV6_SRC] || tb[TCA_FLOWER_KEY_IPV6_DST]) {
1028                 key->control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
1029                 mask->control.addr_type = ~0;
1030                 fl_set_key_val(tb, &key->ipv6.src, TCA_FLOWER_KEY_IPV6_SRC,
1031                                &mask->ipv6.src, TCA_FLOWER_KEY_IPV6_SRC_MASK,
1032                                sizeof(key->ipv6.src));
1033                 fl_set_key_val(tb, &key->ipv6.dst, TCA_FLOWER_KEY_IPV6_DST,
1034                                &mask->ipv6.dst, TCA_FLOWER_KEY_IPV6_DST_MASK,
1035                                sizeof(key->ipv6.dst));
1036         }
1037
1038         if (key->basic.ip_proto == IPPROTO_TCP) {
1039                 fl_set_key_val(tb, &key->tp.src, TCA_FLOWER_KEY_TCP_SRC,
1040                                &mask->tp.src, TCA_FLOWER_KEY_TCP_SRC_MASK,
1041                                sizeof(key->tp.src));
1042                 fl_set_key_val(tb, &key->tp.dst, TCA_FLOWER_KEY_TCP_DST,
1043                                &mask->tp.dst, TCA_FLOWER_KEY_TCP_DST_MASK,
1044                                sizeof(key->tp.dst));
1045                 fl_set_key_val(tb, &key->tcp.flags, TCA_FLOWER_KEY_TCP_FLAGS,
1046                                &mask->tcp.flags, TCA_FLOWER_KEY_TCP_FLAGS_MASK,
1047                                sizeof(key->tcp.flags));
1048         } else if (key->basic.ip_proto == IPPROTO_UDP) {
1049                 fl_set_key_val(tb, &key->tp.src, TCA_FLOWER_KEY_UDP_SRC,
1050                                &mask->tp.src, TCA_FLOWER_KEY_UDP_SRC_MASK,
1051                                sizeof(key->tp.src));
1052                 fl_set_key_val(tb, &key->tp.dst, TCA_FLOWER_KEY_UDP_DST,
1053                                &mask->tp.dst, TCA_FLOWER_KEY_UDP_DST_MASK,
1054                                sizeof(key->tp.dst));
1055         } else if (key->basic.ip_proto == IPPROTO_SCTP) {
1056                 fl_set_key_val(tb, &key->tp.src, TCA_FLOWER_KEY_SCTP_SRC,
1057                                &mask->tp.src, TCA_FLOWER_KEY_SCTP_SRC_MASK,
1058                                sizeof(key->tp.src));
1059                 fl_set_key_val(tb, &key->tp.dst, TCA_FLOWER_KEY_SCTP_DST,
1060                                &mask->tp.dst, TCA_FLOWER_KEY_SCTP_DST_MASK,
1061                                sizeof(key->tp.dst));
1062         } else if (key->basic.n_proto == htons(ETH_P_IP) &&
1063                    key->basic.ip_proto == IPPROTO_ICMP) {
1064                 fl_set_key_val(tb, &key->icmp.type, TCA_FLOWER_KEY_ICMPV4_TYPE,
1065                                &mask->icmp.type,
1066                                TCA_FLOWER_KEY_ICMPV4_TYPE_MASK,
1067                                sizeof(key->icmp.type));
1068                 fl_set_key_val(tb, &key->icmp.code, TCA_FLOWER_KEY_ICMPV4_CODE,
1069                                &mask->icmp.code,
1070                                TCA_FLOWER_KEY_ICMPV4_CODE_MASK,
1071                                sizeof(key->icmp.code));
1072         } else if (key->basic.n_proto == htons(ETH_P_IPV6) &&
1073                    key->basic.ip_proto == IPPROTO_ICMPV6) {
1074                 fl_set_key_val(tb, &key->icmp.type, TCA_FLOWER_KEY_ICMPV6_TYPE,
1075                                &mask->icmp.type,
1076                                TCA_FLOWER_KEY_ICMPV6_TYPE_MASK,
1077                                sizeof(key->icmp.type));
1078                 fl_set_key_val(tb, &key->icmp.code, TCA_FLOWER_KEY_ICMPV6_CODE,
1079                                &mask->icmp.code,
1080                                TCA_FLOWER_KEY_ICMPV6_CODE_MASK,
1081                                sizeof(key->icmp.code));
1082         } else if (key->basic.n_proto == htons(ETH_P_MPLS_UC) ||
1083                    key->basic.n_proto == htons(ETH_P_MPLS_MC)) {
1084                 ret = fl_set_key_mpls(tb, &key->mpls, &mask->mpls);
1085                 if (ret)
1086                         return ret;
1087         } else if (key->basic.n_proto == htons(ETH_P_ARP) ||
1088                    key->basic.n_proto == htons(ETH_P_RARP)) {
1089                 fl_set_key_val(tb, &key->arp.sip, TCA_FLOWER_KEY_ARP_SIP,
1090                                &mask->arp.sip, TCA_FLOWER_KEY_ARP_SIP_MASK,
1091                                sizeof(key->arp.sip));
1092                 fl_set_key_val(tb, &key->arp.tip, TCA_FLOWER_KEY_ARP_TIP,
1093                                &mask->arp.tip, TCA_FLOWER_KEY_ARP_TIP_MASK,
1094                                sizeof(key->arp.tip));
1095                 fl_set_key_val(tb, &key->arp.op, TCA_FLOWER_KEY_ARP_OP,
1096                                &mask->arp.op, TCA_FLOWER_KEY_ARP_OP_MASK,
1097                                sizeof(key->arp.op));
1098                 fl_set_key_val(tb, key->arp.sha, TCA_FLOWER_KEY_ARP_SHA,
1099                                mask->arp.sha, TCA_FLOWER_KEY_ARP_SHA_MASK,
1100                                sizeof(key->arp.sha));
1101                 fl_set_key_val(tb, key->arp.tha, TCA_FLOWER_KEY_ARP_THA,
1102                                mask->arp.tha, TCA_FLOWER_KEY_ARP_THA_MASK,
1103                                sizeof(key->arp.tha));
1104         }
1105
1106         if (key->basic.ip_proto == IPPROTO_TCP ||
1107             key->basic.ip_proto == IPPROTO_UDP ||
1108             key->basic.ip_proto == IPPROTO_SCTP) {
1109                 ret = fl_set_key_port_range(tb, key, mask);
1110                 if (ret)
1111                         return ret;
1112         }
1113
1114         if (tb[TCA_FLOWER_KEY_ENC_IPV4_SRC] ||
1115             tb[TCA_FLOWER_KEY_ENC_IPV4_DST]) {
1116                 key->enc_control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
1117                 mask->enc_control.addr_type = ~0;
1118                 fl_set_key_val(tb, &key->enc_ipv4.src,
1119                                TCA_FLOWER_KEY_ENC_IPV4_SRC,
1120                                &mask->enc_ipv4.src,
1121                                TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK,
1122                                sizeof(key->enc_ipv4.src));
1123                 fl_set_key_val(tb, &key->enc_ipv4.dst,
1124                                TCA_FLOWER_KEY_ENC_IPV4_DST,
1125                                &mask->enc_ipv4.dst,
1126                                TCA_FLOWER_KEY_ENC_IPV4_DST_MASK,
1127                                sizeof(key->enc_ipv4.dst));
1128         }
1129
1130         if (tb[TCA_FLOWER_KEY_ENC_IPV6_SRC] ||
1131             tb[TCA_FLOWER_KEY_ENC_IPV6_DST]) {
1132                 key->enc_control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
1133                 mask->enc_control.addr_type = ~0;
1134                 fl_set_key_val(tb, &key->enc_ipv6.src,
1135                                TCA_FLOWER_KEY_ENC_IPV6_SRC,
1136                                &mask->enc_ipv6.src,
1137                                TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK,
1138                                sizeof(key->enc_ipv6.src));
1139                 fl_set_key_val(tb, &key->enc_ipv6.dst,
1140                                TCA_FLOWER_KEY_ENC_IPV6_DST,
1141                                &mask->enc_ipv6.dst,
1142                                TCA_FLOWER_KEY_ENC_IPV6_DST_MASK,
1143                                sizeof(key->enc_ipv6.dst));
1144         }
1145
1146         fl_set_key_val(tb, &key->enc_key_id.keyid, TCA_FLOWER_KEY_ENC_KEY_ID,
1147                        &mask->enc_key_id.keyid, TCA_FLOWER_UNSPEC,
1148                        sizeof(key->enc_key_id.keyid));
1149
1150         fl_set_key_val(tb, &key->enc_tp.src, TCA_FLOWER_KEY_ENC_UDP_SRC_PORT,
1151                        &mask->enc_tp.src, TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK,
1152                        sizeof(key->enc_tp.src));
1153
1154         fl_set_key_val(tb, &key->enc_tp.dst, TCA_FLOWER_KEY_ENC_UDP_DST_PORT,
1155                        &mask->enc_tp.dst, TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK,
1156                        sizeof(key->enc_tp.dst));
1157
1158         fl_set_key_ip(tb, true, &key->enc_ip, &mask->enc_ip);
1159
1160         if (tb[TCA_FLOWER_KEY_ENC_OPTS]) {
1161                 ret = fl_set_enc_opt(tb, key, mask, extack);
1162                 if (ret)
1163                         return ret;
1164         }
1165
1166         if (tb[TCA_FLOWER_KEY_FLAGS])
1167                 ret = fl_set_key_flags(tb, &key->control.flags, &mask->control.flags);
1168
1169         return ret;
1170 }
1171
1172 static void fl_mask_copy(struct fl_flow_mask *dst,
1173                          struct fl_flow_mask *src)
1174 {
1175         const void *psrc = fl_key_get_start(&src->key, src);
1176         void *pdst = fl_key_get_start(&dst->key, src);
1177
1178         memcpy(pdst, psrc, fl_mask_range(src));
1179         dst->range = src->range;
1180 }
1181
1182 static const struct rhashtable_params fl_ht_params = {
1183         .key_offset = offsetof(struct cls_fl_filter, mkey), /* base offset */
1184         .head_offset = offsetof(struct cls_fl_filter, ht_node),
1185         .automatic_shrinking = true,
1186 };
1187
1188 static int fl_init_mask_hashtable(struct fl_flow_mask *mask)
1189 {
1190         mask->filter_ht_params = fl_ht_params;
1191         mask->filter_ht_params.key_len = fl_mask_range(mask);
1192         mask->filter_ht_params.key_offset += mask->range.start;
1193
1194         return rhashtable_init(&mask->ht, &mask->filter_ht_params);
1195 }
1196
1197 #define FL_KEY_MEMBER_OFFSET(member) offsetof(struct fl_flow_key, member)
1198 #define FL_KEY_MEMBER_SIZE(member) FIELD_SIZEOF(struct fl_flow_key, member)
1199
1200 #define FL_KEY_IS_MASKED(mask, member)                                          \
1201         memchr_inv(((char *)mask) + FL_KEY_MEMBER_OFFSET(member),               \
1202                    0, FL_KEY_MEMBER_SIZE(member))                               \
1203
1204 #define FL_KEY_SET(keys, cnt, id, member)                                       \
1205         do {                                                                    \
1206                 keys[cnt].key_id = id;                                          \
1207                 keys[cnt].offset = FL_KEY_MEMBER_OFFSET(member);                \
1208                 cnt++;                                                          \
1209         } while(0);
1210
1211 #define FL_KEY_SET_IF_MASKED(mask, keys, cnt, id, member)                       \
1212         do {                                                                    \
1213                 if (FL_KEY_IS_MASKED(mask, member))                             \
1214                         FL_KEY_SET(keys, cnt, id, member);                      \
1215         } while(0);
1216
1217 static void fl_init_dissector(struct flow_dissector *dissector,
1218                               struct fl_flow_key *mask)
1219 {
1220         struct flow_dissector_key keys[FLOW_DISSECTOR_KEY_MAX];
1221         size_t cnt = 0;
1222
1223         FL_KEY_SET(keys, cnt, FLOW_DISSECTOR_KEY_CONTROL, control);
1224         FL_KEY_SET(keys, cnt, FLOW_DISSECTOR_KEY_BASIC, basic);
1225         FL_KEY_SET_IF_MASKED(mask, keys, cnt,
1226                              FLOW_DISSECTOR_KEY_ETH_ADDRS, eth);
1227         FL_KEY_SET_IF_MASKED(mask, keys, cnt,
1228                              FLOW_DISSECTOR_KEY_IPV4_ADDRS, ipv4);
1229         FL_KEY_SET_IF_MASKED(mask, keys, cnt,
1230                              FLOW_DISSECTOR_KEY_IPV6_ADDRS, ipv6);
1231         if (FL_KEY_IS_MASKED(mask, tp) ||
1232             FL_KEY_IS_MASKED(mask, tp_min) || FL_KEY_IS_MASKED(mask, tp_max))
1233                 FL_KEY_SET(keys, cnt, FLOW_DISSECTOR_KEY_PORTS, tp);
1234         FL_KEY_SET_IF_MASKED(mask, keys, cnt,
1235                              FLOW_DISSECTOR_KEY_IP, ip);
1236         FL_KEY_SET_IF_MASKED(mask, keys, cnt,
1237                              FLOW_DISSECTOR_KEY_TCP, tcp);
1238         FL_KEY_SET_IF_MASKED(mask, keys, cnt,
1239                              FLOW_DISSECTOR_KEY_ICMP, icmp);
1240         FL_KEY_SET_IF_MASKED(mask, keys, cnt,
1241                              FLOW_DISSECTOR_KEY_ARP, arp);
1242         FL_KEY_SET_IF_MASKED(mask, keys, cnt,
1243                              FLOW_DISSECTOR_KEY_MPLS, mpls);
1244         FL_KEY_SET_IF_MASKED(mask, keys, cnt,
1245                              FLOW_DISSECTOR_KEY_VLAN, vlan);
1246         FL_KEY_SET_IF_MASKED(mask, keys, cnt,
1247                              FLOW_DISSECTOR_KEY_CVLAN, cvlan);
1248         FL_KEY_SET_IF_MASKED(mask, keys, cnt,
1249                              FLOW_DISSECTOR_KEY_ENC_KEYID, enc_key_id);
1250         FL_KEY_SET_IF_MASKED(mask, keys, cnt,
1251                              FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS, enc_ipv4);
1252         FL_KEY_SET_IF_MASKED(mask, keys, cnt,
1253                              FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS, enc_ipv6);
1254         if (FL_KEY_IS_MASKED(mask, enc_ipv4) ||
1255             FL_KEY_IS_MASKED(mask, enc_ipv6))
1256                 FL_KEY_SET(keys, cnt, FLOW_DISSECTOR_KEY_ENC_CONTROL,
1257                            enc_control);
1258         FL_KEY_SET_IF_MASKED(mask, keys, cnt,
1259                              FLOW_DISSECTOR_KEY_ENC_PORTS, enc_tp);
1260         FL_KEY_SET_IF_MASKED(mask, keys, cnt,
1261                              FLOW_DISSECTOR_KEY_ENC_IP, enc_ip);
1262         FL_KEY_SET_IF_MASKED(mask, keys, cnt,
1263                              FLOW_DISSECTOR_KEY_ENC_OPTS, enc_opts);
1264
1265         skb_flow_dissector_init(dissector, keys, cnt);
1266 }
1267
1268 static struct fl_flow_mask *fl_create_new_mask(struct cls_fl_head *head,
1269                                                struct fl_flow_mask *mask)
1270 {
1271         struct fl_flow_mask *newmask;
1272         int err;
1273
1274         newmask = kzalloc(sizeof(*newmask), GFP_KERNEL);
1275         if (!newmask)
1276                 return ERR_PTR(-ENOMEM);
1277
1278         fl_mask_copy(newmask, mask);
1279
1280         if ((newmask->key.tp_min.dst && newmask->key.tp_max.dst) ||
1281             (newmask->key.tp_min.src && newmask->key.tp_max.src))
1282                 newmask->flags |= TCA_FLOWER_MASK_FLAGS_RANGE;
1283
1284         err = fl_init_mask_hashtable(newmask);
1285         if (err)
1286                 goto errout_free;
1287
1288         fl_init_dissector(&newmask->dissector, &newmask->key);
1289
1290         INIT_LIST_HEAD_RCU(&newmask->filters);
1291
1292         err = rhashtable_insert_fast(&head->ht, &newmask->ht_node,
1293                                      mask_ht_params);
1294         if (err)
1295                 goto errout_destroy;
1296
1297         list_add_tail_rcu(&newmask->list, &head->masks);
1298
1299         return newmask;
1300
1301 errout_destroy:
1302         rhashtable_destroy(&newmask->ht);
1303 errout_free:
1304         kfree(newmask);
1305
1306         return ERR_PTR(err);
1307 }
1308
1309 static int fl_check_assign_mask(struct cls_fl_head *head,
1310                                 struct cls_fl_filter *fnew,
1311                                 struct cls_fl_filter *fold,
1312                                 struct fl_flow_mask *mask)
1313 {
1314         struct fl_flow_mask *newmask;
1315
1316         fnew->mask = rhashtable_lookup_fast(&head->ht, mask, mask_ht_params);
1317         if (!fnew->mask) {
1318                 if (fold)
1319                         return -EINVAL;
1320
1321                 newmask = fl_create_new_mask(head, mask);
1322                 if (IS_ERR(newmask))
1323                         return PTR_ERR(newmask);
1324
1325                 fnew->mask = newmask;
1326         } else if (fold && fold->mask != fnew->mask) {
1327                 return -EINVAL;
1328         }
1329
1330         return 0;
1331 }
1332
1333 static int fl_set_parms(struct net *net, struct tcf_proto *tp,
1334                         struct cls_fl_filter *f, struct fl_flow_mask *mask,
1335                         unsigned long base, struct nlattr **tb,
1336                         struct nlattr *est, bool ovr,
1337                         struct fl_flow_tmplt *tmplt,
1338                         struct netlink_ext_ack *extack)
1339 {
1340         int err;
1341
1342         err = tcf_exts_validate(net, tp, tb, est, &f->exts, ovr, true,
1343                                 extack);
1344         if (err < 0)
1345                 return err;
1346
1347         if (tb[TCA_FLOWER_CLASSID]) {
1348                 f->res.classid = nla_get_u32(tb[TCA_FLOWER_CLASSID]);
1349                 tcf_bind_filter(tp, &f->res, base);
1350         }
1351
1352         err = fl_set_key(net, tb, &f->key, &mask->key, extack);
1353         if (err)
1354                 return err;
1355
1356         fl_mask_update_range(mask);
1357         fl_set_masked_key(&f->mkey, &f->key, mask);
1358
1359         if (!fl_mask_fits_tmplt(tmplt, mask)) {
1360                 NL_SET_ERR_MSG_MOD(extack, "Mask does not fit the template");
1361                 return -EINVAL;
1362         }
1363
1364         return 0;
1365 }
1366
1367 static int fl_change(struct net *net, struct sk_buff *in_skb,
1368                      struct tcf_proto *tp, unsigned long base,
1369                      u32 handle, struct nlattr **tca,
1370                      void **arg, bool ovr, bool rtnl_held,
1371                      struct netlink_ext_ack *extack)
1372 {
1373         struct cls_fl_head *head = fl_head_dereference(tp);
1374         struct cls_fl_filter *fold = *arg;
1375         struct cls_fl_filter *fnew;
1376         struct fl_flow_mask *mask;
1377         struct nlattr **tb;
1378         int err;
1379
1380         if (!tca[TCA_OPTIONS]) {
1381                 err = -EINVAL;
1382                 goto errout_fold;
1383         }
1384
1385         mask = kzalloc(sizeof(struct fl_flow_mask), GFP_KERNEL);
1386         if (!mask) {
1387                 err = -ENOBUFS;
1388                 goto errout_fold;
1389         }
1390
1391         tb = kcalloc(TCA_FLOWER_MAX + 1, sizeof(struct nlattr *), GFP_KERNEL);
1392         if (!tb) {
1393                 err = -ENOBUFS;
1394                 goto errout_mask_alloc;
1395         }
1396
1397         err = nla_parse_nested(tb, TCA_FLOWER_MAX, tca[TCA_OPTIONS],
1398                                fl_policy, NULL);
1399         if (err < 0)
1400                 goto errout_tb;
1401
1402         if (fold && handle && fold->handle != handle) {
1403                 err = -EINVAL;
1404                 goto errout_tb;
1405         }
1406
1407         fnew = kzalloc(sizeof(*fnew), GFP_KERNEL);
1408         if (!fnew) {
1409                 err = -ENOBUFS;
1410                 goto errout_tb;
1411         }
1412         refcount_set(&fnew->refcnt, 1);
1413
1414         err = tcf_exts_init(&fnew->exts, net, TCA_FLOWER_ACT, 0);
1415         if (err < 0)
1416                 goto errout;
1417
1418         if (tb[TCA_FLOWER_FLAGS]) {
1419                 fnew->flags = nla_get_u32(tb[TCA_FLOWER_FLAGS]);
1420
1421                 if (!tc_flags_valid(fnew->flags)) {
1422                         err = -EINVAL;
1423                         goto errout;
1424                 }
1425         }
1426
1427         err = fl_set_parms(net, tp, fnew, mask, base, tb, tca[TCA_RATE], ovr,
1428                            tp->chain->tmplt_priv, extack);
1429         if (err)
1430                 goto errout;
1431
1432         err = fl_check_assign_mask(head, fnew, fold, mask);
1433         if (err)
1434                 goto errout;
1435
1436         if (!tc_skip_hw(fnew->flags)) {
1437                 err = fl_hw_replace_filter(tp, fnew, extack);
1438                 if (err)
1439                         goto errout_mask;
1440         }
1441
1442         if (!tc_in_hw(fnew->flags))
1443                 fnew->flags |= TCA_CLS_FLAGS_NOT_IN_HW;
1444
1445         refcount_inc(&fnew->refcnt);
1446         if (fold) {
1447                 fnew->handle = handle;
1448
1449                 err = rhashtable_insert_fast(&fnew->mask->ht, &fnew->ht_node,
1450                                              fnew->mask->filter_ht_params);
1451                 if (err)
1452                         goto errout_hw;
1453
1454                 rhashtable_remove_fast(&fold->mask->ht,
1455                                        &fold->ht_node,
1456                                        fold->mask->filter_ht_params);
1457                 idr_replace(&head->handle_idr, fnew, fnew->handle);
1458                 list_replace_rcu(&fold->list, &fnew->list);
1459
1460                 if (!tc_skip_hw(fold->flags))
1461                         fl_hw_destroy_filter(tp, fold, NULL);
1462                 tcf_unbind_filter(tp, &fold->res);
1463                 tcf_exts_get_net(&fold->exts);
1464                 /* Caller holds reference to fold, so refcnt is always > 0
1465                  * after this.
1466                  */
1467                 refcount_dec(&fold->refcnt);
1468                 __fl_put(fold);
1469         } else {
1470                 if (__fl_lookup(fnew->mask, &fnew->mkey)) {
1471                         err = -EEXIST;
1472                         goto errout_hw;
1473                 }
1474
1475                 if (handle) {
1476                         /* user specifies a handle and it doesn't exist */
1477                         err = idr_alloc_u32(&head->handle_idr, fnew, &handle,
1478                                             handle, GFP_ATOMIC);
1479                 } else {
1480                         handle = 1;
1481                         err = idr_alloc_u32(&head->handle_idr, fnew, &handle,
1482                                             INT_MAX, GFP_ATOMIC);
1483                 }
1484                 if (err)
1485                         goto errout_hw;
1486
1487                 fnew->handle = handle;
1488
1489                 err = rhashtable_insert_fast(&fnew->mask->ht, &fnew->ht_node,
1490                                              fnew->mask->filter_ht_params);
1491                 if (err)
1492                         goto errout_idr;
1493
1494                 list_add_tail_rcu(&fnew->list, &fnew->mask->filters);
1495         }
1496
1497         *arg = fnew;
1498
1499         kfree(tb);
1500         kfree(mask);
1501         return 0;
1502
1503 errout_idr:
1504         idr_remove(&head->handle_idr, fnew->handle);
1505 errout_hw:
1506         if (!tc_skip_hw(fnew->flags))
1507                 fl_hw_destroy_filter(tp, fnew, NULL);
1508 errout_mask:
1509         fl_mask_put(head, fnew->mask, false);
1510 errout:
1511         tcf_exts_destroy(&fnew->exts);
1512         kfree(fnew);
1513 errout_tb:
1514         kfree(tb);
1515 errout_mask_alloc:
1516         kfree(mask);
1517 errout_fold:
1518         if (fold)
1519                 __fl_put(fold);
1520         return err;
1521 }
1522
1523 static int fl_delete(struct tcf_proto *tp, void *arg, bool *last,
1524                      bool rtnl_held, struct netlink_ext_ack *extack)
1525 {
1526         struct cls_fl_head *head = fl_head_dereference(tp);
1527         struct cls_fl_filter *f = arg;
1528
1529         rhashtable_remove_fast(&f->mask->ht, &f->ht_node,
1530                                f->mask->filter_ht_params);
1531         __fl_delete(tp, f, extack);
1532         *last = list_empty(&head->masks);
1533         __fl_put(f);
1534
1535         return 0;
1536 }
1537
1538 static void fl_walk(struct tcf_proto *tp, struct tcf_walker *arg,
1539                     bool rtnl_held)
1540 {
1541         struct cls_fl_filter *f;
1542
1543         arg->count = arg->skip;
1544
1545         while ((f = fl_get_next_filter(tp, &arg->cookie)) != NULL) {
1546                 if (arg->fn(tp, f, arg) < 0) {
1547                         __fl_put(f);
1548                         arg->stop = 1;
1549                         break;
1550                 }
1551                 __fl_put(f);
1552                 arg->cookie++;
1553                 arg->count++;
1554         }
1555 }
1556
1557 static int fl_reoffload(struct tcf_proto *tp, bool add, tc_setup_cb_t *cb,
1558                         void *cb_priv, struct netlink_ext_ack *extack)
1559 {
1560         struct cls_fl_head *head = fl_head_dereference(tp);
1561         struct tc_cls_flower_offload cls_flower = {};
1562         struct tcf_block *block = tp->chain->block;
1563         struct fl_flow_mask *mask;
1564         struct cls_fl_filter *f;
1565         int err;
1566
1567         list_for_each_entry(mask, &head->masks, list) {
1568                 list_for_each_entry(f, &mask->filters, list) {
1569                         if (tc_skip_hw(f->flags))
1570                                 continue;
1571
1572                         cls_flower.rule =
1573                                 flow_rule_alloc(tcf_exts_num_actions(&f->exts));
1574                         if (!cls_flower.rule)
1575                                 return -ENOMEM;
1576
1577                         tc_cls_common_offload_init(&cls_flower.common, tp,
1578                                                    f->flags, extack);
1579                         cls_flower.command = add ?
1580                                 TC_CLSFLOWER_REPLACE : TC_CLSFLOWER_DESTROY;
1581                         cls_flower.cookie = (unsigned long)f;
1582                         cls_flower.rule->match.dissector = &mask->dissector;
1583                         cls_flower.rule->match.mask = &mask->key;
1584                         cls_flower.rule->match.key = &f->mkey;
1585
1586                         err = tc_setup_flow_action(&cls_flower.rule->action,
1587                                                    &f->exts);
1588                         if (err) {
1589                                 kfree(cls_flower.rule);
1590                                 if (tc_skip_sw(f->flags)) {
1591                                         NL_SET_ERR_MSG_MOD(extack, "Failed to setup flow action");
1592                                         return err;
1593                                 }
1594                                 continue;
1595                         }
1596
1597                         cls_flower.classid = f->res.classid;
1598
1599                         err = cb(TC_SETUP_CLSFLOWER, &cls_flower, cb_priv);
1600                         kfree(cls_flower.rule);
1601
1602                         if (err) {
1603                                 if (add && tc_skip_sw(f->flags))
1604                                         return err;
1605                                 continue;
1606                         }
1607
1608                         tc_cls_offload_cnt_update(block, &f->in_hw_count,
1609                                                   &f->flags, add);
1610                 }
1611         }
1612
1613         return 0;
1614 }
1615
1616 static int fl_hw_create_tmplt(struct tcf_chain *chain,
1617                               struct fl_flow_tmplt *tmplt)
1618 {
1619         struct tc_cls_flower_offload cls_flower = {};
1620         struct tcf_block *block = chain->block;
1621
1622         cls_flower.rule = flow_rule_alloc(0);
1623         if (!cls_flower.rule)
1624                 return -ENOMEM;
1625
1626         cls_flower.common.chain_index = chain->index;
1627         cls_flower.command = TC_CLSFLOWER_TMPLT_CREATE;
1628         cls_flower.cookie = (unsigned long) tmplt;
1629         cls_flower.rule->match.dissector = &tmplt->dissector;
1630         cls_flower.rule->match.mask = &tmplt->mask;
1631         cls_flower.rule->match.key = &tmplt->dummy_key;
1632
1633         /* We don't care if driver (any of them) fails to handle this
1634          * call. It serves just as a hint for it.
1635          */
1636         tc_setup_cb_call(block, TC_SETUP_CLSFLOWER, &cls_flower, false);
1637         kfree(cls_flower.rule);
1638
1639         return 0;
1640 }
1641
1642 static void fl_hw_destroy_tmplt(struct tcf_chain *chain,
1643                                 struct fl_flow_tmplt *tmplt)
1644 {
1645         struct tc_cls_flower_offload cls_flower = {};
1646         struct tcf_block *block = chain->block;
1647
1648         cls_flower.common.chain_index = chain->index;
1649         cls_flower.command = TC_CLSFLOWER_TMPLT_DESTROY;
1650         cls_flower.cookie = (unsigned long) tmplt;
1651
1652         tc_setup_cb_call(block, TC_SETUP_CLSFLOWER, &cls_flower, false);
1653 }
1654
1655 static void *fl_tmplt_create(struct net *net, struct tcf_chain *chain,
1656                              struct nlattr **tca,
1657                              struct netlink_ext_ack *extack)
1658 {
1659         struct fl_flow_tmplt *tmplt;
1660         struct nlattr **tb;
1661         int err;
1662
1663         if (!tca[TCA_OPTIONS])
1664                 return ERR_PTR(-EINVAL);
1665
1666         tb = kcalloc(TCA_FLOWER_MAX + 1, sizeof(struct nlattr *), GFP_KERNEL);
1667         if (!tb)
1668                 return ERR_PTR(-ENOBUFS);
1669         err = nla_parse_nested(tb, TCA_FLOWER_MAX, tca[TCA_OPTIONS],
1670                                fl_policy, NULL);
1671         if (err)
1672                 goto errout_tb;
1673
1674         tmplt = kzalloc(sizeof(*tmplt), GFP_KERNEL);
1675         if (!tmplt) {
1676                 err = -ENOMEM;
1677                 goto errout_tb;
1678         }
1679         tmplt->chain = chain;
1680         err = fl_set_key(net, tb, &tmplt->dummy_key, &tmplt->mask, extack);
1681         if (err)
1682                 goto errout_tmplt;
1683
1684         fl_init_dissector(&tmplt->dissector, &tmplt->mask);
1685
1686         err = fl_hw_create_tmplt(chain, tmplt);
1687         if (err)
1688                 goto errout_tmplt;
1689
1690         kfree(tb);
1691         return tmplt;
1692
1693 errout_tmplt:
1694         kfree(tmplt);
1695 errout_tb:
1696         kfree(tb);
1697         return ERR_PTR(err);
1698 }
1699
1700 static void fl_tmplt_destroy(void *tmplt_priv)
1701 {
1702         struct fl_flow_tmplt *tmplt = tmplt_priv;
1703
1704         fl_hw_destroy_tmplt(tmplt->chain, tmplt);
1705         kfree(tmplt);
1706 }
1707
1708 static int fl_dump_key_val(struct sk_buff *skb,
1709                            void *val, int val_type,
1710                            void *mask, int mask_type, int len)
1711 {
1712         int err;
1713
1714         if (!memchr_inv(mask, 0, len))
1715                 return 0;
1716         err = nla_put(skb, val_type, len, val);
1717         if (err)
1718                 return err;
1719         if (mask_type != TCA_FLOWER_UNSPEC) {
1720                 err = nla_put(skb, mask_type, len, mask);
1721                 if (err)
1722                         return err;
1723         }
1724         return 0;
1725 }
1726
1727 static int fl_dump_key_port_range(struct sk_buff *skb, struct fl_flow_key *key,
1728                                   struct fl_flow_key *mask)
1729 {
1730         if (fl_dump_key_val(skb, &key->tp_min.dst, TCA_FLOWER_KEY_PORT_DST_MIN,
1731                             &mask->tp_min.dst, TCA_FLOWER_UNSPEC,
1732                             sizeof(key->tp_min.dst)) ||
1733             fl_dump_key_val(skb, &key->tp_max.dst, TCA_FLOWER_KEY_PORT_DST_MAX,
1734                             &mask->tp_max.dst, TCA_FLOWER_UNSPEC,
1735                             sizeof(key->tp_max.dst)) ||
1736             fl_dump_key_val(skb, &key->tp_min.src, TCA_FLOWER_KEY_PORT_SRC_MIN,
1737                             &mask->tp_min.src, TCA_FLOWER_UNSPEC,
1738                             sizeof(key->tp_min.src)) ||
1739             fl_dump_key_val(skb, &key->tp_max.src, TCA_FLOWER_KEY_PORT_SRC_MAX,
1740                             &mask->tp_max.src, TCA_FLOWER_UNSPEC,
1741                             sizeof(key->tp_max.src)))
1742                 return -1;
1743
1744         return 0;
1745 }
1746
1747 static int fl_dump_key_mpls(struct sk_buff *skb,
1748                             struct flow_dissector_key_mpls *mpls_key,
1749                             struct flow_dissector_key_mpls *mpls_mask)
1750 {
1751         int err;
1752
1753         if (!memchr_inv(mpls_mask, 0, sizeof(*mpls_mask)))
1754                 return 0;
1755         if (mpls_mask->mpls_ttl) {
1756                 err = nla_put_u8(skb, TCA_FLOWER_KEY_MPLS_TTL,
1757                                  mpls_key->mpls_ttl);
1758                 if (err)
1759                         return err;
1760         }
1761         if (mpls_mask->mpls_tc) {
1762                 err = nla_put_u8(skb, TCA_FLOWER_KEY_MPLS_TC,
1763                                  mpls_key->mpls_tc);
1764                 if (err)
1765                         return err;
1766         }
1767         if (mpls_mask->mpls_label) {
1768                 err = nla_put_u32(skb, TCA_FLOWER_KEY_MPLS_LABEL,
1769                                   mpls_key->mpls_label);
1770                 if (err)
1771                         return err;
1772         }
1773         if (mpls_mask->mpls_bos) {
1774                 err = nla_put_u8(skb, TCA_FLOWER_KEY_MPLS_BOS,
1775                                  mpls_key->mpls_bos);
1776                 if (err)
1777                         return err;
1778         }
1779         return 0;
1780 }
1781
1782 static int fl_dump_key_ip(struct sk_buff *skb, bool encap,
1783                           struct flow_dissector_key_ip *key,
1784                           struct flow_dissector_key_ip *mask)
1785 {
1786         int tos_key = encap ? TCA_FLOWER_KEY_ENC_IP_TOS : TCA_FLOWER_KEY_IP_TOS;
1787         int ttl_key = encap ? TCA_FLOWER_KEY_ENC_IP_TTL : TCA_FLOWER_KEY_IP_TTL;
1788         int tos_mask = encap ? TCA_FLOWER_KEY_ENC_IP_TOS_MASK : TCA_FLOWER_KEY_IP_TOS_MASK;
1789         int ttl_mask = encap ? TCA_FLOWER_KEY_ENC_IP_TTL_MASK : TCA_FLOWER_KEY_IP_TTL_MASK;
1790
1791         if (fl_dump_key_val(skb, &key->tos, tos_key, &mask->tos, tos_mask, sizeof(key->tos)) ||
1792             fl_dump_key_val(skb, &key->ttl, ttl_key, &mask->ttl, ttl_mask, sizeof(key->ttl)))
1793                 return -1;
1794
1795         return 0;
1796 }
1797
1798 static int fl_dump_key_vlan(struct sk_buff *skb,
1799                             int vlan_id_key, int vlan_prio_key,
1800                             struct flow_dissector_key_vlan *vlan_key,
1801                             struct flow_dissector_key_vlan *vlan_mask)
1802 {
1803         int err;
1804
1805         if (!memchr_inv(vlan_mask, 0, sizeof(*vlan_mask)))
1806                 return 0;
1807         if (vlan_mask->vlan_id) {
1808                 err = nla_put_u16(skb, vlan_id_key,
1809                                   vlan_key->vlan_id);
1810                 if (err)
1811                         return err;
1812         }
1813         if (vlan_mask->vlan_priority) {
1814                 err = nla_put_u8(skb, vlan_prio_key,
1815                                  vlan_key->vlan_priority);
1816                 if (err)
1817                         return err;
1818         }
1819         return 0;
1820 }
1821
1822 static void fl_get_key_flag(u32 dissector_key, u32 dissector_mask,
1823                             u32 *flower_key, u32 *flower_mask,
1824                             u32 flower_flag_bit, u32 dissector_flag_bit)
1825 {
1826         if (dissector_mask & dissector_flag_bit) {
1827                 *flower_mask |= flower_flag_bit;
1828                 if (dissector_key & dissector_flag_bit)
1829                         *flower_key |= flower_flag_bit;
1830         }
1831 }
1832
1833 static int fl_dump_key_flags(struct sk_buff *skb, u32 flags_key, u32 flags_mask)
1834 {
1835         u32 key, mask;
1836         __be32 _key, _mask;
1837         int err;
1838
1839         if (!memchr_inv(&flags_mask, 0, sizeof(flags_mask)))
1840                 return 0;
1841
1842         key = 0;
1843         mask = 0;
1844
1845         fl_get_key_flag(flags_key, flags_mask, &key, &mask,
1846                         TCA_FLOWER_KEY_FLAGS_IS_FRAGMENT, FLOW_DIS_IS_FRAGMENT);
1847         fl_get_key_flag(flags_key, flags_mask, &key, &mask,
1848                         TCA_FLOWER_KEY_FLAGS_FRAG_IS_FIRST,
1849                         FLOW_DIS_FIRST_FRAG);
1850
1851         _key = cpu_to_be32(key);
1852         _mask = cpu_to_be32(mask);
1853
1854         err = nla_put(skb, TCA_FLOWER_KEY_FLAGS, 4, &_key);
1855         if (err)
1856                 return err;
1857
1858         return nla_put(skb, TCA_FLOWER_KEY_FLAGS_MASK, 4, &_mask);
1859 }
1860
1861 static int fl_dump_key_geneve_opt(struct sk_buff *skb,
1862                                   struct flow_dissector_key_enc_opts *enc_opts)
1863 {
1864         struct geneve_opt *opt;
1865         struct nlattr *nest;
1866         int opt_off = 0;
1867
1868         nest = nla_nest_start(skb, TCA_FLOWER_KEY_ENC_OPTS_GENEVE);
1869         if (!nest)
1870                 goto nla_put_failure;
1871
1872         while (enc_opts->len > opt_off) {
1873                 opt = (struct geneve_opt *)&enc_opts->data[opt_off];
1874
1875                 if (nla_put_be16(skb, TCA_FLOWER_KEY_ENC_OPT_GENEVE_CLASS,
1876                                  opt->opt_class))
1877                         goto nla_put_failure;
1878                 if (nla_put_u8(skb, TCA_FLOWER_KEY_ENC_OPT_GENEVE_TYPE,
1879                                opt->type))
1880                         goto nla_put_failure;
1881                 if (nla_put(skb, TCA_FLOWER_KEY_ENC_OPT_GENEVE_DATA,
1882                             opt->length * 4, opt->opt_data))
1883                         goto nla_put_failure;
1884
1885                 opt_off += sizeof(struct geneve_opt) + opt->length * 4;
1886         }
1887         nla_nest_end(skb, nest);
1888         return 0;
1889
1890 nla_put_failure:
1891         nla_nest_cancel(skb, nest);
1892         return -EMSGSIZE;
1893 }
1894
1895 static int fl_dump_key_options(struct sk_buff *skb, int enc_opt_type,
1896                                struct flow_dissector_key_enc_opts *enc_opts)
1897 {
1898         struct nlattr *nest;
1899         int err;
1900
1901         if (!enc_opts->len)
1902                 return 0;
1903
1904         nest = nla_nest_start(skb, enc_opt_type);
1905         if (!nest)
1906                 goto nla_put_failure;
1907
1908         switch (enc_opts->dst_opt_type) {
1909         case TUNNEL_GENEVE_OPT:
1910                 err = fl_dump_key_geneve_opt(skb, enc_opts);
1911                 if (err)
1912                         goto nla_put_failure;
1913                 break;
1914         default:
1915                 goto nla_put_failure;
1916         }
1917         nla_nest_end(skb, nest);
1918         return 0;
1919
1920 nla_put_failure:
1921         nla_nest_cancel(skb, nest);
1922         return -EMSGSIZE;
1923 }
1924
1925 static int fl_dump_key_enc_opt(struct sk_buff *skb,
1926                                struct flow_dissector_key_enc_opts *key_opts,
1927                                struct flow_dissector_key_enc_opts *msk_opts)
1928 {
1929         int err;
1930
1931         err = fl_dump_key_options(skb, TCA_FLOWER_KEY_ENC_OPTS, key_opts);
1932         if (err)
1933                 return err;
1934
1935         return fl_dump_key_options(skb, TCA_FLOWER_KEY_ENC_OPTS_MASK, msk_opts);
1936 }
1937
1938 static int fl_dump_key(struct sk_buff *skb, struct net *net,
1939                        struct fl_flow_key *key, struct fl_flow_key *mask)
1940 {
1941         if (mask->indev_ifindex) {
1942                 struct net_device *dev;
1943
1944                 dev = __dev_get_by_index(net, key->indev_ifindex);
1945                 if (dev && nla_put_string(skb, TCA_FLOWER_INDEV, dev->name))
1946                         goto nla_put_failure;
1947         }
1948
1949         if (fl_dump_key_val(skb, key->eth.dst, TCA_FLOWER_KEY_ETH_DST,
1950                             mask->eth.dst, TCA_FLOWER_KEY_ETH_DST_MASK,
1951                             sizeof(key->eth.dst)) ||
1952             fl_dump_key_val(skb, key->eth.src, TCA_FLOWER_KEY_ETH_SRC,
1953                             mask->eth.src, TCA_FLOWER_KEY_ETH_SRC_MASK,
1954                             sizeof(key->eth.src)) ||
1955             fl_dump_key_val(skb, &key->basic.n_proto, TCA_FLOWER_KEY_ETH_TYPE,
1956                             &mask->basic.n_proto, TCA_FLOWER_UNSPEC,
1957                             sizeof(key->basic.n_proto)))
1958                 goto nla_put_failure;
1959
1960         if (fl_dump_key_mpls(skb, &key->mpls, &mask->mpls))
1961                 goto nla_put_failure;
1962
1963         if (fl_dump_key_vlan(skb, TCA_FLOWER_KEY_VLAN_ID,
1964                              TCA_FLOWER_KEY_VLAN_PRIO, &key->vlan, &mask->vlan))
1965                 goto nla_put_failure;
1966
1967         if (fl_dump_key_vlan(skb, TCA_FLOWER_KEY_CVLAN_ID,
1968                              TCA_FLOWER_KEY_CVLAN_PRIO,
1969                              &key->cvlan, &mask->cvlan) ||
1970             (mask->cvlan.vlan_tpid &&
1971              nla_put_be16(skb, TCA_FLOWER_KEY_VLAN_ETH_TYPE,
1972                           key->cvlan.vlan_tpid)))
1973                 goto nla_put_failure;
1974
1975         if (mask->basic.n_proto) {
1976                 if (mask->cvlan.vlan_tpid) {
1977                         if (nla_put_be16(skb, TCA_FLOWER_KEY_CVLAN_ETH_TYPE,
1978                                          key->basic.n_proto))
1979                                 goto nla_put_failure;
1980                 } else if (mask->vlan.vlan_tpid) {
1981                         if (nla_put_be16(skb, TCA_FLOWER_KEY_VLAN_ETH_TYPE,
1982                                          key->basic.n_proto))
1983                                 goto nla_put_failure;
1984                 }
1985         }
1986
1987         if ((key->basic.n_proto == htons(ETH_P_IP) ||
1988              key->basic.n_proto == htons(ETH_P_IPV6)) &&
1989             (fl_dump_key_val(skb, &key->basic.ip_proto, TCA_FLOWER_KEY_IP_PROTO,
1990                             &mask->basic.ip_proto, TCA_FLOWER_UNSPEC,
1991                             sizeof(key->basic.ip_proto)) ||
1992             fl_dump_key_ip(skb, false, &key->ip, &mask->ip)))
1993                 goto nla_put_failure;
1994
1995         if (key->control.addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS &&
1996             (fl_dump_key_val(skb, &key->ipv4.src, TCA_FLOWER_KEY_IPV4_SRC,
1997                              &mask->ipv4.src, TCA_FLOWER_KEY_IPV4_SRC_MASK,
1998                              sizeof(key->ipv4.src)) ||
1999              fl_dump_key_val(skb, &key->ipv4.dst, TCA_FLOWER_KEY_IPV4_DST,
2000                              &mask->ipv4.dst, TCA_FLOWER_KEY_IPV4_DST_MASK,
2001                              sizeof(key->ipv4.dst))))
2002                 goto nla_put_failure;
2003         else if (key->control.addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS &&
2004                  (fl_dump_key_val(skb, &key->ipv6.src, TCA_FLOWER_KEY_IPV6_SRC,
2005                                   &mask->ipv6.src, TCA_FLOWER_KEY_IPV6_SRC_MASK,
2006                                   sizeof(key->ipv6.src)) ||
2007                   fl_dump_key_val(skb, &key->ipv6.dst, TCA_FLOWER_KEY_IPV6_DST,
2008                                   &mask->ipv6.dst, TCA_FLOWER_KEY_IPV6_DST_MASK,
2009                                   sizeof(key->ipv6.dst))))
2010                 goto nla_put_failure;
2011
2012         if (key->basic.ip_proto == IPPROTO_TCP &&
2013             (fl_dump_key_val(skb, &key->tp.src, TCA_FLOWER_KEY_TCP_SRC,
2014                              &mask->tp.src, TCA_FLOWER_KEY_TCP_SRC_MASK,
2015                              sizeof(key->tp.src)) ||
2016              fl_dump_key_val(skb, &key->tp.dst, TCA_FLOWER_KEY_TCP_DST,
2017                              &mask->tp.dst, TCA_FLOWER_KEY_TCP_DST_MASK,
2018                              sizeof(key->tp.dst)) ||
2019              fl_dump_key_val(skb, &key->tcp.flags, TCA_FLOWER_KEY_TCP_FLAGS,
2020                              &mask->tcp.flags, TCA_FLOWER_KEY_TCP_FLAGS_MASK,
2021                              sizeof(key->tcp.flags))))
2022                 goto nla_put_failure;
2023         else if (key->basic.ip_proto == IPPROTO_UDP &&
2024                  (fl_dump_key_val(skb, &key->tp.src, TCA_FLOWER_KEY_UDP_SRC,
2025                                   &mask->tp.src, TCA_FLOWER_KEY_UDP_SRC_MASK,
2026                                   sizeof(key->tp.src)) ||
2027                   fl_dump_key_val(skb, &key->tp.dst, TCA_FLOWER_KEY_UDP_DST,
2028                                   &mask->tp.dst, TCA_FLOWER_KEY_UDP_DST_MASK,
2029                                   sizeof(key->tp.dst))))
2030                 goto nla_put_failure;
2031         else if (key->basic.ip_proto == IPPROTO_SCTP &&
2032                  (fl_dump_key_val(skb, &key->tp.src, TCA_FLOWER_KEY_SCTP_SRC,
2033                                   &mask->tp.src, TCA_FLOWER_KEY_SCTP_SRC_MASK,
2034                                   sizeof(key->tp.src)) ||
2035                   fl_dump_key_val(skb, &key->tp.dst, TCA_FLOWER_KEY_SCTP_DST,
2036                                   &mask->tp.dst, TCA_FLOWER_KEY_SCTP_DST_MASK,
2037                                   sizeof(key->tp.dst))))
2038                 goto nla_put_failure;
2039         else if (key->basic.n_proto == htons(ETH_P_IP) &&
2040                  key->basic.ip_proto == IPPROTO_ICMP &&
2041                  (fl_dump_key_val(skb, &key->icmp.type,
2042                                   TCA_FLOWER_KEY_ICMPV4_TYPE, &mask->icmp.type,
2043                                   TCA_FLOWER_KEY_ICMPV4_TYPE_MASK,
2044                                   sizeof(key->icmp.type)) ||
2045                   fl_dump_key_val(skb, &key->icmp.code,
2046                                   TCA_FLOWER_KEY_ICMPV4_CODE, &mask->icmp.code,
2047                                   TCA_FLOWER_KEY_ICMPV4_CODE_MASK,
2048                                   sizeof(key->icmp.code))))
2049                 goto nla_put_failure;
2050         else if (key->basic.n_proto == htons(ETH_P_IPV6) &&
2051                  key->basic.ip_proto == IPPROTO_ICMPV6 &&
2052                  (fl_dump_key_val(skb, &key->icmp.type,
2053                                   TCA_FLOWER_KEY_ICMPV6_TYPE, &mask->icmp.type,
2054                                   TCA_FLOWER_KEY_ICMPV6_TYPE_MASK,
2055                                   sizeof(key->icmp.type)) ||
2056                   fl_dump_key_val(skb, &key->icmp.code,
2057                                   TCA_FLOWER_KEY_ICMPV6_CODE, &mask->icmp.code,
2058                                   TCA_FLOWER_KEY_ICMPV6_CODE_MASK,
2059                                   sizeof(key->icmp.code))))
2060                 goto nla_put_failure;
2061         else if ((key->basic.n_proto == htons(ETH_P_ARP) ||
2062                   key->basic.n_proto == htons(ETH_P_RARP)) &&
2063                  (fl_dump_key_val(skb, &key->arp.sip,
2064                                   TCA_FLOWER_KEY_ARP_SIP, &mask->arp.sip,
2065                                   TCA_FLOWER_KEY_ARP_SIP_MASK,
2066                                   sizeof(key->arp.sip)) ||
2067                   fl_dump_key_val(skb, &key->arp.tip,
2068                                   TCA_FLOWER_KEY_ARP_TIP, &mask->arp.tip,
2069                                   TCA_FLOWER_KEY_ARP_TIP_MASK,
2070                                   sizeof(key->arp.tip)) ||
2071                   fl_dump_key_val(skb, &key->arp.op,
2072                                   TCA_FLOWER_KEY_ARP_OP, &mask->arp.op,
2073                                   TCA_FLOWER_KEY_ARP_OP_MASK,
2074                                   sizeof(key->arp.op)) ||
2075                   fl_dump_key_val(skb, key->arp.sha, TCA_FLOWER_KEY_ARP_SHA,
2076                                   mask->arp.sha, TCA_FLOWER_KEY_ARP_SHA_MASK,
2077                                   sizeof(key->arp.sha)) ||
2078                   fl_dump_key_val(skb, key->arp.tha, TCA_FLOWER_KEY_ARP_THA,
2079                                   mask->arp.tha, TCA_FLOWER_KEY_ARP_THA_MASK,
2080                                   sizeof(key->arp.tha))))
2081                 goto nla_put_failure;
2082
2083         if ((key->basic.ip_proto == IPPROTO_TCP ||
2084              key->basic.ip_proto == IPPROTO_UDP ||
2085              key->basic.ip_proto == IPPROTO_SCTP) &&
2086              fl_dump_key_port_range(skb, key, mask))
2087                 goto nla_put_failure;
2088
2089         if (key->enc_control.addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS &&
2090             (fl_dump_key_val(skb, &key->enc_ipv4.src,
2091                             TCA_FLOWER_KEY_ENC_IPV4_SRC, &mask->enc_ipv4.src,
2092                             TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK,
2093                             sizeof(key->enc_ipv4.src)) ||
2094              fl_dump_key_val(skb, &key->enc_ipv4.dst,
2095                              TCA_FLOWER_KEY_ENC_IPV4_DST, &mask->enc_ipv4.dst,
2096                              TCA_FLOWER_KEY_ENC_IPV4_DST_MASK,
2097                              sizeof(key->enc_ipv4.dst))))
2098                 goto nla_put_failure;
2099         else if (key->enc_control.addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS &&
2100                  (fl_dump_key_val(skb, &key->enc_ipv6.src,
2101                             TCA_FLOWER_KEY_ENC_IPV6_SRC, &mask->enc_ipv6.src,
2102                             TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK,
2103                             sizeof(key->enc_ipv6.src)) ||
2104                  fl_dump_key_val(skb, &key->enc_ipv6.dst,
2105                                  TCA_FLOWER_KEY_ENC_IPV6_DST,
2106                                  &mask->enc_ipv6.dst,
2107                                  TCA_FLOWER_KEY_ENC_IPV6_DST_MASK,
2108                             sizeof(key->enc_ipv6.dst))))
2109                 goto nla_put_failure;
2110
2111         if (fl_dump_key_val(skb, &key->enc_key_id, TCA_FLOWER_KEY_ENC_KEY_ID,
2112                             &mask->enc_key_id, TCA_FLOWER_UNSPEC,
2113                             sizeof(key->enc_key_id)) ||
2114             fl_dump_key_val(skb, &key->enc_tp.src,
2115                             TCA_FLOWER_KEY_ENC_UDP_SRC_PORT,
2116                             &mask->enc_tp.src,
2117                             TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK,
2118                             sizeof(key->enc_tp.src)) ||
2119             fl_dump_key_val(skb, &key->enc_tp.dst,
2120                             TCA_FLOWER_KEY_ENC_UDP_DST_PORT,
2121                             &mask->enc_tp.dst,
2122                             TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK,
2123                             sizeof(key->enc_tp.dst)) ||
2124             fl_dump_key_ip(skb, true, &key->enc_ip, &mask->enc_ip) ||
2125             fl_dump_key_enc_opt(skb, &key->enc_opts, &mask->enc_opts))
2126                 goto nla_put_failure;
2127
2128         if (fl_dump_key_flags(skb, key->control.flags, mask->control.flags))
2129                 goto nla_put_failure;
2130
2131         return 0;
2132
2133 nla_put_failure:
2134         return -EMSGSIZE;
2135 }
2136
2137 static int fl_dump(struct net *net, struct tcf_proto *tp, void *fh,
2138                    struct sk_buff *skb, struct tcmsg *t, bool rtnl_held)
2139 {
2140         struct cls_fl_filter *f = fh;
2141         struct nlattr *nest;
2142         struct fl_flow_key *key, *mask;
2143
2144         if (!f)
2145                 return skb->len;
2146
2147         t->tcm_handle = f->handle;
2148
2149         nest = nla_nest_start(skb, TCA_OPTIONS);
2150         if (!nest)
2151                 goto nla_put_failure;
2152
2153         if (f->res.classid &&
2154             nla_put_u32(skb, TCA_FLOWER_CLASSID, f->res.classid))
2155                 goto nla_put_failure;
2156
2157         key = &f->key;
2158         mask = &f->mask->key;
2159
2160         if (fl_dump_key(skb, net, key, mask))
2161                 goto nla_put_failure;
2162
2163         if (!tc_skip_hw(f->flags))
2164                 fl_hw_update_stats(tp, f);
2165
2166         if (f->flags && nla_put_u32(skb, TCA_FLOWER_FLAGS, f->flags))
2167                 goto nla_put_failure;
2168
2169         if (nla_put_u32(skb, TCA_FLOWER_IN_HW_COUNT, f->in_hw_count))
2170                 goto nla_put_failure;
2171
2172         if (tcf_exts_dump(skb, &f->exts))
2173                 goto nla_put_failure;
2174
2175         nla_nest_end(skb, nest);
2176
2177         if (tcf_exts_dump_stats(skb, &f->exts) < 0)
2178                 goto nla_put_failure;
2179
2180         return skb->len;
2181
2182 nla_put_failure:
2183         nla_nest_cancel(skb, nest);
2184         return -1;
2185 }
2186
2187 static int fl_tmplt_dump(struct sk_buff *skb, struct net *net, void *tmplt_priv)
2188 {
2189         struct fl_flow_tmplt *tmplt = tmplt_priv;
2190         struct fl_flow_key *key, *mask;
2191         struct nlattr *nest;
2192
2193         nest = nla_nest_start(skb, TCA_OPTIONS);
2194         if (!nest)
2195                 goto nla_put_failure;
2196
2197         key = &tmplt->dummy_key;
2198         mask = &tmplt->mask;
2199
2200         if (fl_dump_key(skb, net, key, mask))
2201                 goto nla_put_failure;
2202
2203         nla_nest_end(skb, nest);
2204
2205         return skb->len;
2206
2207 nla_put_failure:
2208         nla_nest_cancel(skb, nest);
2209         return -EMSGSIZE;
2210 }
2211
2212 static void fl_bind_class(void *fh, u32 classid, unsigned long cl)
2213 {
2214         struct cls_fl_filter *f = fh;
2215
2216         if (f && f->res.classid == classid)
2217                 f->res.class = cl;
2218 }
2219
2220 static struct tcf_proto_ops cls_fl_ops __read_mostly = {
2221         .kind           = "flower",
2222         .classify       = fl_classify,
2223         .init           = fl_init,
2224         .destroy        = fl_destroy,
2225         .get            = fl_get,
2226         .put            = fl_put,
2227         .change         = fl_change,
2228         .delete         = fl_delete,
2229         .walk           = fl_walk,
2230         .reoffload      = fl_reoffload,
2231         .dump           = fl_dump,
2232         .bind_class     = fl_bind_class,
2233         .tmplt_create   = fl_tmplt_create,
2234         .tmplt_destroy  = fl_tmplt_destroy,
2235         .tmplt_dump     = fl_tmplt_dump,
2236         .owner          = THIS_MODULE,
2237 };
2238
2239 static int __init cls_fl_init(void)
2240 {
2241         return register_tcf_proto_ops(&cls_fl_ops);
2242 }
2243
2244 static void __exit cls_fl_exit(void)
2245 {
2246         unregister_tcf_proto_ops(&cls_fl_ops);
2247 }
2248
2249 module_init(cls_fl_init);
2250 module_exit(cls_fl_exit);
2251
2252 MODULE_AUTHOR("Jiri Pirko <jiri@resnulli.us>");
2253 MODULE_DESCRIPTION("Flower classifier");
2254 MODULE_LICENSE("GPL v2");