net/sched/cls_api.c

   1 /*
   2  * net/sched/cls_api.c  Packet classifier API.
   3  *
   4  *              This program is free software; you can redistribute it and/or
   5  *              modify it under the terms of the GNU General Public License
   6  *              as published by the Free Software Foundation; either version
   7  *              2 of the License, or (at your option) any later version.
   8  *
   9  * Authors:     Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
  10  *
  11  * Changes:
  12  *
  13  * Eduardo J. Blanco <ejbs@netlabs.com.uy> :990222: kmod support
  14  *
  15  */
  16
  17 #include <linux/module.h>
  18 #include <linux/types.h>
  19 #include <linux/kernel.h>
  20 #include <linux/string.h>
  21 #include <linux/errno.h>
  22 #include <linux/err.h>
  23 #include <linux/skbuff.h>
  24 #include <linux/init.h>
  25 #include <linux/kmod.h>
  26 #include <linux/slab.h>
  27 #include <linux/idr.h>
  28 #include <linux/rhashtable.h>
  29 #include <net/net_namespace.h>
  30 #include <net/sock.h>
  31 #include <net/netlink.h>
  32 #include <net/pkt_sched.h>
  33 #include <net/pkt_cls.h>
  34 #include <net/tc_act/tc_pedit.h>
  35 #include <net/tc_act/tc_mirred.h>
  36 #include <net/tc_act/tc_vlan.h>
  37 #include <net/tc_act/tc_tunnel_key.h>
  38 #include <net/tc_act/tc_csum.h>
  39 #include <net/tc_act/tc_gact.h>
  40 #include <net/tc_act/tc_skbedit.h>
  41 #include <net/tc_act/tc_mirred.h>
  42
  43 extern const struct nla_policy rtm_tca_policy[TCA_MAX + 1];
  44
  45 /* The list of all installed classifier types */
  46 static LIST_HEAD(tcf_proto_base);
  47
  48 /* Protects list of registered TC modules. It is pure SMP lock. */
  49 static DEFINE_RWLOCK(cls_mod_lock);
  50
  51 /* Find classifier type by string name */
  52
  53 static const struct tcf_proto_ops *__tcf_proto_lookup_ops(const char *kind)
  54 {
  55         const struct tcf_proto_ops *t, *res = NULL;
  56
  57         if (kind) {
  58                 read_lock(&cls_mod_lock);
  59                 list_for_each_entry(t, &tcf_proto_base, head) {
  60                         if (strcmp(kind, t->kind) == 0) {
  61                                 if (try_module_get(t->owner))
  62                                         res = t;
  63                                 break;
  64                         }
  65                 }
  66                 read_unlock(&cls_mod_lock);
  67         }
  68         return res;
  69 }
  70
  71 static const struct tcf_proto_ops *
  72 tcf_proto_lookup_ops(const char *kind, struct netlink_ext_ack *extack)
  73 {
  74         const struct tcf_proto_ops *ops;
  75
  76         ops = __tcf_proto_lookup_ops(kind);
  77         if (ops)
  78                 return ops;
  79 #ifdef CONFIG_MODULES
  80         rtnl_unlock();
  81         request_module("cls_%s", kind);
  82         rtnl_lock();
  83         ops = __tcf_proto_lookup_ops(kind);
  84         /* We dropped the RTNL semaphore in order to perform
  85          * the module load. So, even if we succeeded in loading
  86          * the module we have to replay the request. We indicate
  87          * this using -EAGAIN.
  88          */
  89         if (ops) {
  90                 module_put(ops->owner);
  91                 return ERR_PTR(-EAGAIN);
  92         }
  93 #endif
  94         NL_SET_ERR_MSG(extack, "TC classifier not found");
  95         return ERR_PTR(-ENOENT);
  96 }
  97
  98 /* Register(unregister) new classifier type */
  99
 100 int register_tcf_proto_ops(struct tcf_proto_ops *ops)
 101 {
 102         struct tcf_proto_ops *t;
 103         int rc = -EEXIST;
 104
 105         write_lock(&cls_mod_lock);
 106         list_for_each_entry(t, &tcf_proto_base, head)
 107                 if (!strcmp(ops->kind, t->kind))
 108                         goto out;
 109
 110         list_add_tail(&ops->head, &tcf_proto_base);
 111         rc = 0;
 112 out:
 113         write_unlock(&cls_mod_lock);
 114         return rc;
 115 }
 116 EXPORT_SYMBOL(register_tcf_proto_ops);
 117
 118 static struct workqueue_struct *tc_filter_wq;
 119
 120 int unregister_tcf_proto_ops(struct tcf_proto_ops *ops)
 121 {
 122         struct tcf_proto_ops *t;
 123         int rc = -ENOENT;
 124
 125         /* Wait for outstanding call_rcu()s, if any, from a
 126          * tcf_proto_ops's destroy() handler.
 127          */
 128         rcu_barrier();
 129         flush_workqueue(tc_filter_wq);
 130
 131         write_lock(&cls_mod_lock);
 132         list_for_each_entry(t, &tcf_proto_base, head) {
 133                 if (t == ops) {
 134                         list_del(&t->head);
 135                         rc = 0;
 136                         break;
 137                 }
 138         }
 139         write_unlock(&cls_mod_lock);
 140         return rc;
 141 }
 142 EXPORT_SYMBOL(unregister_tcf_proto_ops);
 143
 144 bool tcf_queue_work(struct rcu_work *rwork, work_func_t func)
 145 {
 146         INIT_RCU_WORK(rwork, func);
 147         return queue_rcu_work(tc_filter_wq, rwork);
 148 }
 149 EXPORT_SYMBOL(tcf_queue_work);
 150
 151 /* Select new prio value from the range, managed by kernel. */
 152
 153 static inline u32 tcf_auto_prio(struct tcf_proto *tp)
 154 {
 155         u32 first = TC_H_MAKE(0xC0000000U, 0U);
 156
 157         if (tp)
 158                 first = tp->prio - 1;
 159
 160         return TC_H_MAJ(first);
 161 }
 162
 163 static struct tcf_proto *tcf_proto_create(const char *kind, u32 protocol,
 164                                           u32 prio, struct tcf_chain *chain,
 165                                           struct netlink_ext_ack *extack)
 166 {
 167         struct tcf_proto *tp;
 168         int err;
 169
 170         tp = kzalloc(sizeof(*tp), GFP_KERNEL);
 171         if (!tp)
 172                 return ERR_PTR(-ENOBUFS);
 173
 174         tp->ops = tcf_proto_lookup_ops(kind, extack);
 175         if (IS_ERR(tp->ops)) {
 176                 err = PTR_ERR(tp->ops);
 177                 goto errout;
 178         }
 179         tp->classify = tp->ops->classify;
 180         tp->protocol = protocol;
 181         tp->prio = prio;
 182         tp->chain = chain;
 183
 184         err = tp->ops->init(tp);
 185         if (err) {
 186                 module_put(tp->ops->owner);
 187                 goto errout;
 188         }
 189         return tp;
 190
 191 errout:
 192         kfree(tp);
 193         return ERR_PTR(err);
 194 }
 195
 196 static void tcf_proto_destroy(struct tcf_proto *tp,
 197                               struct netlink_ext_ack *extack)
 198 {
 199         tp->ops->destroy(tp, extack);
 200         module_put(tp->ops->owner);
 201         kfree_rcu(tp, rcu);
 202 }
 203
 204 #define ASSERT_BLOCK_LOCKED(block)                                      \
 205         lockdep_assert_held(&(block)->lock)
 206
 207 struct tcf_filter_chain_list_item {
 208         struct list_head list;
 209         tcf_chain_head_change_t *chain_head_change;
 210         void *chain_head_change_priv;
 211 };
 212
 213 static struct tcf_chain *tcf_chain_create(struct tcf_block *block,
 214                                           u32 chain_index)
 215 {
 216         struct tcf_chain *chain;
 217
 218         ASSERT_BLOCK_LOCKED(block);
 219
 220         chain = kzalloc(sizeof(*chain), GFP_KERNEL);
 221         if (!chain)
 222                 return NULL;
 223         list_add_tail(&chain->list, &block->chain_list);
 224         chain->block = block;
 225         chain->index = chain_index;
 226         chain->refcnt = 1;
 227         if (!chain->index)
 228                 block->chain0.chain = chain;
 229         return chain;
 230 }
 231
 232 static void tcf_chain_head_change_item(struct tcf_filter_chain_list_item *item,
 233                                        struct tcf_proto *tp_head)
 234 {
 235         if (item->chain_head_change)
 236                 item->chain_head_change(tp_head, item->chain_head_change_priv);
 237 }
 238
 239 static void tcf_chain0_head_change(struct tcf_chain *chain,
 240                                    struct tcf_proto *tp_head)
 241 {
 242         struct tcf_filter_chain_list_item *item;
 243         struct tcf_block *block = chain->block;
 244
 245         if (chain->index)
 246                 return;
 247
 248         mutex_lock(&block->lock);
 249         list_for_each_entry(item, &block->chain0.filter_chain_list, list)
 250                 tcf_chain_head_change_item(item, tp_head);
 251         mutex_unlock(&block->lock);
 252 }
 253
 254 /* Returns true if block can be safely freed. */
 255
 256 static bool tcf_chain_detach(struct tcf_chain *chain)
 257 {
 258         struct tcf_block *block = chain->block;
 259
 260         ASSERT_BLOCK_LOCKED(block);
 261
 262         list_del(&chain->list);
 263         if (!chain->index)
 264                 block->chain0.chain = NULL;
 265
 266         if (list_empty(&block->chain_list) &&
 267             refcount_read(&block->refcnt) == 0)
 268                 return true;
 269
 270         return false;
 271 }
 272
 273 static void tcf_block_destroy(struct tcf_block *block)
 274 {
 275         mutex_destroy(&block->lock);
 276         kfree_rcu(block, rcu);
 277 }
 278
 279 static void tcf_chain_destroy(struct tcf_chain *chain, bool free_block)
 280 {
 281         struct tcf_block *block = chain->block;
 282
 283         kfree(chain);
 284         if (free_block)
 285                 tcf_block_destroy(block);
 286 }
 287
 288 static void tcf_chain_hold(struct tcf_chain *chain)
 289 {
 290         ASSERT_BLOCK_LOCKED(chain->block);
 291
 292         ++chain->refcnt;
 293 }
 294
 295 static bool tcf_chain_held_by_acts_only(struct tcf_chain *chain)
 296 {
 297         ASSERT_BLOCK_LOCKED(chain->block);
 298
 299         /* In case all the references are action references, this
 300          * chain should not be shown to the user.
 301          */
 302         return chain->refcnt == chain->action_refcnt;
 303 }
 304
 305 static struct tcf_chain *tcf_chain_lookup(struct tcf_block *block,
 306                                           u32 chain_index)
 307 {
 308         struct tcf_chain *chain;
 309
 310         ASSERT_BLOCK_LOCKED(block);
 311
 312         list_for_each_entry(chain, &block->chain_list, list) {
 313                 if (chain->index == chain_index)
 314                         return chain;
 315         }
 316         return NULL;
 317 }
 318
 319 static int tc_chain_notify(struct tcf_chain *chain, struct sk_buff *oskb,
 320                            u32 seq, u16 flags, int event, bool unicast);
 321
 322 static struct tcf_chain *__tcf_chain_get(struct tcf_block *block,
 323                                          u32 chain_index, bool create,
 324                                          bool by_act)
 325 {
 326         struct tcf_chain *chain = NULL;
 327         bool is_first_reference;
 328
 329         mutex_lock(&block->lock);
 330         chain = tcf_chain_lookup(block, chain_index);
 331         if (chain) {
 332                 tcf_chain_hold(chain);
 333         } else {
 334                 if (!create)
 335                         goto errout;
 336                 chain = tcf_chain_create(block, chain_index);
 337                 if (!chain)
 338                         goto errout;
 339         }
 340
 341         if (by_act)
 342                 ++chain->action_refcnt;
 343         is_first_reference = chain->refcnt - chain->action_refcnt == 1;
 344         mutex_unlock(&block->lock);
 345
 346         /* Send notification only in case we got the first
 347          * non-action reference. Until then, the chain acts only as
 348          * a placeholder for actions pointing to it and user ought
 349          * not know about them.
 350          */
 351         if (is_first_reference && !by_act)
 352                 tc_chain_notify(chain, NULL, 0, NLM_F_CREATE | NLM_F_EXCL,
 353                                 RTM_NEWCHAIN, false);
 354
 355         return chain;
 356
 357 errout:
 358         mutex_unlock(&block->lock);
 359         return chain;
 360 }
 361
 362 static struct tcf_chain *tcf_chain_get(struct tcf_block *block, u32 chain_index,
 363                                        bool create)
 364 {
 365         return __tcf_chain_get(block, chain_index, create, false);
 366 }
 367
 368 struct tcf_chain *tcf_chain_get_by_act(struct tcf_block *block, u32 chain_index)
 369 {
 370         return __tcf_chain_get(block, chain_index, true, true);
 371 }
 372 EXPORT_SYMBOL(tcf_chain_get_by_act);
 373
 374 static void tc_chain_tmplt_del(const struct tcf_proto_ops *tmplt_ops,
 375                                void *tmplt_priv);
 376 static int tc_chain_notify_delete(const struct tcf_proto_ops *tmplt_ops,
 377                                   void *tmplt_priv, u32 chain_index,
 378                                   struct tcf_block *block, struct sk_buff *oskb,
 379                                   u32 seq, u16 flags, bool unicast);
 380
 381 static void __tcf_chain_put(struct tcf_chain *chain, bool by_act,
 382                             bool explicitly_created)
 383 {
 384         struct tcf_block *block = chain->block;
 385         const struct tcf_proto_ops *tmplt_ops;
 386         bool is_last, free_block = false;
 387         unsigned int refcnt;
 388         void *tmplt_priv;
 389         u32 chain_index;
 390
 391         mutex_lock(&block->lock);
 392         if (explicitly_created) {
 393                 if (!chain->explicitly_created) {
 394                         mutex_unlock(&block->lock);
 395                         return;
 396                 }
 397                 chain->explicitly_created = false;
 398         }
 399
 400         if (by_act)
 401                 chain->action_refcnt--;
 402
 403         /* tc_chain_notify_delete can't be called while holding block lock.
 404          * However, when block is unlocked chain can be changed concurrently, so
 405          * save these to temporary variables.
 406          */
 407         refcnt = --chain->refcnt;
 408         is_last = refcnt - chain->action_refcnt == 0;
 409         tmplt_ops = chain->tmplt_ops;
 410         tmplt_priv = chain->tmplt_priv;
 411         chain_index = chain->index;
 412
 413         if (refcnt == 0)
 414                 free_block = tcf_chain_detach(chain);
 415         mutex_unlock(&block->lock);
 416
 417         /* The last dropped non-action reference will trigger notification. */
 418         if (is_last && !by_act)
 419                 tc_chain_notify_delete(tmplt_ops, tmplt_priv, chain_index,
 420                                        block, NULL, 0, 0, false);
 421
 422         if (refcnt == 0) {
 423                 tc_chain_tmplt_del(tmplt_ops, tmplt_priv);
 424                 tcf_chain_destroy(chain, free_block);
 425         }
 426 }
 427
 428 static void tcf_chain_put(struct tcf_chain *chain)
 429 {
 430         __tcf_chain_put(chain, false, false);
 431 }
 432
 433 void tcf_chain_put_by_act(struct tcf_chain *chain)
 434 {
 435         __tcf_chain_put(chain, true, false);
 436 }
 437 EXPORT_SYMBOL(tcf_chain_put_by_act);
 438
 439 static void tcf_chain_put_explicitly_created(struct tcf_chain *chain)
 440 {
 441         __tcf_chain_put(chain, false, true);
 442 }
 443
 444 static void tcf_chain_flush(struct tcf_chain *chain)
 445 {
 446         struct tcf_proto *tp = rtnl_dereference(chain->filter_chain);
 447
 448         tcf_chain0_head_change(chain, NULL);
 449         while (tp) {
 450                 RCU_INIT_POINTER(chain->filter_chain, tp->next);
 451                 tcf_proto_destroy(tp, NULL);
 452                 tp = rtnl_dereference(chain->filter_chain);
 453                 tcf_chain_put(chain);
 454         }
 455 }
 456
 457 static struct tcf_block *tc_dev_ingress_block(struct net_device *dev)
 458 {
 459         const struct Qdisc_class_ops *cops;
 460         struct Qdisc *qdisc;
 461
 462         if (!dev_ingress_queue(dev))
 463                 return NULL;
 464
 465         qdisc = dev_ingress_queue(dev)->qdisc_sleeping;
 466         if (!qdisc)
 467                 return NULL;
 468
 469         cops = qdisc->ops->cl_ops;
 470         if (!cops)
 471                 return NULL;
 472
 473         if (!cops->tcf_block)
 474                 return NULL;
 475
 476         return cops->tcf_block(qdisc, TC_H_MIN_INGRESS, NULL);
 477 }
 478
 479 static struct rhashtable indr_setup_block_ht;
 480
 481 struct tc_indr_block_dev {
 482         struct rhash_head ht_node;
 483         struct net_device *dev;
 484         unsigned int refcnt;
 485         struct list_head cb_list;
 486         struct tcf_block *block;
 487 };
 488
 489 struct tc_indr_block_cb {
 490         struct list_head list;
 491         void *cb_priv;
 492         tc_indr_block_bind_cb_t *cb;
 493         void *cb_ident;
 494 };
 495
 496 static const struct rhashtable_params tc_indr_setup_block_ht_params = {
 497         .key_offset     = offsetof(struct tc_indr_block_dev, dev),
 498         .head_offset    = offsetof(struct tc_indr_block_dev, ht_node),
 499         .key_len        = sizeof(struct net_device *),
 500 };
 501
 502 static struct tc_indr_block_dev *
 503 tc_indr_block_dev_lookup(struct net_device *dev)
 504 {
 505         return rhashtable_lookup_fast(&indr_setup_block_ht, &dev,
 506                                       tc_indr_setup_block_ht_params);
 507 }
 508
 509 static struct tc_indr_block_dev *tc_indr_block_dev_get(struct net_device *dev)
 510 {
 511         struct tc_indr_block_dev *indr_dev;
 512
 513         indr_dev = tc_indr_block_dev_lookup(dev);
 514         if (indr_dev)
 515                 goto inc_ref;
 516
 517         indr_dev = kzalloc(sizeof(*indr_dev), GFP_KERNEL);
 518         if (!indr_dev)
 519                 return NULL;
 520
 521         INIT_LIST_HEAD(&indr_dev->cb_list);
 522         indr_dev->dev = dev;
 523         indr_dev->block = tc_dev_ingress_block(dev);
 524         if (rhashtable_insert_fast(&indr_setup_block_ht, &indr_dev->ht_node,
 525                                    tc_indr_setup_block_ht_params)) {
 526                 kfree(indr_dev);
 527                 return NULL;
 528         }
 529
 530 inc_ref:
 531         indr_dev->refcnt++;
 532         return indr_dev;
 533 }
 534
 535 static void tc_indr_block_dev_put(struct tc_indr_block_dev *indr_dev)
 536 {
 537         if (--indr_dev->refcnt)
 538                 return;
 539
 540         rhashtable_remove_fast(&indr_setup_block_ht, &indr_dev->ht_node,
 541                                tc_indr_setup_block_ht_params);
 542         kfree(indr_dev);
 543 }
 544
 545 static struct tc_indr_block_cb *
 546 tc_indr_block_cb_lookup(struct tc_indr_block_dev *indr_dev,
 547                         tc_indr_block_bind_cb_t *cb, void *cb_ident)
 548 {
 549         struct tc_indr_block_cb *indr_block_cb;
 550
 551         list_for_each_entry(indr_block_cb, &indr_dev->cb_list, list)
 552                 if (indr_block_cb->cb == cb &&
 553                     indr_block_cb->cb_ident == cb_ident)
 554                         return indr_block_cb;
 555         return NULL;
 556 }
 557
 558 static struct tc_indr_block_cb *
 559 tc_indr_block_cb_add(struct tc_indr_block_dev *indr_dev, void *cb_priv,
 560                      tc_indr_block_bind_cb_t *cb, void *cb_ident)
 561 {
 562         struct tc_indr_block_cb *indr_block_cb;
 563
 564         indr_block_cb = tc_indr_block_cb_lookup(indr_dev, cb, cb_ident);
 565         if (indr_block_cb)
 566                 return ERR_PTR(-EEXIST);
 567
 568         indr_block_cb = kzalloc(sizeof(*indr_block_cb), GFP_KERNEL);
 569         if (!indr_block_cb)
 570                 return ERR_PTR(-ENOMEM);
 571
 572         indr_block_cb->cb_priv = cb_priv;
 573         indr_block_cb->cb = cb;
 574         indr_block_cb->cb_ident = cb_ident;
 575         list_add(&indr_block_cb->list, &indr_dev->cb_list);
 576
 577         return indr_block_cb;
 578 }
 579
 580 static void tc_indr_block_cb_del(struct tc_indr_block_cb *indr_block_cb)
 581 {
 582         list_del(&indr_block_cb->list);
 583         kfree(indr_block_cb);
 584 }
 585
 586 static void tc_indr_block_ing_cmd(struct tc_indr_block_dev *indr_dev,
 587                                   struct tc_indr_block_cb *indr_block_cb,
 588                                   enum tc_block_command command)
 589 {
 590         struct tc_block_offload bo = {
 591                 .command        = command,
 592                 .binder_type    = TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS,
 593                 .block          = indr_dev->block,
 594         };
 595
 596         if (!indr_dev->block)
 597                 return;
 598
 599         indr_block_cb->cb(indr_dev->dev, indr_block_cb->cb_priv, TC_SETUP_BLOCK,
 600                           &bo);
 601 }
 602
 603 int __tc_indr_block_cb_register(struct net_device *dev, void *cb_priv,
 604                                 tc_indr_block_bind_cb_t *cb, void *cb_ident)
 605 {
 606         struct tc_indr_block_cb *indr_block_cb;
 607         struct tc_indr_block_dev *indr_dev;
 608         int err;
 609
 610         indr_dev = tc_indr_block_dev_get(dev);
 611         if (!indr_dev)
 612                 return -ENOMEM;
 613
 614         indr_block_cb = tc_indr_block_cb_add(indr_dev, cb_priv, cb, cb_ident);
 615         err = PTR_ERR_OR_ZERO(indr_block_cb);
 616         if (err)
 617                 goto err_dev_put;
 618
 619         tc_indr_block_ing_cmd(indr_dev, indr_block_cb, TC_BLOCK_BIND);
 620         return 0;
 621
 622 err_dev_put:
 623         tc_indr_block_dev_put(indr_dev);
 624         return err;
 625 }
 626 EXPORT_SYMBOL_GPL(__tc_indr_block_cb_register);
 627
 628 int tc_indr_block_cb_register(struct net_device *dev, void *cb_priv,
 629                               tc_indr_block_bind_cb_t *cb, void *cb_ident)
 630 {
 631         int err;
 632
 633         rtnl_lock();
 634         err = __tc_indr_block_cb_register(dev, cb_priv, cb, cb_ident);
 635         rtnl_unlock();
 636
 637         return err;
 638 }
 639 EXPORT_SYMBOL_GPL(tc_indr_block_cb_register);
 640
 641 void __tc_indr_block_cb_unregister(struct net_device *dev,
 642                                    tc_indr_block_bind_cb_t *cb, void *cb_ident)
 643 {
 644         struct tc_indr_block_cb *indr_block_cb;
 645         struct tc_indr_block_dev *indr_dev;
 646
 647         indr_dev = tc_indr_block_dev_lookup(dev);
 648         if (!indr_dev)
 649                 return;
 650
 651         indr_block_cb = tc_indr_block_cb_lookup(indr_dev, cb, cb_ident);
 652         if (!indr_block_cb)
 653                 return;
 654
 655         /* Send unbind message if required to free any block cbs. */
 656         tc_indr_block_ing_cmd(indr_dev, indr_block_cb, TC_BLOCK_UNBIND);
 657         tc_indr_block_cb_del(indr_block_cb);
 658         tc_indr_block_dev_put(indr_dev);
 659 }
 660 EXPORT_SYMBOL_GPL(__tc_indr_block_cb_unregister);
 661
 662 void tc_indr_block_cb_unregister(struct net_device *dev,
 663                                  tc_indr_block_bind_cb_t *cb, void *cb_ident)
 664 {
 665         rtnl_lock();
 666         __tc_indr_block_cb_unregister(dev, cb, cb_ident);
 667         rtnl_unlock();
 668 }
 669 EXPORT_SYMBOL_GPL(tc_indr_block_cb_unregister);
 670
 671 static void tc_indr_block_call(struct tcf_block *block, struct net_device *dev,
 672                                struct tcf_block_ext_info *ei,
 673                                enum tc_block_command command,
 674                                struct netlink_ext_ack *extack)
 675 {
 676         struct tc_indr_block_cb *indr_block_cb;
 677         struct tc_indr_block_dev *indr_dev;
 678         struct tc_block_offload bo = {
 679                 .command        = command,
 680                 .binder_type    = ei->binder_type,
 681                 .block          = block,
 682                 .extack         = extack,
 683         };
 684
 685         indr_dev = tc_indr_block_dev_lookup(dev);
 686         if (!indr_dev)
 687                 return;
 688
 689         indr_dev->block = command == TC_BLOCK_BIND ? block : NULL;
 690
 691         list_for_each_entry(indr_block_cb, &indr_dev->cb_list, list)
 692                 indr_block_cb->cb(dev, indr_block_cb->cb_priv, TC_SETUP_BLOCK,
 693                                   &bo);
 694 }
 695
 696 static bool tcf_block_offload_in_use(struct tcf_block *block)
 697 {
 698         return block->offloadcnt;
 699 }
 700
 701 static int tcf_block_offload_cmd(struct tcf_block *block,
 702                                  struct net_device *dev,
 703                                  struct tcf_block_ext_info *ei,
 704                                  enum tc_block_command command,
 705                                  struct netlink_ext_ack *extack)
 706 {
 707         struct tc_block_offload bo = {};
 708
 709         bo.command = command;
 710         bo.binder_type = ei->binder_type;
 711         bo.block = block;
 712         bo.extack = extack;
 713         return dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_BLOCK, &bo);
 714 }
 715
 716 static int tcf_block_offload_bind(struct tcf_block *block, struct Qdisc *q,
 717                                   struct tcf_block_ext_info *ei,
 718                                   struct netlink_ext_ack *extack)
 719 {
 720         struct net_device *dev = q->dev_queue->dev;
 721         int err;
 722
 723         if (!dev->netdev_ops->ndo_setup_tc)
 724                 goto no_offload_dev_inc;
 725
 726         /* If tc offload feature is disabled and the block we try to bind
 727          * to already has some offloaded filters, forbid to bind.
 728          */
 729         if (!tc_can_offload(dev) && tcf_block_offload_in_use(block)) {
 730                 NL_SET_ERR_MSG(extack, "Bind to offloaded block failed as dev has offload disabled");
 731                 return -EOPNOTSUPP;
 732         }
 733
 734         err = tcf_block_offload_cmd(block, dev, ei, TC_BLOCK_BIND, extack);
 735         if (err == -EOPNOTSUPP)
 736                 goto no_offload_dev_inc;
 737         if (err)
 738                 return err;
 739
 740         tc_indr_block_call(block, dev, ei, TC_BLOCK_BIND, extack);
 741         return 0;
 742
 743 no_offload_dev_inc:
 744         if (tcf_block_offload_in_use(block))
 745                 return -EOPNOTSUPP;
 746         block->nooffloaddevcnt++;
 747         tc_indr_block_call(block, dev, ei, TC_BLOCK_BIND, extack);
 748         return 0;
 749 }
 750
 751 static void tcf_block_offload_unbind(struct tcf_block *block, struct Qdisc *q,
 752                                      struct tcf_block_ext_info *ei)
 753 {
 754         struct net_device *dev = q->dev_queue->dev;
 755         int err;
 756
 757         tc_indr_block_call(block, dev, ei, TC_BLOCK_UNBIND, NULL);
 758
 759         if (!dev->netdev_ops->ndo_setup_tc)
 760                 goto no_offload_dev_dec;
 761         err = tcf_block_offload_cmd(block, dev, ei, TC_BLOCK_UNBIND, NULL);
 762         if (err == -EOPNOTSUPP)
 763                 goto no_offload_dev_dec;
 764         return;
 765
 766 no_offload_dev_dec:
 767         WARN_ON(block->nooffloaddevcnt-- == 0);
 768 }
 769
 770 static int
 771 tcf_chain0_head_change_cb_add(struct tcf_block *block,
 772                               struct tcf_block_ext_info *ei,
 773                               struct netlink_ext_ack *extack)
 774 {
 775         struct tcf_filter_chain_list_item *item;
 776         struct tcf_chain *chain0;
 777
 778         item = kmalloc(sizeof(*item), GFP_KERNEL);
 779         if (!item) {
 780                 NL_SET_ERR_MSG(extack, "Memory allocation for head change callback item failed");
 781                 return -ENOMEM;
 782         }
 783         item->chain_head_change = ei->chain_head_change;
 784         item->chain_head_change_priv = ei->chain_head_change_priv;
 785
 786         mutex_lock(&block->lock);
 787         chain0 = block->chain0.chain;
 788         if (chain0 && chain0->filter_chain)
 789                 tcf_chain_head_change_item(item, chain0->filter_chain);
 790         list_add(&item->list, &block->chain0.filter_chain_list);
 791         mutex_unlock(&block->lock);
 792
 793         return 0;
 794 }
 795
 796 static void
 797 tcf_chain0_head_change_cb_del(struct tcf_block *block,
 798                               struct tcf_block_ext_info *ei)
 799 {
 800         struct tcf_filter_chain_list_item *item;
 801
 802         mutex_lock(&block->lock);
 803         list_for_each_entry(item, &block->chain0.filter_chain_list, list) {
 804                 if ((!ei->chain_head_change && !ei->chain_head_change_priv) ||
 805                     (item->chain_head_change == ei->chain_head_change &&
 806                      item->chain_head_change_priv == ei->chain_head_change_priv)) {
 807                         if (block->chain0.chain)
 808                                 tcf_chain_head_change_item(item, NULL);
 809                         list_del(&item->list);
 810                         mutex_unlock(&block->lock);
 811
 812                         kfree(item);
 813                         return;
 814                 }
 815         }
 816         mutex_unlock(&block->lock);
 817         WARN_ON(1);
 818 }
 819
 820 struct tcf_net {
 821         spinlock_t idr_lock; /* Protects idr */
 822         struct idr idr;
 823 };
 824
 825 static unsigned int tcf_net_id;
 826
 827 static int tcf_block_insert(struct tcf_block *block, struct net *net,
 828                             struct netlink_ext_ack *extack)
 829 {
 830         struct tcf_net *tn = net_generic(net, tcf_net_id);
 831         int err;
 832
 833         idr_preload(GFP_KERNEL);
 834         spin_lock(&tn->idr_lock);
 835         err = idr_alloc_u32(&tn->idr, block, &block->index, block->index,
 836                             GFP_NOWAIT);
 837         spin_unlock(&tn->idr_lock);
 838         idr_preload_end();
 839
 840         return err;
 841 }
 842
 843 static void tcf_block_remove(struct tcf_block *block, struct net *net)
 844 {
 845         struct tcf_net *tn = net_generic(net, tcf_net_id);
 846
 847         spin_lock(&tn->idr_lock);
 848         idr_remove(&tn->idr, block->index);
 849         spin_unlock(&tn->idr_lock);
 850 }
 851
 852 static struct tcf_block *tcf_block_create(struct net *net, struct Qdisc *q,
 853                                           u32 block_index,
 854                                           struct netlink_ext_ack *extack)
 855 {
 856         struct tcf_block *block;
 857
 858         block = kzalloc(sizeof(*block), GFP_KERNEL);
 859         if (!block) {
 860                 NL_SET_ERR_MSG(extack, "Memory allocation for block failed");
 861                 return ERR_PTR(-ENOMEM);
 862         }
 863         mutex_init(&block->lock);
 864         INIT_LIST_HEAD(&block->chain_list);
 865         INIT_LIST_HEAD(&block->cb_list);
 866         INIT_LIST_HEAD(&block->owner_list);
 867         INIT_LIST_HEAD(&block->chain0.filter_chain_list);
 868
 869         refcount_set(&block->refcnt, 1);
 870         block->net = net;
 871         block->index = block_index;
 872
 873         /* Don't store q pointer for blocks which are shared */
 874         if (!tcf_block_shared(block))
 875                 block->q = q;
 876         return block;
 877 }
 878
 879 static struct tcf_block *tcf_block_lookup(struct net *net, u32 block_index)
 880 {
 881         struct tcf_net *tn = net_generic(net, tcf_net_id);
 882
 883         return idr_find(&tn->idr, block_index);
 884 }
 885
 886 static struct tcf_block *tcf_block_refcnt_get(struct net *net, u32 block_index)
 887 {
 888         struct tcf_block *block;
 889
 890         rcu_read_lock();
 891         block = tcf_block_lookup(net, block_index);
 892         if (block && !refcount_inc_not_zero(&block->refcnt))
 893                 block = NULL;
 894         rcu_read_unlock();
 895
 896         return block;
 897 }
 898
 899 static struct tcf_chain *
 900 __tcf_get_next_chain(struct tcf_block *block, struct tcf_chain *chain)
 901 {
 902         mutex_lock(&block->lock);
 903         if (chain)
 904                 chain = list_is_last(&chain->list, &block->chain_list) ?
 905                         NULL : list_next_entry(chain, list);
 906         else
 907                 chain = list_first_entry_or_null(&block->chain_list,
 908                                                  struct tcf_chain, list);
 909
 910         /* skip all action-only chains */
 911         while (chain && tcf_chain_held_by_acts_only(chain))
 912                 chain = list_is_last(&chain->list, &block->chain_list) ?
 913                         NULL : list_next_entry(chain, list);
 914
 915         if (chain)
 916                 tcf_chain_hold(chain);
 917         mutex_unlock(&block->lock);
 918
 919         return chain;
 920 }
 921
 922 /* Function to be used by all clients that want to iterate over all chains on
 923  * block. It properly obtains block->lock and takes reference to chain before
 924  * returning it. Users of this function must be tolerant to concurrent chain
 925  * insertion/deletion or ensure that no concurrent chain modification is
 926  * possible. Note that all netlink dump callbacks cannot guarantee to provide
 927  * consistent dump because rtnl lock is released each time skb is filled with
 928  * data and sent to user-space.
 929  */
 930
 931 struct tcf_chain *
 932 tcf_get_next_chain(struct tcf_block *block, struct tcf_chain *chain)
 933 {
 934         struct tcf_chain *chain_next = __tcf_get_next_chain(block, chain);
 935
 936         if (chain)
 937                 tcf_chain_put(chain);
 938
 939         return chain_next;
 940 }
 941 EXPORT_SYMBOL(tcf_get_next_chain);
 942
 943 static void tcf_block_flush_all_chains(struct tcf_block *block)
 944 {
 945         struct tcf_chain *chain;
 946
 947         /* Last reference to block. At this point chains cannot be added or
 948          * removed concurrently.
 949          */
 950         for (chain = tcf_get_next_chain(block, NULL);
 951              chain;
 952              chain = tcf_get_next_chain(block, chain)) {
 953                 tcf_chain_put_explicitly_created(chain);
 954                 tcf_chain_flush(chain);
 955         }
 956 }
 957
 958 static void __tcf_block_put(struct tcf_block *block, struct Qdisc *q,
 959                             struct tcf_block_ext_info *ei)
 960 {
 961         if (refcount_dec_and_mutex_lock(&block->refcnt, &block->lock)) {
 962                 /* Flushing/putting all chains will cause the block to be
 963                  * deallocated when last chain is freed. However, if chain_list
 964                  * is empty, block has to be manually deallocated. After block
 965                  * reference counter reached 0, it is no longer possible to
 966                  * increment it or add new chains to block.
 967                  */
 968                 bool free_block = list_empty(&block->chain_list);
 969
 970                 mutex_unlock(&block->lock);
 971                 if (tcf_block_shared(block))
 972                         tcf_block_remove(block, block->net);
 973
 974                 if (q)
 975                         tcf_block_offload_unbind(block, q, ei);
 976
 977                 if (free_block)
 978                         tcf_block_destroy(block);
 979                 else
 980                         tcf_block_flush_all_chains(block);
 981         } else if (q) {
 982                 tcf_block_offload_unbind(block, q, ei);
 983         }
 984 }
 985
 986 static void tcf_block_refcnt_put(struct tcf_block *block)
 987 {
 988         __tcf_block_put(block, NULL, NULL);
 989 }
 990
 991 /* Find tcf block.
 992  * Set q, parent, cl when appropriate.
 993  */
 994
 995 static struct tcf_block *tcf_block_find(struct net *net, struct Qdisc **q,
 996                                         u32 *parent, unsigned long *cl,
 997                                         int ifindex, u32 block_index,
 998                                         struct netlink_ext_ack *extack)
 999 {
1000         struct tcf_block *block;
1001         int err = 0;
1002
1003         if (ifindex == TCM_IFINDEX_MAGIC_BLOCK) {
1004                 block = tcf_block_refcnt_get(net, block_index);
1005                 if (!block) {
1006                         NL_SET_ERR_MSG(extack, "Block of given index was not found");
1007                         return ERR_PTR(-EINVAL);
1008                 }
1009         } else {
1010                 const struct Qdisc_class_ops *cops;
1011                 struct net_device *dev;
1012
1013                 rcu_read_lock();
1014
1015                 /* Find link */
1016                 dev = dev_get_by_index_rcu(net, ifindex);
1017                 if (!dev) {
1018                         rcu_read_unlock();
1019                         return ERR_PTR(-ENODEV);
1020                 }
1021
1022                 /* Find qdisc */
1023                 if (!*parent) {
1024                         *q = dev->qdisc;
1025                         *parent = (*q)->handle;
1026                 } else {
1027                         *q = qdisc_lookup_rcu(dev, TC_H_MAJ(*parent));
1028                         if (!*q) {
1029                                 NL_SET_ERR_MSG(extack, "Parent Qdisc doesn't exists");
1030                                 err = -EINVAL;
1031                                 goto errout_rcu;
1032                         }
1033                 }
1034
1035                 *q = qdisc_refcount_inc_nz(*q);
1036                 if (!*q) {
1037                         NL_SET_ERR_MSG(extack, "Parent Qdisc doesn't exists");
1038                         err = -EINVAL;
1039                         goto errout_rcu;
1040                 }
1041
1042                 /* Is it classful? */
1043                 cops = (*q)->ops->cl_ops;
1044                 if (!cops) {
1045                         NL_SET_ERR_MSG(extack, "Qdisc not classful");
1046                         err = -EINVAL;
1047                         goto errout_rcu;
1048                 }
1049
1050                 if (!cops->tcf_block) {
1051                         NL_SET_ERR_MSG(extack, "Class doesn't support blocks");
1052                         err = -EOPNOTSUPP;
1053                         goto errout_rcu;
1054                 }
1055
1056                 /* At this point we know that qdisc is not noop_qdisc,
1057                  * which means that qdisc holds a reference to net_device
1058                  * and we hold a reference to qdisc, so it is safe to release
1059                  * rcu read lock.
1060                  */
1061                 rcu_read_unlock();
1062
1063                 /* Do we search for filter, attached to class? */
1064                 if (TC_H_MIN(*parent)) {
1065                         *cl = cops->find(*q, *parent);
1066                         if (*cl == 0) {
1067                                 NL_SET_ERR_MSG(extack, "Specified class doesn't exist");
1068                                 err = -ENOENT;
1069                                 goto errout_qdisc;
1070                         }
1071                 }
1072
1073                 /* And the last stroke */
1074                 block = cops->tcf_block(*q, *cl, extack);
1075                 if (!block) {
1076                         err = -EINVAL;
1077                         goto errout_qdisc;
1078                 }
1079                 if (tcf_block_shared(block)) {
1080                         NL_SET_ERR_MSG(extack, "This filter block is shared. Please use the block index to manipulate the filters");
1081                         err = -EOPNOTSUPP;
1082                         goto errout_qdisc;
1083                 }
1084
1085                 /* Always take reference to block in order to support execution
1086                  * of rules update path of cls API without rtnl lock. Caller
1087                  * must release block when it is finished using it. 'if' block
1088                  * of this conditional obtain reference to block by calling
1089                  * tcf_block_refcnt_get().
1090                  */
1091                 refcount_inc(&block->refcnt);
1092         }
1093
1094         return block;
1095
1096 errout_rcu:
1097         rcu_read_unlock();
1098 errout_qdisc:
1099         if (*q) {
1100                 qdisc_put(*q);
1101                 *q = NULL;
1102         }
1103         return ERR_PTR(err);
1104 }
1105
1106 static void tcf_block_release(struct Qdisc *q, struct tcf_block *block)
1107 {
1108         if (!IS_ERR_OR_NULL(block))
1109                 tcf_block_refcnt_put(block);
1110
1111         if (q)
1112                 qdisc_put(q);
1113 }
1114
1115 struct tcf_block_owner_item {
1116         struct list_head list;
1117         struct Qdisc *q;
1118         enum tcf_block_binder_type binder_type;
1119 };
1120
1121 static void
1122 tcf_block_owner_netif_keep_dst(struct tcf_block *block,
1123                                struct Qdisc *q,
1124                                enum tcf_block_binder_type binder_type)
1125 {
1126         if (block->keep_dst &&
1127             binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS &&
1128             binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_EGRESS)
1129                 netif_keep_dst(qdisc_dev(q));
1130 }
1131
1132 void tcf_block_netif_keep_dst(struct tcf_block *block)
1133 {
1134         struct tcf_block_owner_item *item;
1135
1136         block->keep_dst = true;
1137         list_for_each_entry(item, &block->owner_list, list)
1138                 tcf_block_owner_netif_keep_dst(block, item->q,
1139                                                item->binder_type);
1140 }
1141 EXPORT_SYMBOL(tcf_block_netif_keep_dst);
1142
1143 static int tcf_block_owner_add(struct tcf_block *block,
1144                                struct Qdisc *q,
1145                                enum tcf_block_binder_type binder_type)
1146 {
1147         struct tcf_block_owner_item *item;
1148
1149         item = kmalloc(sizeof(*item), GFP_KERNEL);
1150         if (!item)
1151                 return -ENOMEM;
1152         item->q = q;
1153         item->binder_type = binder_type;
1154         list_add(&item->list, &block->owner_list);
1155         return 0;
1156 }
1157
1158 static void tcf_block_owner_del(struct tcf_block *block,
1159                                 struct Qdisc *q,
1160                                 enum tcf_block_binder_type binder_type)
1161 {
1162         struct tcf_block_owner_item *item;
1163
1164         list_for_each_entry(item, &block->owner_list, list) {
1165                 if (item->q == q && item->binder_type == binder_type) {
1166                         list_del(&item->list);
1167                         kfree(item);
1168                         return;
1169                 }
1170         }
1171         WARN_ON(1);
1172 }
1173
1174 int tcf_block_get_ext(struct tcf_block **p_block, struct Qdisc *q,
1175                       struct tcf_block_ext_info *ei,
1176                       struct netlink_ext_ack *extack)
1177 {
1178         struct net *net = qdisc_net(q);
1179         struct tcf_block *block = NULL;
1180         int err;
1181
1182         if (ei->block_index)
1183                 /* block_index not 0 means the shared block is requested */
1184                 block = tcf_block_refcnt_get(net, ei->block_index);
1185
1186         if (!block) {
1187                 block = tcf_block_create(net, q, ei->block_index, extack);
1188                 if (IS_ERR(block))
1189                         return PTR_ERR(block);
1190                 if (tcf_block_shared(block)) {
1191                         err = tcf_block_insert(block, net, extack);
1192                         if (err)
1193                                 goto err_block_insert;
1194                 }
1195         }
1196
1197         err = tcf_block_owner_add(block, q, ei->binder_type);
1198         if (err)
1199                 goto err_block_owner_add;
1200
1201         tcf_block_owner_netif_keep_dst(block, q, ei->binder_type);
1202
1203         err = tcf_chain0_head_change_cb_add(block, ei, extack);
1204         if (err)
1205                 goto err_chain0_head_change_cb_add;
1206
1207         err = tcf_block_offload_bind(block, q, ei, extack);
1208         if (err)
1209                 goto err_block_offload_bind;
1210
1211         *p_block = block;
1212         return 0;
1213
1214 err_block_offload_bind:
1215         tcf_chain0_head_change_cb_del(block, ei);
1216 err_chain0_head_change_cb_add:
1217         tcf_block_owner_del(block, q, ei->binder_type);
1218 err_block_owner_add:
1219 err_block_insert:
1220         tcf_block_refcnt_put(block);
1221         return err;
1222 }
1223 EXPORT_SYMBOL(tcf_block_get_ext);
1224
1225 static void tcf_chain_head_change_dflt(struct tcf_proto *tp_head, void *priv)
1226 {
1227         struct tcf_proto __rcu **p_filter_chain = priv;
1228
1229         rcu_assign_pointer(*p_filter_chain, tp_head);
1230 }
1231
1232 int tcf_block_get(struct tcf_block **p_block,
1233                   struct tcf_proto __rcu **p_filter_chain, struct Qdisc *q,
1234                   struct netlink_ext_ack *extack)
1235 {
1236         struct tcf_block_ext_info ei = {
1237                 .chain_head_change = tcf_chain_head_change_dflt,
1238                 .chain_head_change_priv = p_filter_chain,
1239         };
1240
1241         WARN_ON(!p_filter_chain);
1242         return tcf_block_get_ext(p_block, q, &ei, extack);
1243 }
1244 EXPORT_SYMBOL(tcf_block_get);
1245
1246 /* XXX: Standalone actions are not allowed to jump to any chain, and bound
1247  * actions should be all removed after flushing.
1248  */
1249 void tcf_block_put_ext(struct tcf_block *block, struct Qdisc *q,
1250                        struct tcf_block_ext_info *ei)
1251 {
1252         if (!block)
1253                 return;
1254         tcf_chain0_head_change_cb_del(block, ei);
1255         tcf_block_owner_del(block, q, ei->binder_type);
1256
1257         __tcf_block_put(block, q, ei);
1258 }
1259 EXPORT_SYMBOL(tcf_block_put_ext);
1260
1261 void tcf_block_put(struct tcf_block *block)
1262 {
1263         struct tcf_block_ext_info ei = {0, };
1264
1265         if (!block)
1266                 return;
1267         tcf_block_put_ext(block, block->q, &ei);
1268 }
1269
1270 EXPORT_SYMBOL(tcf_block_put);
1271
1272 struct tcf_block_cb {
1273         struct list_head list;
1274         tc_setup_cb_t *cb;
1275         void *cb_ident;
1276         void *cb_priv;
1277         unsigned int refcnt;
1278 };
1279
1280 void *tcf_block_cb_priv(struct tcf_block_cb *block_cb)
1281 {
1282         return block_cb->cb_priv;
1283 }
1284 EXPORT_SYMBOL(tcf_block_cb_priv);
1285
1286 struct tcf_block_cb *tcf_block_cb_lookup(struct tcf_block *block,
1287                                          tc_setup_cb_t *cb, void *cb_ident)
1288 {       struct tcf_block_cb *block_cb;
1289
1290         list_for_each_entry(block_cb, &block->cb_list, list)
1291                 if (block_cb->cb == cb && block_cb->cb_ident == cb_ident)
1292                         return block_cb;
1293         return NULL;
1294 }
1295 EXPORT_SYMBOL(tcf_block_cb_lookup);
1296
1297 void tcf_block_cb_incref(struct tcf_block_cb *block_cb)
1298 {
1299         block_cb->refcnt++;
1300 }
1301 EXPORT_SYMBOL(tcf_block_cb_incref);
1302
1303 unsigned int tcf_block_cb_decref(struct tcf_block_cb *block_cb)
1304 {
1305         return --block_cb->refcnt;
1306 }
1307 EXPORT_SYMBOL(tcf_block_cb_decref);
1308
1309 static int
1310 tcf_block_playback_offloads(struct tcf_block *block, tc_setup_cb_t *cb,
1311                             void *cb_priv, bool add, bool offload_in_use,
1312                             struct netlink_ext_ack *extack)
1313 {
1314         struct tcf_chain *chain, *chain_prev;
1315         struct tcf_proto *tp;
1316         int err;
1317
1318         for (chain = __tcf_get_next_chain(block, NULL);
1319              chain;
1320              chain_prev = chain,
1321                      chain = __tcf_get_next_chain(block, chain),
1322                      tcf_chain_put(chain_prev)) {
1323                 for (tp = rtnl_dereference(chain->filter_chain); tp;
1324                      tp = rtnl_dereference(tp->next)) {
1325                         if (tp->ops->reoffload) {
1326                                 err = tp->ops->reoffload(tp, add, cb, cb_priv,
1327                                                          extack);
1328                                 if (err && add)
1329                                         goto err_playback_remove;
1330                         } else if (add && offload_in_use) {
1331                                 err = -EOPNOTSUPP;
1332                                 NL_SET_ERR_MSG(extack, "Filter HW offload failed - classifier without re-offloading support");
1333                                 goto err_playback_remove;
1334                         }
1335                 }
1336         }
1337
1338         return 0;
1339
1340 err_playback_remove:
1341         tcf_chain_put(chain);
1342         tcf_block_playback_offloads(block, cb, cb_priv, false, offload_in_use,
1343                                     extack);
1344         return err;
1345 }
1346
1347 struct tcf_block_cb *__tcf_block_cb_register(struct tcf_block *block,
1348                                              tc_setup_cb_t *cb, void *cb_ident,
1349                                              void *cb_priv,
1350                                              struct netlink_ext_ack *extack)
1351 {
1352         struct tcf_block_cb *block_cb;
1353         int err;
1354
1355         /* Replay any already present rules */
1356         err = tcf_block_playback_offloads(block, cb, cb_priv, true,
1357                                           tcf_block_offload_in_use(block),
1358                                           extack);
1359         if (err)
1360                 return ERR_PTR(err);
1361
1362         block_cb = kzalloc(sizeof(*block_cb), GFP_KERNEL);
1363         if (!block_cb)
1364                 return ERR_PTR(-ENOMEM);
1365         block_cb->cb = cb;
1366         block_cb->cb_ident = cb_ident;
1367         block_cb->cb_priv = cb_priv;
1368         list_add(&block_cb->list, &block->cb_list);
1369         return block_cb;
1370 }
1371 EXPORT_SYMBOL(__tcf_block_cb_register);
1372
1373 int tcf_block_cb_register(struct tcf_block *block,
1374                           tc_setup_cb_t *cb, void *cb_ident,
1375                           void *cb_priv, struct netlink_ext_ack *extack)
1376 {
1377         struct tcf_block_cb *block_cb;
1378
1379         block_cb = __tcf_block_cb_register(block, cb, cb_ident, cb_priv,
1380                                            extack);
1381         return PTR_ERR_OR_ZERO(block_cb);
1382 }
1383 EXPORT_SYMBOL(tcf_block_cb_register);
1384
1385 void __tcf_block_cb_unregister(struct tcf_block *block,
1386                                struct tcf_block_cb *block_cb)
1387 {
1388         tcf_block_playback_offloads(block, block_cb->cb, block_cb->cb_priv,
1389                                     false, tcf_block_offload_in_use(block),
1390                                     NULL);
1391         list_del(&block_cb->list);
1392         kfree(block_cb);
1393 }
1394 EXPORT_SYMBOL(__tcf_block_cb_unregister);
1395
1396 void tcf_block_cb_unregister(struct tcf_block *block,
1397                              tc_setup_cb_t *cb, void *cb_ident)
1398 {
1399         struct tcf_block_cb *block_cb;
1400
1401         block_cb = tcf_block_cb_lookup(block, cb, cb_ident);
1402         if (!block_cb)
1403                 return;
1404         __tcf_block_cb_unregister(block, block_cb);
1405 }
1406 EXPORT_SYMBOL(tcf_block_cb_unregister);
1407
1408 /* Main classifier routine: scans classifier chain attached
1409  * to this qdisc, (optionally) tests for protocol and asks
1410  * specific classifiers.
1411  */
1412 int tcf_classify(struct sk_buff *skb, const struct tcf_proto *tp,
1413                  struct tcf_result *res, bool compat_mode)
1414 {
1415 #ifdef CONFIG_NET_CLS_ACT
1416         const int max_reclassify_loop = 4;
1417         const struct tcf_proto *orig_tp = tp;
1418         const struct tcf_proto *first_tp;
1419         int limit = 0;
1420
1421 reclassify:
1422 #endif
1423         for (; tp; tp = rcu_dereference_bh(tp->next)) {
1424                 __be16 protocol = tc_skb_protocol(skb);
1425                 int err;
1426
1427                 if (tp->protocol != protocol &&
1428                     tp->protocol != htons(ETH_P_ALL))
1429                         continue;
1430
1431                 err = tp->classify(skb, tp, res);
1432 #ifdef CONFIG_NET_CLS_ACT
1433                 if (unlikely(err == TC_ACT_RECLASSIFY && !compat_mode)) {
1434                         first_tp = orig_tp;
1435                         goto reset;
1436                 } else if (unlikely(TC_ACT_EXT_CMP(err, TC_ACT_GOTO_CHAIN))) {
1437                         first_tp = res->goto_tp;
1438                         goto reset;
1439                 }
1440 #endif
1441                 if (err >= 0)
1442                         return err;
1443         }
1444
1445         return TC_ACT_UNSPEC; /* signal: continue lookup */
1446 #ifdef CONFIG_NET_CLS_ACT
1447 reset:
1448         if (unlikely(limit++ >= max_reclassify_loop)) {
1449                 net_notice_ratelimited("%u: reclassify loop, rule prio %u, protocol %02x\n",
1450                                        tp->chain->block->index,
1451                                        tp->prio & 0xffff,
1452                                        ntohs(tp->protocol));
1453                 return TC_ACT_SHOT;
1454         }
1455
1456         tp = first_tp;
1457         goto reclassify;
1458 #endif
1459 }
1460 EXPORT_SYMBOL(tcf_classify);
1461
1462 struct tcf_chain_info {
1463         struct tcf_proto __rcu **pprev;
1464         struct tcf_proto __rcu *next;
1465 };
1466
1467 static struct tcf_proto *tcf_chain_tp_prev(struct tcf_chain_info *chain_info)
1468 {
1469         return rtnl_dereference(*chain_info->pprev);
1470 }
1471
1472 static void tcf_chain_tp_insert(struct tcf_chain *chain,
1473                                 struct tcf_chain_info *chain_info,
1474                                 struct tcf_proto *tp)
1475 {
1476         if (*chain_info->pprev == chain->filter_chain)
1477                 tcf_chain0_head_change(chain, tp);
1478         RCU_INIT_POINTER(tp->next, tcf_chain_tp_prev(chain_info));
1479         rcu_assign_pointer(*chain_info->pprev, tp);
1480         tcf_chain_hold(chain);
1481 }
1482
1483 static void tcf_chain_tp_remove(struct tcf_chain *chain,
1484                                 struct tcf_chain_info *chain_info,
1485                                 struct tcf_proto *tp)
1486 {
1487         struct tcf_proto *next = rtnl_dereference(chain_info->next);
1488
1489         if (tp == chain->filter_chain)
1490                 tcf_chain0_head_change(chain, next);
1491         RCU_INIT_POINTER(*chain_info->pprev, next);
1492         tcf_chain_put(chain);
1493 }
1494
1495 static struct tcf_proto *tcf_chain_tp_find(struct tcf_chain *chain,
1496                                            struct tcf_chain_info *chain_info,
1497                                            u32 protocol, u32 prio,
1498                                            bool prio_allocate)
1499 {
1500         struct tcf_proto **pprev;
1501         struct tcf_proto *tp;
1502
1503         /* Check the chain for existence of proto-tcf with this priority */
1504         for (pprev = &chain->filter_chain;
1505              (tp = rtnl_dereference(*pprev)); pprev = &tp->next) {
1506                 if (tp->prio >= prio) {
1507                         if (tp->prio == prio) {
1508                                 if (prio_allocate ||
1509                                     (tp->protocol != protocol && protocol))
1510                                         return ERR_PTR(-EINVAL);
1511                         } else {
1512                                 tp = NULL;
1513                         }
1514                         break;
1515                 }
1516         }
1517         chain_info->pprev = pprev;
1518         chain_info->next = tp ? tp->next : NULL;
1519         return tp;
1520 }
1521
1522 static int tcf_fill_node(struct net *net, struct sk_buff *skb,
1523                          struct tcf_proto *tp, struct tcf_block *block,
1524                          struct Qdisc *q, u32 parent, void *fh,
1525                          u32 portid, u32 seq, u16 flags, int event)
1526 {
1527         struct tcmsg *tcm;
1528         struct nlmsghdr  *nlh;
1529         unsigned char *b = skb_tail_pointer(skb);
1530
1531         nlh = nlmsg_put(skb, portid, seq, event, sizeof(*tcm), flags);
1532         if (!nlh)
1533                 goto out_nlmsg_trim;
1534         tcm = nlmsg_data(nlh);
1535         tcm->tcm_family = AF_UNSPEC;
1536         tcm->tcm__pad1 = 0;
1537         tcm->tcm__pad2 = 0;
1538         if (q) {
1539                 tcm->tcm_ifindex = qdisc_dev(q)->ifindex;
1540                 tcm->tcm_parent = parent;
1541         } else {
1542                 tcm->tcm_ifindex = TCM_IFINDEX_MAGIC_BLOCK;
1543                 tcm->tcm_block_index = block->index;
1544         }
1545         tcm->tcm_info = TC_H_MAKE(tp->prio, tp->protocol);
1546         if (nla_put_string(skb, TCA_KIND, tp->ops->kind))
1547                 goto nla_put_failure;
1548         if (nla_put_u32(skb, TCA_CHAIN, tp->chain->index))
1549                 goto nla_put_failure;
1550         if (!fh) {
1551                 tcm->tcm_handle = 0;
1552         } else {
1553                 if (tp->ops->dump && tp->ops->dump(net, tp, fh, skb, tcm) < 0)
1554                         goto nla_put_failure;
1555         }
1556         nlh->nlmsg_len = skb_tail_pointer(skb) - b;
1557         return skb->len;
1558
1559 out_nlmsg_trim:
1560 nla_put_failure:
1561         nlmsg_trim(skb, b);
1562         return -1;
1563 }
1564
1565 static int tfilter_notify(struct net *net, struct sk_buff *oskb,
1566                           struct nlmsghdr *n, struct tcf_proto *tp,
1567                           struct tcf_block *block, struct Qdisc *q,
1568                           u32 parent, void *fh, int event, bool unicast)
1569 {
1570         struct sk_buff *skb;
1571         u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
1572
1573         skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
1574         if (!skb)
1575                 return -ENOBUFS;
1576
1577         if (tcf_fill_node(net, skb, tp, block, q, parent, fh, portid,
1578                           n->nlmsg_seq, n->nlmsg_flags, event) <= 0) {
1579                 kfree_skb(skb);
1580                 return -EINVAL;
1581         }
1582
1583         if (unicast)
1584                 return netlink_unicast(net->rtnl, skb, portid, MSG_DONTWAIT);
1585
1586         return rtnetlink_send(skb, net, portid, RTNLGRP_TC,
1587                               n->nlmsg_flags & NLM_F_ECHO);
1588 }
1589
1590 static int tfilter_del_notify(struct net *net, struct sk_buff *oskb,
1591                               struct nlmsghdr *n, struct tcf_proto *tp,
1592                               struct tcf_block *block, struct Qdisc *q,
1593                               u32 parent, void *fh, bool unicast, bool *last,
1594                               struct netlink_ext_ack *extack)
1595 {
1596         struct sk_buff *skb;
1597         u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
1598         int err;
1599
1600         skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
1601         if (!skb)
1602                 return -ENOBUFS;
1603
1604         if (tcf_fill_node(net, skb, tp, block, q, parent, fh, portid,
1605                           n->nlmsg_seq, n->nlmsg_flags, RTM_DELTFILTER) <= 0) {
1606                 NL_SET_ERR_MSG(extack, "Failed to build del event notification");
1607                 kfree_skb(skb);
1608                 return -EINVAL;
1609         }
1610
1611         err = tp->ops->delete(tp, fh, last, extack);
1612         if (err) {
1613                 kfree_skb(skb);
1614                 return err;
1615         }
1616
1617         if (unicast)
1618                 return netlink_unicast(net->rtnl, skb, portid, MSG_DONTWAIT);
1619
1620         err = rtnetlink_send(skb, net, portid, RTNLGRP_TC,
1621                              n->nlmsg_flags & NLM_F_ECHO);
1622         if (err < 0)
1623                 NL_SET_ERR_MSG(extack, "Failed to send filter delete notification");
1624         return err;
1625 }
1626
1627 static void tfilter_notify_chain(struct net *net, struct sk_buff *oskb,
1628                                  struct tcf_block *block, struct Qdisc *q,
1629                                  u32 parent, struct nlmsghdr *n,
1630                                  struct tcf_chain *chain, int event)
1631 {
1632         struct tcf_proto *tp;
1633
1634         for (tp = rtnl_dereference(chain->filter_chain);
1635              tp; tp = rtnl_dereference(tp->next))
1636                 tfilter_notify(net, oskb, n, tp, block,
1637                                q, parent, NULL, event, false);
1638 }
1639
1640 static int tc_new_tfilter(struct sk_buff *skb, struct nlmsghdr *n,
1641                           struct netlink_ext_ack *extack)
1642 {
1643         struct net *net = sock_net(skb->sk);
1644         struct nlattr *tca[TCA_MAX + 1];
1645         struct tcmsg *t;
1646         u32 protocol;
1647         u32 prio;
1648         bool prio_allocate;
1649         u32 parent;
1650         u32 chain_index;
1651         struct Qdisc *q = NULL;
1652         struct tcf_chain_info chain_info;
1653         struct tcf_chain *chain = NULL;
1654         struct tcf_block *block;
1655         struct tcf_proto *tp;
1656         unsigned long cl;
1657         void *fh;
1658         int err;
1659         int tp_created;
1660
1661         if (!netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN))
1662                 return -EPERM;
1663
1664 replay:
1665         tp_created = 0;
1666
1667         err = nlmsg_parse(n, sizeof(*t), tca, TCA_MAX, rtm_tca_policy, extack);
1668         if (err < 0)
1669                 return err;
1670
1671         t = nlmsg_data(n);
1672         protocol = TC_H_MIN(t->tcm_info);
1673         prio = TC_H_MAJ(t->tcm_info);
1674         prio_allocate = false;
1675         parent = t->tcm_parent;
1676         cl = 0;
1677
1678         if (prio == 0) {
1679                 /* If no priority is provided by the user,
1680                  * we allocate one.
1681                  */
1682                 if (n->nlmsg_flags & NLM_F_CREATE) {
1683                         prio = TC_H_MAKE(0x80000000U, 0U);
1684                         prio_allocate = true;
1685                 } else {
1686                         NL_SET_ERR_MSG(extack, "Invalid filter command with priority of zero");
1687                         return -ENOENT;
1688                 }
1689         }
1690
1691         /* Find head of filter chain. */
1692
1693         block = tcf_block_find(net, &q, &parent, &cl,
1694                                t->tcm_ifindex, t->tcm_block_index, extack);
1695         if (IS_ERR(block)) {
1696                 err = PTR_ERR(block);
1697                 goto errout;
1698         }
1699
1700         chain_index = tca[TCA_CHAIN] ? nla_get_u32(tca[TCA_CHAIN]) : 0;
1701         if (chain_index > TC_ACT_EXT_VAL_MASK) {
1702                 NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit");
1703                 err = -EINVAL;
1704                 goto errout;
1705         }
1706         chain = tcf_chain_get(block, chain_index, true);
1707         if (!chain) {
1708                 NL_SET_ERR_MSG(extack, "Cannot create specified filter chain");
1709                 err = -ENOMEM;
1710                 goto errout;
1711         }
1712
1713         tp = tcf_chain_tp_find(chain, &chain_info, protocol,
1714                                prio, prio_allocate);
1715         if (IS_ERR(tp)) {
1716                 NL_SET_ERR_MSG(extack, "Filter with specified priority/protocol not found");
1717                 err = PTR_ERR(tp);
1718                 goto errout;
1719         }
1720
1721         if (tp == NULL) {
1722                 /* Proto-tcf does not exist, create new one */
1723
1724                 if (tca[TCA_KIND] == NULL || !protocol) {
1725                         NL_SET_ERR_MSG(extack, "Filter kind and protocol must be specified");
1726                         err = -EINVAL;
1727                         goto errout;
1728                 }
1729
1730                 if (!(n->nlmsg_flags & NLM_F_CREATE)) {
1731                         NL_SET_ERR_MSG(extack, "Need both RTM_NEWTFILTER and NLM_F_CREATE to create a new filter");
1732                         err = -ENOENT;
1733                         goto errout;
1734                 }
1735
1736                 if (prio_allocate)
1737                         prio = tcf_auto_prio(tcf_chain_tp_prev(&chain_info));
1738
1739                 tp = tcf_proto_create(nla_data(tca[TCA_KIND]),
1740                                       protocol, prio, chain, extack);
1741                 if (IS_ERR(tp)) {
1742                         err = PTR_ERR(tp);
1743                         goto errout;
1744                 }
1745                 tp_created = 1;
1746         } else if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], tp->ops->kind)) {
1747                 NL_SET_ERR_MSG(extack, "Specified filter kind does not match existing one");
1748                 err = -EINVAL;
1749                 goto errout;
1750         }
1751
1752         fh = tp->ops->get(tp, t->tcm_handle);
1753
1754         if (!fh) {
1755                 if (!(n->nlmsg_flags & NLM_F_CREATE)) {
1756                         NL_SET_ERR_MSG(extack, "Need both RTM_NEWTFILTER and NLM_F_CREATE to create a new filter");
1757                         err = -ENOENT;
1758                         goto errout;
1759                 }
1760         } else if (n->nlmsg_flags & NLM_F_EXCL) {
1761                 NL_SET_ERR_MSG(extack, "Filter already exists");
1762                 err = -EEXIST;
1763                 goto errout;
1764         }
1765
1766         if (chain->tmplt_ops && chain->tmplt_ops != tp->ops) {
1767                 NL_SET_ERR_MSG(extack, "Chain template is set to a different filter kind");
1768                 err = -EINVAL;
1769                 goto errout;
1770         }
1771
1772         err = tp->ops->change(net, skb, tp, cl, t->tcm_handle, tca, &fh,
1773                               n->nlmsg_flags & NLM_F_CREATE ? TCA_ACT_NOREPLACE : TCA_ACT_REPLACE,
1774                               extack);
1775         if (err == 0) {
1776                 if (tp_created)
1777                         tcf_chain_tp_insert(chain, &chain_info, tp);
1778                 tfilter_notify(net, skb, n, tp, block, q, parent, fh,
1779                                RTM_NEWTFILTER, false);
1780         } else {
1781                 if (tp_created)
1782                         tcf_proto_destroy(tp, NULL);
1783         }
1784
1785 errout:
1786         if (chain)
1787                 tcf_chain_put(chain);
1788         tcf_block_release(q, block);
1789         if (err == -EAGAIN)
1790                 /* Replay the request. */
1791                 goto replay;
1792         return err;
1793 }
1794
1795 static int tc_del_tfilter(struct sk_buff *skb, struct nlmsghdr *n,
1796                           struct netlink_ext_ack *extack)
1797 {
1798         struct net *net = sock_net(skb->sk);
1799         struct nlattr *tca[TCA_MAX + 1];
1800         struct tcmsg *t;
1801         u32 protocol;
1802         u32 prio;
1803         u32 parent;
1804         u32 chain_index;
1805         struct Qdisc *q = NULL;
1806         struct tcf_chain_info chain_info;
1807         struct tcf_chain *chain = NULL;
1808         struct tcf_block *block;
1809         struct tcf_proto *tp = NULL;
1810         unsigned long cl = 0;
1811         void *fh = NULL;
1812         int err;
1813
1814         if (!netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN))
1815                 return -EPERM;
1816
1817         err = nlmsg_parse(n, sizeof(*t), tca, TCA_MAX, rtm_tca_policy, extack);
1818         if (err < 0)
1819                 return err;
1820
1821         t = nlmsg_data(n);
1822         protocol = TC_H_MIN(t->tcm_info);
1823         prio = TC_H_MAJ(t->tcm_info);
1824         parent = t->tcm_parent;
1825
1826         if (prio == 0 && (protocol || t->tcm_handle || tca[TCA_KIND])) {
1827                 NL_SET_ERR_MSG(extack, "Cannot flush filters with protocol, handle or kind set");
1828                 return -ENOENT;
1829         }
1830
1831         /* Find head of filter chain. */
1832
1833         block = tcf_block_find(net, &q, &parent, &cl,
1834                                t->tcm_ifindex, t->tcm_block_index, extack);
1835         if (IS_ERR(block)) {
1836                 err = PTR_ERR(block);
1837                 goto errout;
1838         }
1839
1840         chain_index = tca[TCA_CHAIN] ? nla_get_u32(tca[TCA_CHAIN]) : 0;
1841         if (chain_index > TC_ACT_EXT_VAL_MASK) {
1842                 NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit");
1843                 err = -EINVAL;
1844                 goto errout;
1845         }
1846         chain = tcf_chain_get(block, chain_index, false);
1847         if (!chain) {
1848                 /* User requested flush on non-existent chain. Nothing to do,
1849                  * so just return success.
1850                  */
1851                 if (prio == 0) {
1852                         err = 0;
1853                         goto errout;
1854                 }
1855                 NL_SET_ERR_MSG(extack, "Cannot find specified filter chain");
1856                 err = -ENOENT;
1857                 goto errout;
1858         }
1859
1860         if (prio == 0) {
1861                 tfilter_notify_chain(net, skb, block, q, parent, n,
1862                                      chain, RTM_DELTFILTER);
1863                 tcf_chain_flush(chain);
1864                 err = 0;
1865                 goto errout;
1866         }
1867
1868         tp = tcf_chain_tp_find(chain, &chain_info, protocol,
1869                                prio, false);
1870         if (!tp || IS_ERR(tp)) {
1871                 NL_SET_ERR_MSG(extack, "Filter with specified priority/protocol not found");
1872                 err = tp ? PTR_ERR(tp) : -ENOENT;
1873                 goto errout;
1874         } else if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], tp->ops->kind)) {
1875                 NL_SET_ERR_MSG(extack, "Specified filter kind does not match existing one");
1876                 err = -EINVAL;
1877                 goto errout;
1878         }
1879
1880         fh = tp->ops->get(tp, t->tcm_handle);
1881
1882         if (!fh) {
1883                 if (t->tcm_handle == 0) {
1884                         tcf_chain_tp_remove(chain, &chain_info, tp);
1885                         tfilter_notify(net, skb, n, tp, block, q, parent, fh,
1886                                        RTM_DELTFILTER, false);
1887                         tcf_proto_destroy(tp, extack);
1888                         err = 0;
1889                 } else {
1890                         NL_SET_ERR_MSG(extack, "Specified filter handle not found");
1891                         err = -ENOENT;
1892                 }
1893         } else {
1894                 bool last;
1895
1896                 err = tfilter_del_notify(net, skb, n, tp, block,
1897                                          q, parent, fh, false, &last,
1898                                          extack);
1899                 if (err)
1900                         goto errout;
1901                 if (last) {
1902                         tcf_chain_tp_remove(chain, &chain_info, tp);
1903                         tcf_proto_destroy(tp, extack);
1904                 }
1905         }
1906
1907 errout:
1908         if (chain)
1909                 tcf_chain_put(chain);
1910         tcf_block_release(q, block);
1911         return err;
1912 }
1913
1914 static int tc_get_tfilter(struct sk_buff *skb, struct nlmsghdr *n,
1915                           struct netlink_ext_ack *extack)
1916 {
1917         struct net *net = sock_net(skb->sk);
1918         struct nlattr *tca[TCA_MAX + 1];
1919         struct tcmsg *t;
1920         u32 protocol;
1921         u32 prio;
1922         u32 parent;
1923         u32 chain_index;
1924         struct Qdisc *q = NULL;
1925         struct tcf_chain_info chain_info;
1926         struct tcf_chain *chain = NULL;
1927         struct tcf_block *block;
1928         struct tcf_proto *tp = NULL;
1929         unsigned long cl = 0;
1930         void *fh = NULL;
1931         int err;
1932
1933         err = nlmsg_parse(n, sizeof(*t), tca, TCA_MAX, rtm_tca_policy, extack);
1934         if (err < 0)
1935                 return err;
1936
1937         t = nlmsg_data(n);
1938         protocol = TC_H_MIN(t->tcm_info);
1939         prio = TC_H_MAJ(t->tcm_info);
1940         parent = t->tcm_parent;
1941
1942         if (prio == 0) {
1943                 NL_SET_ERR_MSG(extack, "Invalid filter command with priority of zero");
1944                 return -ENOENT;
1945         }
1946
1947         /* Find head of filter chain. */
1948
1949         block = tcf_block_find(net, &q, &parent, &cl,
1950                                t->tcm_ifindex, t->tcm_block_index, extack);
1951         if (IS_ERR(block)) {
1952                 err = PTR_ERR(block);
1953                 goto errout;
1954         }
1955
1956         chain_index = tca[TCA_CHAIN] ? nla_get_u32(tca[TCA_CHAIN]) : 0;
1957         if (chain_index > TC_ACT_EXT_VAL_MASK) {
1958                 NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit");
1959                 err = -EINVAL;
1960                 goto errout;
1961         }
1962         chain = tcf_chain_get(block, chain_index, false);
1963         if (!chain) {
1964                 NL_SET_ERR_MSG(extack, "Cannot find specified filter chain");
1965                 err = -EINVAL;
1966                 goto errout;
1967         }
1968
1969         tp = tcf_chain_tp_find(chain, &chain_info, protocol,
1970                                prio, false);
1971         if (!tp || IS_ERR(tp)) {
1972                 NL_SET_ERR_MSG(extack, "Filter with specified priority/protocol not found");
1973                 err = tp ? PTR_ERR(tp) : -ENOENT;
1974                 goto errout;
1975         } else if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], tp->ops->kind)) {
1976                 NL_SET_ERR_MSG(extack, "Specified filter kind does not match existing one");
1977                 err = -EINVAL;
1978                 goto errout;
1979         }
1980
1981         fh = tp->ops->get(tp, t->tcm_handle);
1982
1983         if (!fh) {
1984                 NL_SET_ERR_MSG(extack, "Specified filter handle not found");
1985                 err = -ENOENT;
1986         } else {
1987                 err = tfilter_notify(net, skb, n, tp, block, q, parent,
1988                                      fh, RTM_NEWTFILTER, true);
1989                 if (err < 0)
1990                         NL_SET_ERR_MSG(extack, "Failed to send filter notify message");
1991         }
1992
1993 errout:
1994         if (chain)
1995                 tcf_chain_put(chain);
1996         tcf_block_release(q, block);
1997         return err;
1998 }
1999
2000 struct tcf_dump_args {
2001         struct tcf_walker w;
2002         struct sk_buff *skb;
2003         struct netlink_callback *cb;
2004         struct tcf_block *block;
2005         struct Qdisc *q;
2006         u32 parent;
2007 };
2008
2009 static int tcf_node_dump(struct tcf_proto *tp, void *n, struct tcf_walker *arg)
2010 {
2011         struct tcf_dump_args *a = (void *)arg;
2012         struct net *net = sock_net(a->skb->sk);
2013
2014         return tcf_fill_node(net, a->skb, tp, a->block, a->q, a->parent,
2015                              n, NETLINK_CB(a->cb->skb).portid,
2016                              a->cb->nlh->nlmsg_seq, NLM_F_MULTI,
2017                              RTM_NEWTFILTER);
2018 }
2019
2020 static bool tcf_chain_dump(struct tcf_chain *chain, struct Qdisc *q, u32 parent,
2021                            struct sk_buff *skb, struct netlink_callback *cb,
2022                            long index_start, long *p_index)
2023 {
2024         struct net *net = sock_net(skb->sk);
2025         struct tcf_block *block = chain->block;
2026         struct tcmsg *tcm = nlmsg_data(cb->nlh);
2027         struct tcf_dump_args arg;
2028         struct tcf_proto *tp;
2029
2030         for (tp = rtnl_dereference(chain->filter_chain);
2031              tp; tp = rtnl_dereference(tp->next), (*p_index)++) {
2032                 if (*p_index < index_start)
2033                         continue;
2034                 if (TC_H_MAJ(tcm->tcm_info) &&
2035                     TC_H_MAJ(tcm->tcm_info) != tp->prio)
2036                         continue;
2037                 if (TC_H_MIN(tcm->tcm_info) &&
2038                     TC_H_MIN(tcm->tcm_info) != tp->protocol)
2039                         continue;
2040                 if (*p_index > index_start)
2041                         memset(&cb->args[1], 0,
2042                                sizeof(cb->args) - sizeof(cb->args[0]));
2043                 if (cb->args[1] == 0) {
2044                         if (tcf_fill_node(net, skb, tp, block, q, parent, NULL,
2045                                           NETLINK_CB(cb->skb).portid,
2046                                           cb->nlh->nlmsg_seq, NLM_F_MULTI,
2047                                           RTM_NEWTFILTER) <= 0)
2048                                 return false;
2049
2050                         cb->args[1] = 1;
2051                 }
2052                 if (!tp->ops->walk)
2053                         continue;
2054                 arg.w.fn = tcf_node_dump;
2055                 arg.skb = skb;
2056                 arg.cb = cb;
2057                 arg.block = block;
2058                 arg.q = q;
2059                 arg.parent = parent;
2060                 arg.w.stop = 0;
2061                 arg.w.skip = cb->args[1] - 1;
2062                 arg.w.count = 0;
2063                 arg.w.cookie = cb->args[2];
2064                 tp->ops->walk(tp, &arg.w);
2065                 cb->args[2] = arg.w.cookie;
2066                 cb->args[1] = arg.w.count + 1;
2067                 if (arg.w.stop)
2068                         return false;
2069         }
2070         return true;
2071 }
2072
2073 /* called with RTNL */
2074 static int tc_dump_tfilter(struct sk_buff *skb, struct netlink_callback *cb)
2075 {
2076         struct tcf_chain *chain, *chain_prev;
2077         struct net *net = sock_net(skb->sk);
2078         struct nlattr *tca[TCA_MAX + 1];
2079         struct Qdisc *q = NULL;
2080         struct tcf_block *block;
2081         struct tcmsg *tcm = nlmsg_data(cb->nlh);
2082         long index_start;
2083         long index;
2084         u32 parent;
2085         int err;
2086
2087         if (nlmsg_len(cb->nlh) < sizeof(*tcm))
2088                 return skb->len;
2089
2090         err = nlmsg_parse(cb->nlh, sizeof(*tcm), tca, TCA_MAX, NULL,
2091                           cb->extack);
2092         if (err)
2093                 return err;
2094
2095         if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK) {
2096                 block = tcf_block_refcnt_get(net, tcm->tcm_block_index);
2097                 if (!block)
2098                         goto out;
2099                 /* If we work with block index, q is NULL and parent value
2100                  * will never be used in the following code. The check
2101                  * in tcf_fill_node prevents it. However, compiler does not
2102                  * see that far, so set parent to zero to silence the warning
2103                  * about parent being uninitialized.
2104                  */
2105                 parent = 0;
2106         } else {
2107                 const struct Qdisc_class_ops *cops;
2108                 struct net_device *dev;
2109                 unsigned long cl = 0;
2110
2111                 dev = __dev_get_by_index(net, tcm->tcm_ifindex);
2112                 if (!dev)
2113                         return skb->len;
2114
2115                 parent = tcm->tcm_parent;
2116                 if (!parent) {
2117                         q = dev->qdisc;
2118                         parent = q->handle;
2119                 } else {
2120                         q = qdisc_lookup(dev, TC_H_MAJ(tcm->tcm_parent));
2121                 }
2122                 if (!q)
2123                         goto out;
2124                 cops = q->ops->cl_ops;
2125                 if (!cops)
2126                         goto out;
2127                 if (!cops->tcf_block)
2128                         goto out;
2129                 if (TC_H_MIN(tcm->tcm_parent)) {
2130                         cl = cops->find(q, tcm->tcm_parent);
2131                         if (cl == 0)
2132                                 goto out;
2133                 }
2134                 block = cops->tcf_block(q, cl, NULL);
2135                 if (!block)
2136                         goto out;
2137                 if (tcf_block_shared(block))
2138                         q = NULL;
2139         }
2140
2141         index_start = cb->args[0];
2142         index = 0;
2143
2144         for (chain = __tcf_get_next_chain(block, NULL);
2145              chain;
2146              chain_prev = chain,
2147                      chain = __tcf_get_next_chain(block, chain),
2148                      tcf_chain_put(chain_prev)) {
2149                 if (tca[TCA_CHAIN] &&
2150                     nla_get_u32(tca[TCA_CHAIN]) != chain->index)
2151                         continue;
2152                 if (!tcf_chain_dump(chain, q, parent, skb, cb,
2153                                     index_start, &index)) {
2154                         tcf_chain_put(chain);
2155                         err = -EMSGSIZE;
2156                         break;
2157                 }
2158         }
2159
2160         if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK)
2161                 tcf_block_refcnt_put(block);
2162         cb->args[0] = index;
2163
2164 out:
2165         /* If we did no progress, the error (EMSGSIZE) is real */
2166         if (skb->len == 0 && err)
2167                 return err;
2168         return skb->len;
2169 }
2170
2171 static int tc_chain_fill_node(const struct tcf_proto_ops *tmplt_ops,
2172                               void *tmplt_priv, u32 chain_index,
2173                               struct net *net, struct sk_buff *skb,
2174                               struct tcf_block *block,
2175                               u32 portid, u32 seq, u16 flags, int event)
2176 {
2177         unsigned char *b = skb_tail_pointer(skb);
2178         const struct tcf_proto_ops *ops;
2179         struct nlmsghdr *nlh;
2180         struct tcmsg *tcm;
2181         void *priv;
2182
2183         ops = tmplt_ops;
2184         priv = tmplt_priv;
2185
2186         nlh = nlmsg_put(skb, portid, seq, event, sizeof(*tcm), flags);
2187         if (!nlh)
2188                 goto out_nlmsg_trim;
2189         tcm = nlmsg_data(nlh);
2190         tcm->tcm_family = AF_UNSPEC;
2191         tcm->tcm__pad1 = 0;
2192         tcm->tcm__pad2 = 0;
2193         tcm->tcm_handle = 0;
2194         if (block->q) {
2195                 tcm->tcm_ifindex = qdisc_dev(block->q)->ifindex;
2196                 tcm->tcm_parent = block->q->handle;
2197         } else {
2198                 tcm->tcm_ifindex = TCM_IFINDEX_MAGIC_BLOCK;
2199                 tcm->tcm_block_index = block->index;
2200         }
2201
2202         if (nla_put_u32(skb, TCA_CHAIN, chain_index))
2203                 goto nla_put_failure;
2204
2205         if (ops) {
2206                 if (nla_put_string(skb, TCA_KIND, ops->kind))
2207                         goto nla_put_failure;
2208                 if (ops->tmplt_dump(skb, net, priv) < 0)
2209                         goto nla_put_failure;
2210         }
2211
2212         nlh->nlmsg_len = skb_tail_pointer(skb) - b;
2213         return skb->len;
2214
2215 out_nlmsg_trim:
2216 nla_put_failure:
2217         nlmsg_trim(skb, b);
2218         return -EMSGSIZE;
2219 }
2220
2221 static int tc_chain_notify(struct tcf_chain *chain, struct sk_buff *oskb,
2222                            u32 seq, u16 flags, int event, bool unicast)
2223 {
2224         u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
2225         struct tcf_block *block = chain->block;
2226         struct net *net = block->net;
2227         struct sk_buff *skb;
2228
2229         skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
2230         if (!skb)
2231                 return -ENOBUFS;
2232
2233         if (tc_chain_fill_node(chain->tmplt_ops, chain->tmplt_priv,
2234                                chain->index, net, skb, block, portid,
2235                                seq, flags, event) <= 0) {
2236                 kfree_skb(skb);
2237                 return -EINVAL;
2238         }
2239
2240         if (unicast)
2241                 return netlink_unicast(net->rtnl, skb, portid, MSG_DONTWAIT);
2242
2243         return rtnetlink_send(skb, net, portid, RTNLGRP_TC, flags & NLM_F_ECHO);
2244 }
2245
2246 static int tc_chain_notify_delete(const struct tcf_proto_ops *tmplt_ops,
2247                                   void *tmplt_priv, u32 chain_index,
2248                                   struct tcf_block *block, struct sk_buff *oskb,
2249                                   u32 seq, u16 flags, bool unicast)
2250 {
2251         u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
2252         struct net *net = block->net;
2253         struct sk_buff *skb;
2254
2255         skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
2256         if (!skb)
2257                 return -ENOBUFS;
2258
2259         if (tc_chain_fill_node(tmplt_ops, tmplt_priv, chain_index, net, skb,
2260                                block, portid, seq, flags, RTM_DELCHAIN) <= 0) {
2261                 kfree_skb(skb);
2262                 return -EINVAL;
2263         }
2264
2265         if (unicast)
2266                 return netlink_unicast(net->rtnl, skb, portid, MSG_DONTWAIT);
2267
2268         return rtnetlink_send(skb, net, portid, RTNLGRP_TC, flags & NLM_F_ECHO);
2269 }
2270
2271 static int tc_chain_tmplt_add(struct tcf_chain *chain, struct net *net,
2272                               struct nlattr **tca,
2273                               struct netlink_ext_ack *extack)
2274 {
2275         const struct tcf_proto_ops *ops;
2276         void *tmplt_priv;
2277
2278         /* If kind is not set, user did not specify template. */
2279         if (!tca[TCA_KIND])
2280                 return 0;
2281
2282         ops = tcf_proto_lookup_ops(nla_data(tca[TCA_KIND]), extack);
2283         if (IS_ERR(ops))
2284                 return PTR_ERR(ops);
2285         if (!ops->tmplt_create || !ops->tmplt_destroy || !ops->tmplt_dump) {
2286                 NL_SET_ERR_MSG(extack, "Chain templates are not supported with specified classifier");
2287                 return -EOPNOTSUPP;
2288         }
2289
2290         tmplt_priv = ops->tmplt_create(net, chain, tca, extack);
2291         if (IS_ERR(tmplt_priv)) {
2292                 module_put(ops->owner);
2293                 return PTR_ERR(tmplt_priv);
2294         }
2295         chain->tmplt_ops = ops;
2296         chain->tmplt_priv = tmplt_priv;
2297         return 0;
2298 }
2299
2300 static void tc_chain_tmplt_del(const struct tcf_proto_ops *tmplt_ops,
2301                                void *tmplt_priv)
2302 {
2303         /* If template ops are set, no work to do for us. */
2304         if (!tmplt_ops)
2305                 return;
2306
2307         tmplt_ops->tmplt_destroy(tmplt_priv);
2308         module_put(tmplt_ops->owner);
2309 }
2310
2311 /* Add/delete/get a chain */
2312
2313 static int tc_ctl_chain(struct sk_buff *skb, struct nlmsghdr *n,
2314                         struct netlink_ext_ack *extack)
2315 {
2316         struct net *net = sock_net(skb->sk);
2317         struct nlattr *tca[TCA_MAX + 1];
2318         struct tcmsg *t;
2319         u32 parent;
2320         u32 chain_index;
2321         struct Qdisc *q = NULL;
2322         struct tcf_chain *chain = NULL;
2323         struct tcf_block *block;
2324         unsigned long cl;
2325         int err;
2326
2327         if (n->nlmsg_type != RTM_GETCHAIN &&
2328             !netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN))
2329                 return -EPERM;
2330
2331 replay:
2332         err = nlmsg_parse(n, sizeof(*t), tca, TCA_MAX, rtm_tca_policy, extack);
2333         if (err < 0)
2334                 return err;
2335
2336         t = nlmsg_data(n);
2337         parent = t->tcm_parent;
2338         cl = 0;
2339
2340         block = tcf_block_find(net, &q, &parent, &cl,
2341                                t->tcm_ifindex, t->tcm_block_index, extack);
2342         if (IS_ERR(block))
2343                 return PTR_ERR(block);
2344
2345         chain_index = tca[TCA_CHAIN] ? nla_get_u32(tca[TCA_CHAIN]) : 0;
2346         if (chain_index > TC_ACT_EXT_VAL_MASK) {
2347                 NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit");
2348                 err = -EINVAL;
2349                 goto errout_block;
2350         }
2351
2352         mutex_lock(&block->lock);
2353         chain = tcf_chain_lookup(block, chain_index);
2354         if (n->nlmsg_type == RTM_NEWCHAIN) {
2355                 if (chain) {
2356                         if (tcf_chain_held_by_acts_only(chain)) {
2357                                 /* The chain exists only because there is
2358                                  * some action referencing it.
2359                                  */
2360                                 tcf_chain_hold(chain);
2361                         } else {
2362                                 NL_SET_ERR_MSG(extack, "Filter chain already exists");
2363                                 err = -EEXIST;
2364                                 goto errout_block_locked;
2365                         }
2366                 } else {
2367                         if (!(n->nlmsg_flags & NLM_F_CREATE)) {
2368                                 NL_SET_ERR_MSG(extack, "Need both RTM_NEWCHAIN and NLM_F_CREATE to create a new chain");
2369                                 err = -ENOENT;
2370                                 goto errout_block_locked;
2371                         }
2372                         chain = tcf_chain_create(block, chain_index);
2373                         if (!chain) {
2374                                 NL_SET_ERR_MSG(extack, "Failed to create filter chain");
2375                                 err = -ENOMEM;
2376                                 goto errout_block_locked;
2377                         }
2378                 }
2379         } else {
2380                 if (!chain || tcf_chain_held_by_acts_only(chain)) {
2381                         NL_SET_ERR_MSG(extack, "Cannot find specified filter chain");
2382                         err = -EINVAL;
2383                         goto errout_block_locked;
2384                 }
2385                 tcf_chain_hold(chain);
2386         }
2387
2388         if (n->nlmsg_type == RTM_NEWCHAIN) {
2389                 /* Modifying chain requires holding parent block lock. In case
2390                  * the chain was successfully added, take a reference to the
2391                  * chain. This ensures that an empty chain does not disappear at
2392                  * the end of this function.
2393                  */
2394                 tcf_chain_hold(chain);
2395                 chain->explicitly_created = true;
2396         }
2397         mutex_unlock(&block->lock);
2398
2399         switch (n->nlmsg_type) {
2400         case RTM_NEWCHAIN:
2401                 err = tc_chain_tmplt_add(chain, net, tca, extack);
2402                 if (err) {
2403                         tcf_chain_put_explicitly_created(chain);
2404                         goto errout;
2405                 }
2406
2407                 tc_chain_notify(chain, NULL, 0, NLM_F_CREATE | NLM_F_EXCL,
2408                                 RTM_NEWCHAIN, false);
2409                 break;
2410         case RTM_DELCHAIN:
2411                 tfilter_notify_chain(net, skb, block, q, parent, n,
2412                                      chain, RTM_DELTFILTER);
2413                 /* Flush the chain first as the user requested chain removal. */
2414                 tcf_chain_flush(chain);
2415                 /* In case the chain was successfully deleted, put a reference
2416                  * to the chain previously taken during addition.
2417                  */
2418                 tcf_chain_put_explicitly_created(chain);
2419                 break;
2420         case RTM_GETCHAIN:
2421                 err = tc_chain_notify(chain, skb, n->nlmsg_seq,
2422                                       n->nlmsg_seq, n->nlmsg_type, true);
2423                 if (err < 0)
2424                         NL_SET_ERR_MSG(extack, "Failed to send chain notify message");
2425                 break;
2426         default:
2427                 err = -EOPNOTSUPP;
2428                 NL_SET_ERR_MSG(extack, "Unsupported message type");
2429                 goto errout;
2430         }
2431
2432 errout:
2433         tcf_chain_put(chain);
2434 errout_block:
2435         tcf_block_release(q, block);
2436         if (err == -EAGAIN)
2437                 /* Replay the request. */
2438                 goto replay;
2439         return err;
2440
2441 errout_block_locked:
2442         mutex_unlock(&block->lock);
2443         goto errout_block;
2444 }
2445
2446 /* called with RTNL */
2447 static int tc_dump_chain(struct sk_buff *skb, struct netlink_callback *cb)
2448 {
2449         struct tcf_chain *chain, *chain_prev;
2450         struct net *net = sock_net(skb->sk);
2451         struct nlattr *tca[TCA_MAX + 1];
2452         struct Qdisc *q = NULL;
2453         struct tcf_block *block;
2454         struct tcmsg *tcm = nlmsg_data(cb->nlh);
2455         long index_start;
2456         long index;
2457         u32 parent;
2458         int err;
2459
2460         if (nlmsg_len(cb->nlh) < sizeof(*tcm))
2461                 return skb->len;
2462
2463         err = nlmsg_parse(cb->nlh, sizeof(*tcm), tca, TCA_MAX, rtm_tca_policy,
2464                           cb->extack);
2465         if (err)
2466                 return err;
2467
2468         if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK) {
2469                 block = tcf_block_refcnt_get(net, tcm->tcm_block_index);
2470                 if (!block)
2471                         goto out;
2472                 /* If we work with block index, q is NULL and parent value
2473                  * will never be used in the following code. The check
2474                  * in tcf_fill_node prevents it. However, compiler does not
2475                  * see that far, so set parent to zero to silence the warning
2476                  * about parent being uninitialized.
2477                  */
2478                 parent = 0;
2479         } else {
2480                 const struct Qdisc_class_ops *cops;
2481                 struct net_device *dev;
2482                 unsigned long cl = 0;
2483
2484                 dev = __dev_get_by_index(net, tcm->tcm_ifindex);
2485                 if (!dev)
2486                         return skb->len;
2487
2488                 parent = tcm->tcm_parent;
2489                 if (!parent) {
2490                         q = dev->qdisc;
2491                         parent = q->handle;
2492                 } else {
2493                         q = qdisc_lookup(dev, TC_H_MAJ(tcm->tcm_parent));
2494                 }
2495                 if (!q)
2496                         goto out;
2497                 cops = q->ops->cl_ops;
2498                 if (!cops)
2499                         goto out;
2500                 if (!cops->tcf_block)
2501                         goto out;
2502                 if (TC_H_MIN(tcm->tcm_parent)) {
2503                         cl = cops->find(q, tcm->tcm_parent);
2504                         if (cl == 0)
2505                                 goto out;
2506                 }
2507                 block = cops->tcf_block(q, cl, NULL);
2508                 if (!block)
2509                         goto out;
2510                 if (tcf_block_shared(block))
2511                         q = NULL;
2512         }
2513
2514         index_start = cb->args[0];
2515         index = 0;
2516
2517         for (chain = __tcf_get_next_chain(block, NULL);
2518              chain;
2519              chain_prev = chain,
2520                      chain = __tcf_get_next_chain(block, chain),
2521                      tcf_chain_put(chain_prev)) {
2522                 if ((tca[TCA_CHAIN] &&
2523                      nla_get_u32(tca[TCA_CHAIN]) != chain->index))
2524                         continue;
2525                 if (index < index_start) {
2526                         index++;
2527                         continue;
2528                 }
2529                 err = tc_chain_fill_node(chain->tmplt_ops, chain->tmplt_priv,
2530                                          chain->index, net, skb, block,
2531                                          NETLINK_CB(cb->skb).portid,
2532                                          cb->nlh->nlmsg_seq, NLM_F_MULTI,
2533                                          RTM_NEWCHAIN);
2534                 if (err <= 0) {
2535                         tcf_chain_put(chain);
2536                         break;
2537                 }
2538                 index++;
2539         }
2540
2541         if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK)
2542                 tcf_block_refcnt_put(block);
2543         cb->args[0] = index;
2544
2545 out:
2546         /* If we did no progress, the error (EMSGSIZE) is real */
2547         if (skb->len == 0 && err)
2548                 return err;
2549         return skb->len;
2550 }
2551
2552 void tcf_exts_destroy(struct tcf_exts *exts)
2553 {
2554 #ifdef CONFIG_NET_CLS_ACT
2555         tcf_action_destroy(exts->actions, TCA_ACT_UNBIND);
2556         kfree(exts->actions);
2557         exts->nr_actions = 0;
2558 #endif
2559 }
2560 EXPORT_SYMBOL(tcf_exts_destroy);
2561
2562 int tcf_exts_validate(struct net *net, struct tcf_proto *tp, struct nlattr **tb,
2563                       struct nlattr *rate_tlv, struct tcf_exts *exts, bool ovr,
2564                       struct netlink_ext_ack *extack)
2565 {
2566 #ifdef CONFIG_NET_CLS_ACT
2567         {
2568                 struct tc_action *act;
2569                 size_t attr_size = 0;
2570
2571                 if (exts->police && tb[exts->police]) {
2572                         act = tcf_action_init_1(net, tp, tb[exts->police],
2573                                                 rate_tlv, "police", ovr,
2574                                                 TCA_ACT_BIND, true, extack);
2575                         if (IS_ERR(act))
2576                                 return PTR_ERR(act);
2577
2578                         act->type = exts->type = TCA_OLD_COMPAT;
2579                         exts->actions[0] = act;
2580                         exts->nr_actions = 1;
2581                 } else if (exts->action && tb[exts->action]) {
2582                         int err;
2583
2584                         err = tcf_action_init(net, tp, tb[exts->action],
2585                                               rate_tlv, NULL, ovr, TCA_ACT_BIND,
2586                                               exts->actions, &attr_size, true,
2587                                               extack);
2588                         if (err < 0)
2589                                 return err;
2590                         exts->nr_actions = err;
2591                 }
2592                 exts->net = net;
2593         }
2594 #else
2595         if ((exts->action && tb[exts->action]) ||
2596             (exts->police && tb[exts->police])) {
2597                 NL_SET_ERR_MSG(extack, "Classifier actions are not supported per compile options (CONFIG_NET_CLS_ACT)");
2598                 return -EOPNOTSUPP;
2599         }
2600 #endif
2601
2602         return 0;
2603 }
2604 EXPORT_SYMBOL(tcf_exts_validate);
2605
2606 void tcf_exts_change(struct tcf_exts *dst, struct tcf_exts *src)
2607 {
2608 #ifdef CONFIG_NET_CLS_ACT
2609         struct tcf_exts old = *dst;
2610
2611         *dst = *src;
2612         tcf_exts_destroy(&old);
2613 #endif
2614 }
2615 EXPORT_SYMBOL(tcf_exts_change);
2616
2617 #ifdef CONFIG_NET_CLS_ACT
2618 static struct tc_action *tcf_exts_first_act(struct tcf_exts *exts)
2619 {
2620         if (exts->nr_actions == 0)
2621                 return NULL;
2622         else
2623                 return exts->actions[0];
2624 }
2625 #endif
2626
2627 int tcf_exts_dump(struct sk_buff *skb, struct tcf_exts *exts)
2628 {
2629 #ifdef CONFIG_NET_CLS_ACT
2630         struct nlattr *nest;
2631
2632         if (exts->action && tcf_exts_has_actions(exts)) {
2633                 /*
2634                  * again for backward compatible mode - we want
2635                  * to work with both old and new modes of entering
2636                  * tc data even if iproute2  was newer - jhs
2637                  */
2638                 if (exts->type != TCA_OLD_COMPAT) {
2639                         nest = nla_nest_start(skb, exts->action);
2640                         if (nest == NULL)
2641                                 goto nla_put_failure;
2642
2643                         if (tcf_action_dump(skb, exts->actions, 0, 0) < 0)
2644                                 goto nla_put_failure;
2645                         nla_nest_end(skb, nest);
2646                 } else if (exts->police) {
2647                         struct tc_action *act = tcf_exts_first_act(exts);
2648                         nest = nla_nest_start(skb, exts->police);
2649                         if (nest == NULL || !act)
2650                                 goto nla_put_failure;
2651                         if (tcf_action_dump_old(skb, act, 0, 0) < 0)
2652                                 goto nla_put_failure;
2653                         nla_nest_end(skb, nest);
2654                 }
2655         }
2656         return 0;
2657
2658 nla_put_failure:
2659         nla_nest_cancel(skb, nest);
2660         return -1;
2661 #else
2662         return 0;
2663 #endif
2664 }
2665 EXPORT_SYMBOL(tcf_exts_dump);
2666
2667
2668 int tcf_exts_dump_stats(struct sk_buff *skb, struct tcf_exts *exts)
2669 {
2670 #ifdef CONFIG_NET_CLS_ACT
2671         struct tc_action *a = tcf_exts_first_act(exts);
2672         if (a != NULL && tcf_action_copy_stats(skb, a, 1) < 0)
2673                 return -1;
2674 #endif
2675         return 0;
2676 }
2677 EXPORT_SYMBOL(tcf_exts_dump_stats);
2678
2679 int tc_setup_cb_call(struct tcf_block *block, enum tc_setup_type type,
2680                      void *type_data, bool err_stop)
2681 {
2682         struct tcf_block_cb *block_cb;
2683         int ok_count = 0;
2684         int err;
2685
2686         /* Make sure all netdevs sharing this block are offload-capable. */
2687         if (block->nooffloaddevcnt && err_stop)
2688                 return -EOPNOTSUPP;
2689
2690         list_for_each_entry(block_cb, &block->cb_list, list) {
2691                 err = block_cb->cb(type, type_data, block_cb->cb_priv);
2692                 if (err) {
2693                         if (err_stop)
2694                                 return err;
2695                 } else {
2696                         ok_count++;
2697                 }
2698         }
2699         return ok_count;
2700 }
2701 EXPORT_SYMBOL(tc_setup_cb_call);
2702
2703 int tc_setup_flow_action(struct flow_action *flow_action,
2704                          const struct tcf_exts *exts)
2705 {
2706         const struct tc_action *act;
2707         int i, j, k;
2708
2709         if (!exts)
2710                 return 0;
2711
2712         j = 0;
2713         tcf_exts_for_each_action(i, act, exts) {
2714                 struct flow_action_entry *entry;
2715
2716                 entry = &flow_action->entries[j];
2717                 if (is_tcf_gact_ok(act)) {
2718                         entry->id = FLOW_ACTION_ACCEPT;
2719                 } else if (is_tcf_gact_shot(act)) {
2720                         entry->id = FLOW_ACTION_DROP;
2721                 } else if (is_tcf_gact_trap(act)) {
2722                         entry->id = FLOW_ACTION_TRAP;
2723                 } else if (is_tcf_gact_goto_chain(act)) {
2724                         entry->id = FLOW_ACTION_GOTO;
2725                         entry->chain_index = tcf_gact_goto_chain_index(act);
2726                 } else if (is_tcf_mirred_egress_redirect(act)) {
2727                         entry->id = FLOW_ACTION_REDIRECT;
2728                         entry->dev = tcf_mirred_dev(act);
2729                 } else if (is_tcf_mirred_egress_mirror(act)) {
2730                         entry->id = FLOW_ACTION_MIRRED;
2731                         entry->dev = tcf_mirred_dev(act);
2732                 } else if (is_tcf_vlan(act)) {
2733                         switch (tcf_vlan_action(act)) {
2734                         case TCA_VLAN_ACT_PUSH:
2735                                 entry->id = FLOW_ACTION_VLAN_PUSH;
2736                                 entry->vlan.vid = tcf_vlan_push_vid(act);
2737                                 entry->vlan.proto = tcf_vlan_push_proto(act);
2738                                 entry->vlan.prio = tcf_vlan_push_prio(act);
2739                                 break;
2740                         case TCA_VLAN_ACT_POP:
2741                                 entry->id = FLOW_ACTION_VLAN_POP;
2742                                 break;
2743                         case TCA_VLAN_ACT_MODIFY:
2744                                 entry->id = FLOW_ACTION_VLAN_MANGLE;
2745                                 entry->vlan.vid = tcf_vlan_push_vid(act);
2746                                 entry->vlan.proto = tcf_vlan_push_proto(act);
2747                                 entry->vlan.prio = tcf_vlan_push_prio(act);
2748                                 break;
2749                         default:
2750                                 goto err_out;
2751                         }
2752                 } else if (is_tcf_tunnel_set(act)) {
2753                         entry->id = FLOW_ACTION_TUNNEL_ENCAP;
2754                         entry->tunnel = tcf_tunnel_info(act);
2755                 } else if (is_tcf_tunnel_release(act)) {
2756                         entry->id = FLOW_ACTION_TUNNEL_DECAP;
2757                         entry->tunnel = tcf_tunnel_info(act);
2758                 } else if (is_tcf_pedit(act)) {
2759                         for (k = 0; k < tcf_pedit_nkeys(act); k++) {
2760                                 switch (tcf_pedit_cmd(act, k)) {
2761                                 case TCA_PEDIT_KEY_EX_CMD_SET:
2762                                         entry->id = FLOW_ACTION_MANGLE;
2763                                         break;
2764                                 case TCA_PEDIT_KEY_EX_CMD_ADD:
2765                                         entry->id = FLOW_ACTION_ADD;
2766                                         break;
2767                                 default:
2768                                         goto err_out;
2769                                 }
2770                                 entry->mangle.htype = tcf_pedit_htype(act, k);
2771                                 entry->mangle.mask = tcf_pedit_mask(act, k);
2772                                 entry->mangle.val = tcf_pedit_val(act, k);
2773                                 entry->mangle.offset = tcf_pedit_offset(act, k);
2774                                 entry = &flow_action->entries[++j];
2775                         }
2776                 } else if (is_tcf_csum(act)) {
2777                         entry->id = FLOW_ACTION_CSUM;
2778                         entry->csum_flags = tcf_csum_update_flags(act);
2779                 } else if (is_tcf_skbedit_mark(act)) {
2780                         entry->id = FLOW_ACTION_MARK;
2781                         entry->mark = tcf_skbedit_mark(act);
2782                 } else {
2783                         goto err_out;
2784                 }
2785
2786                 if (!is_tcf_pedit(act))
2787                         j++;
2788         }
2789         return 0;
2790 err_out:
2791         return -EOPNOTSUPP;
2792 }
2793 EXPORT_SYMBOL(tc_setup_flow_action);
2794
2795 unsigned int tcf_exts_num_actions(struct tcf_exts *exts)
2796 {
2797         unsigned int num_acts = 0;
2798         struct tc_action *act;
2799         int i;
2800
2801         tcf_exts_for_each_action(i, act, exts) {
2802                 if (is_tcf_pedit(act))
2803                         num_acts += tcf_pedit_nkeys(act);
2804                 else
2805                         num_acts++;
2806         }
2807         return num_acts;
2808 }
2809 EXPORT_SYMBOL(tcf_exts_num_actions);
2810
2811 static __net_init int tcf_net_init(struct net *net)
2812 {
2813         struct tcf_net *tn = net_generic(net, tcf_net_id);
2814
2815         spin_lock_init(&tn->idr_lock);
2816         idr_init(&tn->idr);
2817         return 0;
2818 }
2819
2820 static void __net_exit tcf_net_exit(struct net *net)
2821 {
2822         struct tcf_net *tn = net_generic(net, tcf_net_id);
2823
2824         idr_destroy(&tn->idr);
2825 }
2826
2827 static struct pernet_operations tcf_net_ops = {
2828         .init = tcf_net_init,
2829         .exit = tcf_net_exit,
2830         .id   = &tcf_net_id,
2831         .size = sizeof(struct tcf_net),
2832 };
2833
2834 static int __init tc_filter_init(void)
2835 {
2836         int err;
2837
2838         tc_filter_wq = alloc_ordered_workqueue("tc_filter_workqueue", 0);
2839         if (!tc_filter_wq)
2840                 return -ENOMEM;
2841
2842         err = register_pernet_subsys(&tcf_net_ops);
2843         if (err)
2844                 goto err_register_pernet_subsys;
2845
2846         err = rhashtable_init(&indr_setup_block_ht,
2847                               &tc_indr_setup_block_ht_params);
2848         if (err)
2849                 goto err_rhash_setup_block_ht;
2850
2851         rtnl_register(PF_UNSPEC, RTM_NEWTFILTER, tc_new_tfilter, NULL, 0);
2852         rtnl_register(PF_UNSPEC, RTM_DELTFILTER, tc_del_tfilter, NULL, 0);
2853         rtnl_register(PF_UNSPEC, RTM_GETTFILTER, tc_get_tfilter,
2854                       tc_dump_tfilter, 0);
2855         rtnl_register(PF_UNSPEC, RTM_NEWCHAIN, tc_ctl_chain, NULL, 0);
2856         rtnl_register(PF_UNSPEC, RTM_DELCHAIN, tc_ctl_chain, NULL, 0);
2857         rtnl_register(PF_UNSPEC, RTM_GETCHAIN, tc_ctl_chain,
2858                       tc_dump_chain, 0);
2859
2860         return 0;
2861
2862 err_rhash_setup_block_ht:
2863         unregister_pernet_subsys(&tcf_net_ops);
2864 err_register_pernet_subsys:
2865         destroy_workqueue(tc_filter_wq);
2866         return err;
2867 }
2868
2869 subsys_initcall(tc_filter_init);