net/sched/cls_api.c

   1 /*
   2  * net/sched/cls_api.c  Packet classifier API.
   3  *
   4  *              This program is free software; you can redistribute it and/or
   5  *              modify it under the terms of the GNU General Public License
   6  *              as published by the Free Software Foundation; either version
   7  *              2 of the License, or (at your option) any later version.
   8  *
   9  * Authors:     Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
  10  *
  11  * Changes:
  12  *
  13  * Eduardo J. Blanco <ejbs@netlabs.com.uy> :990222: kmod support
  14  *
  15  */
  16
  17 #include <linux/module.h>
  18 #include <linux/types.h>
  19 #include <linux/kernel.h>
  20 #include <linux/string.h>
  21 #include <linux/errno.h>
  22 #include <linux/err.h>
  23 #include <linux/skbuff.h>
  24 #include <linux/init.h>
  25 #include <linux/kmod.h>
  26 #include <linux/slab.h>
  27 #include <linux/idr.h>
  28 #include <linux/rhashtable.h>
  29 #include <net/net_namespace.h>
  30 #include <net/sock.h>
  31 #include <net/netlink.h>
  32 #include <net/pkt_sched.h>
  33 #include <net/pkt_cls.h>
  34 #include <net/tc_act/tc_pedit.h>
  35 #include <net/tc_act/tc_mirred.h>
  36 #include <net/tc_act/tc_vlan.h>
  37 #include <net/tc_act/tc_tunnel_key.h>
  38 #include <net/tc_act/tc_csum.h>
  39 #include <net/tc_act/tc_gact.h>
  40 #include <net/tc_act/tc_skbedit.h>
  41 #include <net/tc_act/tc_mirred.h>
  42
  43 extern const struct nla_policy rtm_tca_policy[TCA_MAX + 1];
  44
  45 /* The list of all installed classifier types */
  46 static LIST_HEAD(tcf_proto_base);
  47
  48 /* Protects list of registered TC modules. It is pure SMP lock. */
  49 static DEFINE_RWLOCK(cls_mod_lock);
  50
  51 /* Find classifier type by string name */
  52
  53 static const struct tcf_proto_ops *__tcf_proto_lookup_ops(const char *kind)
  54 {
  55         const struct tcf_proto_ops *t, *res = NULL;
  56
  57         if (kind) {
  58                 read_lock(&cls_mod_lock);
  59                 list_for_each_entry(t, &tcf_proto_base, head) {
  60                         if (strcmp(kind, t->kind) == 0) {
  61                                 if (try_module_get(t->owner))
  62                                         res = t;
  63                                 break;
  64                         }
  65                 }
  66                 read_unlock(&cls_mod_lock);
  67         }
  68         return res;
  69 }
  70
  71 static const struct tcf_proto_ops *
  72 tcf_proto_lookup_ops(const char *kind, struct netlink_ext_ack *extack)
  73 {
  74         const struct tcf_proto_ops *ops;
  75
  76         ops = __tcf_proto_lookup_ops(kind);
  77         if (ops)
  78                 return ops;
  79 #ifdef CONFIG_MODULES
  80         rtnl_unlock();
  81         request_module("cls_%s", kind);
  82         rtnl_lock();
  83         ops = __tcf_proto_lookup_ops(kind);
  84         /* We dropped the RTNL semaphore in order to perform
  85          * the module load. So, even if we succeeded in loading
  86          * the module we have to replay the request. We indicate
  87          * this using -EAGAIN.
  88          */
  89         if (ops) {
  90                 module_put(ops->owner);
  91                 return ERR_PTR(-EAGAIN);
  92         }
  93 #endif
  94         NL_SET_ERR_MSG(extack, "TC classifier not found");
  95         return ERR_PTR(-ENOENT);
  96 }
  97
  98 /* Register(unregister) new classifier type */
  99
 100 int register_tcf_proto_ops(struct tcf_proto_ops *ops)
 101 {
 102         struct tcf_proto_ops *t;
 103         int rc = -EEXIST;
 104
 105         write_lock(&cls_mod_lock);
 106         list_for_each_entry(t, &tcf_proto_base, head)
 107                 if (!strcmp(ops->kind, t->kind))
 108                         goto out;
 109
 110         list_add_tail(&ops->head, &tcf_proto_base);
 111         rc = 0;
 112 out:
 113         write_unlock(&cls_mod_lock);
 114         return rc;
 115 }
 116 EXPORT_SYMBOL(register_tcf_proto_ops);
 117
 118 static struct workqueue_struct *tc_filter_wq;
 119
 120 int unregister_tcf_proto_ops(struct tcf_proto_ops *ops)
 121 {
 122         struct tcf_proto_ops *t;
 123         int rc = -ENOENT;
 124
 125         /* Wait for outstanding call_rcu()s, if any, from a
 126          * tcf_proto_ops's destroy() handler.
 127          */
 128         rcu_barrier();
 129         flush_workqueue(tc_filter_wq);
 130
 131         write_lock(&cls_mod_lock);
 132         list_for_each_entry(t, &tcf_proto_base, head) {
 133                 if (t == ops) {
 134                         list_del(&t->head);
 135                         rc = 0;
 136                         break;
 137                 }
 138         }
 139         write_unlock(&cls_mod_lock);
 140         return rc;
 141 }
 142 EXPORT_SYMBOL(unregister_tcf_proto_ops);
 143
 144 bool tcf_queue_work(struct rcu_work *rwork, work_func_t func)
 145 {
 146         INIT_RCU_WORK(rwork, func);
 147         return queue_rcu_work(tc_filter_wq, rwork);
 148 }
 149 EXPORT_SYMBOL(tcf_queue_work);
 150
 151 /* Select new prio value from the range, managed by kernel. */
 152
 153 static inline u32 tcf_auto_prio(struct tcf_proto *tp)
 154 {
 155         u32 first = TC_H_MAKE(0xC0000000U, 0U);
 156
 157         if (tp)
 158                 first = tp->prio - 1;
 159
 160         return TC_H_MAJ(first);
 161 }
 162
 163 static struct tcf_proto *tcf_proto_create(const char *kind, u32 protocol,
 164                                           u32 prio, struct tcf_chain *chain,
 165                                           struct netlink_ext_ack *extack)
 166 {
 167         struct tcf_proto *tp;
 168         int err;
 169
 170         tp = kzalloc(sizeof(*tp), GFP_KERNEL);
 171         if (!tp)
 172                 return ERR_PTR(-ENOBUFS);
 173
 174         tp->ops = tcf_proto_lookup_ops(kind, extack);
 175         if (IS_ERR(tp->ops)) {
 176                 err = PTR_ERR(tp->ops);
 177                 goto errout;
 178         }
 179         tp->classify = tp->ops->classify;
 180         tp->protocol = protocol;
 181         tp->prio = prio;
 182         tp->chain = chain;
 183         spin_lock_init(&tp->lock);
 184         refcount_set(&tp->refcnt, 1);
 185
 186         err = tp->ops->init(tp);
 187         if (err) {
 188                 module_put(tp->ops->owner);
 189                 goto errout;
 190         }
 191         return tp;
 192
 193 errout:
 194         kfree(tp);
 195         return ERR_PTR(err);
 196 }
 197
 198 static void tcf_proto_get(struct tcf_proto *tp)
 199 {
 200         refcount_inc(&tp->refcnt);
 201 }
 202
 203 static void tcf_chain_put(struct tcf_chain *chain);
 204
 205 static void tcf_proto_destroy(struct tcf_proto *tp,
 206                               struct netlink_ext_ack *extack)
 207 {
 208         tp->ops->destroy(tp, extack);
 209         tcf_chain_put(tp->chain);
 210         module_put(tp->ops->owner);
 211         kfree_rcu(tp, rcu);
 212 }
 213
 214 static void tcf_proto_put(struct tcf_proto *tp,
 215                           struct netlink_ext_ack *extack)
 216 {
 217         if (refcount_dec_and_test(&tp->refcnt))
 218                 tcf_proto_destroy(tp, extack);
 219 }
 220
 221 static int walker_noop(struct tcf_proto *tp, void *d, struct tcf_walker *arg)
 222 {
 223         return -1;
 224 }
 225
 226 static bool tcf_proto_is_empty(struct tcf_proto *tp)
 227 {
 228         struct tcf_walker walker = { .fn = walker_noop, };
 229
 230         if (tp->ops->walk) {
 231                 tp->ops->walk(tp, &walker);
 232                 return !walker.stop;
 233         }
 234         return true;
 235 }
 236
 237 static bool tcf_proto_check_delete(struct tcf_proto *tp)
 238 {
 239         spin_lock(&tp->lock);
 240         if (tcf_proto_is_empty(tp))
 241                 tp->deleting = true;
 242         spin_unlock(&tp->lock);
 243         return tp->deleting;
 244 }
 245
 246 static void tcf_proto_mark_delete(struct tcf_proto *tp)
 247 {
 248         spin_lock(&tp->lock);
 249         tp->deleting = true;
 250         spin_unlock(&tp->lock);
 251 }
 252
 253 static bool tcf_proto_is_deleting(struct tcf_proto *tp)
 254 {
 255         bool deleting;
 256
 257         spin_lock(&tp->lock);
 258         deleting = tp->deleting;
 259         spin_unlock(&tp->lock);
 260
 261         return deleting;
 262 }
 263
 264 #define ASSERT_BLOCK_LOCKED(block)                                      \
 265         lockdep_assert_held(&(block)->lock)
 266
 267 struct tcf_filter_chain_list_item {
 268         struct list_head list;
 269         tcf_chain_head_change_t *chain_head_change;
 270         void *chain_head_change_priv;
 271 };
 272
 273 static struct tcf_chain *tcf_chain_create(struct tcf_block *block,
 274                                           u32 chain_index)
 275 {
 276         struct tcf_chain *chain;
 277
 278         ASSERT_BLOCK_LOCKED(block);
 279
 280         chain = kzalloc(sizeof(*chain), GFP_KERNEL);
 281         if (!chain)
 282                 return NULL;
 283         list_add_tail(&chain->list, &block->chain_list);
 284         mutex_init(&chain->filter_chain_lock);
 285         chain->block = block;
 286         chain->index = chain_index;
 287         chain->refcnt = 1;
 288         if (!chain->index)
 289                 block->chain0.chain = chain;
 290         return chain;
 291 }
 292
 293 static void tcf_chain_head_change_item(struct tcf_filter_chain_list_item *item,
 294                                        struct tcf_proto *tp_head)
 295 {
 296         if (item->chain_head_change)
 297                 item->chain_head_change(tp_head, item->chain_head_change_priv);
 298 }
 299
 300 static void tcf_chain0_head_change(struct tcf_chain *chain,
 301                                    struct tcf_proto *tp_head)
 302 {
 303         struct tcf_filter_chain_list_item *item;
 304         struct tcf_block *block = chain->block;
 305
 306         if (chain->index)
 307                 return;
 308
 309         mutex_lock(&block->lock);
 310         list_for_each_entry(item, &block->chain0.filter_chain_list, list)
 311                 tcf_chain_head_change_item(item, tp_head);
 312         mutex_unlock(&block->lock);
 313 }
 314
 315 /* Returns true if block can be safely freed. */
 316
 317 static bool tcf_chain_detach(struct tcf_chain *chain)
 318 {
 319         struct tcf_block *block = chain->block;
 320
 321         ASSERT_BLOCK_LOCKED(block);
 322
 323         list_del(&chain->list);
 324         if (!chain->index)
 325                 block->chain0.chain = NULL;
 326
 327         if (list_empty(&block->chain_list) &&
 328             refcount_read(&block->refcnt) == 0)
 329                 return true;
 330
 331         return false;
 332 }
 333
 334 static void tcf_block_destroy(struct tcf_block *block)
 335 {
 336         mutex_destroy(&block->lock);
 337         kfree_rcu(block, rcu);
 338 }
 339
 340 static void tcf_chain_destroy(struct tcf_chain *chain, bool free_block)
 341 {
 342         struct tcf_block *block = chain->block;
 343
 344         mutex_destroy(&chain->filter_chain_lock);
 345         kfree(chain);
 346         if (free_block)
 347                 tcf_block_destroy(block);
 348 }
 349
 350 static void tcf_chain_hold(struct tcf_chain *chain)
 351 {
 352         ASSERT_BLOCK_LOCKED(chain->block);
 353
 354         ++chain->refcnt;
 355 }
 356
 357 static bool tcf_chain_held_by_acts_only(struct tcf_chain *chain)
 358 {
 359         ASSERT_BLOCK_LOCKED(chain->block);
 360
 361         /* In case all the references are action references, this
 362          * chain should not be shown to the user.
 363          */
 364         return chain->refcnt == chain->action_refcnt;
 365 }
 366
 367 static struct tcf_chain *tcf_chain_lookup(struct tcf_block *block,
 368                                           u32 chain_index)
 369 {
 370         struct tcf_chain *chain;
 371
 372         ASSERT_BLOCK_LOCKED(block);
 373
 374         list_for_each_entry(chain, &block->chain_list, list) {
 375                 if (chain->index == chain_index)
 376                         return chain;
 377         }
 378         return NULL;
 379 }
 380
 381 static int tc_chain_notify(struct tcf_chain *chain, struct sk_buff *oskb,
 382                            u32 seq, u16 flags, int event, bool unicast);
 383
 384 static struct tcf_chain *__tcf_chain_get(struct tcf_block *block,
 385                                          u32 chain_index, bool create,
 386                                          bool by_act)
 387 {
 388         struct tcf_chain *chain = NULL;
 389         bool is_first_reference;
 390
 391         mutex_lock(&block->lock);
 392         chain = tcf_chain_lookup(block, chain_index);
 393         if (chain) {
 394                 tcf_chain_hold(chain);
 395         } else {
 396                 if (!create)
 397                         goto errout;
 398                 chain = tcf_chain_create(block, chain_index);
 399                 if (!chain)
 400                         goto errout;
 401         }
 402
 403         if (by_act)
 404                 ++chain->action_refcnt;
 405         is_first_reference = chain->refcnt - chain->action_refcnt == 1;
 406         mutex_unlock(&block->lock);
 407
 408         /* Send notification only in case we got the first
 409          * non-action reference. Until then, the chain acts only as
 410          * a placeholder for actions pointing to it and user ought
 411          * not know about them.
 412          */
 413         if (is_first_reference && !by_act)
 414                 tc_chain_notify(chain, NULL, 0, NLM_F_CREATE | NLM_F_EXCL,
 415                                 RTM_NEWCHAIN, false);
 416
 417         return chain;
 418
 419 errout:
 420         mutex_unlock(&block->lock);
 421         return chain;
 422 }
 423
 424 static struct tcf_chain *tcf_chain_get(struct tcf_block *block, u32 chain_index,
 425                                        bool create)
 426 {
 427         return __tcf_chain_get(block, chain_index, create, false);
 428 }
 429
 430 struct tcf_chain *tcf_chain_get_by_act(struct tcf_block *block, u32 chain_index)
 431 {
 432         return __tcf_chain_get(block, chain_index, true, true);
 433 }
 434 EXPORT_SYMBOL(tcf_chain_get_by_act);
 435
 436 static void tc_chain_tmplt_del(const struct tcf_proto_ops *tmplt_ops,
 437                                void *tmplt_priv);
 438 static int tc_chain_notify_delete(const struct tcf_proto_ops *tmplt_ops,
 439                                   void *tmplt_priv, u32 chain_index,
 440                                   struct tcf_block *block, struct sk_buff *oskb,
 441                                   u32 seq, u16 flags, bool unicast);
 442
 443 static void __tcf_chain_put(struct tcf_chain *chain, bool by_act,
 444                             bool explicitly_created)
 445 {
 446         struct tcf_block *block = chain->block;
 447         const struct tcf_proto_ops *tmplt_ops;
 448         bool is_last, free_block = false;
 449         unsigned int refcnt;
 450         void *tmplt_priv;
 451         u32 chain_index;
 452
 453         mutex_lock(&block->lock);
 454         if (explicitly_created) {
 455                 if (!chain->explicitly_created) {
 456                         mutex_unlock(&block->lock);
 457                         return;
 458                 }
 459                 chain->explicitly_created = false;
 460         }
 461
 462         if (by_act)
 463                 chain->action_refcnt--;
 464
 465         /* tc_chain_notify_delete can't be called while holding block lock.
 466          * However, when block is unlocked chain can be changed concurrently, so
 467          * save these to temporary variables.
 468          */
 469         refcnt = --chain->refcnt;
 470         is_last = refcnt - chain->action_refcnt == 0;
 471         tmplt_ops = chain->tmplt_ops;
 472         tmplt_priv = chain->tmplt_priv;
 473         chain_index = chain->index;
 474
 475         if (refcnt == 0)
 476                 free_block = tcf_chain_detach(chain);
 477         mutex_unlock(&block->lock);
 478
 479         /* The last dropped non-action reference will trigger notification. */
 480         if (is_last && !by_act)
 481                 tc_chain_notify_delete(tmplt_ops, tmplt_priv, chain_index,
 482                                        block, NULL, 0, 0, false);
 483
 484         if (refcnt == 0) {
 485                 tc_chain_tmplt_del(tmplt_ops, tmplt_priv);
 486                 tcf_chain_destroy(chain, free_block);
 487         }
 488 }
 489
 490 static void tcf_chain_put(struct tcf_chain *chain)
 491 {
 492         __tcf_chain_put(chain, false, false);
 493 }
 494
 495 void tcf_chain_put_by_act(struct tcf_chain *chain)
 496 {
 497         __tcf_chain_put(chain, true, false);
 498 }
 499 EXPORT_SYMBOL(tcf_chain_put_by_act);
 500
 501 static void tcf_chain_put_explicitly_created(struct tcf_chain *chain)
 502 {
 503         __tcf_chain_put(chain, false, true);
 504 }
 505
 506 static void tcf_chain_flush(struct tcf_chain *chain)
 507 {
 508         struct tcf_proto *tp, *tp_next;
 509
 510         mutex_lock(&chain->filter_chain_lock);
 511         tp = tcf_chain_dereference(chain->filter_chain, chain);
 512         RCU_INIT_POINTER(chain->filter_chain, NULL);
 513         tcf_chain0_head_change(chain, NULL);
 514         mutex_unlock(&chain->filter_chain_lock);
 515
 516         while (tp) {
 517                 tp_next = rcu_dereference_protected(tp->next, 1);
 518                 tcf_proto_put(tp, NULL);
 519                 tp = tp_next;
 520         }
 521 }
 522
 523 static struct tcf_block *tc_dev_ingress_block(struct net_device *dev)
 524 {
 525         const struct Qdisc_class_ops *cops;
 526         struct Qdisc *qdisc;
 527
 528         if (!dev_ingress_queue(dev))
 529                 return NULL;
 530
 531         qdisc = dev_ingress_queue(dev)->qdisc_sleeping;
 532         if (!qdisc)
 533                 return NULL;
 534
 535         cops = qdisc->ops->cl_ops;
 536         if (!cops)
 537                 return NULL;
 538
 539         if (!cops->tcf_block)
 540                 return NULL;
 541
 542         return cops->tcf_block(qdisc, TC_H_MIN_INGRESS, NULL);
 543 }
 544
 545 static struct rhashtable indr_setup_block_ht;
 546
 547 struct tc_indr_block_dev {
 548         struct rhash_head ht_node;
 549         struct net_device *dev;
 550         unsigned int refcnt;
 551         struct list_head cb_list;
 552         struct tcf_block *block;
 553 };
 554
 555 struct tc_indr_block_cb {
 556         struct list_head list;
 557         void *cb_priv;
 558         tc_indr_block_bind_cb_t *cb;
 559         void *cb_ident;
 560 };
 561
 562 static const struct rhashtable_params tc_indr_setup_block_ht_params = {
 563         .key_offset     = offsetof(struct tc_indr_block_dev, dev),
 564         .head_offset    = offsetof(struct tc_indr_block_dev, ht_node),
 565         .key_len        = sizeof(struct net_device *),
 566 };
 567
 568 static struct tc_indr_block_dev *
 569 tc_indr_block_dev_lookup(struct net_device *dev)
 570 {
 571         return rhashtable_lookup_fast(&indr_setup_block_ht, &dev,
 572                                       tc_indr_setup_block_ht_params);
 573 }
 574
 575 static struct tc_indr_block_dev *tc_indr_block_dev_get(struct net_device *dev)
 576 {
 577         struct tc_indr_block_dev *indr_dev;
 578
 579         indr_dev = tc_indr_block_dev_lookup(dev);
 580         if (indr_dev)
 581                 goto inc_ref;
 582
 583         indr_dev = kzalloc(sizeof(*indr_dev), GFP_KERNEL);
 584         if (!indr_dev)
 585                 return NULL;
 586
 587         INIT_LIST_HEAD(&indr_dev->cb_list);
 588         indr_dev->dev = dev;
 589         indr_dev->block = tc_dev_ingress_block(dev);
 590         if (rhashtable_insert_fast(&indr_setup_block_ht, &indr_dev->ht_node,
 591                                    tc_indr_setup_block_ht_params)) {
 592                 kfree(indr_dev);
 593                 return NULL;
 594         }
 595
 596 inc_ref:
 597         indr_dev->refcnt++;
 598         return indr_dev;
 599 }
 600
 601 static void tc_indr_block_dev_put(struct tc_indr_block_dev *indr_dev)
 602 {
 603         if (--indr_dev->refcnt)
 604                 return;
 605
 606         rhashtable_remove_fast(&indr_setup_block_ht, &indr_dev->ht_node,
 607                                tc_indr_setup_block_ht_params);
 608         kfree(indr_dev);
 609 }
 610
 611 static struct tc_indr_block_cb *
 612 tc_indr_block_cb_lookup(struct tc_indr_block_dev *indr_dev,
 613                         tc_indr_block_bind_cb_t *cb, void *cb_ident)
 614 {
 615         struct tc_indr_block_cb *indr_block_cb;
 616
 617         list_for_each_entry(indr_block_cb, &indr_dev->cb_list, list)
 618                 if (indr_block_cb->cb == cb &&
 619                     indr_block_cb->cb_ident == cb_ident)
 620                         return indr_block_cb;
 621         return NULL;
 622 }
 623
 624 static struct tc_indr_block_cb *
 625 tc_indr_block_cb_add(struct tc_indr_block_dev *indr_dev, void *cb_priv,
 626                      tc_indr_block_bind_cb_t *cb, void *cb_ident)
 627 {
 628         struct tc_indr_block_cb *indr_block_cb;
 629
 630         indr_block_cb = tc_indr_block_cb_lookup(indr_dev, cb, cb_ident);
 631         if (indr_block_cb)
 632                 return ERR_PTR(-EEXIST);
 633
 634         indr_block_cb = kzalloc(sizeof(*indr_block_cb), GFP_KERNEL);
 635         if (!indr_block_cb)
 636                 return ERR_PTR(-ENOMEM);
 637
 638         indr_block_cb->cb_priv = cb_priv;
 639         indr_block_cb->cb = cb;
 640         indr_block_cb->cb_ident = cb_ident;
 641         list_add(&indr_block_cb->list, &indr_dev->cb_list);
 642
 643         return indr_block_cb;
 644 }
 645
 646 static void tc_indr_block_cb_del(struct tc_indr_block_cb *indr_block_cb)
 647 {
 648         list_del(&indr_block_cb->list);
 649         kfree(indr_block_cb);
 650 }
 651
 652 static void tc_indr_block_ing_cmd(struct tc_indr_block_dev *indr_dev,
 653                                   struct tc_indr_block_cb *indr_block_cb,
 654                                   enum tc_block_command command)
 655 {
 656         struct tc_block_offload bo = {
 657                 .command        = command,
 658                 .binder_type    = TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS,
 659                 .block          = indr_dev->block,
 660         };
 661
 662         if (!indr_dev->block)
 663                 return;
 664
 665         indr_block_cb->cb(indr_dev->dev, indr_block_cb->cb_priv, TC_SETUP_BLOCK,
 666                           &bo);
 667 }
 668
 669 int __tc_indr_block_cb_register(struct net_device *dev, void *cb_priv,
 670                                 tc_indr_block_bind_cb_t *cb, void *cb_ident)
 671 {
 672         struct tc_indr_block_cb *indr_block_cb;
 673         struct tc_indr_block_dev *indr_dev;
 674         int err;
 675
 676         indr_dev = tc_indr_block_dev_get(dev);
 677         if (!indr_dev)
 678                 return -ENOMEM;
 679
 680         indr_block_cb = tc_indr_block_cb_add(indr_dev, cb_priv, cb, cb_ident);
 681         err = PTR_ERR_OR_ZERO(indr_block_cb);
 682         if (err)
 683                 goto err_dev_put;
 684
 685         tc_indr_block_ing_cmd(indr_dev, indr_block_cb, TC_BLOCK_BIND);
 686         return 0;
 687
 688 err_dev_put:
 689         tc_indr_block_dev_put(indr_dev);
 690         return err;
 691 }
 692 EXPORT_SYMBOL_GPL(__tc_indr_block_cb_register);
 693
 694 int tc_indr_block_cb_register(struct net_device *dev, void *cb_priv,
 695                               tc_indr_block_bind_cb_t *cb, void *cb_ident)
 696 {
 697         int err;
 698
 699         rtnl_lock();
 700         err = __tc_indr_block_cb_register(dev, cb_priv, cb, cb_ident);
 701         rtnl_unlock();
 702
 703         return err;
 704 }
 705 EXPORT_SYMBOL_GPL(tc_indr_block_cb_register);
 706
 707 void __tc_indr_block_cb_unregister(struct net_device *dev,
 708                                    tc_indr_block_bind_cb_t *cb, void *cb_ident)
 709 {
 710         struct tc_indr_block_cb *indr_block_cb;
 711         struct tc_indr_block_dev *indr_dev;
 712
 713         indr_dev = tc_indr_block_dev_lookup(dev);
 714         if (!indr_dev)
 715                 return;
 716
 717         indr_block_cb = tc_indr_block_cb_lookup(indr_dev, cb, cb_ident);
 718         if (!indr_block_cb)
 719                 return;
 720
 721         /* Send unbind message if required to free any block cbs. */
 722         tc_indr_block_ing_cmd(indr_dev, indr_block_cb, TC_BLOCK_UNBIND);
 723         tc_indr_block_cb_del(indr_block_cb);
 724         tc_indr_block_dev_put(indr_dev);
 725 }
 726 EXPORT_SYMBOL_GPL(__tc_indr_block_cb_unregister);
 727
 728 void tc_indr_block_cb_unregister(struct net_device *dev,
 729                                  tc_indr_block_bind_cb_t *cb, void *cb_ident)
 730 {
 731         rtnl_lock();
 732         __tc_indr_block_cb_unregister(dev, cb, cb_ident);
 733         rtnl_unlock();
 734 }
 735 EXPORT_SYMBOL_GPL(tc_indr_block_cb_unregister);
 736
 737 static void tc_indr_block_call(struct tcf_block *block, struct net_device *dev,
 738                                struct tcf_block_ext_info *ei,
 739                                enum tc_block_command command,
 740                                struct netlink_ext_ack *extack)
 741 {
 742         struct tc_indr_block_cb *indr_block_cb;
 743         struct tc_indr_block_dev *indr_dev;
 744         struct tc_block_offload bo = {
 745                 .command        = command,
 746                 .binder_type    = ei->binder_type,
 747                 .block          = block,
 748                 .extack         = extack,
 749         };
 750
 751         indr_dev = tc_indr_block_dev_lookup(dev);
 752         if (!indr_dev)
 753                 return;
 754
 755         indr_dev->block = command == TC_BLOCK_BIND ? block : NULL;
 756
 757         list_for_each_entry(indr_block_cb, &indr_dev->cb_list, list)
 758                 indr_block_cb->cb(dev, indr_block_cb->cb_priv, TC_SETUP_BLOCK,
 759                                   &bo);
 760 }
 761
 762 static bool tcf_block_offload_in_use(struct tcf_block *block)
 763 {
 764         return block->offloadcnt;
 765 }
 766
 767 static int tcf_block_offload_cmd(struct tcf_block *block,
 768                                  struct net_device *dev,
 769                                  struct tcf_block_ext_info *ei,
 770                                  enum tc_block_command command,
 771                                  struct netlink_ext_ack *extack)
 772 {
 773         struct tc_block_offload bo = {};
 774
 775         bo.command = command;
 776         bo.binder_type = ei->binder_type;
 777         bo.block = block;
 778         bo.extack = extack;
 779         return dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_BLOCK, &bo);
 780 }
 781
 782 static int tcf_block_offload_bind(struct tcf_block *block, struct Qdisc *q,
 783                                   struct tcf_block_ext_info *ei,
 784                                   struct netlink_ext_ack *extack)
 785 {
 786         struct net_device *dev = q->dev_queue->dev;
 787         int err;
 788
 789         if (!dev->netdev_ops->ndo_setup_tc)
 790                 goto no_offload_dev_inc;
 791
 792         /* If tc offload feature is disabled and the block we try to bind
 793          * to already has some offloaded filters, forbid to bind.
 794          */
 795         if (!tc_can_offload(dev) && tcf_block_offload_in_use(block)) {
 796                 NL_SET_ERR_MSG(extack, "Bind to offloaded block failed as dev has offload disabled");
 797                 return -EOPNOTSUPP;
 798         }
 799
 800         err = tcf_block_offload_cmd(block, dev, ei, TC_BLOCK_BIND, extack);
 801         if (err == -EOPNOTSUPP)
 802                 goto no_offload_dev_inc;
 803         if (err)
 804                 return err;
 805
 806         tc_indr_block_call(block, dev, ei, TC_BLOCK_BIND, extack);
 807         return 0;
 808
 809 no_offload_dev_inc:
 810         if (tcf_block_offload_in_use(block))
 811                 return -EOPNOTSUPP;
 812         block->nooffloaddevcnt++;
 813         tc_indr_block_call(block, dev, ei, TC_BLOCK_BIND, extack);
 814         return 0;
 815 }
 816
 817 static void tcf_block_offload_unbind(struct tcf_block *block, struct Qdisc *q,
 818                                      struct tcf_block_ext_info *ei)
 819 {
 820         struct net_device *dev = q->dev_queue->dev;
 821         int err;
 822
 823         tc_indr_block_call(block, dev, ei, TC_BLOCK_UNBIND, NULL);
 824
 825         if (!dev->netdev_ops->ndo_setup_tc)
 826                 goto no_offload_dev_dec;
 827         err = tcf_block_offload_cmd(block, dev, ei, TC_BLOCK_UNBIND, NULL);
 828         if (err == -EOPNOTSUPP)
 829                 goto no_offload_dev_dec;
 830         return;
 831
 832 no_offload_dev_dec:
 833         WARN_ON(block->nooffloaddevcnt-- == 0);
 834 }
 835
 836 static int
 837 tcf_chain0_head_change_cb_add(struct tcf_block *block,
 838                               struct tcf_block_ext_info *ei,
 839                               struct netlink_ext_ack *extack)
 840 {
 841         struct tcf_filter_chain_list_item *item;
 842         struct tcf_chain *chain0;
 843
 844         item = kmalloc(sizeof(*item), GFP_KERNEL);
 845         if (!item) {
 846                 NL_SET_ERR_MSG(extack, "Memory allocation for head change callback item failed");
 847                 return -ENOMEM;
 848         }
 849         item->chain_head_change = ei->chain_head_change;
 850         item->chain_head_change_priv = ei->chain_head_change_priv;
 851
 852         mutex_lock(&block->lock);
 853         chain0 = block->chain0.chain;
 854         if (chain0)
 855                 tcf_chain_hold(chain0);
 856         else
 857                 list_add(&item->list, &block->chain0.filter_chain_list);
 858         mutex_unlock(&block->lock);
 859
 860         if (chain0) {
 861                 struct tcf_proto *tp_head;
 862
 863                 mutex_lock(&chain0->filter_chain_lock);
 864
 865                 tp_head = tcf_chain_dereference(chain0->filter_chain, chain0);
 866                 if (tp_head)
 867                         tcf_chain_head_change_item(item, tp_head);
 868
 869                 mutex_lock(&block->lock);
 870                 list_add(&item->list, &block->chain0.filter_chain_list);
 871                 mutex_unlock(&block->lock);
 872
 873                 mutex_unlock(&chain0->filter_chain_lock);
 874                 tcf_chain_put(chain0);
 875         }
 876
 877         return 0;
 878 }
 879
 880 static void
 881 tcf_chain0_head_change_cb_del(struct tcf_block *block,
 882                               struct tcf_block_ext_info *ei)
 883 {
 884         struct tcf_filter_chain_list_item *item;
 885
 886         mutex_lock(&block->lock);
 887         list_for_each_entry(item, &block->chain0.filter_chain_list, list) {
 888                 if ((!ei->chain_head_change && !ei->chain_head_change_priv) ||
 889                     (item->chain_head_change == ei->chain_head_change &&
 890                      item->chain_head_change_priv == ei->chain_head_change_priv)) {
 891                         if (block->chain0.chain)
 892                                 tcf_chain_head_change_item(item, NULL);
 893                         list_del(&item->list);
 894                         mutex_unlock(&block->lock);
 895
 896                         kfree(item);
 897                         return;
 898                 }
 899         }
 900         mutex_unlock(&block->lock);
 901         WARN_ON(1);
 902 }
 903
 904 struct tcf_net {
 905         spinlock_t idr_lock; /* Protects idr */
 906         struct idr idr;
 907 };
 908
 909 static unsigned int tcf_net_id;
 910
 911 static int tcf_block_insert(struct tcf_block *block, struct net *net,
 912                             struct netlink_ext_ack *extack)
 913 {
 914         struct tcf_net *tn = net_generic(net, tcf_net_id);
 915         int err;
 916
 917         idr_preload(GFP_KERNEL);
 918         spin_lock(&tn->idr_lock);
 919         err = idr_alloc_u32(&tn->idr, block, &block->index, block->index,
 920                             GFP_NOWAIT);
 921         spin_unlock(&tn->idr_lock);
 922         idr_preload_end();
 923
 924         return err;
 925 }
 926
 927 static void tcf_block_remove(struct tcf_block *block, struct net *net)
 928 {
 929         struct tcf_net *tn = net_generic(net, tcf_net_id);
 930
 931         spin_lock(&tn->idr_lock);
 932         idr_remove(&tn->idr, block->index);
 933         spin_unlock(&tn->idr_lock);
 934 }
 935
 936 static struct tcf_block *tcf_block_create(struct net *net, struct Qdisc *q,
 937                                           u32 block_index,
 938                                           struct netlink_ext_ack *extack)
 939 {
 940         struct tcf_block *block;
 941
 942         block = kzalloc(sizeof(*block), GFP_KERNEL);
 943         if (!block) {
 944                 NL_SET_ERR_MSG(extack, "Memory allocation for block failed");
 945                 return ERR_PTR(-ENOMEM);
 946         }
 947         mutex_init(&block->lock);
 948         INIT_LIST_HEAD(&block->chain_list);
 949         INIT_LIST_HEAD(&block->cb_list);
 950         INIT_LIST_HEAD(&block->owner_list);
 951         INIT_LIST_HEAD(&block->chain0.filter_chain_list);
 952
 953         refcount_set(&block->refcnt, 1);
 954         block->net = net;
 955         block->index = block_index;
 956
 957         /* Don't store q pointer for blocks which are shared */
 958         if (!tcf_block_shared(block))
 959                 block->q = q;
 960         return block;
 961 }
 962
 963 static struct tcf_block *tcf_block_lookup(struct net *net, u32 block_index)
 964 {
 965         struct tcf_net *tn = net_generic(net, tcf_net_id);
 966
 967         return idr_find(&tn->idr, block_index);
 968 }
 969
 970 static struct tcf_block *tcf_block_refcnt_get(struct net *net, u32 block_index)
 971 {
 972         struct tcf_block *block;
 973
 974         rcu_read_lock();
 975         block = tcf_block_lookup(net, block_index);
 976         if (block && !refcount_inc_not_zero(&block->refcnt))
 977                 block = NULL;
 978         rcu_read_unlock();
 979
 980         return block;
 981 }
 982
 983 static struct tcf_chain *
 984 __tcf_get_next_chain(struct tcf_block *block, struct tcf_chain *chain)
 985 {
 986         mutex_lock(&block->lock);
 987         if (chain)
 988                 chain = list_is_last(&chain->list, &block->chain_list) ?
 989                         NULL : list_next_entry(chain, list);
 990         else
 991                 chain = list_first_entry_or_null(&block->chain_list,
 992                                                  struct tcf_chain, list);
 993
 994         /* skip all action-only chains */
 995         while (chain && tcf_chain_held_by_acts_only(chain))
 996                 chain = list_is_last(&chain->list, &block->chain_list) ?
 997                         NULL : list_next_entry(chain, list);
 998
 999         if (chain)
1000                 tcf_chain_hold(chain);
1001         mutex_unlock(&block->lock);
1002
1003         return chain;
1004 }
1005
1006 /* Function to be used by all clients that want to iterate over all chains on
1007  * block. It properly obtains block->lock and takes reference to chain before
1008  * returning it. Users of this function must be tolerant to concurrent chain
1009  * insertion/deletion or ensure that no concurrent chain modification is
1010  * possible. Note that all netlink dump callbacks cannot guarantee to provide
1011  * consistent dump because rtnl lock is released each time skb is filled with
1012  * data and sent to user-space.
1013  */
1014
1015 struct tcf_chain *
1016 tcf_get_next_chain(struct tcf_block *block, struct tcf_chain *chain)
1017 {
1018         struct tcf_chain *chain_next = __tcf_get_next_chain(block, chain);
1019
1020         if (chain)
1021                 tcf_chain_put(chain);
1022
1023         return chain_next;
1024 }
1025 EXPORT_SYMBOL(tcf_get_next_chain);
1026
1027 static struct tcf_proto *
1028 __tcf_get_next_proto(struct tcf_chain *chain, struct tcf_proto *tp)
1029 {
1030         u32 prio = 0;
1031
1032         ASSERT_RTNL();
1033         mutex_lock(&chain->filter_chain_lock);
1034
1035         if (!tp) {
1036                 tp = tcf_chain_dereference(chain->filter_chain, chain);
1037         } else if (tcf_proto_is_deleting(tp)) {
1038                 /* 'deleting' flag is set and chain->filter_chain_lock was
1039                  * unlocked, which means next pointer could be invalid. Restart
1040                  * search.
1041                  */
1042                 prio = tp->prio + 1;
1043                 tp = tcf_chain_dereference(chain->filter_chain, chain);
1044
1045                 for (; tp; tp = tcf_chain_dereference(tp->next, chain))
1046                         if (!tp->deleting && tp->prio >= prio)
1047                                 break;
1048         } else {
1049                 tp = tcf_chain_dereference(tp->next, chain);
1050         }
1051
1052         if (tp)
1053                 tcf_proto_get(tp);
1054
1055         mutex_unlock(&chain->filter_chain_lock);
1056
1057         return tp;
1058 }
1059
1060 /* Function to be used by all clients that want to iterate over all tp's on
1061  * chain. Users of this function must be tolerant to concurrent tp
1062  * insertion/deletion or ensure that no concurrent chain modification is
1063  * possible. Note that all netlink dump callbacks cannot guarantee to provide
1064  * consistent dump because rtnl lock is released each time skb is filled with
1065  * data and sent to user-space.
1066  */
1067
1068 struct tcf_proto *
1069 tcf_get_next_proto(struct tcf_chain *chain, struct tcf_proto *tp)
1070 {
1071         struct tcf_proto *tp_next = __tcf_get_next_proto(chain, tp);
1072
1073         if (tp)
1074                 tcf_proto_put(tp, NULL);
1075
1076         return tp_next;
1077 }
1078 EXPORT_SYMBOL(tcf_get_next_proto);
1079
1080 static void tcf_block_flush_all_chains(struct tcf_block *block)
1081 {
1082         struct tcf_chain *chain;
1083
1084         /* Last reference to block. At this point chains cannot be added or
1085          * removed concurrently.
1086          */
1087         for (chain = tcf_get_next_chain(block, NULL);
1088              chain;
1089              chain = tcf_get_next_chain(block, chain)) {
1090                 tcf_chain_put_explicitly_created(chain);
1091                 tcf_chain_flush(chain);
1092         }
1093 }
1094
1095 static void __tcf_block_put(struct tcf_block *block, struct Qdisc *q,
1096                             struct tcf_block_ext_info *ei)
1097 {
1098         if (refcount_dec_and_mutex_lock(&block->refcnt, &block->lock)) {
1099                 /* Flushing/putting all chains will cause the block to be
1100                  * deallocated when last chain is freed. However, if chain_list
1101                  * is empty, block has to be manually deallocated. After block
1102                  * reference counter reached 0, it is no longer possible to
1103                  * increment it or add new chains to block.
1104                  */
1105                 bool free_block = list_empty(&block->chain_list);
1106
1107                 mutex_unlock(&block->lock);
1108                 if (tcf_block_shared(block))
1109                         tcf_block_remove(block, block->net);
1110
1111                 if (q)
1112                         tcf_block_offload_unbind(block, q, ei);
1113
1114                 if (free_block)
1115                         tcf_block_destroy(block);
1116                 else
1117                         tcf_block_flush_all_chains(block);
1118         } else if (q) {
1119                 tcf_block_offload_unbind(block, q, ei);
1120         }
1121 }
1122
1123 static void tcf_block_refcnt_put(struct tcf_block *block)
1124 {
1125         __tcf_block_put(block, NULL, NULL);
1126 }
1127
1128 /* Find tcf block.
1129  * Set q, parent, cl when appropriate.
1130  */
1131
1132 static struct tcf_block *tcf_block_find(struct net *net, struct Qdisc **q,
1133                                         u32 *parent, unsigned long *cl,
1134                                         int ifindex, u32 block_index,
1135                                         struct netlink_ext_ack *extack)
1136 {
1137         struct tcf_block *block;
1138         int err = 0;
1139
1140         if (ifindex == TCM_IFINDEX_MAGIC_BLOCK) {
1141                 block = tcf_block_refcnt_get(net, block_index);
1142                 if (!block) {
1143                         NL_SET_ERR_MSG(extack, "Block of given index was not found");
1144                         return ERR_PTR(-EINVAL);
1145                 }
1146         } else {
1147                 const struct Qdisc_class_ops *cops;
1148                 struct net_device *dev;
1149
1150                 rcu_read_lock();
1151
1152                 /* Find link */
1153                 dev = dev_get_by_index_rcu(net, ifindex);
1154                 if (!dev) {
1155                         rcu_read_unlock();
1156                         return ERR_PTR(-ENODEV);
1157                 }
1158
1159                 /* Find qdisc */
1160                 if (!*parent) {
1161                         *q = dev->qdisc;
1162                         *parent = (*q)->handle;
1163                 } else {
1164                         *q = qdisc_lookup_rcu(dev, TC_H_MAJ(*parent));
1165                         if (!*q) {
1166                                 NL_SET_ERR_MSG(extack, "Parent Qdisc doesn't exists");
1167                                 err = -EINVAL;
1168                                 goto errout_rcu;
1169                         }
1170                 }
1171
1172                 *q = qdisc_refcount_inc_nz(*q);
1173                 if (!*q) {
1174                         NL_SET_ERR_MSG(extack, "Parent Qdisc doesn't exists");
1175                         err = -EINVAL;
1176                         goto errout_rcu;
1177                 }
1178
1179                 /* Is it classful? */
1180                 cops = (*q)->ops->cl_ops;
1181                 if (!cops) {
1182                         NL_SET_ERR_MSG(extack, "Qdisc not classful");
1183                         err = -EINVAL;
1184                         goto errout_rcu;
1185                 }
1186
1187                 if (!cops->tcf_block) {
1188                         NL_SET_ERR_MSG(extack, "Class doesn't support blocks");
1189                         err = -EOPNOTSUPP;
1190                         goto errout_rcu;
1191                 }
1192
1193                 /* At this point we know that qdisc is not noop_qdisc,
1194                  * which means that qdisc holds a reference to net_device
1195                  * and we hold a reference to qdisc, so it is safe to release
1196                  * rcu read lock.
1197                  */
1198                 rcu_read_unlock();
1199
1200                 /* Do we search for filter, attached to class? */
1201                 if (TC_H_MIN(*parent)) {
1202                         *cl = cops->find(*q, *parent);
1203                         if (*cl == 0) {
1204                                 NL_SET_ERR_MSG(extack, "Specified class doesn't exist");
1205                                 err = -ENOENT;
1206                                 goto errout_qdisc;
1207                         }
1208                 }
1209
1210                 /* And the last stroke */
1211                 block = cops->tcf_block(*q, *cl, extack);
1212                 if (!block) {
1213                         err = -EINVAL;
1214                         goto errout_qdisc;
1215                 }
1216                 if (tcf_block_shared(block)) {
1217                         NL_SET_ERR_MSG(extack, "This filter block is shared. Please use the block index to manipulate the filters");
1218                         err = -EOPNOTSUPP;
1219                         goto errout_qdisc;
1220                 }
1221
1222                 /* Always take reference to block in order to support execution
1223                  * of rules update path of cls API without rtnl lock. Caller
1224                  * must release block when it is finished using it. 'if' block
1225                  * of this conditional obtain reference to block by calling
1226                  * tcf_block_refcnt_get().
1227                  */
1228                 refcount_inc(&block->refcnt);
1229         }
1230
1231         return block;
1232
1233 errout_rcu:
1234         rcu_read_unlock();
1235 errout_qdisc:
1236         if (*q) {
1237                 qdisc_put(*q);
1238                 *q = NULL;
1239         }
1240         return ERR_PTR(err);
1241 }
1242
1243 static void tcf_block_release(struct Qdisc *q, struct tcf_block *block)
1244 {
1245         if (!IS_ERR_OR_NULL(block))
1246                 tcf_block_refcnt_put(block);
1247
1248         if (q)
1249                 qdisc_put(q);
1250 }
1251
1252 struct tcf_block_owner_item {
1253         struct list_head list;
1254         struct Qdisc *q;
1255         enum tcf_block_binder_type binder_type;
1256 };
1257
1258 static void
1259 tcf_block_owner_netif_keep_dst(struct tcf_block *block,
1260                                struct Qdisc *q,
1261                                enum tcf_block_binder_type binder_type)
1262 {
1263         if (block->keep_dst &&
1264             binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS &&
1265             binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_EGRESS)
1266                 netif_keep_dst(qdisc_dev(q));
1267 }
1268
1269 void tcf_block_netif_keep_dst(struct tcf_block *block)
1270 {
1271         struct tcf_block_owner_item *item;
1272
1273         block->keep_dst = true;
1274         list_for_each_entry(item, &block->owner_list, list)
1275                 tcf_block_owner_netif_keep_dst(block, item->q,
1276                                                item->binder_type);
1277 }
1278 EXPORT_SYMBOL(tcf_block_netif_keep_dst);
1279
1280 static int tcf_block_owner_add(struct tcf_block *block,
1281                                struct Qdisc *q,
1282                                enum tcf_block_binder_type binder_type)
1283 {
1284         struct tcf_block_owner_item *item;
1285
1286         item = kmalloc(sizeof(*item), GFP_KERNEL);
1287         if (!item)
1288                 return -ENOMEM;
1289         item->q = q;
1290         item->binder_type = binder_type;
1291         list_add(&item->list, &block->owner_list);
1292         return 0;
1293 }
1294
1295 static void tcf_block_owner_del(struct tcf_block *block,
1296                                 struct Qdisc *q,
1297                                 enum tcf_block_binder_type binder_type)
1298 {
1299         struct tcf_block_owner_item *item;
1300
1301         list_for_each_entry(item, &block->owner_list, list) {
1302                 if (item->q == q && item->binder_type == binder_type) {
1303                         list_del(&item->list);
1304                         kfree(item);
1305                         return;
1306                 }
1307         }
1308         WARN_ON(1);
1309 }
1310
1311 int tcf_block_get_ext(struct tcf_block **p_block, struct Qdisc *q,
1312                       struct tcf_block_ext_info *ei,
1313                       struct netlink_ext_ack *extack)
1314 {
1315         struct net *net = qdisc_net(q);
1316         struct tcf_block *block = NULL;
1317         int err;
1318
1319         if (ei->block_index)
1320                 /* block_index not 0 means the shared block is requested */
1321                 block = tcf_block_refcnt_get(net, ei->block_index);
1322
1323         if (!block) {
1324                 block = tcf_block_create(net, q, ei->block_index, extack);
1325                 if (IS_ERR(block))
1326                         return PTR_ERR(block);
1327                 if (tcf_block_shared(block)) {
1328                         err = tcf_block_insert(block, net, extack);
1329                         if (err)
1330                                 goto err_block_insert;
1331                 }
1332         }
1333
1334         err = tcf_block_owner_add(block, q, ei->binder_type);
1335         if (err)
1336                 goto err_block_owner_add;
1337
1338         tcf_block_owner_netif_keep_dst(block, q, ei->binder_type);
1339
1340         err = tcf_chain0_head_change_cb_add(block, ei, extack);
1341         if (err)
1342                 goto err_chain0_head_change_cb_add;
1343
1344         err = tcf_block_offload_bind(block, q, ei, extack);
1345         if (err)
1346                 goto err_block_offload_bind;
1347
1348         *p_block = block;
1349         return 0;
1350
1351 err_block_offload_bind:
1352         tcf_chain0_head_change_cb_del(block, ei);
1353 err_chain0_head_change_cb_add:
1354         tcf_block_owner_del(block, q, ei->binder_type);
1355 err_block_owner_add:
1356 err_block_insert:
1357         tcf_block_refcnt_put(block);
1358         return err;
1359 }
1360 EXPORT_SYMBOL(tcf_block_get_ext);
1361
1362 static void tcf_chain_head_change_dflt(struct tcf_proto *tp_head, void *priv)
1363 {
1364         struct tcf_proto __rcu **p_filter_chain = priv;
1365
1366         rcu_assign_pointer(*p_filter_chain, tp_head);
1367 }
1368
1369 int tcf_block_get(struct tcf_block **p_block,
1370                   struct tcf_proto __rcu **p_filter_chain, struct Qdisc *q,
1371                   struct netlink_ext_ack *extack)
1372 {
1373         struct tcf_block_ext_info ei = {
1374                 .chain_head_change = tcf_chain_head_change_dflt,
1375                 .chain_head_change_priv = p_filter_chain,
1376         };
1377
1378         WARN_ON(!p_filter_chain);
1379         return tcf_block_get_ext(p_block, q, &ei, extack);
1380 }
1381 EXPORT_SYMBOL(tcf_block_get);
1382
1383 /* XXX: Standalone actions are not allowed to jump to any chain, and bound
1384  * actions should be all removed after flushing.
1385  */
1386 void tcf_block_put_ext(struct tcf_block *block, struct Qdisc *q,
1387                        struct tcf_block_ext_info *ei)
1388 {
1389         if (!block)
1390                 return;
1391         tcf_chain0_head_change_cb_del(block, ei);
1392         tcf_block_owner_del(block, q, ei->binder_type);
1393
1394         __tcf_block_put(block, q, ei);
1395 }
1396 EXPORT_SYMBOL(tcf_block_put_ext);
1397
1398 void tcf_block_put(struct tcf_block *block)
1399 {
1400         struct tcf_block_ext_info ei = {0, };
1401
1402         if (!block)
1403                 return;
1404         tcf_block_put_ext(block, block->q, &ei);
1405 }
1406
1407 EXPORT_SYMBOL(tcf_block_put);
1408
1409 struct tcf_block_cb {
1410         struct list_head list;
1411         tc_setup_cb_t *cb;
1412         void *cb_ident;
1413         void *cb_priv;
1414         unsigned int refcnt;
1415 };
1416
1417 void *tcf_block_cb_priv(struct tcf_block_cb *block_cb)
1418 {
1419         return block_cb->cb_priv;
1420 }
1421 EXPORT_SYMBOL(tcf_block_cb_priv);
1422
1423 struct tcf_block_cb *tcf_block_cb_lookup(struct tcf_block *block,
1424                                          tc_setup_cb_t *cb, void *cb_ident)
1425 {       struct tcf_block_cb *block_cb;
1426
1427         list_for_each_entry(block_cb, &block->cb_list, list)
1428                 if (block_cb->cb == cb && block_cb->cb_ident == cb_ident)
1429                         return block_cb;
1430         return NULL;
1431 }
1432 EXPORT_SYMBOL(tcf_block_cb_lookup);
1433
1434 void tcf_block_cb_incref(struct tcf_block_cb *block_cb)
1435 {
1436         block_cb->refcnt++;
1437 }
1438 EXPORT_SYMBOL(tcf_block_cb_incref);
1439
1440 unsigned int tcf_block_cb_decref(struct tcf_block_cb *block_cb)
1441 {
1442         return --block_cb->refcnt;
1443 }
1444 EXPORT_SYMBOL(tcf_block_cb_decref);
1445
1446 static int
1447 tcf_block_playback_offloads(struct tcf_block *block, tc_setup_cb_t *cb,
1448                             void *cb_priv, bool add, bool offload_in_use,
1449                             struct netlink_ext_ack *extack)
1450 {
1451         struct tcf_chain *chain, *chain_prev;
1452         struct tcf_proto *tp, *tp_prev;
1453         int err;
1454
1455         for (chain = __tcf_get_next_chain(block, NULL);
1456              chain;
1457              chain_prev = chain,
1458                      chain = __tcf_get_next_chain(block, chain),
1459                      tcf_chain_put(chain_prev)) {
1460                 for (tp = __tcf_get_next_proto(chain, NULL); tp;
1461                      tp_prev = tp,
1462                              tp = __tcf_get_next_proto(chain, tp),
1463                              tcf_proto_put(tp_prev, NULL)) {
1464                         if (tp->ops->reoffload) {
1465                                 err = tp->ops->reoffload(tp, add, cb, cb_priv,
1466                                                          extack);
1467                                 if (err && add)
1468                                         goto err_playback_remove;
1469                         } else if (add && offload_in_use) {
1470                                 err = -EOPNOTSUPP;
1471                                 NL_SET_ERR_MSG(extack, "Filter HW offload failed - classifier without re-offloading support");
1472                                 goto err_playback_remove;
1473                         }
1474                 }
1475         }
1476
1477         return 0;
1478
1479 err_playback_remove:
1480         tcf_proto_put(tp, NULL);
1481         tcf_chain_put(chain);
1482         tcf_block_playback_offloads(block, cb, cb_priv, false, offload_in_use,
1483                                     extack);
1484         return err;
1485 }
1486
1487 struct tcf_block_cb *__tcf_block_cb_register(struct tcf_block *block,
1488                                              tc_setup_cb_t *cb, void *cb_ident,
1489                                              void *cb_priv,
1490                                              struct netlink_ext_ack *extack)
1491 {
1492         struct tcf_block_cb *block_cb;
1493         int err;
1494
1495         /* Replay any already present rules */
1496         err = tcf_block_playback_offloads(block, cb, cb_priv, true,
1497                                           tcf_block_offload_in_use(block),
1498                                           extack);
1499         if (err)
1500                 return ERR_PTR(err);
1501
1502         block_cb = kzalloc(sizeof(*block_cb), GFP_KERNEL);
1503         if (!block_cb)
1504                 return ERR_PTR(-ENOMEM);
1505         block_cb->cb = cb;
1506         block_cb->cb_ident = cb_ident;
1507         block_cb->cb_priv = cb_priv;
1508         list_add(&block_cb->list, &block->cb_list);
1509         return block_cb;
1510 }
1511 EXPORT_SYMBOL(__tcf_block_cb_register);
1512
1513 int tcf_block_cb_register(struct tcf_block *block,
1514                           tc_setup_cb_t *cb, void *cb_ident,
1515                           void *cb_priv, struct netlink_ext_ack *extack)
1516 {
1517         struct tcf_block_cb *block_cb;
1518
1519         block_cb = __tcf_block_cb_register(block, cb, cb_ident, cb_priv,
1520                                            extack);
1521         return PTR_ERR_OR_ZERO(block_cb);
1522 }
1523 EXPORT_SYMBOL(tcf_block_cb_register);
1524
1525 void __tcf_block_cb_unregister(struct tcf_block *block,
1526                                struct tcf_block_cb *block_cb)
1527 {
1528         tcf_block_playback_offloads(block, block_cb->cb, block_cb->cb_priv,
1529                                     false, tcf_block_offload_in_use(block),
1530                                     NULL);
1531         list_del(&block_cb->list);
1532         kfree(block_cb);
1533 }
1534 EXPORT_SYMBOL(__tcf_block_cb_unregister);
1535
1536 void tcf_block_cb_unregister(struct tcf_block *block,
1537                              tc_setup_cb_t *cb, void *cb_ident)
1538 {
1539         struct tcf_block_cb *block_cb;
1540
1541         block_cb = tcf_block_cb_lookup(block, cb, cb_ident);
1542         if (!block_cb)
1543                 return;
1544         __tcf_block_cb_unregister(block, block_cb);
1545 }
1546 EXPORT_SYMBOL(tcf_block_cb_unregister);
1547
1548 /* Main classifier routine: scans classifier chain attached
1549  * to this qdisc, (optionally) tests for protocol and asks
1550  * specific classifiers.
1551  */
1552 int tcf_classify(struct sk_buff *skb, const struct tcf_proto *tp,
1553                  struct tcf_result *res, bool compat_mode)
1554 {
1555 #ifdef CONFIG_NET_CLS_ACT
1556         const int max_reclassify_loop = 4;
1557         const struct tcf_proto *orig_tp = tp;
1558         const struct tcf_proto *first_tp;
1559         int limit = 0;
1560
1561 reclassify:
1562 #endif
1563         for (; tp; tp = rcu_dereference_bh(tp->next)) {
1564                 __be16 protocol = tc_skb_protocol(skb);
1565                 int err;
1566
1567                 if (tp->protocol != protocol &&
1568                     tp->protocol != htons(ETH_P_ALL))
1569                         continue;
1570
1571                 err = tp->classify(skb, tp, res);
1572 #ifdef CONFIG_NET_CLS_ACT
1573                 if (unlikely(err == TC_ACT_RECLASSIFY && !compat_mode)) {
1574                         first_tp = orig_tp;
1575                         goto reset;
1576                 } else if (unlikely(TC_ACT_EXT_CMP(err, TC_ACT_GOTO_CHAIN))) {
1577                         first_tp = res->goto_tp;
1578                         goto reset;
1579                 }
1580 #endif
1581                 if (err >= 0)
1582                         return err;
1583         }
1584
1585         return TC_ACT_UNSPEC; /* signal: continue lookup */
1586 #ifdef CONFIG_NET_CLS_ACT
1587 reset:
1588         if (unlikely(limit++ >= max_reclassify_loop)) {
1589                 net_notice_ratelimited("%u: reclassify loop, rule prio %u, protocol %02x\n",
1590                                        tp->chain->block->index,
1591                                        tp->prio & 0xffff,
1592                                        ntohs(tp->protocol));
1593                 return TC_ACT_SHOT;
1594         }
1595
1596         tp = first_tp;
1597         goto reclassify;
1598 #endif
1599 }
1600 EXPORT_SYMBOL(tcf_classify);
1601
1602 struct tcf_chain_info {
1603         struct tcf_proto __rcu **pprev;
1604         struct tcf_proto __rcu *next;
1605 };
1606
1607 static struct tcf_proto *tcf_chain_tp_prev(struct tcf_chain *chain,
1608                                            struct tcf_chain_info *chain_info)
1609 {
1610         return tcf_chain_dereference(*chain_info->pprev, chain);
1611 }
1612
1613 static void tcf_chain_tp_insert(struct tcf_chain *chain,
1614                                 struct tcf_chain_info *chain_info,
1615                                 struct tcf_proto *tp)
1616 {
1617         if (*chain_info->pprev == chain->filter_chain)
1618                 tcf_chain0_head_change(chain, tp);
1619         tcf_proto_get(tp);
1620         RCU_INIT_POINTER(tp->next, tcf_chain_tp_prev(chain, chain_info));
1621         rcu_assign_pointer(*chain_info->pprev, tp);
1622 }
1623
1624 static void tcf_chain_tp_remove(struct tcf_chain *chain,
1625                                 struct tcf_chain_info *chain_info,
1626                                 struct tcf_proto *tp)
1627 {
1628         struct tcf_proto *next = tcf_chain_dereference(chain_info->next, chain);
1629
1630         tcf_proto_mark_delete(tp);
1631         if (tp == chain->filter_chain)
1632                 tcf_chain0_head_change(chain, next);
1633         RCU_INIT_POINTER(*chain_info->pprev, next);
1634 }
1635
1636 static struct tcf_proto *tcf_chain_tp_find(struct tcf_chain *chain,
1637                                            struct tcf_chain_info *chain_info,
1638                                            u32 protocol, u32 prio,
1639                                            bool prio_allocate);
1640
1641 /* Try to insert new proto.
1642  * If proto with specified priority already exists, free new proto
1643  * and return existing one.
1644  */
1645
1646 static struct tcf_proto *tcf_chain_tp_insert_unique(struct tcf_chain *chain,
1647                                                     struct tcf_proto *tp_new,
1648                                                     u32 protocol, u32 prio)
1649 {
1650         struct tcf_chain_info chain_info;
1651         struct tcf_proto *tp;
1652
1653         mutex_lock(&chain->filter_chain_lock);
1654
1655         tp = tcf_chain_tp_find(chain, &chain_info,
1656                                protocol, prio, false);
1657         if (!tp)
1658                 tcf_chain_tp_insert(chain, &chain_info, tp_new);
1659         mutex_unlock(&chain->filter_chain_lock);
1660
1661         if (tp) {
1662                 tcf_proto_destroy(tp_new, NULL);
1663                 tp_new = tp;
1664         }
1665
1666         return tp_new;
1667 }
1668
1669 static void tcf_chain_tp_delete_empty(struct tcf_chain *chain,
1670                                       struct tcf_proto *tp,
1671                                       struct netlink_ext_ack *extack)
1672 {
1673         struct tcf_chain_info chain_info;
1674         struct tcf_proto *tp_iter;
1675         struct tcf_proto **pprev;
1676         struct tcf_proto *next;
1677
1678         mutex_lock(&chain->filter_chain_lock);
1679
1680         /* Atomically find and remove tp from chain. */
1681         for (pprev = &chain->filter_chain;
1682              (tp_iter = tcf_chain_dereference(*pprev, chain));
1683              pprev = &tp_iter->next) {
1684                 if (tp_iter == tp) {
1685                         chain_info.pprev = pprev;
1686                         chain_info.next = tp_iter->next;
1687                         WARN_ON(tp_iter->deleting);
1688                         break;
1689                 }
1690         }
1691         /* Verify that tp still exists and no new filters were inserted
1692          * concurrently.
1693          * Mark tp for deletion if it is empty.
1694          */
1695         if (!tp_iter || !tcf_proto_check_delete(tp)) {
1696                 mutex_unlock(&chain->filter_chain_lock);
1697                 return;
1698         }
1699
1700         next = tcf_chain_dereference(chain_info.next, chain);
1701         if (tp == chain->filter_chain)
1702                 tcf_chain0_head_change(chain, next);
1703         RCU_INIT_POINTER(*chain_info.pprev, next);
1704         mutex_unlock(&chain->filter_chain_lock);
1705
1706         tcf_proto_put(tp, extack);
1707 }
1708
1709 static struct tcf_proto *tcf_chain_tp_find(struct tcf_chain *chain,
1710                                            struct tcf_chain_info *chain_info,
1711                                            u32 protocol, u32 prio,
1712                                            bool prio_allocate)
1713 {
1714         struct tcf_proto **pprev;
1715         struct tcf_proto *tp;
1716
1717         /* Check the chain for existence of proto-tcf with this priority */
1718         for (pprev = &chain->filter_chain;
1719              (tp = tcf_chain_dereference(*pprev, chain));
1720              pprev = &tp->next) {
1721                 if (tp->prio >= prio) {
1722                         if (tp->prio == prio) {
1723                                 if (prio_allocate ||
1724                                     (tp->protocol != protocol && protocol))
1725                                         return ERR_PTR(-EINVAL);
1726                         } else {
1727                                 tp = NULL;
1728                         }
1729                         break;
1730                 }
1731         }
1732         chain_info->pprev = pprev;
1733         if (tp) {
1734                 chain_info->next = tp->next;
1735                 tcf_proto_get(tp);
1736         } else {
1737                 chain_info->next = NULL;
1738         }
1739         return tp;
1740 }
1741
1742 static int tcf_fill_node(struct net *net, struct sk_buff *skb,
1743                          struct tcf_proto *tp, struct tcf_block *block,
1744                          struct Qdisc *q, u32 parent, void *fh,
1745                          u32 portid, u32 seq, u16 flags, int event)
1746 {
1747         struct tcmsg *tcm;
1748         struct nlmsghdr  *nlh;
1749         unsigned char *b = skb_tail_pointer(skb);
1750
1751         nlh = nlmsg_put(skb, portid, seq, event, sizeof(*tcm), flags);
1752         if (!nlh)
1753                 goto out_nlmsg_trim;
1754         tcm = nlmsg_data(nlh);
1755         tcm->tcm_family = AF_UNSPEC;
1756         tcm->tcm__pad1 = 0;
1757         tcm->tcm__pad2 = 0;
1758         if (q) {
1759                 tcm->tcm_ifindex = qdisc_dev(q)->ifindex;
1760                 tcm->tcm_parent = parent;
1761         } else {
1762                 tcm->tcm_ifindex = TCM_IFINDEX_MAGIC_BLOCK;
1763                 tcm->tcm_block_index = block->index;
1764         }
1765         tcm->tcm_info = TC_H_MAKE(tp->prio, tp->protocol);
1766         if (nla_put_string(skb, TCA_KIND, tp->ops->kind))
1767                 goto nla_put_failure;
1768         if (nla_put_u32(skb, TCA_CHAIN, tp->chain->index))
1769                 goto nla_put_failure;
1770         if (!fh) {
1771                 tcm->tcm_handle = 0;
1772         } else {
1773                 if (tp->ops->dump && tp->ops->dump(net, tp, fh, skb, tcm) < 0)
1774                         goto nla_put_failure;
1775         }
1776         nlh->nlmsg_len = skb_tail_pointer(skb) - b;
1777         return skb->len;
1778
1779 out_nlmsg_trim:
1780 nla_put_failure:
1781         nlmsg_trim(skb, b);
1782         return -1;
1783 }
1784
1785 static int tfilter_notify(struct net *net, struct sk_buff *oskb,
1786                           struct nlmsghdr *n, struct tcf_proto *tp,
1787                           struct tcf_block *block, struct Qdisc *q,
1788                           u32 parent, void *fh, int event, bool unicast)
1789 {
1790         struct sk_buff *skb;
1791         u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
1792
1793         skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
1794         if (!skb)
1795                 return -ENOBUFS;
1796
1797         if (tcf_fill_node(net, skb, tp, block, q, parent, fh, portid,
1798                           n->nlmsg_seq, n->nlmsg_flags, event) <= 0) {
1799                 kfree_skb(skb);
1800                 return -EINVAL;
1801         }
1802
1803         if (unicast)
1804                 return netlink_unicast(net->rtnl, skb, portid, MSG_DONTWAIT);
1805
1806         return rtnetlink_send(skb, net, portid, RTNLGRP_TC,
1807                               n->nlmsg_flags & NLM_F_ECHO);
1808 }
1809
1810 static int tfilter_del_notify(struct net *net, struct sk_buff *oskb,
1811                               struct nlmsghdr *n, struct tcf_proto *tp,
1812                               struct tcf_block *block, struct Qdisc *q,
1813                               u32 parent, void *fh, bool unicast, bool *last,
1814                               struct netlink_ext_ack *extack)
1815 {
1816         struct sk_buff *skb;
1817         u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
1818         int err;
1819
1820         skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
1821         if (!skb)
1822                 return -ENOBUFS;
1823
1824         if (tcf_fill_node(net, skb, tp, block, q, parent, fh, portid,
1825                           n->nlmsg_seq, n->nlmsg_flags, RTM_DELTFILTER) <= 0) {
1826                 NL_SET_ERR_MSG(extack, "Failed to build del event notification");
1827                 kfree_skb(skb);
1828                 return -EINVAL;
1829         }
1830
1831         err = tp->ops->delete(tp, fh, last, extack);
1832         if (err) {
1833                 kfree_skb(skb);
1834                 return err;
1835         }
1836
1837         if (unicast)
1838                 return netlink_unicast(net->rtnl, skb, portid, MSG_DONTWAIT);
1839
1840         err = rtnetlink_send(skb, net, portid, RTNLGRP_TC,
1841                              n->nlmsg_flags & NLM_F_ECHO);
1842         if (err < 0)
1843                 NL_SET_ERR_MSG(extack, "Failed to send filter delete notification");
1844         return err;
1845 }
1846
1847 static void tfilter_notify_chain(struct net *net, struct sk_buff *oskb,
1848                                  struct tcf_block *block, struct Qdisc *q,
1849                                  u32 parent, struct nlmsghdr *n,
1850                                  struct tcf_chain *chain, int event)
1851 {
1852         struct tcf_proto *tp;
1853
1854         for (tp = tcf_get_next_proto(chain, NULL);
1855              tp; tp = tcf_get_next_proto(chain, tp))
1856                 tfilter_notify(net, oskb, n, tp, block,
1857                                q, parent, NULL, event, false);
1858 }
1859
1860 static int tc_new_tfilter(struct sk_buff *skb, struct nlmsghdr *n,
1861                           struct netlink_ext_ack *extack)
1862 {
1863         struct net *net = sock_net(skb->sk);
1864         struct nlattr *tca[TCA_MAX + 1];
1865         struct tcmsg *t;
1866         u32 protocol;
1867         u32 prio;
1868         bool prio_allocate;
1869         u32 parent;
1870         u32 chain_index;
1871         struct Qdisc *q = NULL;
1872         struct tcf_chain_info chain_info;
1873         struct tcf_chain *chain = NULL;
1874         struct tcf_block *block;
1875         struct tcf_proto *tp;
1876         unsigned long cl;
1877         void *fh;
1878         int err;
1879         int tp_created;
1880
1881         if (!netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN))
1882                 return -EPERM;
1883
1884 replay:
1885         tp_created = 0;
1886
1887         err = nlmsg_parse(n, sizeof(*t), tca, TCA_MAX, rtm_tca_policy, extack);
1888         if (err < 0)
1889                 return err;
1890
1891         t = nlmsg_data(n);
1892         protocol = TC_H_MIN(t->tcm_info);
1893         prio = TC_H_MAJ(t->tcm_info);
1894         prio_allocate = false;
1895         parent = t->tcm_parent;
1896         tp = NULL;
1897         cl = 0;
1898
1899         if (prio == 0) {
1900                 /* If no priority is provided by the user,
1901                  * we allocate one.
1902                  */
1903                 if (n->nlmsg_flags & NLM_F_CREATE) {
1904                         prio = TC_H_MAKE(0x80000000U, 0U);
1905                         prio_allocate = true;
1906                 } else {
1907                         NL_SET_ERR_MSG(extack, "Invalid filter command with priority of zero");
1908                         return -ENOENT;
1909                 }
1910         }
1911
1912         /* Find head of filter chain. */
1913
1914         block = tcf_block_find(net, &q, &parent, &cl,
1915                                t->tcm_ifindex, t->tcm_block_index, extack);
1916         if (IS_ERR(block)) {
1917                 err = PTR_ERR(block);
1918                 goto errout;
1919         }
1920
1921         chain_index = tca[TCA_CHAIN] ? nla_get_u32(tca[TCA_CHAIN]) : 0;
1922         if (chain_index > TC_ACT_EXT_VAL_MASK) {
1923                 NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit");
1924                 err = -EINVAL;
1925                 goto errout;
1926         }
1927         chain = tcf_chain_get(block, chain_index, true);
1928         if (!chain) {
1929                 NL_SET_ERR_MSG(extack, "Cannot create specified filter chain");
1930                 err = -ENOMEM;
1931                 goto errout;
1932         }
1933
1934         mutex_lock(&chain->filter_chain_lock);
1935         tp = tcf_chain_tp_find(chain, &chain_info, protocol,
1936                                prio, prio_allocate);
1937         if (IS_ERR(tp)) {
1938                 NL_SET_ERR_MSG(extack, "Filter with specified priority/protocol not found");
1939                 err = PTR_ERR(tp);
1940                 goto errout_locked;
1941         }
1942
1943         if (tp == NULL) {
1944                 struct tcf_proto *tp_new = NULL;
1945
1946                 /* Proto-tcf does not exist, create new one */
1947
1948                 if (tca[TCA_KIND] == NULL || !protocol) {
1949                         NL_SET_ERR_MSG(extack, "Filter kind and protocol must be specified");
1950                         err = -EINVAL;
1951                         goto errout_locked;
1952                 }
1953
1954                 if (!(n->nlmsg_flags & NLM_F_CREATE)) {
1955                         NL_SET_ERR_MSG(extack, "Need both RTM_NEWTFILTER and NLM_F_CREATE to create a new filter");
1956                         err = -ENOENT;
1957                         goto errout_locked;
1958                 }
1959
1960                 if (prio_allocate)
1961                         prio = tcf_auto_prio(tcf_chain_tp_prev(chain,
1962                                                                &chain_info));
1963
1964                 mutex_unlock(&chain->filter_chain_lock);
1965                 tp_new = tcf_proto_create(nla_data(tca[TCA_KIND]),
1966                                           protocol, prio, chain, extack);
1967                 if (IS_ERR(tp_new)) {
1968                         err = PTR_ERR(tp_new);
1969                         goto errout;
1970                 }
1971
1972                 tp_created = 1;
1973                 tp = tcf_chain_tp_insert_unique(chain, tp_new, protocol, prio);
1974         } else {
1975                 mutex_unlock(&chain->filter_chain_lock);
1976         }
1977
1978         if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], tp->ops->kind)) {
1979                 NL_SET_ERR_MSG(extack, "Specified filter kind does not match existing one");
1980                 err = -EINVAL;
1981                 goto errout;
1982         }
1983
1984         fh = tp->ops->get(tp, t->tcm_handle);
1985
1986         if (!fh) {
1987                 if (!(n->nlmsg_flags & NLM_F_CREATE)) {
1988                         NL_SET_ERR_MSG(extack, "Need both RTM_NEWTFILTER and NLM_F_CREATE to create a new filter");
1989                         err = -ENOENT;
1990                         goto errout;
1991                 }
1992         } else if (n->nlmsg_flags & NLM_F_EXCL) {
1993                 NL_SET_ERR_MSG(extack, "Filter already exists");
1994                 err = -EEXIST;
1995                 goto errout;
1996         }
1997
1998         if (chain->tmplt_ops && chain->tmplt_ops != tp->ops) {
1999                 NL_SET_ERR_MSG(extack, "Chain template is set to a different filter kind");
2000                 err = -EINVAL;
2001                 goto errout;
2002         }
2003
2004         err = tp->ops->change(net, skb, tp, cl, t->tcm_handle, tca, &fh,
2005                               n->nlmsg_flags & NLM_F_CREATE ? TCA_ACT_NOREPLACE : TCA_ACT_REPLACE,
2006                               extack);
2007         if (err == 0)
2008                 tfilter_notify(net, skb, n, tp, block, q, parent, fh,
2009                                RTM_NEWTFILTER, false);
2010
2011 errout:
2012         if (err && tp_created)
2013                 tcf_chain_tp_delete_empty(chain, tp, NULL);
2014         if (chain) {
2015                 if (tp && !IS_ERR(tp))
2016                         tcf_proto_put(tp, NULL);
2017                 if (!tp_created)
2018                         tcf_chain_put(chain);
2019         }
2020         tcf_block_release(q, block);
2021         if (err == -EAGAIN)
2022                 /* Replay the request. */
2023                 goto replay;
2024         return err;
2025
2026 errout_locked:
2027         mutex_unlock(&chain->filter_chain_lock);
2028         goto errout;
2029 }
2030
2031 static int tc_del_tfilter(struct sk_buff *skb, struct nlmsghdr *n,
2032                           struct netlink_ext_ack *extack)
2033 {
2034         struct net *net = sock_net(skb->sk);
2035         struct nlattr *tca[TCA_MAX + 1];
2036         struct tcmsg *t;
2037         u32 protocol;
2038         u32 prio;
2039         u32 parent;
2040         u32 chain_index;
2041         struct Qdisc *q = NULL;
2042         struct tcf_chain_info chain_info;
2043         struct tcf_chain *chain = NULL;
2044         struct tcf_block *block;
2045         struct tcf_proto *tp = NULL;
2046         unsigned long cl = 0;
2047         void *fh = NULL;
2048         int err;
2049
2050         if (!netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN))
2051                 return -EPERM;
2052
2053         err = nlmsg_parse(n, sizeof(*t), tca, TCA_MAX, rtm_tca_policy, extack);
2054         if (err < 0)
2055                 return err;
2056
2057         t = nlmsg_data(n);
2058         protocol = TC_H_MIN(t->tcm_info);
2059         prio = TC_H_MAJ(t->tcm_info);
2060         parent = t->tcm_parent;
2061
2062         if (prio == 0 && (protocol || t->tcm_handle || tca[TCA_KIND])) {
2063                 NL_SET_ERR_MSG(extack, "Cannot flush filters with protocol, handle or kind set");
2064                 return -ENOENT;
2065         }
2066
2067         /* Find head of filter chain. */
2068
2069         block = tcf_block_find(net, &q, &parent, &cl,
2070                                t->tcm_ifindex, t->tcm_block_index, extack);
2071         if (IS_ERR(block)) {
2072                 err = PTR_ERR(block);
2073                 goto errout;
2074         }
2075
2076         chain_index = tca[TCA_CHAIN] ? nla_get_u32(tca[TCA_CHAIN]) : 0;
2077         if (chain_index > TC_ACT_EXT_VAL_MASK) {
2078                 NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit");
2079                 err = -EINVAL;
2080                 goto errout;
2081         }
2082         chain = tcf_chain_get(block, chain_index, false);
2083         if (!chain) {
2084                 /* User requested flush on non-existent chain. Nothing to do,
2085                  * so just return success.
2086                  */
2087                 if (prio == 0) {
2088                         err = 0;
2089                         goto errout;
2090                 }
2091                 NL_SET_ERR_MSG(extack, "Cannot find specified filter chain");
2092                 err = -ENOENT;
2093                 goto errout;
2094         }
2095
2096         if (prio == 0) {
2097                 tfilter_notify_chain(net, skb, block, q, parent, n,
2098                                      chain, RTM_DELTFILTER);
2099                 tcf_chain_flush(chain);
2100                 err = 0;
2101                 goto errout;
2102         }
2103
2104         mutex_lock(&chain->filter_chain_lock);
2105         tp = tcf_chain_tp_find(chain, &chain_info, protocol,
2106                                prio, false);
2107         if (!tp || IS_ERR(tp)) {
2108                 NL_SET_ERR_MSG(extack, "Filter with specified priority/protocol not found");
2109                 err = tp ? PTR_ERR(tp) : -ENOENT;
2110                 goto errout_locked;
2111         } else if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], tp->ops->kind)) {
2112                 NL_SET_ERR_MSG(extack, "Specified filter kind does not match existing one");
2113                 err = -EINVAL;
2114                 goto errout_locked;
2115         } else if (t->tcm_handle == 0) {
2116                 tcf_chain_tp_remove(chain, &chain_info, tp);
2117                 mutex_unlock(&chain->filter_chain_lock);
2118
2119                 tcf_proto_put(tp, NULL);
2120                 tfilter_notify(net, skb, n, tp, block, q, parent, fh,
2121                                RTM_DELTFILTER, false);
2122                 err = 0;
2123                 goto errout;
2124         }
2125         mutex_unlock(&chain->filter_chain_lock);
2126
2127         fh = tp->ops->get(tp, t->tcm_handle);
2128
2129         if (!fh) {
2130                 NL_SET_ERR_MSG(extack, "Specified filter handle not found");
2131                 err = -ENOENT;
2132         } else {
2133                 bool last;
2134
2135                 err = tfilter_del_notify(net, skb, n, tp, block,
2136                                          q, parent, fh, false, &last,
2137                                          extack);
2138                 if (err)
2139                         goto errout;
2140                 if (last)
2141                         tcf_chain_tp_delete_empty(chain, tp, extack);
2142         }
2143
2144 errout:
2145         if (chain) {
2146                 if (tp && !IS_ERR(tp))
2147                         tcf_proto_put(tp, NULL);
2148                 tcf_chain_put(chain);
2149         }
2150         tcf_block_release(q, block);
2151         return err;
2152
2153 errout_locked:
2154         mutex_unlock(&chain->filter_chain_lock);
2155         goto errout;
2156 }
2157
2158 static int tc_get_tfilter(struct sk_buff *skb, struct nlmsghdr *n,
2159                           struct netlink_ext_ack *extack)
2160 {
2161         struct net *net = sock_net(skb->sk);
2162         struct nlattr *tca[TCA_MAX + 1];
2163         struct tcmsg *t;
2164         u32 protocol;
2165         u32 prio;
2166         u32 parent;
2167         u32 chain_index;
2168         struct Qdisc *q = NULL;
2169         struct tcf_chain_info chain_info;
2170         struct tcf_chain *chain = NULL;
2171         struct tcf_block *block;
2172         struct tcf_proto *tp = NULL;
2173         unsigned long cl = 0;
2174         void *fh = NULL;
2175         int err;
2176
2177         err = nlmsg_parse(n, sizeof(*t), tca, TCA_MAX, rtm_tca_policy, extack);
2178         if (err < 0)
2179                 return err;
2180
2181         t = nlmsg_data(n);
2182         protocol = TC_H_MIN(t->tcm_info);
2183         prio = TC_H_MAJ(t->tcm_info);
2184         parent = t->tcm_parent;
2185
2186         if (prio == 0) {
2187                 NL_SET_ERR_MSG(extack, "Invalid filter command with priority of zero");
2188                 return -ENOENT;
2189         }
2190
2191         /* Find head of filter chain. */
2192
2193         block = tcf_block_find(net, &q, &parent, &cl,
2194                                t->tcm_ifindex, t->tcm_block_index, extack);
2195         if (IS_ERR(block)) {
2196                 err = PTR_ERR(block);
2197                 goto errout;
2198         }
2199
2200         chain_index = tca[TCA_CHAIN] ? nla_get_u32(tca[TCA_CHAIN]) : 0;
2201         if (chain_index > TC_ACT_EXT_VAL_MASK) {
2202                 NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit");
2203                 err = -EINVAL;
2204                 goto errout;
2205         }
2206         chain = tcf_chain_get(block, chain_index, false);
2207         if (!chain) {
2208                 NL_SET_ERR_MSG(extack, "Cannot find specified filter chain");
2209                 err = -EINVAL;
2210                 goto errout;
2211         }
2212
2213         mutex_lock(&chain->filter_chain_lock);
2214         tp = tcf_chain_tp_find(chain, &chain_info, protocol,
2215                                prio, false);
2216         mutex_unlock(&chain->filter_chain_lock);
2217         if (!tp || IS_ERR(tp)) {
2218                 NL_SET_ERR_MSG(extack, "Filter with specified priority/protocol not found");
2219                 err = tp ? PTR_ERR(tp) : -ENOENT;
2220                 goto errout;
2221         } else if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], tp->ops->kind)) {
2222                 NL_SET_ERR_MSG(extack, "Specified filter kind does not match existing one");
2223                 err = -EINVAL;
2224                 goto errout;
2225         }
2226
2227         fh = tp->ops->get(tp, t->tcm_handle);
2228
2229         if (!fh) {
2230                 NL_SET_ERR_MSG(extack, "Specified filter handle not found");
2231                 err = -ENOENT;
2232         } else {
2233                 err = tfilter_notify(net, skb, n, tp, block, q, parent,
2234                                      fh, RTM_NEWTFILTER, true);
2235                 if (err < 0)
2236                         NL_SET_ERR_MSG(extack, "Failed to send filter notify message");
2237         }
2238
2239 errout:
2240         if (chain) {
2241                 if (tp && !IS_ERR(tp))
2242                         tcf_proto_put(tp, NULL);
2243                 tcf_chain_put(chain);
2244         }
2245         tcf_block_release(q, block);
2246         return err;
2247 }
2248
2249 struct tcf_dump_args {
2250         struct tcf_walker w;
2251         struct sk_buff *skb;
2252         struct netlink_callback *cb;
2253         struct tcf_block *block;
2254         struct Qdisc *q;
2255         u32 parent;
2256 };
2257
2258 static int tcf_node_dump(struct tcf_proto *tp, void *n, struct tcf_walker *arg)
2259 {
2260         struct tcf_dump_args *a = (void *)arg;
2261         struct net *net = sock_net(a->skb->sk);
2262
2263         return tcf_fill_node(net, a->skb, tp, a->block, a->q, a->parent,
2264                              n, NETLINK_CB(a->cb->skb).portid,
2265                              a->cb->nlh->nlmsg_seq, NLM_F_MULTI,
2266                              RTM_NEWTFILTER);
2267 }
2268
2269 static bool tcf_chain_dump(struct tcf_chain *chain, struct Qdisc *q, u32 parent,
2270                            struct sk_buff *skb, struct netlink_callback *cb,
2271                            long index_start, long *p_index)
2272 {
2273         struct net *net = sock_net(skb->sk);
2274         struct tcf_block *block = chain->block;
2275         struct tcmsg *tcm = nlmsg_data(cb->nlh);
2276         struct tcf_proto *tp, *tp_prev;
2277         struct tcf_dump_args arg;
2278
2279         for (tp = __tcf_get_next_proto(chain, NULL);
2280              tp;
2281              tp_prev = tp,
2282                      tp = __tcf_get_next_proto(chain, tp),
2283                      tcf_proto_put(tp_prev, NULL),
2284                      (*p_index)++) {
2285                 if (*p_index < index_start)
2286                         continue;
2287                 if (TC_H_MAJ(tcm->tcm_info) &&
2288                     TC_H_MAJ(tcm->tcm_info) != tp->prio)
2289                         continue;
2290                 if (TC_H_MIN(tcm->tcm_info) &&
2291                     TC_H_MIN(tcm->tcm_info) != tp->protocol)
2292                         continue;
2293                 if (*p_index > index_start)
2294                         memset(&cb->args[1], 0,
2295                                sizeof(cb->args) - sizeof(cb->args[0]));
2296                 if (cb->args[1] == 0) {
2297                         if (tcf_fill_node(net, skb, tp, block, q, parent, NULL,
2298                                           NETLINK_CB(cb->skb).portid,
2299                                           cb->nlh->nlmsg_seq, NLM_F_MULTI,
2300                                           RTM_NEWTFILTER) <= 0)
2301                                 goto errout;
2302
2303                         cb->args[1] = 1;
2304                 }
2305                 if (!tp->ops->walk)
2306                         continue;
2307                 arg.w.fn = tcf_node_dump;
2308                 arg.skb = skb;
2309                 arg.cb = cb;
2310                 arg.block = block;
2311                 arg.q = q;
2312                 arg.parent = parent;
2313                 arg.w.stop = 0;
2314                 arg.w.skip = cb->args[1] - 1;
2315                 arg.w.count = 0;
2316                 arg.w.cookie = cb->args[2];
2317                 tp->ops->walk(tp, &arg.w);
2318                 cb->args[2] = arg.w.cookie;
2319                 cb->args[1] = arg.w.count + 1;
2320                 if (arg.w.stop)
2321                         goto errout;
2322         }
2323         return true;
2324
2325 errout:
2326         tcf_proto_put(tp, NULL);
2327         return false;
2328 }
2329
2330 /* called with RTNL */
2331 static int tc_dump_tfilter(struct sk_buff *skb, struct netlink_callback *cb)
2332 {
2333         struct tcf_chain *chain, *chain_prev;
2334         struct net *net = sock_net(skb->sk);
2335         struct nlattr *tca[TCA_MAX + 1];
2336         struct Qdisc *q = NULL;
2337         struct tcf_block *block;
2338         struct tcmsg *tcm = nlmsg_data(cb->nlh);
2339         long index_start;
2340         long index;
2341         u32 parent;
2342         int err;
2343
2344         if (nlmsg_len(cb->nlh) < sizeof(*tcm))
2345                 return skb->len;
2346
2347         err = nlmsg_parse(cb->nlh, sizeof(*tcm), tca, TCA_MAX, NULL,
2348                           cb->extack);
2349         if (err)
2350                 return err;
2351
2352         if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK) {
2353                 block = tcf_block_refcnt_get(net, tcm->tcm_block_index);
2354                 if (!block)
2355                         goto out;
2356                 /* If we work with block index, q is NULL and parent value
2357                  * will never be used in the following code. The check
2358                  * in tcf_fill_node prevents it. However, compiler does not
2359                  * see that far, so set parent to zero to silence the warning
2360                  * about parent being uninitialized.
2361                  */
2362                 parent = 0;
2363         } else {
2364                 const struct Qdisc_class_ops *cops;
2365                 struct net_device *dev;
2366                 unsigned long cl = 0;
2367
2368                 dev = __dev_get_by_index(net, tcm->tcm_ifindex);
2369                 if (!dev)
2370                         return skb->len;
2371
2372                 parent = tcm->tcm_parent;
2373                 if (!parent) {
2374                         q = dev->qdisc;
2375                         parent = q->handle;
2376                 } else {
2377                         q = qdisc_lookup(dev, TC_H_MAJ(tcm->tcm_parent));
2378                 }
2379                 if (!q)
2380                         goto out;
2381                 cops = q->ops->cl_ops;
2382                 if (!cops)
2383                         goto out;
2384                 if (!cops->tcf_block)
2385                         goto out;
2386                 if (TC_H_MIN(tcm->tcm_parent)) {
2387                         cl = cops->find(q, tcm->tcm_parent);
2388                         if (cl == 0)
2389                                 goto out;
2390                 }
2391                 block = cops->tcf_block(q, cl, NULL);
2392                 if (!block)
2393                         goto out;
2394                 if (tcf_block_shared(block))
2395                         q = NULL;
2396         }
2397
2398         index_start = cb->args[0];
2399         index = 0;
2400
2401         for (chain = __tcf_get_next_chain(block, NULL);
2402              chain;
2403              chain_prev = chain,
2404                      chain = __tcf_get_next_chain(block, chain),
2405                      tcf_chain_put(chain_prev)) {
2406                 if (tca[TCA_CHAIN] &&
2407                     nla_get_u32(tca[TCA_CHAIN]) != chain->index)
2408                         continue;
2409                 if (!tcf_chain_dump(chain, q, parent, skb, cb,
2410                                     index_start, &index)) {
2411                         tcf_chain_put(chain);
2412                         err = -EMSGSIZE;
2413                         break;
2414                 }
2415         }
2416
2417         if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK)
2418                 tcf_block_refcnt_put(block);
2419         cb->args[0] = index;
2420
2421 out:
2422         /* If we did no progress, the error (EMSGSIZE) is real */
2423         if (skb->len == 0 && err)
2424                 return err;
2425         return skb->len;
2426 }
2427
2428 static int tc_chain_fill_node(const struct tcf_proto_ops *tmplt_ops,
2429                               void *tmplt_priv, u32 chain_index,
2430                               struct net *net, struct sk_buff *skb,
2431                               struct tcf_block *block,
2432                               u32 portid, u32 seq, u16 flags, int event)
2433 {
2434         unsigned char *b = skb_tail_pointer(skb);
2435         const struct tcf_proto_ops *ops;
2436         struct nlmsghdr *nlh;
2437         struct tcmsg *tcm;
2438         void *priv;
2439
2440         ops = tmplt_ops;
2441         priv = tmplt_priv;
2442
2443         nlh = nlmsg_put(skb, portid, seq, event, sizeof(*tcm), flags);
2444         if (!nlh)
2445                 goto out_nlmsg_trim;
2446         tcm = nlmsg_data(nlh);
2447         tcm->tcm_family = AF_UNSPEC;
2448         tcm->tcm__pad1 = 0;
2449         tcm->tcm__pad2 = 0;
2450         tcm->tcm_handle = 0;
2451         if (block->q) {
2452                 tcm->tcm_ifindex = qdisc_dev(block->q)->ifindex;
2453                 tcm->tcm_parent = block->q->handle;
2454         } else {
2455                 tcm->tcm_ifindex = TCM_IFINDEX_MAGIC_BLOCK;
2456                 tcm->tcm_block_index = block->index;
2457         }
2458
2459         if (nla_put_u32(skb, TCA_CHAIN, chain_index))
2460                 goto nla_put_failure;
2461
2462         if (ops) {
2463                 if (nla_put_string(skb, TCA_KIND, ops->kind))
2464                         goto nla_put_failure;
2465                 if (ops->tmplt_dump(skb, net, priv) < 0)
2466                         goto nla_put_failure;
2467         }
2468
2469         nlh->nlmsg_len = skb_tail_pointer(skb) - b;
2470         return skb->len;
2471
2472 out_nlmsg_trim:
2473 nla_put_failure:
2474         nlmsg_trim(skb, b);
2475         return -EMSGSIZE;
2476 }
2477
2478 static int tc_chain_notify(struct tcf_chain *chain, struct sk_buff *oskb,
2479                            u32 seq, u16 flags, int event, bool unicast)
2480 {
2481         u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
2482         struct tcf_block *block = chain->block;
2483         struct net *net = block->net;
2484         struct sk_buff *skb;
2485
2486         skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
2487         if (!skb)
2488                 return -ENOBUFS;
2489
2490         if (tc_chain_fill_node(chain->tmplt_ops, chain->tmplt_priv,
2491                                chain->index, net, skb, block, portid,
2492                                seq, flags, event) <= 0) {
2493                 kfree_skb(skb);
2494                 return -EINVAL;
2495         }
2496
2497         if (unicast)
2498                 return netlink_unicast(net->rtnl, skb, portid, MSG_DONTWAIT);
2499
2500         return rtnetlink_send(skb, net, portid, RTNLGRP_TC, flags & NLM_F_ECHO);
2501 }
2502
2503 static int tc_chain_notify_delete(const struct tcf_proto_ops *tmplt_ops,
2504                                   void *tmplt_priv, u32 chain_index,
2505                                   struct tcf_block *block, struct sk_buff *oskb,
2506                                   u32 seq, u16 flags, bool unicast)
2507 {
2508         u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
2509         struct net *net = block->net;
2510         struct sk_buff *skb;
2511
2512         skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
2513         if (!skb)
2514                 return -ENOBUFS;
2515
2516         if (tc_chain_fill_node(tmplt_ops, tmplt_priv, chain_index, net, skb,
2517                                block, portid, seq, flags, RTM_DELCHAIN) <= 0) {
2518                 kfree_skb(skb);
2519                 return -EINVAL;
2520         }
2521
2522         if (unicast)
2523                 return netlink_unicast(net->rtnl, skb, portid, MSG_DONTWAIT);
2524
2525         return rtnetlink_send(skb, net, portid, RTNLGRP_TC, flags & NLM_F_ECHO);
2526 }
2527
2528 static int tc_chain_tmplt_add(struct tcf_chain *chain, struct net *net,
2529                               struct nlattr **tca,
2530                               struct netlink_ext_ack *extack)
2531 {
2532         const struct tcf_proto_ops *ops;
2533         void *tmplt_priv;
2534
2535         /* If kind is not set, user did not specify template. */
2536         if (!tca[TCA_KIND])
2537                 return 0;
2538
2539         ops = tcf_proto_lookup_ops(nla_data(tca[TCA_KIND]), extack);
2540         if (IS_ERR(ops))
2541                 return PTR_ERR(ops);
2542         if (!ops->tmplt_create || !ops->tmplt_destroy || !ops->tmplt_dump) {
2543                 NL_SET_ERR_MSG(extack, "Chain templates are not supported with specified classifier");
2544                 return -EOPNOTSUPP;
2545         }
2546
2547         tmplt_priv = ops->tmplt_create(net, chain, tca, extack);
2548         if (IS_ERR(tmplt_priv)) {
2549                 module_put(ops->owner);
2550                 return PTR_ERR(tmplt_priv);
2551         }
2552         chain->tmplt_ops = ops;
2553         chain->tmplt_priv = tmplt_priv;
2554         return 0;
2555 }
2556
2557 static void tc_chain_tmplt_del(const struct tcf_proto_ops *tmplt_ops,
2558                                void *tmplt_priv)
2559 {
2560         /* If template ops are set, no work to do for us. */
2561         if (!tmplt_ops)
2562                 return;
2563
2564         tmplt_ops->tmplt_destroy(tmplt_priv);
2565         module_put(tmplt_ops->owner);
2566 }
2567
2568 /* Add/delete/get a chain */
2569
2570 static int tc_ctl_chain(struct sk_buff *skb, struct nlmsghdr *n,
2571                         struct netlink_ext_ack *extack)
2572 {
2573         struct net *net = sock_net(skb->sk);
2574         struct nlattr *tca[TCA_MAX + 1];
2575         struct tcmsg *t;
2576         u32 parent;
2577         u32 chain_index;
2578         struct Qdisc *q = NULL;
2579         struct tcf_chain *chain = NULL;
2580         struct tcf_block *block;
2581         unsigned long cl;
2582         int err;
2583
2584         if (n->nlmsg_type != RTM_GETCHAIN &&
2585             !netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN))
2586                 return -EPERM;
2587
2588 replay:
2589         err = nlmsg_parse(n, sizeof(*t), tca, TCA_MAX, rtm_tca_policy, extack);
2590         if (err < 0)
2591                 return err;
2592
2593         t = nlmsg_data(n);
2594         parent = t->tcm_parent;
2595         cl = 0;
2596
2597         block = tcf_block_find(net, &q, &parent, &cl,
2598                                t->tcm_ifindex, t->tcm_block_index, extack);
2599         if (IS_ERR(block))
2600                 return PTR_ERR(block);
2601
2602         chain_index = tca[TCA_CHAIN] ? nla_get_u32(tca[TCA_CHAIN]) : 0;
2603         if (chain_index > TC_ACT_EXT_VAL_MASK) {
2604                 NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit");
2605                 err = -EINVAL;
2606                 goto errout_block;
2607         }
2608
2609         mutex_lock(&block->lock);
2610         chain = tcf_chain_lookup(block, chain_index);
2611         if (n->nlmsg_type == RTM_NEWCHAIN) {
2612                 if (chain) {
2613                         if (tcf_chain_held_by_acts_only(chain)) {
2614                                 /* The chain exists only because there is
2615                                  * some action referencing it.
2616                                  */
2617                                 tcf_chain_hold(chain);
2618                         } else {
2619                                 NL_SET_ERR_MSG(extack, "Filter chain already exists");
2620                                 err = -EEXIST;
2621                                 goto errout_block_locked;
2622                         }
2623                 } else {
2624                         if (!(n->nlmsg_flags & NLM_F_CREATE)) {
2625                                 NL_SET_ERR_MSG(extack, "Need both RTM_NEWCHAIN and NLM_F_CREATE to create a new chain");
2626                                 err = -ENOENT;
2627                                 goto errout_block_locked;
2628                         }
2629                         chain = tcf_chain_create(block, chain_index);
2630                         if (!chain) {
2631                                 NL_SET_ERR_MSG(extack, "Failed to create filter chain");
2632                                 err = -ENOMEM;
2633                                 goto errout_block_locked;
2634                         }
2635                 }
2636         } else {
2637                 if (!chain || tcf_chain_held_by_acts_only(chain)) {
2638                         NL_SET_ERR_MSG(extack, "Cannot find specified filter chain");
2639                         err = -EINVAL;
2640                         goto errout_block_locked;
2641                 }
2642                 tcf_chain_hold(chain);
2643         }
2644
2645         if (n->nlmsg_type == RTM_NEWCHAIN) {
2646                 /* Modifying chain requires holding parent block lock. In case
2647                  * the chain was successfully added, take a reference to the
2648                  * chain. This ensures that an empty chain does not disappear at
2649                  * the end of this function.
2650                  */
2651                 tcf_chain_hold(chain);
2652                 chain->explicitly_created = true;
2653         }
2654         mutex_unlock(&block->lock);
2655
2656         switch (n->nlmsg_type) {
2657         case RTM_NEWCHAIN:
2658                 err = tc_chain_tmplt_add(chain, net, tca, extack);
2659                 if (err) {
2660                         tcf_chain_put_explicitly_created(chain);
2661                         goto errout;
2662                 }
2663
2664                 tc_chain_notify(chain, NULL, 0, NLM_F_CREATE | NLM_F_EXCL,
2665                                 RTM_NEWCHAIN, false);
2666                 break;
2667         case RTM_DELCHAIN:
2668                 tfilter_notify_chain(net, skb, block, q, parent, n,
2669                                      chain, RTM_DELTFILTER);
2670                 /* Flush the chain first as the user requested chain removal. */
2671                 tcf_chain_flush(chain);
2672                 /* In case the chain was successfully deleted, put a reference
2673                  * to the chain previously taken during addition.
2674                  */
2675                 tcf_chain_put_explicitly_created(chain);
2676                 break;
2677         case RTM_GETCHAIN:
2678                 err = tc_chain_notify(chain, skb, n->nlmsg_seq,
2679                                       n->nlmsg_seq, n->nlmsg_type, true);
2680                 if (err < 0)
2681                         NL_SET_ERR_MSG(extack, "Failed to send chain notify message");
2682                 break;
2683         default:
2684                 err = -EOPNOTSUPP;
2685                 NL_SET_ERR_MSG(extack, "Unsupported message type");
2686                 goto errout;
2687         }
2688
2689 errout:
2690         tcf_chain_put(chain);
2691 errout_block:
2692         tcf_block_release(q, block);
2693         if (err == -EAGAIN)
2694                 /* Replay the request. */
2695                 goto replay;
2696         return err;
2697
2698 errout_block_locked:
2699         mutex_unlock(&block->lock);
2700         goto errout_block;
2701 }
2702
2703 /* called with RTNL */
2704 static int tc_dump_chain(struct sk_buff *skb, struct netlink_callback *cb)
2705 {
2706         struct tcf_chain *chain, *chain_prev;
2707         struct net *net = sock_net(skb->sk);
2708         struct nlattr *tca[TCA_MAX + 1];
2709         struct Qdisc *q = NULL;
2710         struct tcf_block *block;
2711         struct tcmsg *tcm = nlmsg_data(cb->nlh);
2712         long index_start;
2713         long index;
2714         u32 parent;
2715         int err;
2716
2717         if (nlmsg_len(cb->nlh) < sizeof(*tcm))
2718                 return skb->len;
2719
2720         err = nlmsg_parse(cb->nlh, sizeof(*tcm), tca, TCA_MAX, rtm_tca_policy,
2721                           cb->extack);
2722         if (err)
2723                 return err;
2724
2725         if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK) {
2726                 block = tcf_block_refcnt_get(net, tcm->tcm_block_index);
2727                 if (!block)
2728                         goto out;
2729                 /* If we work with block index, q is NULL and parent value
2730                  * will never be used in the following code. The check
2731                  * in tcf_fill_node prevents it. However, compiler does not
2732                  * see that far, so set parent to zero to silence the warning
2733                  * about parent being uninitialized.
2734                  */
2735                 parent = 0;
2736         } else {
2737                 const struct Qdisc_class_ops *cops;
2738                 struct net_device *dev;
2739                 unsigned long cl = 0;
2740
2741                 dev = __dev_get_by_index(net, tcm->tcm_ifindex);
2742                 if (!dev)
2743                         return skb->len;
2744
2745                 parent = tcm->tcm_parent;
2746                 if (!parent) {
2747                         q = dev->qdisc;
2748                         parent = q->handle;
2749                 } else {
2750                         q = qdisc_lookup(dev, TC_H_MAJ(tcm->tcm_parent));
2751                 }
2752                 if (!q)
2753                         goto out;
2754                 cops = q->ops->cl_ops;
2755                 if (!cops)
2756                         goto out;
2757                 if (!cops->tcf_block)
2758                         goto out;
2759                 if (TC_H_MIN(tcm->tcm_parent)) {
2760                         cl = cops->find(q, tcm->tcm_parent);
2761                         if (cl == 0)
2762                                 goto out;
2763                 }
2764                 block = cops->tcf_block(q, cl, NULL);
2765                 if (!block)
2766                         goto out;
2767                 if (tcf_block_shared(block))
2768                         q = NULL;
2769         }
2770
2771         index_start = cb->args[0];
2772         index = 0;
2773
2774         for (chain = __tcf_get_next_chain(block, NULL);
2775              chain;
2776              chain_prev = chain,
2777                      chain = __tcf_get_next_chain(block, chain),
2778                      tcf_chain_put(chain_prev)) {
2779                 if ((tca[TCA_CHAIN] &&
2780                      nla_get_u32(tca[TCA_CHAIN]) != chain->index))
2781                         continue;
2782                 if (index < index_start) {
2783                         index++;
2784                         continue;
2785                 }
2786                 err = tc_chain_fill_node(chain->tmplt_ops, chain->tmplt_priv,
2787                                          chain->index, net, skb, block,
2788                                          NETLINK_CB(cb->skb).portid,
2789                                          cb->nlh->nlmsg_seq, NLM_F_MULTI,
2790                                          RTM_NEWCHAIN);
2791                 if (err <= 0) {
2792                         tcf_chain_put(chain);
2793                         break;
2794                 }
2795                 index++;
2796         }
2797
2798         if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK)
2799                 tcf_block_refcnt_put(block);
2800         cb->args[0] = index;
2801
2802 out:
2803         /* If we did no progress, the error (EMSGSIZE) is real */
2804         if (skb->len == 0 && err)
2805                 return err;
2806         return skb->len;
2807 }
2808
2809 void tcf_exts_destroy(struct tcf_exts *exts)
2810 {
2811 #ifdef CONFIG_NET_CLS_ACT
2812         tcf_action_destroy(exts->actions, TCA_ACT_UNBIND);
2813         kfree(exts->actions);
2814         exts->nr_actions = 0;
2815 #endif
2816 }
2817 EXPORT_SYMBOL(tcf_exts_destroy);
2818
2819 int tcf_exts_validate(struct net *net, struct tcf_proto *tp, struct nlattr **tb,
2820                       struct nlattr *rate_tlv, struct tcf_exts *exts, bool ovr,
2821                       struct netlink_ext_ack *extack)
2822 {
2823 #ifdef CONFIG_NET_CLS_ACT
2824         {
2825                 struct tc_action *act;
2826                 size_t attr_size = 0;
2827
2828                 if (exts->police && tb[exts->police]) {
2829                         act = tcf_action_init_1(net, tp, tb[exts->police],
2830                                                 rate_tlv, "police", ovr,
2831                                                 TCA_ACT_BIND, true, extack);
2832                         if (IS_ERR(act))
2833                                 return PTR_ERR(act);
2834
2835                         act->type = exts->type = TCA_OLD_COMPAT;
2836                         exts->actions[0] = act;
2837                         exts->nr_actions = 1;
2838                 } else if (exts->action && tb[exts->action]) {
2839                         int err;
2840
2841                         err = tcf_action_init(net, tp, tb[exts->action],
2842                                               rate_tlv, NULL, ovr, TCA_ACT_BIND,
2843                                               exts->actions, &attr_size, true,
2844                                               extack);
2845                         if (err < 0)
2846                                 return err;
2847                         exts->nr_actions = err;
2848                 }
2849                 exts->net = net;
2850         }
2851 #else
2852         if ((exts->action && tb[exts->action]) ||
2853             (exts->police && tb[exts->police])) {
2854                 NL_SET_ERR_MSG(extack, "Classifier actions are not supported per compile options (CONFIG_NET_CLS_ACT)");
2855                 return -EOPNOTSUPP;
2856         }
2857 #endif
2858
2859         return 0;
2860 }
2861 EXPORT_SYMBOL(tcf_exts_validate);
2862
2863 void tcf_exts_change(struct tcf_exts *dst, struct tcf_exts *src)
2864 {
2865 #ifdef CONFIG_NET_CLS_ACT
2866         struct tcf_exts old = *dst;
2867
2868         *dst = *src;
2869         tcf_exts_destroy(&old);
2870 #endif
2871 }
2872 EXPORT_SYMBOL(tcf_exts_change);
2873
2874 #ifdef CONFIG_NET_CLS_ACT
2875 static struct tc_action *tcf_exts_first_act(struct tcf_exts *exts)
2876 {
2877         if (exts->nr_actions == 0)
2878                 return NULL;
2879         else
2880                 return exts->actions[0];
2881 }
2882 #endif
2883
2884 int tcf_exts_dump(struct sk_buff *skb, struct tcf_exts *exts)
2885 {
2886 #ifdef CONFIG_NET_CLS_ACT
2887         struct nlattr *nest;
2888
2889         if (exts->action && tcf_exts_has_actions(exts)) {
2890                 /*
2891                  * again for backward compatible mode - we want
2892                  * to work with both old and new modes of entering
2893                  * tc data even if iproute2  was newer - jhs
2894                  */
2895                 if (exts->type != TCA_OLD_COMPAT) {
2896                         nest = nla_nest_start(skb, exts->action);
2897                         if (nest == NULL)
2898                                 goto nla_put_failure;
2899
2900                         if (tcf_action_dump(skb, exts->actions, 0, 0) < 0)
2901                                 goto nla_put_failure;
2902                         nla_nest_end(skb, nest);
2903                 } else if (exts->police) {
2904                         struct tc_action *act = tcf_exts_first_act(exts);
2905                         nest = nla_nest_start(skb, exts->police);
2906                         if (nest == NULL || !act)
2907                                 goto nla_put_failure;
2908                         if (tcf_action_dump_old(skb, act, 0, 0) < 0)
2909                                 goto nla_put_failure;
2910                         nla_nest_end(skb, nest);
2911                 }
2912         }
2913         return 0;
2914
2915 nla_put_failure:
2916         nla_nest_cancel(skb, nest);
2917         return -1;
2918 #else
2919         return 0;
2920 #endif
2921 }
2922 EXPORT_SYMBOL(tcf_exts_dump);
2923
2924
2925 int tcf_exts_dump_stats(struct sk_buff *skb, struct tcf_exts *exts)
2926 {
2927 #ifdef CONFIG_NET_CLS_ACT
2928         struct tc_action *a = tcf_exts_first_act(exts);
2929         if (a != NULL && tcf_action_copy_stats(skb, a, 1) < 0)
2930                 return -1;
2931 #endif
2932         return 0;
2933 }
2934 EXPORT_SYMBOL(tcf_exts_dump_stats);
2935
2936 int tc_setup_cb_call(struct tcf_block *block, enum tc_setup_type type,
2937                      void *type_data, bool err_stop)
2938 {
2939         struct tcf_block_cb *block_cb;
2940         int ok_count = 0;
2941         int err;
2942
2943         /* Make sure all netdevs sharing this block are offload-capable. */
2944         if (block->nooffloaddevcnt && err_stop)
2945                 return -EOPNOTSUPP;
2946
2947         list_for_each_entry(block_cb, &block->cb_list, list) {
2948                 err = block_cb->cb(type, type_data, block_cb->cb_priv);
2949                 if (err) {
2950                         if (err_stop)
2951                                 return err;
2952                 } else {
2953                         ok_count++;
2954                 }
2955         }
2956         return ok_count;
2957 }
2958 EXPORT_SYMBOL(tc_setup_cb_call);
2959
2960 int tc_setup_flow_action(struct flow_action *flow_action,
2961                          const struct tcf_exts *exts)
2962 {
2963         const struct tc_action *act;
2964         int i, j, k;
2965
2966         if (!exts)
2967                 return 0;
2968
2969         j = 0;
2970         tcf_exts_for_each_action(i, act, exts) {
2971                 struct flow_action_entry *entry;
2972
2973                 entry = &flow_action->entries[j];
2974                 if (is_tcf_gact_ok(act)) {
2975                         entry->id = FLOW_ACTION_ACCEPT;
2976                 } else if (is_tcf_gact_shot(act)) {
2977                         entry->id = FLOW_ACTION_DROP;
2978                 } else if (is_tcf_gact_trap(act)) {
2979                         entry->id = FLOW_ACTION_TRAP;
2980                 } else if (is_tcf_gact_goto_chain(act)) {
2981                         entry->id = FLOW_ACTION_GOTO;
2982                         entry->chain_index = tcf_gact_goto_chain_index(act);
2983                 } else if (is_tcf_mirred_egress_redirect(act)) {
2984                         entry->id = FLOW_ACTION_REDIRECT;
2985                         entry->dev = tcf_mirred_dev(act);
2986                 } else if (is_tcf_mirred_egress_mirror(act)) {
2987                         entry->id = FLOW_ACTION_MIRRED;
2988                         entry->dev = tcf_mirred_dev(act);
2989                 } else if (is_tcf_vlan(act)) {
2990                         switch (tcf_vlan_action(act)) {
2991                         case TCA_VLAN_ACT_PUSH:
2992                                 entry->id = FLOW_ACTION_VLAN_PUSH;
2993                                 entry->vlan.vid = tcf_vlan_push_vid(act);
2994                                 entry->vlan.proto = tcf_vlan_push_proto(act);
2995                                 entry->vlan.prio = tcf_vlan_push_prio(act);
2996                                 break;
2997                         case TCA_VLAN_ACT_POP:
2998                                 entry->id = FLOW_ACTION_VLAN_POP;
2999                                 break;
3000                         case TCA_VLAN_ACT_MODIFY:
3001                                 entry->id = FLOW_ACTION_VLAN_MANGLE;
3002                                 entry->vlan.vid = tcf_vlan_push_vid(act);
3003                                 entry->vlan.proto = tcf_vlan_push_proto(act);
3004                                 entry->vlan.prio = tcf_vlan_push_prio(act);
3005                                 break;
3006                         default:
3007                                 goto err_out;
3008                         }
3009                 } else if (is_tcf_tunnel_set(act)) {
3010                         entry->id = FLOW_ACTION_TUNNEL_ENCAP;
3011                         entry->tunnel = tcf_tunnel_info(act);
3012                 } else if (is_tcf_tunnel_release(act)) {
3013                         entry->id = FLOW_ACTION_TUNNEL_DECAP;
3014                         entry->tunnel = tcf_tunnel_info(act);
3015                 } else if (is_tcf_pedit(act)) {
3016                         for (k = 0; k < tcf_pedit_nkeys(act); k++) {
3017                                 switch (tcf_pedit_cmd(act, k)) {
3018                                 case TCA_PEDIT_KEY_EX_CMD_SET:
3019                                         entry->id = FLOW_ACTION_MANGLE;
3020                                         break;
3021                                 case TCA_PEDIT_KEY_EX_CMD_ADD:
3022                                         entry->id = FLOW_ACTION_ADD;
3023                                         break;
3024                                 default:
3025                                         goto err_out;
3026                                 }
3027                                 entry->mangle.htype = tcf_pedit_htype(act, k);
3028                                 entry->mangle.mask = tcf_pedit_mask(act, k);
3029                                 entry->mangle.val = tcf_pedit_val(act, k);
3030                                 entry->mangle.offset = tcf_pedit_offset(act, k);
3031                                 entry = &flow_action->entries[++j];
3032                         }
3033                 } else if (is_tcf_csum(act)) {
3034                         entry->id = FLOW_ACTION_CSUM;
3035                         entry->csum_flags = tcf_csum_update_flags(act);
3036                 } else if (is_tcf_skbedit_mark(act)) {
3037                         entry->id = FLOW_ACTION_MARK;
3038                         entry->mark = tcf_skbedit_mark(act);
3039                 } else {
3040                         goto err_out;
3041                 }
3042
3043                 if (!is_tcf_pedit(act))
3044                         j++;
3045         }
3046         return 0;
3047 err_out:
3048         return -EOPNOTSUPP;
3049 }
3050 EXPORT_SYMBOL(tc_setup_flow_action);
3051
3052 unsigned int tcf_exts_num_actions(struct tcf_exts *exts)
3053 {
3054         unsigned int num_acts = 0;
3055         struct tc_action *act;
3056         int i;
3057
3058         tcf_exts_for_each_action(i, act, exts) {
3059                 if (is_tcf_pedit(act))
3060                         num_acts += tcf_pedit_nkeys(act);
3061                 else
3062                         num_acts++;
3063         }
3064         return num_acts;
3065 }
3066 EXPORT_SYMBOL(tcf_exts_num_actions);
3067
3068 static __net_init int tcf_net_init(struct net *net)
3069 {
3070         struct tcf_net *tn = net_generic(net, tcf_net_id);
3071
3072         spin_lock_init(&tn->idr_lock);
3073         idr_init(&tn->idr);
3074         return 0;
3075 }
3076
3077 static void __net_exit tcf_net_exit(struct net *net)
3078 {
3079         struct tcf_net *tn = net_generic(net, tcf_net_id);
3080
3081         idr_destroy(&tn->idr);
3082 }
3083
3084 static struct pernet_operations tcf_net_ops = {
3085         .init = tcf_net_init,
3086         .exit = tcf_net_exit,
3087         .id   = &tcf_net_id,
3088         .size = sizeof(struct tcf_net),
3089 };
3090
3091 static int __init tc_filter_init(void)
3092 {
3093         int err;
3094
3095         tc_filter_wq = alloc_ordered_workqueue("tc_filter_workqueue", 0);
3096         if (!tc_filter_wq)
3097                 return -ENOMEM;
3098
3099         err = register_pernet_subsys(&tcf_net_ops);
3100         if (err)
3101                 goto err_register_pernet_subsys;
3102
3103         err = rhashtable_init(&indr_setup_block_ht,
3104                               &tc_indr_setup_block_ht_params);
3105         if (err)
3106                 goto err_rhash_setup_block_ht;
3107
3108         rtnl_register(PF_UNSPEC, RTM_NEWTFILTER, tc_new_tfilter, NULL, 0);
3109         rtnl_register(PF_UNSPEC, RTM_DELTFILTER, tc_del_tfilter, NULL, 0);
3110         rtnl_register(PF_UNSPEC, RTM_GETTFILTER, tc_get_tfilter,
3111                       tc_dump_tfilter, 0);
3112         rtnl_register(PF_UNSPEC, RTM_NEWCHAIN, tc_ctl_chain, NULL, 0);
3113         rtnl_register(PF_UNSPEC, RTM_DELCHAIN, tc_ctl_chain, NULL, 0);
3114         rtnl_register(PF_UNSPEC, RTM_GETCHAIN, tc_ctl_chain,
3115                       tc_dump_chain, 0);
3116
3117         return 0;
3118
3119 err_rhash_setup_block_ht:
3120         unregister_pernet_subsys(&tcf_net_ops);
3121 err_register_pernet_subsys:
3122         destroy_workqueue(tc_filter_wq);
3123         return err;
3124 }
3125
3126 subsys_initcall(tc_filter_init);