2 * net/sched/sch_api.c Packet scheduler API.
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
9 * Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
13 * Rani Assaf <rani@magic.metawire.com> :980802: JIFFIES and CPU clock sources are repaired.
14 * Eduardo J. Blanco <ejbs@netlabs.com.uy> :990222: kmod support
15 * Jamal Hadi Salim <hadi@nortelnetworks.com>: 990601: ingress support
18 #include <linux/module.h>
19 #include <linux/types.h>
20 #include <linux/kernel.h>
21 #include <linux/string.h>
22 #include <linux/errno.h>
23 #include <linux/skbuff.h>
24 #include <linux/init.h>
25 #include <linux/proc_fs.h>
26 #include <linux/seq_file.h>
27 #include <linux/kmod.h>
28 #include <linux/list.h>
29 #include <linux/hrtimer.h>
30 #include <linux/slab.h>
31 #include <linux/hashtable.h>
33 #include <net/net_namespace.h>
35 #include <net/netlink.h>
36 #include <net/pkt_sched.h>
37 #include <net/pkt_cls.h>
44 This file consists of two interrelated parts:
46 1. queueing disciplines manager frontend.
47 2. traffic classes manager frontend.
49 Generally, queueing discipline ("qdisc") is a black box,
50 which is able to enqueue packets and to dequeue them (when
51 device is ready to send something) in order and at times
52 determined by algorithm hidden in it.
54 qdisc's are divided to two categories:
55 - "queues", which have no internal structure visible from outside.
56 - "schedulers", which split all the packets to "traffic classes",
57 using "packet classifiers" (look at cls_api.c)
59 In turn, classes may have child qdiscs (as rule, queues)
60 attached to them etc. etc. etc.
62 The goal of the routines in this file is to translate
63 information supplied by user in the form of handles
64 to more intelligible for kernel form, to make some sanity
65 checks and part of work, which is common to all qdiscs
66 and to provide rtnetlink notifications.
68 All real intelligent work is done inside qdisc modules.
72 Every discipline has two major routines: enqueue and dequeue.
76 dequeue usually returns a skb to send. It is allowed to return NULL,
77 but it does not mean that queue is empty, it just means that
78 discipline does not want to send anything this time.
79 Queue is really empty if q->q.qlen == 0.
80 For complicated disciplines with multiple queues q->q is not
81 real packet queue, but however q->q.qlen must be valid.
85 enqueue returns 0, if packet was enqueued successfully.
86 If packet (this one or another one) was dropped, it returns
88 NET_XMIT_DROP - this packet dropped
89 Expected action: do not backoff, but wait until queue will clear.
90 NET_XMIT_CN - probably this packet enqueued, but another one dropped.
91 Expected action: backoff or ignore
97 like dequeue but without removing a packet from the queue
101 returns qdisc to initial state: purge all buffers, clear all
102 timers, counters (except for statistics) etc.
106 initializes newly created qdisc.
110 destroys resources allocated by init and during lifetime of qdisc.
114 changes qdisc parameters.
117 /* Protects list of registered TC modules. It is pure SMP lock. */
118 static DEFINE_RWLOCK(qdisc_mod_lock);
121 /************************************************
122 * Queueing disciplines manipulation. *
123 ************************************************/
126 /* The list of all installed queueing disciplines. */
128 static struct Qdisc_ops *qdisc_base;
130 /* Register/unregister queueing discipline */
132 int register_qdisc(struct Qdisc_ops *qops)
134 struct Qdisc_ops *q, **qp;
137 write_lock(&qdisc_mod_lock);
138 for (qp = &qdisc_base; (q = *qp) != NULL; qp = &q->next)
139 if (!strcmp(qops->id, q->id))
142 if (qops->enqueue == NULL)
143 qops->enqueue = noop_qdisc_ops.enqueue;
144 if (qops->peek == NULL) {
145 if (qops->dequeue == NULL)
146 qops->peek = noop_qdisc_ops.peek;
150 if (qops->dequeue == NULL)
151 qops->dequeue = noop_qdisc_ops.dequeue;
154 const struct Qdisc_class_ops *cops = qops->cl_ops;
156 if (!(cops->find && cops->walk && cops->leaf))
159 if (cops->tcf_block && !(cops->bind_tcf && cops->unbind_tcf))
167 write_unlock(&qdisc_mod_lock);
174 EXPORT_SYMBOL(register_qdisc);
176 int unregister_qdisc(struct Qdisc_ops *qops)
178 struct Qdisc_ops *q, **qp;
181 write_lock(&qdisc_mod_lock);
182 for (qp = &qdisc_base; (q = *qp) != NULL; qp = &q->next)
190 write_unlock(&qdisc_mod_lock);
193 EXPORT_SYMBOL(unregister_qdisc);
195 /* Get default qdisc if not otherwise specified */
196 void qdisc_get_default(char *name, size_t len)
198 read_lock(&qdisc_mod_lock);
199 strlcpy(name, default_qdisc_ops->id, len);
200 read_unlock(&qdisc_mod_lock);
203 static struct Qdisc_ops *qdisc_lookup_default(const char *name)
205 struct Qdisc_ops *q = NULL;
207 for (q = qdisc_base; q; q = q->next) {
208 if (!strcmp(name, q->id)) {
209 if (!try_module_get(q->owner))
218 /* Set new default qdisc to use */
219 int qdisc_set_default(const char *name)
221 const struct Qdisc_ops *ops;
223 if (!capable(CAP_NET_ADMIN))
226 write_lock(&qdisc_mod_lock);
227 ops = qdisc_lookup_default(name);
229 /* Not found, drop lock and try to load module */
230 write_unlock(&qdisc_mod_lock);
231 request_module("sch_%s", name);
232 write_lock(&qdisc_mod_lock);
234 ops = qdisc_lookup_default(name);
238 /* Set new default */
239 module_put(default_qdisc_ops->owner);
240 default_qdisc_ops = ops;
242 write_unlock(&qdisc_mod_lock);
244 return ops ? 0 : -ENOENT;
247 #ifdef CONFIG_NET_SCH_DEFAULT
248 /* Set default value from kernel config */
249 static int __init sch_default_qdisc(void)
251 return qdisc_set_default(CONFIG_DEFAULT_NET_SCH);
253 late_initcall(sch_default_qdisc);
256 /* We know handle. Find qdisc among all qdisc's attached to device
257 * (root qdisc, all its children, children of children etc.)
258 * Note: caller either uses rtnl or rcu_read_lock()
261 static struct Qdisc *qdisc_match_from_root(struct Qdisc *root, u32 handle)
265 if (!qdisc_dev(root))
266 return (root->handle == handle ? root : NULL);
268 if (!(root->flags & TCQ_F_BUILTIN) &&
269 root->handle == handle)
272 hash_for_each_possible_rcu(qdisc_dev(root)->qdisc_hash, q, hash, handle) {
273 if (q->handle == handle)
279 void qdisc_hash_add(struct Qdisc *q, bool invisible)
281 if ((q->parent != TC_H_ROOT) && !(q->flags & TCQ_F_INGRESS)) {
283 hash_add_rcu(qdisc_dev(q)->qdisc_hash, &q->hash, q->handle);
285 q->flags |= TCQ_F_INVISIBLE;
288 EXPORT_SYMBOL(qdisc_hash_add);
290 void qdisc_hash_del(struct Qdisc *q)
292 if ((q->parent != TC_H_ROOT) && !(q->flags & TCQ_F_INGRESS)) {
294 hash_del_rcu(&q->hash);
297 EXPORT_SYMBOL(qdisc_hash_del);
299 struct Qdisc *qdisc_lookup(struct net_device *dev, u32 handle)
305 q = qdisc_match_from_root(dev->qdisc, handle);
309 if (dev_ingress_queue(dev))
310 q = qdisc_match_from_root(
311 dev_ingress_queue(dev)->qdisc_sleeping,
317 struct Qdisc *qdisc_lookup_rcu(struct net_device *dev, u32 handle)
319 struct netdev_queue *nq;
324 q = qdisc_match_from_root(dev->qdisc, handle);
328 nq = dev_ingress_queue_rcu(dev);
330 q = qdisc_match_from_root(nq->qdisc_sleeping, handle);
335 static struct Qdisc *qdisc_leaf(struct Qdisc *p, u32 classid)
338 const struct Qdisc_class_ops *cops = p->ops->cl_ops;
342 cl = cops->find(p, classid);
346 return cops->leaf(p, cl);
349 /* Find queueing discipline by name */
351 static struct Qdisc_ops *qdisc_lookup_ops(struct nlattr *kind)
353 struct Qdisc_ops *q = NULL;
356 read_lock(&qdisc_mod_lock);
357 for (q = qdisc_base; q; q = q->next) {
358 if (nla_strcmp(kind, q->id) == 0) {
359 if (!try_module_get(q->owner))
364 read_unlock(&qdisc_mod_lock);
369 /* The linklayer setting were not transferred from iproute2, in older
370 * versions, and the rate tables lookup systems have been dropped in
371 * the kernel. To keep backward compatible with older iproute2 tc
372 * utils, we detect the linklayer setting by detecting if the rate
373 * table were modified.
375 * For linklayer ATM table entries, the rate table will be aligned to
376 * 48 bytes, thus some table entries will contain the same value. The
377 * mpu (min packet unit) is also encoded into the old rate table, thus
378 * starting from the mpu, we find low and high table entries for
379 * mapping this cell. If these entries contain the same value, when
380 * the rate tables have been modified for linklayer ATM.
382 * This is done by rounding mpu to the nearest 48 bytes cell/entry,
383 * and then roundup to the next cell, calc the table entry one below,
386 static __u8 __detect_linklayer(struct tc_ratespec *r, __u32 *rtab)
388 int low = roundup(r->mpu, 48);
389 int high = roundup(low+1, 48);
390 int cell_low = low >> r->cell_log;
391 int cell_high = (high >> r->cell_log) - 1;
393 /* rtab is too inaccurate at rates > 100Mbit/s */
394 if ((r->rate > (100000000/8)) || (rtab[0] == 0)) {
395 pr_debug("TC linklayer: Giving up ATM detection\n");
396 return TC_LINKLAYER_ETHERNET;
399 if ((cell_high > cell_low) && (cell_high < 256)
400 && (rtab[cell_low] == rtab[cell_high])) {
401 pr_debug("TC linklayer: Detected ATM, low(%d)=high(%d)=%u\n",
402 cell_low, cell_high, rtab[cell_high]);
403 return TC_LINKLAYER_ATM;
405 return TC_LINKLAYER_ETHERNET;
408 static struct qdisc_rate_table *qdisc_rtab_list;
410 struct qdisc_rate_table *qdisc_get_rtab(struct tc_ratespec *r,
412 struct netlink_ext_ack *extack)
414 struct qdisc_rate_table *rtab;
416 if (tab == NULL || r->rate == 0 || r->cell_log == 0 ||
417 nla_len(tab) != TC_RTAB_SIZE) {
418 NL_SET_ERR_MSG(extack, "Invalid rate table parameters for searching");
422 for (rtab = qdisc_rtab_list; rtab; rtab = rtab->next) {
423 if (!memcmp(&rtab->rate, r, sizeof(struct tc_ratespec)) &&
424 !memcmp(&rtab->data, nla_data(tab), 1024)) {
430 rtab = kmalloc(sizeof(*rtab), GFP_KERNEL);
434 memcpy(rtab->data, nla_data(tab), 1024);
435 if (r->linklayer == TC_LINKLAYER_UNAWARE)
436 r->linklayer = __detect_linklayer(r, rtab->data);
437 rtab->next = qdisc_rtab_list;
438 qdisc_rtab_list = rtab;
440 NL_SET_ERR_MSG(extack, "Failed to allocate new qdisc rate table");
444 EXPORT_SYMBOL(qdisc_get_rtab);
446 void qdisc_put_rtab(struct qdisc_rate_table *tab)
448 struct qdisc_rate_table *rtab, **rtabp;
450 if (!tab || --tab->refcnt)
453 for (rtabp = &qdisc_rtab_list;
454 (rtab = *rtabp) != NULL;
455 rtabp = &rtab->next) {
463 EXPORT_SYMBOL(qdisc_put_rtab);
465 static LIST_HEAD(qdisc_stab_list);
467 static const struct nla_policy stab_policy[TCA_STAB_MAX + 1] = {
468 [TCA_STAB_BASE] = { .len = sizeof(struct tc_sizespec) },
469 [TCA_STAB_DATA] = { .type = NLA_BINARY },
472 static struct qdisc_size_table *qdisc_get_stab(struct nlattr *opt,
473 struct netlink_ext_ack *extack)
475 struct nlattr *tb[TCA_STAB_MAX + 1];
476 struct qdisc_size_table *stab;
477 struct tc_sizespec *s;
478 unsigned int tsize = 0;
482 err = nla_parse_nested_deprecated(tb, TCA_STAB_MAX, opt, stab_policy,
486 if (!tb[TCA_STAB_BASE]) {
487 NL_SET_ERR_MSG(extack, "Size table base attribute is missing");
488 return ERR_PTR(-EINVAL);
491 s = nla_data(tb[TCA_STAB_BASE]);
494 if (!tb[TCA_STAB_DATA]) {
495 NL_SET_ERR_MSG(extack, "Size table data attribute is missing");
496 return ERR_PTR(-EINVAL);
498 tab = nla_data(tb[TCA_STAB_DATA]);
499 tsize = nla_len(tb[TCA_STAB_DATA]) / sizeof(u16);
502 if (tsize != s->tsize || (!tab && tsize > 0)) {
503 NL_SET_ERR_MSG(extack, "Invalid size of size table");
504 return ERR_PTR(-EINVAL);
507 list_for_each_entry(stab, &qdisc_stab_list, list) {
508 if (memcmp(&stab->szopts, s, sizeof(*s)))
510 if (tsize > 0 && memcmp(stab->data, tab, tsize * sizeof(u16)))
516 stab = kmalloc(sizeof(*stab) + tsize * sizeof(u16), GFP_KERNEL);
518 return ERR_PTR(-ENOMEM);
523 memcpy(stab->data, tab, tsize * sizeof(u16));
525 list_add_tail(&stab->list, &qdisc_stab_list);
530 void qdisc_put_stab(struct qdisc_size_table *tab)
535 if (--tab->refcnt == 0) {
536 list_del(&tab->list);
540 EXPORT_SYMBOL(qdisc_put_stab);
542 static int qdisc_dump_stab(struct sk_buff *skb, struct qdisc_size_table *stab)
546 nest = nla_nest_start_noflag(skb, TCA_STAB);
548 goto nla_put_failure;
549 if (nla_put(skb, TCA_STAB_BASE, sizeof(stab->szopts), &stab->szopts))
550 goto nla_put_failure;
551 nla_nest_end(skb, nest);
559 void __qdisc_calculate_pkt_len(struct sk_buff *skb,
560 const struct qdisc_size_table *stab)
564 pkt_len = skb->len + stab->szopts.overhead;
565 if (unlikely(!stab->szopts.tsize))
568 slot = pkt_len + stab->szopts.cell_align;
569 if (unlikely(slot < 0))
572 slot >>= stab->szopts.cell_log;
573 if (likely(slot < stab->szopts.tsize))
574 pkt_len = stab->data[slot];
576 pkt_len = stab->data[stab->szopts.tsize - 1] *
577 (slot / stab->szopts.tsize) +
578 stab->data[slot % stab->szopts.tsize];
580 pkt_len <<= stab->szopts.size_log;
582 if (unlikely(pkt_len < 1))
584 qdisc_skb_cb(skb)->pkt_len = pkt_len;
586 EXPORT_SYMBOL(__qdisc_calculate_pkt_len);
588 void qdisc_warn_nonwc(const char *txt, struct Qdisc *qdisc)
590 if (!(qdisc->flags & TCQ_F_WARN_NONWC)) {
591 pr_warn("%s: %s qdisc %X: is non-work-conserving?\n",
592 txt, qdisc->ops->id, qdisc->handle >> 16);
593 qdisc->flags |= TCQ_F_WARN_NONWC;
596 EXPORT_SYMBOL(qdisc_warn_nonwc);
598 static enum hrtimer_restart qdisc_watchdog(struct hrtimer *timer)
600 struct qdisc_watchdog *wd = container_of(timer, struct qdisc_watchdog,
604 __netif_schedule(qdisc_root(wd->qdisc));
607 return HRTIMER_NORESTART;
610 void qdisc_watchdog_init_clockid(struct qdisc_watchdog *wd, struct Qdisc *qdisc,
613 hrtimer_init(&wd->timer, clockid, HRTIMER_MODE_ABS_PINNED);
614 wd->timer.function = qdisc_watchdog;
617 EXPORT_SYMBOL(qdisc_watchdog_init_clockid);
619 void qdisc_watchdog_init(struct qdisc_watchdog *wd, struct Qdisc *qdisc)
621 qdisc_watchdog_init_clockid(wd, qdisc, CLOCK_MONOTONIC);
623 EXPORT_SYMBOL(qdisc_watchdog_init);
625 void qdisc_watchdog_schedule_ns(struct qdisc_watchdog *wd, u64 expires)
627 if (test_bit(__QDISC_STATE_DEACTIVATED,
628 &qdisc_root_sleeping(wd->qdisc)->state))
631 if (wd->last_expires == expires)
634 wd->last_expires = expires;
635 hrtimer_start(&wd->timer,
636 ns_to_ktime(expires),
637 HRTIMER_MODE_ABS_PINNED);
639 EXPORT_SYMBOL(qdisc_watchdog_schedule_ns);
641 void qdisc_watchdog_cancel(struct qdisc_watchdog *wd)
643 hrtimer_cancel(&wd->timer);
645 EXPORT_SYMBOL(qdisc_watchdog_cancel);
647 static struct hlist_head *qdisc_class_hash_alloc(unsigned int n)
649 struct hlist_head *h;
652 h = kvmalloc_array(n, sizeof(struct hlist_head), GFP_KERNEL);
655 for (i = 0; i < n; i++)
656 INIT_HLIST_HEAD(&h[i]);
661 void qdisc_class_hash_grow(struct Qdisc *sch, struct Qdisc_class_hash *clhash)
663 struct Qdisc_class_common *cl;
664 struct hlist_node *next;
665 struct hlist_head *nhash, *ohash;
666 unsigned int nsize, nmask, osize;
669 /* Rehash when load factor exceeds 0.75 */
670 if (clhash->hashelems * 4 <= clhash->hashsize * 3)
672 nsize = clhash->hashsize * 2;
674 nhash = qdisc_class_hash_alloc(nsize);
678 ohash = clhash->hash;
679 osize = clhash->hashsize;
682 for (i = 0; i < osize; i++) {
683 hlist_for_each_entry_safe(cl, next, &ohash[i], hnode) {
684 h = qdisc_class_hash(cl->classid, nmask);
685 hlist_add_head(&cl->hnode, &nhash[h]);
688 clhash->hash = nhash;
689 clhash->hashsize = nsize;
690 clhash->hashmask = nmask;
691 sch_tree_unlock(sch);
695 EXPORT_SYMBOL(qdisc_class_hash_grow);
697 int qdisc_class_hash_init(struct Qdisc_class_hash *clhash)
699 unsigned int size = 4;
701 clhash->hash = qdisc_class_hash_alloc(size);
704 clhash->hashsize = size;
705 clhash->hashmask = size - 1;
706 clhash->hashelems = 0;
709 EXPORT_SYMBOL(qdisc_class_hash_init);
711 void qdisc_class_hash_destroy(struct Qdisc_class_hash *clhash)
713 kvfree(clhash->hash);
715 EXPORT_SYMBOL(qdisc_class_hash_destroy);
717 void qdisc_class_hash_insert(struct Qdisc_class_hash *clhash,
718 struct Qdisc_class_common *cl)
722 INIT_HLIST_NODE(&cl->hnode);
723 h = qdisc_class_hash(cl->classid, clhash->hashmask);
724 hlist_add_head(&cl->hnode, &clhash->hash[h]);
727 EXPORT_SYMBOL(qdisc_class_hash_insert);
729 void qdisc_class_hash_remove(struct Qdisc_class_hash *clhash,
730 struct Qdisc_class_common *cl)
732 hlist_del(&cl->hnode);
735 EXPORT_SYMBOL(qdisc_class_hash_remove);
737 /* Allocate an unique handle from space managed by kernel
738 * Possible range is [8000-FFFF]:0000 (0x8000 values)
740 static u32 qdisc_alloc_handle(struct net_device *dev)
743 static u32 autohandle = TC_H_MAKE(0x80000000U, 0);
746 autohandle += TC_H_MAKE(0x10000U, 0);
747 if (autohandle == TC_H_MAKE(TC_H_ROOT, 0))
748 autohandle = TC_H_MAKE(0x80000000U, 0);
749 if (!qdisc_lookup(dev, autohandle))
757 void qdisc_tree_reduce_backlog(struct Qdisc *sch, int n, int len)
759 bool qdisc_is_offloaded = sch->flags & TCQ_F_OFFLOADED;
760 const struct Qdisc_class_ops *cops;
766 if (n == 0 && len == 0)
768 drops = max_t(int, n, 0);
770 while ((parentid = sch->parent)) {
771 if (TC_H_MAJ(parentid) == TC_H_MAJ(TC_H_INGRESS))
774 if (sch->flags & TCQ_F_NOPARENT)
776 /* Notify parent qdisc only if child qdisc becomes empty.
778 * If child was empty even before update then backlog
779 * counter is screwed and we skip notification because
780 * parent class is already passive.
782 * If the original child was offloaded then it is allowed
783 * to be seem as empty, so the parent is notified anyway.
785 notify = !sch->q.qlen && !WARN_ON_ONCE(!n &&
786 !qdisc_is_offloaded);
787 /* TODO: perform the search on a per txq basis */
788 sch = qdisc_lookup(qdisc_dev(sch), TC_H_MAJ(parentid));
790 WARN_ON_ONCE(parentid != TC_H_ROOT);
793 cops = sch->ops->cl_ops;
794 if (notify && cops->qlen_notify) {
795 cl = cops->find(sch, parentid);
796 cops->qlen_notify(sch, cl);
799 sch->qstats.backlog -= len;
800 __qdisc_qstats_drop(sch, drops);
804 EXPORT_SYMBOL(qdisc_tree_reduce_backlog);
806 int qdisc_offload_dump_helper(struct Qdisc *sch, enum tc_setup_type type,
809 struct net_device *dev = qdisc_dev(sch);
812 sch->flags &= ~TCQ_F_OFFLOADED;
813 if (!tc_can_offload(dev) || !dev->netdev_ops->ndo_setup_tc)
816 err = dev->netdev_ops->ndo_setup_tc(dev, type, type_data);
817 if (err == -EOPNOTSUPP)
821 sch->flags |= TCQ_F_OFFLOADED;
825 EXPORT_SYMBOL(qdisc_offload_dump_helper);
827 void qdisc_offload_graft_helper(struct net_device *dev, struct Qdisc *sch,
828 struct Qdisc *new, struct Qdisc *old,
829 enum tc_setup_type type, void *type_data,
830 struct netlink_ext_ack *extack)
832 bool any_qdisc_is_offloaded;
835 if (!tc_can_offload(dev) || !dev->netdev_ops->ndo_setup_tc)
838 err = dev->netdev_ops->ndo_setup_tc(dev, type, type_data);
840 /* Don't report error if the graft is part of destroy operation. */
841 if (!err || !new || new == &noop_qdisc)
844 /* Don't report error if the parent, the old child and the new
845 * one are not offloaded.
847 any_qdisc_is_offloaded = new->flags & TCQ_F_OFFLOADED;
848 any_qdisc_is_offloaded |= sch && sch->flags & TCQ_F_OFFLOADED;
849 any_qdisc_is_offloaded |= old && old->flags & TCQ_F_OFFLOADED;
851 if (any_qdisc_is_offloaded)
852 NL_SET_ERR_MSG(extack, "Offloading graft operation failed.");
854 EXPORT_SYMBOL(qdisc_offload_graft_helper);
856 static void qdisc_offload_graft_root(struct net_device *dev,
857 struct Qdisc *new, struct Qdisc *old,
858 struct netlink_ext_ack *extack)
860 struct tc_root_qopt_offload graft_offload = {
861 .command = TC_ROOT_GRAFT,
862 .handle = new ? new->handle : 0,
863 .ingress = (new && new->flags & TCQ_F_INGRESS) ||
864 (old && old->flags & TCQ_F_INGRESS),
867 qdisc_offload_graft_helper(dev, NULL, new, old,
868 TC_SETUP_ROOT_QDISC, &graft_offload, extack);
871 static int tc_fill_qdisc(struct sk_buff *skb, struct Qdisc *q, u32 clid,
872 u32 portid, u32 seq, u16 flags, int event)
874 struct gnet_stats_basic_cpu __percpu *cpu_bstats = NULL;
875 struct gnet_stats_queue __percpu *cpu_qstats = NULL;
877 struct nlmsghdr *nlh;
878 unsigned char *b = skb_tail_pointer(skb);
880 struct qdisc_size_table *stab;
885 nlh = nlmsg_put(skb, portid, seq, event, sizeof(*tcm), flags);
888 tcm = nlmsg_data(nlh);
889 tcm->tcm_family = AF_UNSPEC;
892 tcm->tcm_ifindex = qdisc_dev(q)->ifindex;
893 tcm->tcm_parent = clid;
894 tcm->tcm_handle = q->handle;
895 tcm->tcm_info = refcount_read(&q->refcnt);
896 if (nla_put_string(skb, TCA_KIND, q->ops->id))
897 goto nla_put_failure;
898 if (q->ops->ingress_block_get) {
899 block_index = q->ops->ingress_block_get(q);
901 nla_put_u32(skb, TCA_INGRESS_BLOCK, block_index))
902 goto nla_put_failure;
904 if (q->ops->egress_block_get) {
905 block_index = q->ops->egress_block_get(q);
907 nla_put_u32(skb, TCA_EGRESS_BLOCK, block_index))
908 goto nla_put_failure;
910 if (q->ops->dump && q->ops->dump(q, skb) < 0)
911 goto nla_put_failure;
912 if (nla_put_u8(skb, TCA_HW_OFFLOAD, !!(q->flags & TCQ_F_OFFLOADED)))
913 goto nla_put_failure;
914 qlen = qdisc_qlen_sum(q);
916 stab = rtnl_dereference(q->stab);
917 if (stab && qdisc_dump_stab(skb, stab) < 0)
918 goto nla_put_failure;
920 if (gnet_stats_start_copy_compat(skb, TCA_STATS2, TCA_STATS, TCA_XSTATS,
921 NULL, &d, TCA_PAD) < 0)
922 goto nla_put_failure;
924 if (q->ops->dump_stats && q->ops->dump_stats(q, &d) < 0)
925 goto nla_put_failure;
927 if (qdisc_is_percpu_stats(q)) {
928 cpu_bstats = q->cpu_bstats;
929 cpu_qstats = q->cpu_qstats;
932 if (gnet_stats_copy_basic(qdisc_root_sleeping_running(q),
933 &d, cpu_bstats, &q->bstats) < 0 ||
934 gnet_stats_copy_rate_est(&d, &q->rate_est) < 0 ||
935 gnet_stats_copy_queue(&d, cpu_qstats, &q->qstats, qlen) < 0)
936 goto nla_put_failure;
938 if (gnet_stats_finish_copy(&d) < 0)
939 goto nla_put_failure;
941 nlh->nlmsg_len = skb_tail_pointer(skb) - b;
950 static bool tc_qdisc_dump_ignore(struct Qdisc *q, bool dump_invisible)
952 if (q->flags & TCQ_F_BUILTIN)
954 if ((q->flags & TCQ_F_INVISIBLE) && !dump_invisible)
960 static int qdisc_notify(struct net *net, struct sk_buff *oskb,
961 struct nlmsghdr *n, u32 clid,
962 struct Qdisc *old, struct Qdisc *new)
965 u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
967 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
971 if (old && !tc_qdisc_dump_ignore(old, false)) {
972 if (tc_fill_qdisc(skb, old, clid, portid, n->nlmsg_seq,
973 0, RTM_DELQDISC) < 0)
976 if (new && !tc_qdisc_dump_ignore(new, false)) {
977 if (tc_fill_qdisc(skb, new, clid, portid, n->nlmsg_seq,
978 old ? NLM_F_REPLACE : 0, RTM_NEWQDISC) < 0)
983 return rtnetlink_send(skb, net, portid, RTNLGRP_TC,
984 n->nlmsg_flags & NLM_F_ECHO);
991 static void notify_and_destroy(struct net *net, struct sk_buff *skb,
992 struct nlmsghdr *n, u32 clid,
993 struct Qdisc *old, struct Qdisc *new)
996 qdisc_notify(net, skb, n, clid, old, new);
1002 static void qdisc_clear_nolock(struct Qdisc *sch)
1004 sch->flags &= ~TCQ_F_NOLOCK;
1005 if (!(sch->flags & TCQ_F_CPUSTATS))
1008 free_percpu(sch->cpu_bstats);
1009 free_percpu(sch->cpu_qstats);
1010 sch->cpu_bstats = NULL;
1011 sch->cpu_qstats = NULL;
1012 sch->flags &= ~TCQ_F_CPUSTATS;
1015 /* Graft qdisc "new" to class "classid" of qdisc "parent" or
1018 * When appropriate send a netlink notification using 'skb'
1021 * On success, destroy old qdisc.
1024 static int qdisc_graft(struct net_device *dev, struct Qdisc *parent,
1025 struct sk_buff *skb, struct nlmsghdr *n, u32 classid,
1026 struct Qdisc *new, struct Qdisc *old,
1027 struct netlink_ext_ack *extack)
1029 struct Qdisc *q = old;
1030 struct net *net = dev_net(dev);
1032 if (parent == NULL) {
1033 unsigned int i, num_q, ingress;
1036 num_q = dev->num_tx_queues;
1037 if ((q && q->flags & TCQ_F_INGRESS) ||
1038 (new && new->flags & TCQ_F_INGRESS)) {
1041 if (!dev_ingress_queue(dev)) {
1042 NL_SET_ERR_MSG(extack, "Device does not have an ingress queue");
1047 if (dev->flags & IFF_UP)
1048 dev_deactivate(dev);
1050 qdisc_offload_graft_root(dev, new, old, extack);
1052 if (new && new->ops->attach)
1055 for (i = 0; i < num_q; i++) {
1056 struct netdev_queue *dev_queue = dev_ingress_queue(dev);
1059 dev_queue = netdev_get_tx_queue(dev, i);
1061 old = dev_graft_qdisc(dev_queue, new);
1063 qdisc_refcount_inc(new);
1071 notify_and_destroy(net, skb, n, classid,
1073 if (new && !new->ops->attach)
1074 qdisc_refcount_inc(new);
1075 dev->qdisc = new ? : &noop_qdisc;
1077 if (new && new->ops->attach)
1078 new->ops->attach(new);
1080 notify_and_destroy(net, skb, n, classid, old, new);
1083 if (dev->flags & IFF_UP)
1086 const struct Qdisc_class_ops *cops = parent->ops->cl_ops;
1090 /* Only support running class lockless if parent is lockless */
1091 if (new && (new->flags & TCQ_F_NOLOCK) &&
1092 parent && !(parent->flags & TCQ_F_NOLOCK))
1093 qdisc_clear_nolock(new);
1095 if (!cops || !cops->graft)
1098 cl = cops->find(parent, classid);
1100 NL_SET_ERR_MSG(extack, "Specified class not found");
1104 err = cops->graft(parent, cl, new, &old, extack);
1107 notify_and_destroy(net, skb, n, classid, old, new);
1112 static int qdisc_block_indexes_set(struct Qdisc *sch, struct nlattr **tca,
1113 struct netlink_ext_ack *extack)
1117 if (tca[TCA_INGRESS_BLOCK]) {
1118 block_index = nla_get_u32(tca[TCA_INGRESS_BLOCK]);
1121 NL_SET_ERR_MSG(extack, "Ingress block index cannot be 0");
1124 if (!sch->ops->ingress_block_set) {
1125 NL_SET_ERR_MSG(extack, "Ingress block sharing is not supported");
1128 sch->ops->ingress_block_set(sch, block_index);
1130 if (tca[TCA_EGRESS_BLOCK]) {
1131 block_index = nla_get_u32(tca[TCA_EGRESS_BLOCK]);
1134 NL_SET_ERR_MSG(extack, "Egress block index cannot be 0");
1137 if (!sch->ops->egress_block_set) {
1138 NL_SET_ERR_MSG(extack, "Egress block sharing is not supported");
1141 sch->ops->egress_block_set(sch, block_index);
1147 Allocate and initialize new qdisc.
1149 Parameters are passed via opt.
1152 static struct Qdisc *qdisc_create(struct net_device *dev,
1153 struct netdev_queue *dev_queue,
1154 struct Qdisc *p, u32 parent, u32 handle,
1155 struct nlattr **tca, int *errp,
1156 struct netlink_ext_ack *extack)
1159 struct nlattr *kind = tca[TCA_KIND];
1161 struct Qdisc_ops *ops;
1162 struct qdisc_size_table *stab;
1164 ops = qdisc_lookup_ops(kind);
1165 #ifdef CONFIG_MODULES
1166 if (ops == NULL && kind != NULL) {
1167 char name[IFNAMSIZ];
1168 if (nla_strlcpy(name, kind, IFNAMSIZ) < IFNAMSIZ) {
1169 /* We dropped the RTNL semaphore in order to
1170 * perform the module load. So, even if we
1171 * succeeded in loading the module we have to
1172 * tell the caller to replay the request. We
1173 * indicate this using -EAGAIN.
1174 * We replay the request because the device may
1175 * go away in the mean time.
1178 request_module("sch_%s", name);
1180 ops = qdisc_lookup_ops(kind);
1182 /* We will try again qdisc_lookup_ops,
1183 * so don't keep a reference.
1185 module_put(ops->owner);
1195 NL_SET_ERR_MSG(extack, "Specified qdisc not found");
1199 sch = qdisc_alloc(dev_queue, ops, extack);
1205 sch->parent = parent;
1207 if (handle == TC_H_INGRESS) {
1208 sch->flags |= TCQ_F_INGRESS;
1209 handle = TC_H_MAKE(TC_H_INGRESS, 0);
1212 handle = qdisc_alloc_handle(dev);
1214 NL_SET_ERR_MSG(extack, "Maximum number of qdisc handles was exceeded");
1219 if (!netif_is_multiqueue(dev))
1220 sch->flags |= TCQ_F_ONETXQUEUE;
1223 sch->handle = handle;
1225 /* This exist to keep backward compatible with a userspace
1226 * loophole, what allowed userspace to get IFF_NO_QUEUE
1227 * facility on older kernels by setting tx_queue_len=0 (prior
1228 * to qdisc init), and then forgot to reinit tx_queue_len
1229 * before again attaching a qdisc.
1231 if ((dev->priv_flags & IFF_NO_QUEUE) && (dev->tx_queue_len == 0)) {
1232 dev->tx_queue_len = DEFAULT_TX_QUEUE_LEN;
1233 netdev_info(dev, "Caught tx_queue_len zero misconfig\n");
1236 err = qdisc_block_indexes_set(sch, tca, extack);
1241 err = ops->init(sch, tca[TCA_OPTIONS], extack);
1246 if (tca[TCA_STAB]) {
1247 stab = qdisc_get_stab(tca[TCA_STAB], extack);
1249 err = PTR_ERR(stab);
1252 rcu_assign_pointer(sch->stab, stab);
1254 if (tca[TCA_RATE]) {
1255 seqcount_t *running;
1258 if (sch->flags & TCQ_F_MQROOT) {
1259 NL_SET_ERR_MSG(extack, "Cannot attach rate estimator to a multi-queue root qdisc");
1263 if (sch->parent != TC_H_ROOT &&
1264 !(sch->flags & TCQ_F_INGRESS) &&
1265 (!p || !(p->flags & TCQ_F_MQROOT)))
1266 running = qdisc_root_sleeping_running(sch);
1268 running = &sch->running;
1270 err = gen_new_estimator(&sch->bstats,
1277 NL_SET_ERR_MSG(extack, "Failed to generate new estimator");
1282 qdisc_hash_add(sch, false);
1287 /* ops->init() failed, we call ->destroy() like qdisc_create_dflt() */
1294 module_put(ops->owner);
1301 * Any broken qdiscs that would require a ops->reset() here?
1302 * The qdisc was never in action so it shouldn't be necessary.
1304 qdisc_put_stab(rtnl_dereference(sch->stab));
1310 static int qdisc_change(struct Qdisc *sch, struct nlattr **tca,
1311 struct netlink_ext_ack *extack)
1313 struct qdisc_size_table *ostab, *stab = NULL;
1316 if (tca[TCA_OPTIONS]) {
1317 if (!sch->ops->change) {
1318 NL_SET_ERR_MSG(extack, "Change operation not supported by specified qdisc");
1321 if (tca[TCA_INGRESS_BLOCK] || tca[TCA_EGRESS_BLOCK]) {
1322 NL_SET_ERR_MSG(extack, "Change of blocks is not supported");
1325 err = sch->ops->change(sch, tca[TCA_OPTIONS], extack);
1330 if (tca[TCA_STAB]) {
1331 stab = qdisc_get_stab(tca[TCA_STAB], extack);
1333 return PTR_ERR(stab);
1336 ostab = rtnl_dereference(sch->stab);
1337 rcu_assign_pointer(sch->stab, stab);
1338 qdisc_put_stab(ostab);
1340 if (tca[TCA_RATE]) {
1341 /* NB: ignores errors from replace_estimator
1342 because change can't be undone. */
1343 if (sch->flags & TCQ_F_MQROOT)
1345 gen_replace_estimator(&sch->bstats,
1349 qdisc_root_sleeping_running(sch),
1356 struct check_loop_arg {
1357 struct qdisc_walker w;
1362 static int check_loop_fn(struct Qdisc *q, unsigned long cl,
1363 struct qdisc_walker *w);
1365 static int check_loop(struct Qdisc *q, struct Qdisc *p, int depth)
1367 struct check_loop_arg arg;
1369 if (q->ops->cl_ops == NULL)
1372 arg.w.stop = arg.w.skip = arg.w.count = 0;
1373 arg.w.fn = check_loop_fn;
1376 q->ops->cl_ops->walk(q, &arg.w);
1377 return arg.w.stop ? -ELOOP : 0;
1381 check_loop_fn(struct Qdisc *q, unsigned long cl, struct qdisc_walker *w)
1384 const struct Qdisc_class_ops *cops = q->ops->cl_ops;
1385 struct check_loop_arg *arg = (struct check_loop_arg *)w;
1387 leaf = cops->leaf(q, cl);
1389 if (leaf == arg->p || arg->depth > 7)
1391 return check_loop(leaf, arg->p, arg->depth + 1);
1396 const struct nla_policy rtm_tca_policy[TCA_MAX + 1] = {
1397 [TCA_KIND] = { .type = NLA_STRING },
1398 [TCA_RATE] = { .type = NLA_BINARY,
1399 .len = sizeof(struct tc_estimator) },
1400 [TCA_STAB] = { .type = NLA_NESTED },
1401 [TCA_DUMP_INVISIBLE] = { .type = NLA_FLAG },
1402 [TCA_CHAIN] = { .type = NLA_U32 },
1403 [TCA_INGRESS_BLOCK] = { .type = NLA_U32 },
1404 [TCA_EGRESS_BLOCK] = { .type = NLA_U32 },
1411 static int tc_get_qdisc(struct sk_buff *skb, struct nlmsghdr *n,
1412 struct netlink_ext_ack *extack)
1414 struct net *net = sock_net(skb->sk);
1415 struct tcmsg *tcm = nlmsg_data(n);
1416 struct nlattr *tca[TCA_MAX + 1];
1417 struct net_device *dev;
1419 struct Qdisc *q = NULL;
1420 struct Qdisc *p = NULL;
1423 if ((n->nlmsg_type != RTM_GETQDISC) &&
1424 !netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN))
1427 err = nlmsg_parse_deprecated(n, sizeof(*tcm), tca, TCA_MAX,
1428 rtm_tca_policy, extack);
1432 dev = __dev_get_by_index(net, tcm->tcm_ifindex);
1436 clid = tcm->tcm_parent;
1438 if (clid != TC_H_ROOT) {
1439 if (TC_H_MAJ(clid) != TC_H_MAJ(TC_H_INGRESS)) {
1440 p = qdisc_lookup(dev, TC_H_MAJ(clid));
1442 NL_SET_ERR_MSG(extack, "Failed to find qdisc with specified classid");
1445 q = qdisc_leaf(p, clid);
1446 } else if (dev_ingress_queue(dev)) {
1447 q = dev_ingress_queue(dev)->qdisc_sleeping;
1453 NL_SET_ERR_MSG(extack, "Cannot find specified qdisc on specified device");
1457 if (tcm->tcm_handle && q->handle != tcm->tcm_handle) {
1458 NL_SET_ERR_MSG(extack, "Invalid handle");
1462 q = qdisc_lookup(dev, tcm->tcm_handle);
1464 NL_SET_ERR_MSG(extack, "Failed to find qdisc with specified handle");
1469 if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], q->ops->id)) {
1470 NL_SET_ERR_MSG(extack, "Invalid qdisc name");
1474 if (n->nlmsg_type == RTM_DELQDISC) {
1476 NL_SET_ERR_MSG(extack, "Classid cannot be zero");
1479 if (q->handle == 0) {
1480 NL_SET_ERR_MSG(extack, "Cannot delete qdisc with handle of zero");
1483 err = qdisc_graft(dev, p, skb, n, clid, NULL, q, extack);
1487 qdisc_notify(net, skb, n, clid, NULL, q);
1493 * Create/change qdisc.
1496 static int tc_modify_qdisc(struct sk_buff *skb, struct nlmsghdr *n,
1497 struct netlink_ext_ack *extack)
1499 struct net *net = sock_net(skb->sk);
1501 struct nlattr *tca[TCA_MAX + 1];
1502 struct net_device *dev;
1504 struct Qdisc *q, *p;
1507 if (!netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN))
1511 /* Reinit, just in case something touches this. */
1512 err = nlmsg_parse_deprecated(n, sizeof(*tcm), tca, TCA_MAX,
1513 rtm_tca_policy, extack);
1517 tcm = nlmsg_data(n);
1518 clid = tcm->tcm_parent;
1521 dev = __dev_get_by_index(net, tcm->tcm_ifindex);
1527 if (clid != TC_H_ROOT) {
1528 if (clid != TC_H_INGRESS) {
1529 p = qdisc_lookup(dev, TC_H_MAJ(clid));
1531 NL_SET_ERR_MSG(extack, "Failed to find specified qdisc");
1534 q = qdisc_leaf(p, clid);
1535 } else if (dev_ingress_queue_create(dev)) {
1536 q = dev_ingress_queue(dev)->qdisc_sleeping;
1542 /* It may be default qdisc, ignore it */
1543 if (q && q->handle == 0)
1546 if (!q || !tcm->tcm_handle || q->handle != tcm->tcm_handle) {
1547 if (tcm->tcm_handle) {
1548 if (q && !(n->nlmsg_flags & NLM_F_REPLACE)) {
1549 NL_SET_ERR_MSG(extack, "NLM_F_REPLACE needed to override");
1552 if (TC_H_MIN(tcm->tcm_handle)) {
1553 NL_SET_ERR_MSG(extack, "Invalid minor handle");
1556 q = qdisc_lookup(dev, tcm->tcm_handle);
1558 goto create_n_graft;
1559 if (n->nlmsg_flags & NLM_F_EXCL) {
1560 NL_SET_ERR_MSG(extack, "Exclusivity flag on, cannot override");
1563 if (tca[TCA_KIND] &&
1564 nla_strcmp(tca[TCA_KIND], q->ops->id)) {
1565 NL_SET_ERR_MSG(extack, "Invalid qdisc name");
1569 (p && check_loop(q, p, 0))) {
1570 NL_SET_ERR_MSG(extack, "Qdisc parent/child loop detected");
1573 qdisc_refcount_inc(q);
1577 goto create_n_graft;
1579 /* This magic test requires explanation.
1581 * We know, that some child q is already
1582 * attached to this parent and have choice:
1583 * either to change it or to create/graft new one.
1585 * 1. We are allowed to create/graft only
1586 * if CREATE and REPLACE flags are set.
1588 * 2. If EXCL is set, requestor wanted to say,
1589 * that qdisc tcm_handle is not expected
1590 * to exist, so that we choose create/graft too.
1592 * 3. The last case is when no flags are set.
1593 * Alas, it is sort of hole in API, we
1594 * cannot decide what to do unambiguously.
1595 * For now we select create/graft, if
1596 * user gave KIND, which does not match existing.
1598 if ((n->nlmsg_flags & NLM_F_CREATE) &&
1599 (n->nlmsg_flags & NLM_F_REPLACE) &&
1600 ((n->nlmsg_flags & NLM_F_EXCL) ||
1602 nla_strcmp(tca[TCA_KIND], q->ops->id))))
1603 goto create_n_graft;
1607 if (!tcm->tcm_handle) {
1608 NL_SET_ERR_MSG(extack, "Handle cannot be zero");
1611 q = qdisc_lookup(dev, tcm->tcm_handle);
1614 /* Change qdisc parameters */
1616 NL_SET_ERR_MSG(extack, "Specified qdisc not found");
1619 if (n->nlmsg_flags & NLM_F_EXCL) {
1620 NL_SET_ERR_MSG(extack, "Exclusivity flag on, cannot modify");
1623 if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], q->ops->id)) {
1624 NL_SET_ERR_MSG(extack, "Invalid qdisc name");
1627 err = qdisc_change(q, tca, extack);
1629 qdisc_notify(net, skb, n, clid, NULL, q);
1633 if (!(n->nlmsg_flags & NLM_F_CREATE)) {
1634 NL_SET_ERR_MSG(extack, "Qdisc not found. To create specify NLM_F_CREATE flag");
1637 if (clid == TC_H_INGRESS) {
1638 if (dev_ingress_queue(dev)) {
1639 q = qdisc_create(dev, dev_ingress_queue(dev), p,
1640 tcm->tcm_parent, tcm->tcm_parent,
1643 NL_SET_ERR_MSG(extack, "Cannot find ingress queue for specified device");
1647 struct netdev_queue *dev_queue;
1649 if (p && p->ops->cl_ops && p->ops->cl_ops->select_queue)
1650 dev_queue = p->ops->cl_ops->select_queue(p, tcm);
1652 dev_queue = p->dev_queue;
1654 dev_queue = netdev_get_tx_queue(dev, 0);
1656 q = qdisc_create(dev, dev_queue, p,
1657 tcm->tcm_parent, tcm->tcm_handle,
1667 err = qdisc_graft(dev, p, skb, n, clid, q, NULL, extack);
1677 static int tc_dump_qdisc_root(struct Qdisc *root, struct sk_buff *skb,
1678 struct netlink_callback *cb,
1679 int *q_idx_p, int s_q_idx, bool recur,
1680 bool dump_invisible)
1682 int ret = 0, q_idx = *q_idx_p;
1690 if (q_idx < s_q_idx) {
1693 if (!tc_qdisc_dump_ignore(q, dump_invisible) &&
1694 tc_fill_qdisc(skb, q, q->parent, NETLINK_CB(cb->skb).portid,
1695 cb->nlh->nlmsg_seq, NLM_F_MULTI,
1701 /* If dumping singletons, there is no qdisc_dev(root) and the singleton
1702 * itself has already been dumped.
1704 * If we've already dumped the top-level (ingress) qdisc above and the global
1705 * qdisc hashtable, we don't want to hit it again
1707 if (!qdisc_dev(root) || !recur)
1710 hash_for_each(qdisc_dev(root)->qdisc_hash, b, q, hash) {
1711 if (q_idx < s_q_idx) {
1715 if (!tc_qdisc_dump_ignore(q, dump_invisible) &&
1716 tc_fill_qdisc(skb, q, q->parent, NETLINK_CB(cb->skb).portid,
1717 cb->nlh->nlmsg_seq, NLM_F_MULTI,
1731 static int tc_dump_qdisc(struct sk_buff *skb, struct netlink_callback *cb)
1733 struct net *net = sock_net(skb->sk);
1736 struct net_device *dev;
1737 const struct nlmsghdr *nlh = cb->nlh;
1738 struct nlattr *tca[TCA_MAX + 1];
1741 s_idx = cb->args[0];
1742 s_q_idx = q_idx = cb->args[1];
1747 err = nlmsg_parse_deprecated(nlh, sizeof(struct tcmsg), tca, TCA_MAX,
1748 rtm_tca_policy, cb->extack);
1752 for_each_netdev(net, dev) {
1753 struct netdev_queue *dev_queue;
1761 if (tc_dump_qdisc_root(dev->qdisc, skb, cb, &q_idx, s_q_idx,
1762 true, tca[TCA_DUMP_INVISIBLE]) < 0)
1765 dev_queue = dev_ingress_queue(dev);
1767 tc_dump_qdisc_root(dev_queue->qdisc_sleeping, skb, cb,
1768 &q_idx, s_q_idx, false,
1769 tca[TCA_DUMP_INVISIBLE]) < 0)
1778 cb->args[1] = q_idx;
1785 /************************************************
1786 * Traffic classes manipulation. *
1787 ************************************************/
1789 static int tc_fill_tclass(struct sk_buff *skb, struct Qdisc *q,
1791 u32 portid, u32 seq, u16 flags, int event)
1794 struct nlmsghdr *nlh;
1795 unsigned char *b = skb_tail_pointer(skb);
1797 const struct Qdisc_class_ops *cl_ops = q->ops->cl_ops;
1800 nlh = nlmsg_put(skb, portid, seq, event, sizeof(*tcm), flags);
1802 goto out_nlmsg_trim;
1803 tcm = nlmsg_data(nlh);
1804 tcm->tcm_family = AF_UNSPEC;
1807 tcm->tcm_ifindex = qdisc_dev(q)->ifindex;
1808 tcm->tcm_parent = q->handle;
1809 tcm->tcm_handle = q->handle;
1811 if (nla_put_string(skb, TCA_KIND, q->ops->id))
1812 goto nla_put_failure;
1813 if (cl_ops->dump && cl_ops->dump(q, cl, skb, tcm) < 0)
1814 goto nla_put_failure;
1816 if (gnet_stats_start_copy_compat(skb, TCA_STATS2, TCA_STATS, TCA_XSTATS,
1817 NULL, &d, TCA_PAD) < 0)
1818 goto nla_put_failure;
1820 if (cl_ops->dump_stats && cl_ops->dump_stats(q, cl, &d) < 0)
1821 goto nla_put_failure;
1823 if (gnet_stats_finish_copy(&d) < 0)
1824 goto nla_put_failure;
1826 nlh->nlmsg_len = skb_tail_pointer(skb) - b;
1835 static int tclass_notify(struct net *net, struct sk_buff *oskb,
1836 struct nlmsghdr *n, struct Qdisc *q,
1837 unsigned long cl, int event)
1839 struct sk_buff *skb;
1840 u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
1843 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
1847 if (tc_fill_tclass(skb, q, cl, portid, n->nlmsg_seq, 0, event) < 0) {
1852 err = rtnetlink_send(skb, net, portid, RTNLGRP_TC,
1853 n->nlmsg_flags & NLM_F_ECHO);
1859 static int tclass_del_notify(struct net *net,
1860 const struct Qdisc_class_ops *cops,
1861 struct sk_buff *oskb, struct nlmsghdr *n,
1862 struct Qdisc *q, unsigned long cl)
1864 u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
1865 struct sk_buff *skb;
1871 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
1875 if (tc_fill_tclass(skb, q, cl, portid, n->nlmsg_seq, 0,
1876 RTM_DELTCLASS) < 0) {
1881 err = cops->delete(q, cl);
1887 err = rtnetlink_send(skb, net, portid, RTNLGRP_TC,
1888 n->nlmsg_flags & NLM_F_ECHO);
1894 #ifdef CONFIG_NET_CLS
1896 struct tcf_bind_args {
1897 struct tcf_walker w;
1902 static int tcf_node_bind(struct tcf_proto *tp, void *n, struct tcf_walker *arg)
1904 struct tcf_bind_args *a = (void *)arg;
1906 if (tp->ops->bind_class) {
1907 struct Qdisc *q = tcf_block_q(tp->chain->block);
1910 tp->ops->bind_class(n, a->classid, a->cl);
1916 static void tc_bind_tclass(struct Qdisc *q, u32 portid, u32 clid,
1917 unsigned long new_cl)
1919 const struct Qdisc_class_ops *cops = q->ops->cl_ops;
1920 struct tcf_block *block;
1921 struct tcf_chain *chain;
1924 cl = cops->find(q, portid);
1927 block = cops->tcf_block(q, cl, NULL);
1930 for (chain = tcf_get_next_chain(block, NULL);
1932 chain = tcf_get_next_chain(block, chain)) {
1933 struct tcf_proto *tp;
1935 for (tp = tcf_get_next_proto(chain, NULL, true);
1936 tp; tp = tcf_get_next_proto(chain, tp, true)) {
1937 struct tcf_bind_args arg = {};
1939 arg.w.fn = tcf_node_bind;
1942 tp->ops->walk(tp, &arg.w, true);
1949 static void tc_bind_tclass(struct Qdisc *q, u32 portid, u32 clid,
1950 unsigned long new_cl)
1956 static int tc_ctl_tclass(struct sk_buff *skb, struct nlmsghdr *n,
1957 struct netlink_ext_ack *extack)
1959 struct net *net = sock_net(skb->sk);
1960 struct tcmsg *tcm = nlmsg_data(n);
1961 struct nlattr *tca[TCA_MAX + 1];
1962 struct net_device *dev;
1963 struct Qdisc *q = NULL;
1964 const struct Qdisc_class_ops *cops;
1965 unsigned long cl = 0;
1966 unsigned long new_cl;
1972 if ((n->nlmsg_type != RTM_GETTCLASS) &&
1973 !netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN))
1976 err = nlmsg_parse_deprecated(n, sizeof(*tcm), tca, TCA_MAX,
1977 rtm_tca_policy, extack);
1981 dev = __dev_get_by_index(net, tcm->tcm_ifindex);
1986 parent == TC_H_UNSPEC - unspecified parent.
1987 parent == TC_H_ROOT - class is root, which has no parent.
1988 parent == X:0 - parent is root class.
1989 parent == X:Y - parent is a node in hierarchy.
1990 parent == 0:Y - parent is X:Y, where X:0 is qdisc.
1992 handle == 0:0 - generate handle from kernel pool.
1993 handle == 0:Y - class is X:Y, where X:0 is qdisc.
1994 handle == X:Y - clear.
1995 handle == X:0 - root class.
1998 /* Step 1. Determine qdisc handle X:0 */
2000 portid = tcm->tcm_parent;
2001 clid = tcm->tcm_handle;
2002 qid = TC_H_MAJ(clid);
2004 if (portid != TC_H_ROOT) {
2005 u32 qid1 = TC_H_MAJ(portid);
2008 /* If both majors are known, they must be identical. */
2013 } else if (qid == 0)
2014 qid = dev->qdisc->handle;
2016 /* Now qid is genuine qdisc handle consistent
2017 * both with parent and child.
2019 * TC_H_MAJ(portid) still may be unspecified, complete it now.
2022 portid = TC_H_MAKE(qid, portid);
2025 qid = dev->qdisc->handle;
2028 /* OK. Locate qdisc */
2029 q = qdisc_lookup(dev, qid);
2033 /* An check that it supports classes */
2034 cops = q->ops->cl_ops;
2038 /* Now try to get class */
2040 if (portid == TC_H_ROOT)
2043 clid = TC_H_MAKE(qid, clid);
2046 cl = cops->find(q, clid);
2050 if (n->nlmsg_type != RTM_NEWTCLASS ||
2051 !(n->nlmsg_flags & NLM_F_CREATE))
2054 switch (n->nlmsg_type) {
2057 if (n->nlmsg_flags & NLM_F_EXCL)
2061 err = tclass_del_notify(net, cops, skb, n, q, cl);
2062 /* Unbind the class with flilters with 0 */
2063 tc_bind_tclass(q, portid, clid, 0);
2066 err = tclass_notify(net, skb, n, q, cl, RTM_NEWTCLASS);
2074 if (tca[TCA_INGRESS_BLOCK] || tca[TCA_EGRESS_BLOCK]) {
2075 NL_SET_ERR_MSG(extack, "Shared blocks are not supported for classes");
2082 err = cops->change(q, clid, portid, tca, &new_cl, extack);
2084 tclass_notify(net, skb, n, q, new_cl, RTM_NEWTCLASS);
2085 /* We just create a new class, need to do reverse binding. */
2087 tc_bind_tclass(q, portid, clid, new_cl);
2093 struct qdisc_dump_args {
2094 struct qdisc_walker w;
2095 struct sk_buff *skb;
2096 struct netlink_callback *cb;
2099 static int qdisc_class_dump(struct Qdisc *q, unsigned long cl,
2100 struct qdisc_walker *arg)
2102 struct qdisc_dump_args *a = (struct qdisc_dump_args *)arg;
2104 return tc_fill_tclass(a->skb, q, cl, NETLINK_CB(a->cb->skb).portid,
2105 a->cb->nlh->nlmsg_seq, NLM_F_MULTI,
2109 static int tc_dump_tclass_qdisc(struct Qdisc *q, struct sk_buff *skb,
2110 struct tcmsg *tcm, struct netlink_callback *cb,
2113 struct qdisc_dump_args arg;
2115 if (tc_qdisc_dump_ignore(q, false) ||
2116 *t_p < s_t || !q->ops->cl_ops ||
2118 TC_H_MAJ(tcm->tcm_parent) != q->handle)) {
2123 memset(&cb->args[1], 0, sizeof(cb->args)-sizeof(cb->args[0]));
2124 arg.w.fn = qdisc_class_dump;
2128 arg.w.skip = cb->args[1];
2130 q->ops->cl_ops->walk(q, &arg.w);
2131 cb->args[1] = arg.w.count;
2138 static int tc_dump_tclass_root(struct Qdisc *root, struct sk_buff *skb,
2139 struct tcmsg *tcm, struct netlink_callback *cb,
2148 if (tc_dump_tclass_qdisc(root, skb, tcm, cb, t_p, s_t) < 0)
2151 if (!qdisc_dev(root))
2154 if (tcm->tcm_parent) {
2155 q = qdisc_match_from_root(root, TC_H_MAJ(tcm->tcm_parent));
2156 if (q && q != root &&
2157 tc_dump_tclass_qdisc(q, skb, tcm, cb, t_p, s_t) < 0)
2161 hash_for_each(qdisc_dev(root)->qdisc_hash, b, q, hash) {
2162 if (tc_dump_tclass_qdisc(q, skb, tcm, cb, t_p, s_t) < 0)
2169 static int tc_dump_tclass(struct sk_buff *skb, struct netlink_callback *cb)
2171 struct tcmsg *tcm = nlmsg_data(cb->nlh);
2172 struct net *net = sock_net(skb->sk);
2173 struct netdev_queue *dev_queue;
2174 struct net_device *dev;
2177 if (nlmsg_len(cb->nlh) < sizeof(*tcm))
2179 dev = dev_get_by_index(net, tcm->tcm_ifindex);
2186 if (tc_dump_tclass_root(dev->qdisc, skb, tcm, cb, &t, s_t) < 0)
2189 dev_queue = dev_ingress_queue(dev);
2191 tc_dump_tclass_root(dev_queue->qdisc_sleeping, skb, tcm, cb,
2202 #ifdef CONFIG_PROC_FS
2203 static int psched_show(struct seq_file *seq, void *v)
2205 seq_printf(seq, "%08x %08x %08x %08x\n",
2206 (u32)NSEC_PER_USEC, (u32)PSCHED_TICKS2NS(1),
2208 (u32)NSEC_PER_SEC / hrtimer_resolution);
2213 static int __net_init psched_net_init(struct net *net)
2215 struct proc_dir_entry *e;
2217 e = proc_create_single("psched", 0, net->proc_net, psched_show);
2224 static void __net_exit psched_net_exit(struct net *net)
2226 remove_proc_entry("psched", net->proc_net);
2229 static int __net_init psched_net_init(struct net *net)
2234 static void __net_exit psched_net_exit(struct net *net)
2239 static struct pernet_operations psched_net_ops = {
2240 .init = psched_net_init,
2241 .exit = psched_net_exit,
2244 static int __init pktsched_init(void)
2248 err = register_pernet_subsys(&psched_net_ops);
2250 pr_err("pktsched_init: "
2251 "cannot initialize per netns operations\n");
2255 register_qdisc(&pfifo_fast_ops);
2256 register_qdisc(&pfifo_qdisc_ops);
2257 register_qdisc(&bfifo_qdisc_ops);
2258 register_qdisc(&pfifo_head_drop_qdisc_ops);
2259 register_qdisc(&mq_qdisc_ops);
2260 register_qdisc(&noqueue_qdisc_ops);
2262 rtnl_register(PF_UNSPEC, RTM_NEWQDISC, tc_modify_qdisc, NULL, 0);
2263 rtnl_register(PF_UNSPEC, RTM_DELQDISC, tc_get_qdisc, NULL, 0);
2264 rtnl_register(PF_UNSPEC, RTM_GETQDISC, tc_get_qdisc, tc_dump_qdisc,
2266 rtnl_register(PF_UNSPEC, RTM_NEWTCLASS, tc_ctl_tclass, NULL, 0);
2267 rtnl_register(PF_UNSPEC, RTM_DELTCLASS, tc_ctl_tclass, NULL, 0);
2268 rtnl_register(PF_UNSPEC, RTM_GETTCLASS, tc_ctl_tclass, tc_dump_tclass,
2274 subsys_initcall(pktsched_init);