2 * net/sched/cls_flower.c Flower classifier
4 * Copyright (c) 2015 Jiri Pirko <jiri@resnulli.us>
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
12 #include <linux/kernel.h>
13 #include <linux/init.h>
14 #include <linux/module.h>
15 #include <linux/rhashtable.h>
16 #include <linux/workqueue.h>
18 #include <linux/if_ether.h>
19 #include <linux/in6.h>
22 #include <net/sch_generic.h>
23 #include <net/pkt_cls.h>
25 #include <net/flow_dissector.h>
28 #include <net/dst_metadata.h>
32 struct flow_dissector_key_control control;
33 struct flow_dissector_key_control enc_control;
34 struct flow_dissector_key_basic basic;
35 struct flow_dissector_key_eth_addrs eth;
36 struct flow_dissector_key_vlan vlan;
38 struct flow_dissector_key_ipv4_addrs ipv4;
39 struct flow_dissector_key_ipv6_addrs ipv6;
41 struct flow_dissector_key_ports tp;
42 struct flow_dissector_key_keyid enc_key_id;
44 struct flow_dissector_key_ipv4_addrs enc_ipv4;
45 struct flow_dissector_key_ipv6_addrs enc_ipv6;
47 struct flow_dissector_key_ports enc_tp;
48 } __aligned(BITS_PER_LONG / 8); /* Ensure that we can do comparisons as longs. */
50 struct fl_flow_mask_range {
51 unsigned short int start;
52 unsigned short int end;
56 struct fl_flow_key key;
57 struct fl_flow_mask_range range;
63 struct fl_flow_mask mask;
64 struct flow_dissector dissector;
67 struct list_head filters;
68 struct rhashtable_params ht_params;
70 struct work_struct work;
75 struct cls_fl_filter {
76 struct rhash_head ht_node;
77 struct fl_flow_key mkey;
79 struct tcf_result res;
80 struct fl_flow_key key;
81 struct list_head list;
85 struct tc_to_netdev tc;
86 struct net_device *hw_dev;
89 static unsigned short int fl_mask_range(const struct fl_flow_mask *mask)
91 return mask->range.end - mask->range.start;
94 static void fl_mask_update_range(struct fl_flow_mask *mask)
96 const u8 *bytes = (const u8 *) &mask->key;
97 size_t size = sizeof(mask->key);
98 size_t i, first = 0, last = size - 1;
100 for (i = 0; i < sizeof(mask->key); i++) {
107 mask->range.start = rounddown(first, sizeof(long));
108 mask->range.end = roundup(last + 1, sizeof(long));
111 static void *fl_key_get_start(struct fl_flow_key *key,
112 const struct fl_flow_mask *mask)
114 return (u8 *) key + mask->range.start;
117 static void fl_set_masked_key(struct fl_flow_key *mkey, struct fl_flow_key *key,
118 struct fl_flow_mask *mask)
120 const long *lkey = fl_key_get_start(key, mask);
121 const long *lmask = fl_key_get_start(&mask->key, mask);
122 long *lmkey = fl_key_get_start(mkey, mask);
125 for (i = 0; i < fl_mask_range(mask); i += sizeof(long))
126 *lmkey++ = *lkey++ & *lmask++;
129 static void fl_clear_masked_range(struct fl_flow_key *key,
130 struct fl_flow_mask *mask)
132 memset(fl_key_get_start(key, mask), 0, fl_mask_range(mask));
135 static int fl_classify(struct sk_buff *skb, const struct tcf_proto *tp,
136 struct tcf_result *res)
138 struct cls_fl_head *head = rcu_dereference_bh(tp->root);
139 struct cls_fl_filter *f;
140 struct fl_flow_key skb_key;
141 struct fl_flow_key skb_mkey;
142 struct ip_tunnel_info *info;
144 if (!atomic_read(&head->ht.nelems))
147 fl_clear_masked_range(&skb_key, &head->mask);
149 info = skb_tunnel_info(skb);
151 struct ip_tunnel_key *key = &info->key;
153 switch (ip_tunnel_info_af(info)) {
155 skb_key.enc_ipv4.src = key->u.ipv4.src;
156 skb_key.enc_ipv4.dst = key->u.ipv4.dst;
159 skb_key.enc_ipv6.src = key->u.ipv6.src;
160 skb_key.enc_ipv6.dst = key->u.ipv6.dst;
164 skb_key.enc_key_id.keyid = tunnel_id_to_key32(key->tun_id);
165 skb_key.enc_tp.src = key->tp_src;
166 skb_key.enc_tp.dst = key->tp_dst;
169 skb_key.indev_ifindex = skb->skb_iif;
170 /* skb_flow_dissect() does not set n_proto in case an unknown protocol,
171 * so do it rather here.
173 skb_key.basic.n_proto = skb->protocol;
174 skb_flow_dissect(skb, &head->dissector, &skb_key, 0);
176 fl_set_masked_key(&skb_mkey, &skb_key, &head->mask);
178 f = rhashtable_lookup_fast(&head->ht,
179 fl_key_get_start(&skb_mkey, &head->mask),
181 if (f && !tc_skip_sw(f->flags)) {
183 return tcf_exts_exec(skb, &f->exts, res);
188 static int fl_init(struct tcf_proto *tp)
190 struct cls_fl_head *head;
192 head = kzalloc(sizeof(*head), GFP_KERNEL);
196 INIT_LIST_HEAD_RCU(&head->filters);
197 rcu_assign_pointer(tp->root, head);
202 static void fl_destroy_filter(struct rcu_head *head)
204 struct cls_fl_filter *f = container_of(head, struct cls_fl_filter, rcu);
206 tcf_exts_destroy(&f->exts);
210 static void fl_hw_destroy_filter(struct tcf_proto *tp, struct cls_fl_filter *f)
212 struct tc_cls_flower_offload offload = {0};
213 struct net_device *dev = f->hw_dev;
214 struct tc_to_netdev *tc = &f->tc;
216 if (!tc_can_offload(dev, tp))
219 offload.command = TC_CLSFLOWER_DESTROY;
220 offload.cookie = (unsigned long)f;
222 tc->type = TC_SETUP_CLSFLOWER;
223 tc->cls_flower = &offload;
225 dev->netdev_ops->ndo_setup_tc(dev, tp->q->handle, tp->protocol, tc);
228 static int fl_hw_replace_filter(struct tcf_proto *tp,
229 struct flow_dissector *dissector,
230 struct fl_flow_key *mask,
231 struct cls_fl_filter *f)
233 struct net_device *dev = tp->q->dev_queue->dev;
234 struct tc_cls_flower_offload offload = {0};
235 struct tc_to_netdev *tc = &f->tc;
238 if (!tc_can_offload(dev, tp)) {
239 if (tcf_exts_get_dev(dev, &f->exts, &f->hw_dev) ||
240 (f->hw_dev && !tc_can_offload(f->hw_dev, tp))) {
242 return tc_skip_sw(f->flags) ? -EINVAL : 0;
245 tc->egress_dev = true;
250 offload.command = TC_CLSFLOWER_REPLACE;
251 offload.cookie = (unsigned long)f;
252 offload.dissector = dissector;
254 offload.key = &f->key;
255 offload.exts = &f->exts;
257 tc->type = TC_SETUP_CLSFLOWER;
258 tc->cls_flower = &offload;
260 err = dev->netdev_ops->ndo_setup_tc(dev, tp->q->handle, tp->protocol,
263 if (tc_skip_sw(f->flags))
268 static void fl_hw_update_stats(struct tcf_proto *tp, struct cls_fl_filter *f)
270 struct tc_cls_flower_offload offload = {0};
271 struct net_device *dev = f->hw_dev;
272 struct tc_to_netdev *tc = &f->tc;
274 if (!tc_can_offload(dev, tp))
277 offload.command = TC_CLSFLOWER_STATS;
278 offload.cookie = (unsigned long)f;
279 offload.exts = &f->exts;
281 tc->type = TC_SETUP_CLSFLOWER;
282 tc->cls_flower = &offload;
284 dev->netdev_ops->ndo_setup_tc(dev, tp->q->handle, tp->protocol, tc);
287 static void __fl_delete(struct tcf_proto *tp, struct cls_fl_filter *f)
289 list_del_rcu(&f->list);
290 if (!tc_skip_hw(f->flags))
291 fl_hw_destroy_filter(tp, f);
292 tcf_unbind_filter(tp, &f->res);
293 call_rcu(&f->rcu, fl_destroy_filter);
296 static void fl_destroy_sleepable(struct work_struct *work)
298 struct cls_fl_head *head = container_of(work, struct cls_fl_head,
300 if (head->mask_assigned)
301 rhashtable_destroy(&head->ht);
303 module_put(THIS_MODULE);
306 static void fl_destroy_rcu(struct rcu_head *rcu)
308 struct cls_fl_head *head = container_of(rcu, struct cls_fl_head, rcu);
310 INIT_WORK(&head->work, fl_destroy_sleepable);
311 schedule_work(&head->work);
314 static bool fl_destroy(struct tcf_proto *tp, bool force)
316 struct cls_fl_head *head = rtnl_dereference(tp->root);
317 struct cls_fl_filter *f, *next;
319 if (!force && !list_empty(&head->filters))
322 list_for_each_entry_safe(f, next, &head->filters, list)
325 __module_get(THIS_MODULE);
326 call_rcu(&head->rcu, fl_destroy_rcu);
331 static unsigned long fl_get(struct tcf_proto *tp, u32 handle)
333 struct cls_fl_head *head = rtnl_dereference(tp->root);
334 struct cls_fl_filter *f;
336 list_for_each_entry(f, &head->filters, list)
337 if (f->handle == handle)
338 return (unsigned long) f;
342 static const struct nla_policy fl_policy[TCA_FLOWER_MAX + 1] = {
343 [TCA_FLOWER_UNSPEC] = { .type = NLA_UNSPEC },
344 [TCA_FLOWER_CLASSID] = { .type = NLA_U32 },
345 [TCA_FLOWER_INDEV] = { .type = NLA_STRING,
347 [TCA_FLOWER_KEY_ETH_DST] = { .len = ETH_ALEN },
348 [TCA_FLOWER_KEY_ETH_DST_MASK] = { .len = ETH_ALEN },
349 [TCA_FLOWER_KEY_ETH_SRC] = { .len = ETH_ALEN },
350 [TCA_FLOWER_KEY_ETH_SRC_MASK] = { .len = ETH_ALEN },
351 [TCA_FLOWER_KEY_ETH_TYPE] = { .type = NLA_U16 },
352 [TCA_FLOWER_KEY_IP_PROTO] = { .type = NLA_U8 },
353 [TCA_FLOWER_KEY_IPV4_SRC] = { .type = NLA_U32 },
354 [TCA_FLOWER_KEY_IPV4_SRC_MASK] = { .type = NLA_U32 },
355 [TCA_FLOWER_KEY_IPV4_DST] = { .type = NLA_U32 },
356 [TCA_FLOWER_KEY_IPV4_DST_MASK] = { .type = NLA_U32 },
357 [TCA_FLOWER_KEY_IPV6_SRC] = { .len = sizeof(struct in6_addr) },
358 [TCA_FLOWER_KEY_IPV6_SRC_MASK] = { .len = sizeof(struct in6_addr) },
359 [TCA_FLOWER_KEY_IPV6_DST] = { .len = sizeof(struct in6_addr) },
360 [TCA_FLOWER_KEY_IPV6_DST_MASK] = { .len = sizeof(struct in6_addr) },
361 [TCA_FLOWER_KEY_TCP_SRC] = { .type = NLA_U16 },
362 [TCA_FLOWER_KEY_TCP_DST] = { .type = NLA_U16 },
363 [TCA_FLOWER_KEY_UDP_SRC] = { .type = NLA_U16 },
364 [TCA_FLOWER_KEY_UDP_DST] = { .type = NLA_U16 },
365 [TCA_FLOWER_KEY_VLAN_ID] = { .type = NLA_U16 },
366 [TCA_FLOWER_KEY_VLAN_PRIO] = { .type = NLA_U8 },
367 [TCA_FLOWER_KEY_VLAN_ETH_TYPE] = { .type = NLA_U16 },
368 [TCA_FLOWER_KEY_ENC_KEY_ID] = { .type = NLA_U32 },
369 [TCA_FLOWER_KEY_ENC_IPV4_SRC] = { .type = NLA_U32 },
370 [TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK] = { .type = NLA_U32 },
371 [TCA_FLOWER_KEY_ENC_IPV4_DST] = { .type = NLA_U32 },
372 [TCA_FLOWER_KEY_ENC_IPV4_DST_MASK] = { .type = NLA_U32 },
373 [TCA_FLOWER_KEY_ENC_IPV6_SRC] = { .len = sizeof(struct in6_addr) },
374 [TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK] = { .len = sizeof(struct in6_addr) },
375 [TCA_FLOWER_KEY_ENC_IPV6_DST] = { .len = sizeof(struct in6_addr) },
376 [TCA_FLOWER_KEY_ENC_IPV6_DST_MASK] = { .len = sizeof(struct in6_addr) },
377 [TCA_FLOWER_KEY_TCP_SRC_MASK] = { .type = NLA_U16 },
378 [TCA_FLOWER_KEY_TCP_DST_MASK] = { .type = NLA_U16 },
379 [TCA_FLOWER_KEY_UDP_SRC_MASK] = { .type = NLA_U16 },
380 [TCA_FLOWER_KEY_UDP_DST_MASK] = { .type = NLA_U16 },
381 [TCA_FLOWER_KEY_SCTP_SRC_MASK] = { .type = NLA_U16 },
382 [TCA_FLOWER_KEY_SCTP_DST_MASK] = { .type = NLA_U16 },
383 [TCA_FLOWER_KEY_SCTP_SRC] = { .type = NLA_U16 },
384 [TCA_FLOWER_KEY_SCTP_DST] = { .type = NLA_U16 },
385 [TCA_FLOWER_KEY_ENC_UDP_SRC_PORT] = { .type = NLA_U16 },
386 [TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK] = { .type = NLA_U16 },
387 [TCA_FLOWER_KEY_ENC_UDP_DST_PORT] = { .type = NLA_U16 },
388 [TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK] = { .type = NLA_U16 },
391 static void fl_set_key_val(struct nlattr **tb,
392 void *val, int val_type,
393 void *mask, int mask_type, int len)
397 memcpy(val, nla_data(tb[val_type]), len);
398 if (mask_type == TCA_FLOWER_UNSPEC || !tb[mask_type])
399 memset(mask, 0xff, len);
401 memcpy(mask, nla_data(tb[mask_type]), len);
404 static void fl_set_key_vlan(struct nlattr **tb,
405 struct flow_dissector_key_vlan *key_val,
406 struct flow_dissector_key_vlan *key_mask)
408 #define VLAN_PRIORITY_MASK 0x7
410 if (tb[TCA_FLOWER_KEY_VLAN_ID]) {
412 nla_get_u16(tb[TCA_FLOWER_KEY_VLAN_ID]) & VLAN_VID_MASK;
413 key_mask->vlan_id = VLAN_VID_MASK;
415 if (tb[TCA_FLOWER_KEY_VLAN_PRIO]) {
416 key_val->vlan_priority =
417 nla_get_u8(tb[TCA_FLOWER_KEY_VLAN_PRIO]) &
419 key_mask->vlan_priority = VLAN_PRIORITY_MASK;
423 static int fl_set_key(struct net *net, struct nlattr **tb,
424 struct fl_flow_key *key, struct fl_flow_key *mask)
427 #ifdef CONFIG_NET_CLS_IND
428 if (tb[TCA_FLOWER_INDEV]) {
429 int err = tcf_change_indev(net, tb[TCA_FLOWER_INDEV]);
432 key->indev_ifindex = err;
433 mask->indev_ifindex = 0xffffffff;
437 fl_set_key_val(tb, key->eth.dst, TCA_FLOWER_KEY_ETH_DST,
438 mask->eth.dst, TCA_FLOWER_KEY_ETH_DST_MASK,
439 sizeof(key->eth.dst));
440 fl_set_key_val(tb, key->eth.src, TCA_FLOWER_KEY_ETH_SRC,
441 mask->eth.src, TCA_FLOWER_KEY_ETH_SRC_MASK,
442 sizeof(key->eth.src));
444 if (tb[TCA_FLOWER_KEY_ETH_TYPE]) {
445 ethertype = nla_get_be16(tb[TCA_FLOWER_KEY_ETH_TYPE]);
447 if (ethertype == htons(ETH_P_8021Q)) {
448 fl_set_key_vlan(tb, &key->vlan, &mask->vlan);
449 fl_set_key_val(tb, &key->basic.n_proto,
450 TCA_FLOWER_KEY_VLAN_ETH_TYPE,
451 &mask->basic.n_proto, TCA_FLOWER_UNSPEC,
452 sizeof(key->basic.n_proto));
454 key->basic.n_proto = ethertype;
455 mask->basic.n_proto = cpu_to_be16(~0);
459 if (key->basic.n_proto == htons(ETH_P_IP) ||
460 key->basic.n_proto == htons(ETH_P_IPV6)) {
461 fl_set_key_val(tb, &key->basic.ip_proto, TCA_FLOWER_KEY_IP_PROTO,
462 &mask->basic.ip_proto, TCA_FLOWER_UNSPEC,
463 sizeof(key->basic.ip_proto));
466 if (tb[TCA_FLOWER_KEY_IPV4_SRC] || tb[TCA_FLOWER_KEY_IPV4_DST]) {
467 key->control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
468 fl_set_key_val(tb, &key->ipv4.src, TCA_FLOWER_KEY_IPV4_SRC,
469 &mask->ipv4.src, TCA_FLOWER_KEY_IPV4_SRC_MASK,
470 sizeof(key->ipv4.src));
471 fl_set_key_val(tb, &key->ipv4.dst, TCA_FLOWER_KEY_IPV4_DST,
472 &mask->ipv4.dst, TCA_FLOWER_KEY_IPV4_DST_MASK,
473 sizeof(key->ipv4.dst));
474 } else if (tb[TCA_FLOWER_KEY_IPV6_SRC] || tb[TCA_FLOWER_KEY_IPV6_DST]) {
475 key->control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
476 fl_set_key_val(tb, &key->ipv6.src, TCA_FLOWER_KEY_IPV6_SRC,
477 &mask->ipv6.src, TCA_FLOWER_KEY_IPV6_SRC_MASK,
478 sizeof(key->ipv6.src));
479 fl_set_key_val(tb, &key->ipv6.dst, TCA_FLOWER_KEY_IPV6_DST,
480 &mask->ipv6.dst, TCA_FLOWER_KEY_IPV6_DST_MASK,
481 sizeof(key->ipv6.dst));
484 if (key->basic.ip_proto == IPPROTO_TCP) {
485 fl_set_key_val(tb, &key->tp.src, TCA_FLOWER_KEY_TCP_SRC,
486 &mask->tp.src, TCA_FLOWER_KEY_TCP_SRC_MASK,
487 sizeof(key->tp.src));
488 fl_set_key_val(tb, &key->tp.dst, TCA_FLOWER_KEY_TCP_DST,
489 &mask->tp.dst, TCA_FLOWER_KEY_TCP_DST_MASK,
490 sizeof(key->tp.dst));
491 } else if (key->basic.ip_proto == IPPROTO_UDP) {
492 fl_set_key_val(tb, &key->tp.src, TCA_FLOWER_KEY_UDP_SRC,
493 &mask->tp.src, TCA_FLOWER_KEY_UDP_SRC_MASK,
494 sizeof(key->tp.src));
495 fl_set_key_val(tb, &key->tp.dst, TCA_FLOWER_KEY_UDP_DST,
496 &mask->tp.dst, TCA_FLOWER_KEY_UDP_DST_MASK,
497 sizeof(key->tp.dst));
498 } else if (key->basic.ip_proto == IPPROTO_SCTP) {
499 fl_set_key_val(tb, &key->tp.src, TCA_FLOWER_KEY_SCTP_SRC,
500 &mask->tp.src, TCA_FLOWER_KEY_SCTP_SRC_MASK,
501 sizeof(key->tp.src));
502 fl_set_key_val(tb, &key->tp.dst, TCA_FLOWER_KEY_SCTP_DST,
503 &mask->tp.dst, TCA_FLOWER_KEY_SCTP_DST_MASK,
504 sizeof(key->tp.dst));
507 if (tb[TCA_FLOWER_KEY_ENC_IPV4_SRC] ||
508 tb[TCA_FLOWER_KEY_ENC_IPV4_DST]) {
509 key->enc_control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
510 fl_set_key_val(tb, &key->enc_ipv4.src,
511 TCA_FLOWER_KEY_ENC_IPV4_SRC,
513 TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK,
514 sizeof(key->enc_ipv4.src));
515 fl_set_key_val(tb, &key->enc_ipv4.dst,
516 TCA_FLOWER_KEY_ENC_IPV4_DST,
518 TCA_FLOWER_KEY_ENC_IPV4_DST_MASK,
519 sizeof(key->enc_ipv4.dst));
522 if (tb[TCA_FLOWER_KEY_ENC_IPV6_SRC] ||
523 tb[TCA_FLOWER_KEY_ENC_IPV6_DST]) {
524 key->enc_control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
525 fl_set_key_val(tb, &key->enc_ipv6.src,
526 TCA_FLOWER_KEY_ENC_IPV6_SRC,
528 TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK,
529 sizeof(key->enc_ipv6.src));
530 fl_set_key_val(tb, &key->enc_ipv6.dst,
531 TCA_FLOWER_KEY_ENC_IPV6_DST,
533 TCA_FLOWER_KEY_ENC_IPV6_DST_MASK,
534 sizeof(key->enc_ipv6.dst));
537 fl_set_key_val(tb, &key->enc_key_id.keyid, TCA_FLOWER_KEY_ENC_KEY_ID,
538 &mask->enc_key_id.keyid, TCA_FLOWER_UNSPEC,
539 sizeof(key->enc_key_id.keyid));
541 fl_set_key_val(tb, &key->enc_tp.src, TCA_FLOWER_KEY_ENC_UDP_SRC_PORT,
542 &mask->enc_tp.src, TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK,
543 sizeof(key->enc_tp.src));
545 fl_set_key_val(tb, &key->enc_tp.dst, TCA_FLOWER_KEY_ENC_UDP_DST_PORT,
546 &mask->enc_tp.dst, TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK,
547 sizeof(key->enc_tp.dst));
552 static bool fl_mask_eq(struct fl_flow_mask *mask1,
553 struct fl_flow_mask *mask2)
555 const long *lmask1 = fl_key_get_start(&mask1->key, mask1);
556 const long *lmask2 = fl_key_get_start(&mask2->key, mask2);
558 return !memcmp(&mask1->range, &mask2->range, sizeof(mask1->range)) &&
559 !memcmp(lmask1, lmask2, fl_mask_range(mask1));
562 static const struct rhashtable_params fl_ht_params = {
563 .key_offset = offsetof(struct cls_fl_filter, mkey), /* base offset */
564 .head_offset = offsetof(struct cls_fl_filter, ht_node),
565 .automatic_shrinking = true,
568 static int fl_init_hashtable(struct cls_fl_head *head,
569 struct fl_flow_mask *mask)
571 head->ht_params = fl_ht_params;
572 head->ht_params.key_len = fl_mask_range(mask);
573 head->ht_params.key_offset += mask->range.start;
575 return rhashtable_init(&head->ht, &head->ht_params);
578 #define FL_KEY_MEMBER_OFFSET(member) offsetof(struct fl_flow_key, member)
579 #define FL_KEY_MEMBER_SIZE(member) (sizeof(((struct fl_flow_key *) 0)->member))
581 #define FL_KEY_IS_MASKED(mask, member) \
582 memchr_inv(((char *)mask) + FL_KEY_MEMBER_OFFSET(member), \
583 0, FL_KEY_MEMBER_SIZE(member)) \
585 #define FL_KEY_SET(keys, cnt, id, member) \
587 keys[cnt].key_id = id; \
588 keys[cnt].offset = FL_KEY_MEMBER_OFFSET(member); \
592 #define FL_KEY_SET_IF_MASKED(mask, keys, cnt, id, member) \
594 if (FL_KEY_IS_MASKED(mask, member)) \
595 FL_KEY_SET(keys, cnt, id, member); \
598 static void fl_init_dissector(struct cls_fl_head *head,
599 struct fl_flow_mask *mask)
601 struct flow_dissector_key keys[FLOW_DISSECTOR_KEY_MAX];
604 FL_KEY_SET(keys, cnt, FLOW_DISSECTOR_KEY_CONTROL, control);
605 FL_KEY_SET(keys, cnt, FLOW_DISSECTOR_KEY_BASIC, basic);
606 FL_KEY_SET_IF_MASKED(&mask->key, keys, cnt,
607 FLOW_DISSECTOR_KEY_ETH_ADDRS, eth);
608 FL_KEY_SET_IF_MASKED(&mask->key, keys, cnt,
609 FLOW_DISSECTOR_KEY_IPV4_ADDRS, ipv4);
610 FL_KEY_SET_IF_MASKED(&mask->key, keys, cnt,
611 FLOW_DISSECTOR_KEY_IPV6_ADDRS, ipv6);
612 FL_KEY_SET_IF_MASKED(&mask->key, keys, cnt,
613 FLOW_DISSECTOR_KEY_PORTS, tp);
614 FL_KEY_SET_IF_MASKED(&mask->key, keys, cnt,
615 FLOW_DISSECTOR_KEY_VLAN, vlan);
616 FL_KEY_SET_IF_MASKED(&mask->key, keys, cnt,
617 FLOW_DISSECTOR_KEY_ENC_KEYID, enc_key_id);
618 FL_KEY_SET_IF_MASKED(&mask->key, keys, cnt,
619 FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS, enc_ipv4);
620 FL_KEY_SET_IF_MASKED(&mask->key, keys, cnt,
621 FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS, enc_ipv6);
622 if (FL_KEY_IS_MASKED(&mask->key, enc_ipv4) ||
623 FL_KEY_IS_MASKED(&mask->key, enc_ipv6))
624 FL_KEY_SET(keys, cnt, FLOW_DISSECTOR_KEY_ENC_CONTROL,
626 FL_KEY_SET_IF_MASKED(&mask->key, keys, cnt,
627 FLOW_DISSECTOR_KEY_ENC_PORTS, enc_tp);
629 skb_flow_dissector_init(&head->dissector, keys, cnt);
632 static int fl_check_assign_mask(struct cls_fl_head *head,
633 struct fl_flow_mask *mask)
637 if (head->mask_assigned) {
638 if (!fl_mask_eq(&head->mask, mask))
644 /* Mask is not assigned yet. So assign it and init hashtable
647 err = fl_init_hashtable(head, mask);
650 memcpy(&head->mask, mask, sizeof(head->mask));
651 head->mask_assigned = true;
653 fl_init_dissector(head, mask);
658 static int fl_set_parms(struct net *net, struct tcf_proto *tp,
659 struct cls_fl_filter *f, struct fl_flow_mask *mask,
660 unsigned long base, struct nlattr **tb,
661 struct nlattr *est, bool ovr)
666 err = tcf_exts_init(&e, TCA_FLOWER_ACT, 0);
669 err = tcf_exts_validate(net, tp, tb, est, &e, ovr);
673 if (tb[TCA_FLOWER_CLASSID]) {
674 f->res.classid = nla_get_u32(tb[TCA_FLOWER_CLASSID]);
675 tcf_bind_filter(tp, &f->res, base);
678 err = fl_set_key(net, tb, &f->key, &mask->key);
682 fl_mask_update_range(mask);
683 fl_set_masked_key(&f->mkey, &f->key, mask);
685 tcf_exts_change(tp, &f->exts, &e);
689 tcf_exts_destroy(&e);
693 static u32 fl_grab_new_handle(struct tcf_proto *tp,
694 struct cls_fl_head *head)
696 unsigned int i = 0x80000000;
700 if (++head->hgen == 0x7FFFFFFF)
702 } while (--i > 0 && fl_get(tp, head->hgen));
704 if (unlikely(i == 0)) {
705 pr_err("Insufficient number of handles\n");
714 static int fl_change(struct net *net, struct sk_buff *in_skb,
715 struct tcf_proto *tp, unsigned long base,
716 u32 handle, struct nlattr **tca,
717 unsigned long *arg, bool ovr)
719 struct cls_fl_head *head = rtnl_dereference(tp->root);
720 struct cls_fl_filter *fold = (struct cls_fl_filter *) *arg;
721 struct cls_fl_filter *fnew;
722 struct nlattr *tb[TCA_FLOWER_MAX + 1];
723 struct fl_flow_mask mask = {};
726 if (!tca[TCA_OPTIONS])
729 err = nla_parse_nested(tb, TCA_FLOWER_MAX, tca[TCA_OPTIONS], fl_policy);
733 if (fold && handle && fold->handle != handle)
736 fnew = kzalloc(sizeof(*fnew), GFP_KERNEL);
740 err = tcf_exts_init(&fnew->exts, TCA_FLOWER_ACT, 0);
745 handle = fl_grab_new_handle(tp, head);
751 fnew->handle = handle;
753 if (tb[TCA_FLOWER_FLAGS]) {
754 fnew->flags = nla_get_u32(tb[TCA_FLOWER_FLAGS]);
756 if (!tc_flags_valid(fnew->flags)) {
762 err = fl_set_parms(net, tp, fnew, &mask, base, tb, tca[TCA_RATE], ovr);
766 err = fl_check_assign_mask(head, &mask);
770 if (!tc_skip_sw(fnew->flags)) {
771 err = rhashtable_insert_fast(&head->ht, &fnew->ht_node,
777 if (!tc_skip_hw(fnew->flags)) {
778 err = fl_hw_replace_filter(tp,
787 if (!tc_skip_sw(fold->flags))
788 rhashtable_remove_fast(&head->ht, &fold->ht_node,
790 if (!tc_skip_hw(fold->flags))
791 fl_hw_destroy_filter(tp, fold);
794 *arg = (unsigned long) fnew;
797 list_replace_rcu(&fold->list, &fnew->list);
798 tcf_unbind_filter(tp, &fold->res);
799 call_rcu(&fold->rcu, fl_destroy_filter);
801 list_add_tail_rcu(&fnew->list, &head->filters);
807 tcf_exts_destroy(&fnew->exts);
812 static int fl_delete(struct tcf_proto *tp, unsigned long arg)
814 struct cls_fl_head *head = rtnl_dereference(tp->root);
815 struct cls_fl_filter *f = (struct cls_fl_filter *) arg;
817 if (!tc_skip_sw(f->flags))
818 rhashtable_remove_fast(&head->ht, &f->ht_node,
824 static void fl_walk(struct tcf_proto *tp, struct tcf_walker *arg)
826 struct cls_fl_head *head = rtnl_dereference(tp->root);
827 struct cls_fl_filter *f;
829 list_for_each_entry_rcu(f, &head->filters, list) {
830 if (arg->count < arg->skip)
832 if (arg->fn(tp, (unsigned long) f, arg) < 0) {
841 static int fl_dump_key_val(struct sk_buff *skb,
842 void *val, int val_type,
843 void *mask, int mask_type, int len)
847 if (!memchr_inv(mask, 0, len))
849 err = nla_put(skb, val_type, len, val);
852 if (mask_type != TCA_FLOWER_UNSPEC) {
853 err = nla_put(skb, mask_type, len, mask);
860 static int fl_dump_key_vlan(struct sk_buff *skb,
861 struct flow_dissector_key_vlan *vlan_key,
862 struct flow_dissector_key_vlan *vlan_mask)
866 if (!memchr_inv(vlan_mask, 0, sizeof(*vlan_mask)))
868 if (vlan_mask->vlan_id) {
869 err = nla_put_u16(skb, TCA_FLOWER_KEY_VLAN_ID,
874 if (vlan_mask->vlan_priority) {
875 err = nla_put_u8(skb, TCA_FLOWER_KEY_VLAN_PRIO,
876 vlan_key->vlan_priority);
883 static int fl_dump(struct net *net, struct tcf_proto *tp, unsigned long fh,
884 struct sk_buff *skb, struct tcmsg *t)
886 struct cls_fl_head *head = rtnl_dereference(tp->root);
887 struct cls_fl_filter *f = (struct cls_fl_filter *) fh;
889 struct fl_flow_key *key, *mask;
894 t->tcm_handle = f->handle;
896 nest = nla_nest_start(skb, TCA_OPTIONS);
898 goto nla_put_failure;
900 if (f->res.classid &&
901 nla_put_u32(skb, TCA_FLOWER_CLASSID, f->res.classid))
902 goto nla_put_failure;
905 mask = &head->mask.key;
907 if (mask->indev_ifindex) {
908 struct net_device *dev;
910 dev = __dev_get_by_index(net, key->indev_ifindex);
911 if (dev && nla_put_string(skb, TCA_FLOWER_INDEV, dev->name))
912 goto nla_put_failure;
915 if (!tc_skip_hw(f->flags))
916 fl_hw_update_stats(tp, f);
918 if (fl_dump_key_val(skb, key->eth.dst, TCA_FLOWER_KEY_ETH_DST,
919 mask->eth.dst, TCA_FLOWER_KEY_ETH_DST_MASK,
920 sizeof(key->eth.dst)) ||
921 fl_dump_key_val(skb, key->eth.src, TCA_FLOWER_KEY_ETH_SRC,
922 mask->eth.src, TCA_FLOWER_KEY_ETH_SRC_MASK,
923 sizeof(key->eth.src)) ||
924 fl_dump_key_val(skb, &key->basic.n_proto, TCA_FLOWER_KEY_ETH_TYPE,
925 &mask->basic.n_proto, TCA_FLOWER_UNSPEC,
926 sizeof(key->basic.n_proto)))
927 goto nla_put_failure;
929 if (fl_dump_key_vlan(skb, &key->vlan, &mask->vlan))
930 goto nla_put_failure;
932 if ((key->basic.n_proto == htons(ETH_P_IP) ||
933 key->basic.n_proto == htons(ETH_P_IPV6)) &&
934 fl_dump_key_val(skb, &key->basic.ip_proto, TCA_FLOWER_KEY_IP_PROTO,
935 &mask->basic.ip_proto, TCA_FLOWER_UNSPEC,
936 sizeof(key->basic.ip_proto)))
937 goto nla_put_failure;
939 if (key->control.addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS &&
940 (fl_dump_key_val(skb, &key->ipv4.src, TCA_FLOWER_KEY_IPV4_SRC,
941 &mask->ipv4.src, TCA_FLOWER_KEY_IPV4_SRC_MASK,
942 sizeof(key->ipv4.src)) ||
943 fl_dump_key_val(skb, &key->ipv4.dst, TCA_FLOWER_KEY_IPV4_DST,
944 &mask->ipv4.dst, TCA_FLOWER_KEY_IPV4_DST_MASK,
945 sizeof(key->ipv4.dst))))
946 goto nla_put_failure;
947 else if (key->control.addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS &&
948 (fl_dump_key_val(skb, &key->ipv6.src, TCA_FLOWER_KEY_IPV6_SRC,
949 &mask->ipv6.src, TCA_FLOWER_KEY_IPV6_SRC_MASK,
950 sizeof(key->ipv6.src)) ||
951 fl_dump_key_val(skb, &key->ipv6.dst, TCA_FLOWER_KEY_IPV6_DST,
952 &mask->ipv6.dst, TCA_FLOWER_KEY_IPV6_DST_MASK,
953 sizeof(key->ipv6.dst))))
954 goto nla_put_failure;
956 if (key->basic.ip_proto == IPPROTO_TCP &&
957 (fl_dump_key_val(skb, &key->tp.src, TCA_FLOWER_KEY_TCP_SRC,
958 &mask->tp.src, TCA_FLOWER_KEY_TCP_SRC_MASK,
959 sizeof(key->tp.src)) ||
960 fl_dump_key_val(skb, &key->tp.dst, TCA_FLOWER_KEY_TCP_DST,
961 &mask->tp.dst, TCA_FLOWER_KEY_TCP_DST_MASK,
962 sizeof(key->tp.dst))))
963 goto nla_put_failure;
964 else if (key->basic.ip_proto == IPPROTO_UDP &&
965 (fl_dump_key_val(skb, &key->tp.src, TCA_FLOWER_KEY_UDP_SRC,
966 &mask->tp.src, TCA_FLOWER_KEY_UDP_SRC_MASK,
967 sizeof(key->tp.src)) ||
968 fl_dump_key_val(skb, &key->tp.dst, TCA_FLOWER_KEY_UDP_DST,
969 &mask->tp.dst, TCA_FLOWER_KEY_UDP_DST_MASK,
970 sizeof(key->tp.dst))))
971 goto nla_put_failure;
972 else if (key->basic.ip_proto == IPPROTO_SCTP &&
973 (fl_dump_key_val(skb, &key->tp.src, TCA_FLOWER_KEY_SCTP_SRC,
974 &mask->tp.src, TCA_FLOWER_KEY_SCTP_SRC_MASK,
975 sizeof(key->tp.src)) ||
976 fl_dump_key_val(skb, &key->tp.dst, TCA_FLOWER_KEY_SCTP_DST,
977 &mask->tp.dst, TCA_FLOWER_KEY_SCTP_DST_MASK,
978 sizeof(key->tp.dst))))
979 goto nla_put_failure;
981 if (key->enc_control.addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS &&
982 (fl_dump_key_val(skb, &key->enc_ipv4.src,
983 TCA_FLOWER_KEY_ENC_IPV4_SRC, &mask->enc_ipv4.src,
984 TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK,
985 sizeof(key->enc_ipv4.src)) ||
986 fl_dump_key_val(skb, &key->enc_ipv4.dst,
987 TCA_FLOWER_KEY_ENC_IPV4_DST, &mask->enc_ipv4.dst,
988 TCA_FLOWER_KEY_ENC_IPV4_DST_MASK,
989 sizeof(key->enc_ipv4.dst))))
990 goto nla_put_failure;
991 else if (key->enc_control.addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS &&
992 (fl_dump_key_val(skb, &key->enc_ipv6.src,
993 TCA_FLOWER_KEY_ENC_IPV6_SRC, &mask->enc_ipv6.src,
994 TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK,
995 sizeof(key->enc_ipv6.src)) ||
996 fl_dump_key_val(skb, &key->enc_ipv6.dst,
997 TCA_FLOWER_KEY_ENC_IPV6_DST,
999 TCA_FLOWER_KEY_ENC_IPV6_DST_MASK,
1000 sizeof(key->enc_ipv6.dst))))
1001 goto nla_put_failure;
1003 if (fl_dump_key_val(skb, &key->enc_key_id, TCA_FLOWER_KEY_ENC_KEY_ID,
1004 &mask->enc_key_id, TCA_FLOWER_UNSPEC,
1005 sizeof(key->enc_key_id)) ||
1006 fl_dump_key_val(skb, &key->enc_tp.src,
1007 TCA_FLOWER_KEY_ENC_UDP_SRC_PORT,
1009 TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK,
1010 sizeof(key->enc_tp.src)) ||
1011 fl_dump_key_val(skb, &key->enc_tp.dst,
1012 TCA_FLOWER_KEY_ENC_UDP_DST_PORT,
1014 TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK,
1015 sizeof(key->enc_tp.dst)))
1016 goto nla_put_failure;
1018 nla_put_u32(skb, TCA_FLOWER_FLAGS, f->flags);
1020 if (tcf_exts_dump(skb, &f->exts))
1021 goto nla_put_failure;
1023 nla_nest_end(skb, nest);
1025 if (tcf_exts_dump_stats(skb, &f->exts) < 0)
1026 goto nla_put_failure;
1031 nla_nest_cancel(skb, nest);
1035 static struct tcf_proto_ops cls_fl_ops __read_mostly = {
1037 .classify = fl_classify,
1039 .destroy = fl_destroy,
1041 .change = fl_change,
1042 .delete = fl_delete,
1045 .owner = THIS_MODULE,
1048 static int __init cls_fl_init(void)
1050 return register_tcf_proto_ops(&cls_fl_ops);
1053 static void __exit cls_fl_exit(void)
1055 unregister_tcf_proto_ops(&cls_fl_ops);
1058 module_init(cls_fl_init);
1059 module_exit(cls_fl_exit);
1061 MODULE_AUTHOR("Jiri Pirko <jiri@resnulli.us>");
1062 MODULE_DESCRIPTION("Flower classifier");
1063 MODULE_LICENSE("GPL v2");