net/netfilter/nf_flow_table_offload.c

   1 #include <linux/kernel.h>
   2 #include <linux/init.h>
   3 #include <linux/module.h>
   4 #include <linux/netfilter.h>
   5 #include <linux/rhashtable.h>
   6 #include <linux/netdevice.h>
   7 #include <linux/tc_act/tc_csum.h>
   8 #include <net/flow_offload.h>
   9 #include <net/netfilter/nf_flow_table.h>
  10 #include <net/netfilter/nf_conntrack.h>
  11 #include <net/netfilter/nf_conntrack_core.h>
  12 #include <net/netfilter/nf_conntrack_tuple.h>
  13
  14 static struct work_struct nf_flow_offload_work;
  15 static DEFINE_SPINLOCK(flow_offload_pending_list_lock);
  16 static LIST_HEAD(flow_offload_pending_list);
  17
  18 struct flow_offload_work {
  19         struct list_head        list;
  20         enum flow_cls_command   cmd;
  21         int                     priority;
  22         struct nf_flowtable     *flowtable;
  23         struct flow_offload     *flow;
  24 };
  25
  26 struct nf_flow_key {
  27         struct flow_dissector_key_control               control;
  28         struct flow_dissector_key_basic                 basic;
  29         union {
  30                 struct flow_dissector_key_ipv4_addrs    ipv4;
  31         };
  32         struct flow_dissector_key_tcp                   tcp;
  33         struct flow_dissector_key_ports                 tp;
  34 } __aligned(BITS_PER_LONG / 8); /* Ensure that we can do comparisons as longs. */
  35
  36 struct nf_flow_match {
  37         struct flow_dissector   dissector;
  38         struct nf_flow_key      key;
  39         struct nf_flow_key      mask;
  40 };
  41
  42 struct nf_flow_rule {
  43         struct nf_flow_match    match;
  44         struct flow_rule        *rule;
  45 };
  46
  47 #define NF_FLOW_DISSECTOR(__match, __type, __field)     \
  48         (__match)->dissector.offset[__type] =           \
  49                 offsetof(struct nf_flow_key, __field)
  50
  51 static int nf_flow_rule_match(struct nf_flow_match *match,
  52                               const struct flow_offload_tuple *tuple)
  53 {
  54         struct nf_flow_key *mask = &match->mask;
  55         struct nf_flow_key *key = &match->key;
  56
  57         NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_CONTROL, control);
  58         NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_BASIC, basic);
  59         NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_IPV4_ADDRS, ipv4);
  60         NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_TCP, tcp);
  61         NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_PORTS, tp);
  62
  63         switch (tuple->l3proto) {
  64         case AF_INET:
  65                 key->control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
  66                 key->basic.n_proto = htons(ETH_P_IP);
  67                 key->ipv4.src = tuple->src_v4.s_addr;
  68                 mask->ipv4.src = 0xffffffff;
  69                 key->ipv4.dst = tuple->dst_v4.s_addr;
  70                 mask->ipv4.dst = 0xffffffff;
  71                 break;
  72         default:
  73                 return -EOPNOTSUPP;
  74         }
  75         mask->basic.n_proto = 0xffff;
  76
  77         switch (tuple->l4proto) {
  78         case IPPROTO_TCP:
  79                 key->tcp.flags = 0;
  80                 mask->tcp.flags = TCP_FLAG_RST | TCP_FLAG_FIN;
  81                 match->dissector.used_keys |= BIT(FLOW_DISSECTOR_KEY_TCP);
  82                 break;
  83         case IPPROTO_UDP:
  84                 break;
  85         default:
  86                 return -EOPNOTSUPP;
  87         }
  88
  89         key->basic.ip_proto = tuple->l4proto;
  90         mask->basic.ip_proto = 0xff;
  91
  92         key->tp.src = tuple->src_port;
  93         mask->tp.src = 0xffff;
  94         key->tp.dst = tuple->dst_port;
  95         mask->tp.dst = 0xffff;
  96
  97         match->dissector.used_keys |= BIT(FLOW_DISSECTOR_KEY_CONTROL) |
  98                                       BIT(FLOW_DISSECTOR_KEY_BASIC) |
  99                                       BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |
 100                                       BIT(FLOW_DISSECTOR_KEY_PORTS);
 101         return 0;
 102 }
 103
 104 static void flow_offload_mangle(struct flow_action_entry *entry,
 105                                 enum flow_action_mangle_base htype,
 106                                 u32 offset, u8 *value, u8 *mask)
 107 {
 108         entry->id = FLOW_ACTION_MANGLE;
 109         entry->mangle.htype = htype;
 110         entry->mangle.offset = offset;
 111         memcpy(&entry->mangle.mask, mask, sizeof(u32));
 112         memcpy(&entry->mangle.val, value, sizeof(u32));
 113 }
 114
 115 static inline struct flow_action_entry *
 116 flow_action_entry_next(struct nf_flow_rule *flow_rule)
 117 {
 118         int i = flow_rule->rule->action.num_entries++;
 119
 120         return &flow_rule->rule->action.entries[i];
 121 }
 122
 123 static int flow_offload_eth_src(struct net *net,
 124                                 const struct flow_offload *flow,
 125                                 enum flow_offload_tuple_dir dir,
 126                                 struct nf_flow_rule *flow_rule)
 127 {
 128         const struct flow_offload_tuple *tuple = &flow->tuplehash[!dir].tuple;
 129         struct flow_action_entry *entry0 = flow_action_entry_next(flow_rule);
 130         struct flow_action_entry *entry1 = flow_action_entry_next(flow_rule);
 131         struct net_device *dev;
 132         u32 mask, val;
 133         u16 val16;
 134
 135         dev = dev_get_by_index(net, tuple->iifidx);
 136         if (!dev)
 137                 return -ENOENT;
 138
 139         mask = ~0xffff0000;
 140         memcpy(&val16, dev->dev_addr, 2);
 141         val = val16 << 16;
 142         flow_offload_mangle(entry0, FLOW_ACT_MANGLE_HDR_TYPE_ETH, 4,
 143                             (u8 *)&val, (u8 *)&mask);
 144
 145         mask = ~0xffffffff;
 146         memcpy(&val, dev->dev_addr + 2, 4);
 147         flow_offload_mangle(entry1, FLOW_ACT_MANGLE_HDR_TYPE_ETH, 8,
 148                             (u8 *)&val, (u8 *)&mask);
 149         dev_put(dev);
 150
 151         return 0;
 152 }
 153
 154 static int flow_offload_eth_dst(struct net *net,
 155                                 const struct flow_offload *flow,
 156                                 enum flow_offload_tuple_dir dir,
 157                                 struct nf_flow_rule *flow_rule)
 158 {
 159         const struct flow_offload_tuple *tuple = &flow->tuplehash[dir].tuple;
 160         struct flow_action_entry *entry0 = flow_action_entry_next(flow_rule);
 161         struct flow_action_entry *entry1 = flow_action_entry_next(flow_rule);
 162         struct neighbour *n;
 163         u32 mask, val;
 164         u16 val16;
 165
 166         n = dst_neigh_lookup(tuple->dst_cache, &tuple->dst_v4);
 167         if (!n)
 168                 return -ENOENT;
 169
 170         mask = ~0xffffffff;
 171         memcpy(&val, n->ha, 4);
 172         flow_offload_mangle(entry0, FLOW_ACT_MANGLE_HDR_TYPE_ETH, 0,
 173                             (u8 *)&val, (u8 *)&mask);
 174
 175         mask = ~0x0000ffff;
 176         memcpy(&val16, n->ha + 4, 2);
 177         val = val16;
 178         flow_offload_mangle(entry1, FLOW_ACT_MANGLE_HDR_TYPE_ETH, 4,
 179                             (u8 *)&val, (u8 *)&mask);
 180         neigh_release(n);
 181
 182         return 0;
 183 }
 184
 185 static void flow_offload_ipv4_snat(struct net *net,
 186                                    const struct flow_offload *flow,
 187                                    enum flow_offload_tuple_dir dir,
 188                                    struct nf_flow_rule *flow_rule)
 189 {
 190         struct flow_action_entry *entry = flow_action_entry_next(flow_rule);
 191         u32 mask = ~htonl(0xffffffff);
 192         __be32 addr;
 193         u32 offset;
 194
 195         switch (dir) {
 196         case FLOW_OFFLOAD_DIR_ORIGINAL:
 197                 addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.dst_v4.s_addr;
 198                 offset = offsetof(struct iphdr, saddr);
 199                 break;
 200         case FLOW_OFFLOAD_DIR_REPLY:
 201                 addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.src_v4.s_addr;
 202                 offset = offsetof(struct iphdr, daddr);
 203                 break;
 204         default:
 205                 return;
 206         }
 207
 208         flow_offload_mangle(entry, FLOW_ACT_MANGLE_HDR_TYPE_IP4, offset,
 209                             (u8 *)&addr, (u8 *)&mask);
 210 }
 211
 212 static void flow_offload_ipv4_dnat(struct net *net,
 213                                    const struct flow_offload *flow,
 214                                    enum flow_offload_tuple_dir dir,
 215                                    struct nf_flow_rule *flow_rule)
 216 {
 217         struct flow_action_entry *entry = flow_action_entry_next(flow_rule);
 218         u32 mask = ~htonl(0xffffffff);
 219         __be32 addr;
 220         u32 offset;
 221
 222         switch (dir) {
 223         case FLOW_OFFLOAD_DIR_ORIGINAL:
 224                 addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.src_v4.s_addr;
 225                 offset = offsetof(struct iphdr, daddr);
 226                 break;
 227         case FLOW_OFFLOAD_DIR_REPLY:
 228                 addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.dst_v4.s_addr;
 229                 offset = offsetof(struct iphdr, saddr);
 230                 break;
 231         default:
 232                 return;
 233         }
 234
 235         flow_offload_mangle(entry, FLOW_ACT_MANGLE_HDR_TYPE_IP4, offset,
 236                             (u8 *)&addr, (u8 *)&mask);
 237 }
 238
 239 static void flow_offload_ipv6_mangle(struct nf_flow_rule *flow_rule,
 240                                      unsigned int offset,
 241                                      u8 *addr, u8 *mask)
 242 {
 243         struct flow_action_entry *entry;
 244         int i;
 245
 246         for (i = 0; i < sizeof(struct in6_addr) / sizeof(u32); i += sizeof(u32)) {
 247                 entry = flow_action_entry_next(flow_rule);
 248                 flow_offload_mangle(entry, FLOW_ACT_MANGLE_HDR_TYPE_IP6,
 249                                     offset + i,
 250                                     &addr[i], mask);
 251         }
 252 }
 253
 254 static void flow_offload_ipv6_snat(struct net *net,
 255                                    const struct flow_offload *flow,
 256                                    enum flow_offload_tuple_dir dir,
 257                                    struct nf_flow_rule *flow_rule)
 258 {
 259         u32 mask = ~htonl(0xffffffff);
 260         const u8 *addr;
 261         u32 offset;
 262
 263         switch (dir) {
 264         case FLOW_OFFLOAD_DIR_ORIGINAL:
 265                 addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.dst_v6.s6_addr;
 266                 offset = offsetof(struct ipv6hdr, saddr);
 267                 break;
 268         case FLOW_OFFLOAD_DIR_REPLY:
 269                 addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.src_v6.s6_addr;
 270                 offset = offsetof(struct ipv6hdr, daddr);
 271                 break;
 272         default:
 273                 return;
 274         }
 275
 276         flow_offload_ipv6_mangle(flow_rule, offset, (u8 *)addr, (u8 *)&mask);
 277 }
 278
 279 static void flow_offload_ipv6_dnat(struct net *net,
 280                                    const struct flow_offload *flow,
 281                                    enum flow_offload_tuple_dir dir,
 282                                    struct nf_flow_rule *flow_rule)
 283 {
 284         u32 mask = ~htonl(0xffffffff);
 285         const u8 *addr;
 286         u32 offset;
 287
 288         switch (dir) {
 289         case FLOW_OFFLOAD_DIR_ORIGINAL:
 290                 addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.src_v6.s6_addr;
 291                 offset = offsetof(struct ipv6hdr, daddr);
 292                 break;
 293         case FLOW_OFFLOAD_DIR_REPLY:
 294                 addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.dst_v6.s6_addr;
 295                 offset = offsetof(struct ipv6hdr, saddr);
 296                 break;
 297         default:
 298                 return;
 299         }
 300
 301         flow_offload_ipv6_mangle(flow_rule, offset, (u8 *)addr, (u8 *)&mask);
 302 }
 303
 304 static int flow_offload_l4proto(const struct flow_offload *flow)
 305 {
 306         u8 protonum = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.l4proto;
 307         u8 type = 0;
 308
 309         switch (protonum) {
 310         case IPPROTO_TCP:
 311                 type = FLOW_ACT_MANGLE_HDR_TYPE_TCP;
 312                 break;
 313         case IPPROTO_UDP:
 314                 type = FLOW_ACT_MANGLE_HDR_TYPE_UDP;
 315                 break;
 316         default:
 317                 break;
 318         }
 319
 320         return type;
 321 }
 322
 323 static void flow_offload_port_snat(struct net *net,
 324                                    const struct flow_offload *flow,
 325                                    enum flow_offload_tuple_dir dir,
 326                                    struct nf_flow_rule *flow_rule)
 327 {
 328         struct flow_action_entry *entry = flow_action_entry_next(flow_rule);
 329         u32 mask = ~htonl(0xffff0000);
 330         __be16 port;
 331         u32 offset;
 332
 333         switch (dir) {
 334         case FLOW_OFFLOAD_DIR_ORIGINAL:
 335                 port = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.dst_port;
 336                 offset = 0; /* offsetof(struct tcphdr, source); */
 337                 break;
 338         case FLOW_OFFLOAD_DIR_REPLY:
 339                 port = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.src_port;
 340                 offset = 0; /* offsetof(struct tcphdr, dest); */
 341                 break;
 342         default:
 343                 break;
 344         }
 345
 346         flow_offload_mangle(entry, flow_offload_l4proto(flow), offset,
 347                             (u8 *)&port, (u8 *)&mask);
 348 }
 349
 350 static void flow_offload_port_dnat(struct net *net,
 351                                    const struct flow_offload *flow,
 352                                    enum flow_offload_tuple_dir dir,
 353                                    struct nf_flow_rule *flow_rule)
 354 {
 355         struct flow_action_entry *entry = flow_action_entry_next(flow_rule);
 356         u32 mask = ~htonl(0xffff);
 357         __be16 port;
 358         u32 offset;
 359
 360         switch (dir) {
 361         case FLOW_OFFLOAD_DIR_ORIGINAL:
 362                 port = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.dst_port;
 363                 offset = 0; /* offsetof(struct tcphdr, source); */
 364                 break;
 365         case FLOW_OFFLOAD_DIR_REPLY:
 366                 port = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.src_port;
 367                 offset = 0; /* offsetof(struct tcphdr, dest); */
 368                 break;
 369         default:
 370                 break;
 371         }
 372
 373         flow_offload_mangle(entry, flow_offload_l4proto(flow), offset,
 374                             (u8 *)&port, (u8 *)&mask);
 375 }
 376
 377 static void flow_offload_ipv4_checksum(struct net *net,
 378                                        const struct flow_offload *flow,
 379                                        struct nf_flow_rule *flow_rule)
 380 {
 381         u8 protonum = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.l4proto;
 382         struct flow_action_entry *entry = flow_action_entry_next(flow_rule);
 383
 384         entry->id = FLOW_ACTION_CSUM;
 385         entry->csum_flags = TCA_CSUM_UPDATE_FLAG_IPV4HDR;
 386
 387         switch (protonum) {
 388         case IPPROTO_TCP:
 389                 entry->csum_flags |= TCA_CSUM_UPDATE_FLAG_TCP;
 390                 break;
 391         case IPPROTO_UDP:
 392                 entry->csum_flags |= TCA_CSUM_UPDATE_FLAG_UDP;
 393                 break;
 394         }
 395 }
 396
 397 static void flow_offload_redirect(const struct flow_offload *flow,
 398                                   enum flow_offload_tuple_dir dir,
 399                                   struct nf_flow_rule *flow_rule)
 400 {
 401         struct flow_action_entry *entry = flow_action_entry_next(flow_rule);
 402         struct rtable *rt;
 403
 404         rt = (struct rtable *)flow->tuplehash[dir].tuple.dst_cache;
 405         entry->id = FLOW_ACTION_REDIRECT;
 406         entry->dev = rt->dst.dev;
 407         dev_hold(rt->dst.dev);
 408 }
 409
 410 int nf_flow_rule_route_ipv4(struct net *net, const struct flow_offload *flow,
 411                             enum flow_offload_tuple_dir dir,
 412                             struct nf_flow_rule *flow_rule)
 413 {
 414         if (flow_offload_eth_src(net, flow, dir, flow_rule) < 0 ||
 415             flow_offload_eth_dst(net, flow, dir, flow_rule) < 0)
 416                 return -1;
 417
 418         if (flow->flags & FLOW_OFFLOAD_SNAT) {
 419                 flow_offload_ipv4_snat(net, flow, dir, flow_rule);
 420                 flow_offload_port_snat(net, flow, dir, flow_rule);
 421         }
 422         if (flow->flags & FLOW_OFFLOAD_DNAT) {
 423                 flow_offload_ipv4_dnat(net, flow, dir, flow_rule);
 424                 flow_offload_port_dnat(net, flow, dir, flow_rule);
 425         }
 426         if (flow->flags & FLOW_OFFLOAD_SNAT ||
 427             flow->flags & FLOW_OFFLOAD_DNAT)
 428                 flow_offload_ipv4_checksum(net, flow, flow_rule);
 429
 430         flow_offload_redirect(flow, dir, flow_rule);
 431
 432         return 0;
 433 }
 434 EXPORT_SYMBOL_GPL(nf_flow_rule_route_ipv4);
 435
 436 int nf_flow_rule_route_ipv6(struct net *net, const struct flow_offload *flow,
 437                             enum flow_offload_tuple_dir dir,
 438                             struct nf_flow_rule *flow_rule)
 439 {
 440         if (flow_offload_eth_src(net, flow, dir, flow_rule) < 0 ||
 441             flow_offload_eth_dst(net, flow, dir, flow_rule) < 0)
 442                 return -1;
 443
 444         if (flow->flags & FLOW_OFFLOAD_SNAT) {
 445                 flow_offload_ipv6_snat(net, flow, dir, flow_rule);
 446                 flow_offload_port_snat(net, flow, dir, flow_rule);
 447         }
 448         if (flow->flags & FLOW_OFFLOAD_DNAT) {
 449                 flow_offload_ipv6_dnat(net, flow, dir, flow_rule);
 450                 flow_offload_port_dnat(net, flow, dir, flow_rule);
 451         }
 452
 453         flow_offload_redirect(flow, dir, flow_rule);
 454
 455         return 0;
 456 }
 457 EXPORT_SYMBOL_GPL(nf_flow_rule_route_ipv6);
 458
 459 #define NF_FLOW_RULE_ACTION_MAX 16
 460
 461 static struct nf_flow_rule *
 462 nf_flow_offload_rule_alloc(struct net *net,
 463                            const struct flow_offload_work *offload,
 464                            enum flow_offload_tuple_dir dir)
 465 {
 466         const struct nf_flowtable *flowtable = offload->flowtable;
 467         const struct flow_offload *flow = offload->flow;
 468         const struct flow_offload_tuple *tuple;
 469         struct nf_flow_rule *flow_rule;
 470         int err = -ENOMEM;
 471
 472         flow_rule = kzalloc(sizeof(*flow_rule), GFP_KERNEL);
 473         if (!flow_rule)
 474                 goto err_flow;
 475
 476         flow_rule->rule = flow_rule_alloc(NF_FLOW_RULE_ACTION_MAX);
 477         if (!flow_rule->rule)
 478                 goto err_flow_rule;
 479
 480         flow_rule->rule->match.dissector = &flow_rule->match.dissector;
 481         flow_rule->rule->match.mask = &flow_rule->match.mask;
 482         flow_rule->rule->match.key = &flow_rule->match.key;
 483
 484         tuple = &flow->tuplehash[dir].tuple;
 485         err = nf_flow_rule_match(&flow_rule->match, tuple);
 486         if (err < 0)
 487                 goto err_flow_match;
 488
 489         flow_rule->rule->action.num_entries = 0;
 490         if (flowtable->type->action(net, flow, dir, flow_rule) < 0)
 491                 goto err_flow_match;
 492
 493         return flow_rule;
 494
 495 err_flow_match:
 496         kfree(flow_rule->rule);
 497 err_flow_rule:
 498         kfree(flow_rule);
 499 err_flow:
 500         return NULL;
 501 }
 502
 503 static void __nf_flow_offload_destroy(struct nf_flow_rule *flow_rule)
 504 {
 505         struct flow_action_entry *entry;
 506         int i;
 507
 508         for (i = 0; i < flow_rule->rule->action.num_entries; i++) {
 509                 entry = &flow_rule->rule->action.entries[i];
 510                 if (entry->id != FLOW_ACTION_REDIRECT)
 511                         continue;
 512
 513                 dev_put(entry->dev);
 514         }
 515         kfree(flow_rule->rule);
 516         kfree(flow_rule);
 517 }
 518
 519 static void nf_flow_offload_destroy(struct nf_flow_rule *flow_rule[])
 520 {
 521         int i;
 522
 523         for (i = 0; i < FLOW_OFFLOAD_DIR_MAX; i++)
 524                 __nf_flow_offload_destroy(flow_rule[i]);
 525 }
 526
 527 static int nf_flow_offload_alloc(const struct flow_offload_work *offload,
 528                                  struct nf_flow_rule *flow_rule[])
 529 {
 530         struct net *net = read_pnet(&offload->flowtable->net);
 531
 532         flow_rule[0] = nf_flow_offload_rule_alloc(net, offload,
 533                                                   FLOW_OFFLOAD_DIR_ORIGINAL);
 534         if (!flow_rule[0])
 535                 return -ENOMEM;
 536
 537         flow_rule[1] = nf_flow_offload_rule_alloc(net, offload,
 538                                                   FLOW_OFFLOAD_DIR_REPLY);
 539         if (!flow_rule[1]) {
 540                 __nf_flow_offload_destroy(flow_rule[0]);
 541                 return -ENOMEM;
 542         }
 543
 544         return 0;
 545 }
 546
 547 static void nf_flow_offload_init(struct flow_cls_offload *cls_flow,
 548                                  __be16 proto, int priority,
 549                                  enum flow_cls_command cmd,
 550                                  const struct flow_offload_tuple *tuple,
 551                                  struct netlink_ext_ack *extack)
 552 {
 553         cls_flow->common.protocol = proto;
 554         cls_flow->common.prio = priority;
 555         cls_flow->common.extack = extack;
 556         cls_flow->command = cmd;
 557         cls_flow->cookie = (unsigned long)tuple;
 558 }
 559
 560 static int flow_offload_tuple_add(struct flow_offload_work *offload,
 561                                   struct nf_flow_rule *flow_rule,
 562                                   enum flow_offload_tuple_dir dir)
 563 {
 564         struct nf_flowtable *flowtable = offload->flowtable;
 565         struct flow_cls_offload cls_flow = {};
 566         struct flow_block_cb *block_cb;
 567         struct netlink_ext_ack extack;
 568         __be16 proto = ETH_P_ALL;
 569         int err, i = 0;
 570
 571         nf_flow_offload_init(&cls_flow, proto, offload->priority,
 572                              FLOW_CLS_REPLACE,
 573                              &offload->flow->tuplehash[dir].tuple, &extack);
 574         cls_flow.rule = flow_rule->rule;
 575
 576         list_for_each_entry(block_cb, &flowtable->flow_block.cb_list, list) {
 577                 err = block_cb->cb(TC_SETUP_FT, &cls_flow,
 578                                    block_cb->cb_priv);
 579                 if (err < 0)
 580                         continue;
 581
 582                 i++;
 583         }
 584
 585         return i;
 586 }
 587
 588 static void flow_offload_tuple_del(struct flow_offload_work *offload,
 589                                    enum flow_offload_tuple_dir dir)
 590 {
 591         struct nf_flowtable *flowtable = offload->flowtable;
 592         struct flow_cls_offload cls_flow = {};
 593         struct flow_block_cb *block_cb;
 594         struct netlink_ext_ack extack;
 595         __be16 proto = ETH_P_ALL;
 596
 597         nf_flow_offload_init(&cls_flow, proto, offload->priority,
 598                              FLOW_CLS_DESTROY,
 599                              &offload->flow->tuplehash[dir].tuple, &extack);
 600
 601         list_for_each_entry(block_cb, &flowtable->flow_block.cb_list, list)
 602                 block_cb->cb(TC_SETUP_FT, &cls_flow, block_cb->cb_priv);
 603
 604         offload->flow->flags |= FLOW_OFFLOAD_HW_DEAD;
 605 }
 606
 607 static int flow_offload_rule_add(struct flow_offload_work *offload,
 608                                  struct nf_flow_rule *flow_rule[])
 609 {
 610         int ok_count = 0;
 611
 612         ok_count += flow_offload_tuple_add(offload, flow_rule[0],
 613                                            FLOW_OFFLOAD_DIR_ORIGINAL);
 614         ok_count += flow_offload_tuple_add(offload, flow_rule[1],
 615                                            FLOW_OFFLOAD_DIR_REPLY);
 616         if (ok_count == 0)
 617                 return -ENOENT;
 618
 619         return 0;
 620 }
 621
 622 static int flow_offload_work_add(struct flow_offload_work *offload)
 623 {
 624         struct nf_flow_rule *flow_rule[FLOW_OFFLOAD_DIR_MAX];
 625         int err;
 626
 627         err = nf_flow_offload_alloc(offload, flow_rule);
 628         if (err < 0)
 629                 return -ENOMEM;
 630
 631         err = flow_offload_rule_add(offload, flow_rule);
 632
 633         nf_flow_offload_destroy(flow_rule);
 634
 635         return err;
 636 }
 637
 638 static void flow_offload_work_del(struct flow_offload_work *offload)
 639 {
 640         flow_offload_tuple_del(offload, FLOW_OFFLOAD_DIR_ORIGINAL);
 641         flow_offload_tuple_del(offload, FLOW_OFFLOAD_DIR_REPLY);
 642 }
 643
 644 static void flow_offload_tuple_stats(struct flow_offload_work *offload,
 645                                      enum flow_offload_tuple_dir dir,
 646                                      struct flow_stats *stats)
 647 {
 648         struct nf_flowtable *flowtable = offload->flowtable;
 649         struct flow_cls_offload cls_flow = {};
 650         struct flow_block_cb *block_cb;
 651         struct netlink_ext_ack extack;
 652         __be16 proto = ETH_P_ALL;
 653
 654         nf_flow_offload_init(&cls_flow, proto, offload->priority,
 655                              FLOW_CLS_STATS,
 656                              &offload->flow->tuplehash[dir].tuple, &extack);
 657
 658         list_for_each_entry(block_cb, &flowtable->flow_block.cb_list, list)
 659                 block_cb->cb(TC_SETUP_FT, &cls_flow, block_cb->cb_priv);
 660         memcpy(stats, &cls_flow.stats, sizeof(*stats));
 661 }
 662
 663 static void flow_offload_work_stats(struct flow_offload_work *offload)
 664 {
 665         struct flow_stats stats[FLOW_OFFLOAD_DIR_MAX] = {};
 666         u64 lastused;
 667
 668         flow_offload_tuple_stats(offload, FLOW_OFFLOAD_DIR_ORIGINAL, &stats[0]);
 669         flow_offload_tuple_stats(offload, FLOW_OFFLOAD_DIR_REPLY, &stats[1]);
 670
 671         lastused = max_t(u64, stats[0].lastused, stats[1].lastused);
 672         offload->flow->timeout = max_t(u64, offload->flow->timeout,
 673                                        lastused + NF_FLOW_TIMEOUT);
 674 }
 675
 676 static void flow_offload_work_handler(struct work_struct *work)
 677 {
 678         struct flow_offload_work *offload, *next;
 679         LIST_HEAD(offload_pending_list);
 680         int ret;
 681
 682         spin_lock_bh(&flow_offload_pending_list_lock);
 683         list_replace_init(&flow_offload_pending_list, &offload_pending_list);
 684         spin_unlock_bh(&flow_offload_pending_list_lock);
 685
 686         list_for_each_entry_safe(offload, next, &offload_pending_list, list) {
 687                 switch (offload->cmd) {
 688                 case FLOW_CLS_REPLACE:
 689                         ret = flow_offload_work_add(offload);
 690                         if (ret < 0)
 691                                 offload->flow->flags &= ~FLOW_OFFLOAD_HW;
 692                         break;
 693                 case FLOW_CLS_DESTROY:
 694                         flow_offload_work_del(offload);
 695                         break;
 696                 case FLOW_CLS_STATS:
 697                         flow_offload_work_stats(offload);
 698                         break;
 699                 default:
 700                         WARN_ON_ONCE(1);
 701                 }
 702                 list_del(&offload->list);
 703                 kfree(offload);
 704         }
 705 }
 706
 707 static void flow_offload_queue_work(struct flow_offload_work *offload)
 708 {
 709         spin_lock_bh(&flow_offload_pending_list_lock);
 710         list_add_tail(&offload->list, &flow_offload_pending_list);
 711         spin_unlock_bh(&flow_offload_pending_list_lock);
 712
 713         schedule_work(&nf_flow_offload_work);
 714 }
 715
 716 void nf_flow_offload_add(struct nf_flowtable *flowtable,
 717                          struct flow_offload *flow)
 718 {
 719         struct flow_offload_work *offload;
 720
 721         offload = kmalloc(sizeof(struct flow_offload_work), GFP_ATOMIC);
 722         if (!offload)
 723                 return;
 724
 725         offload->cmd = FLOW_CLS_REPLACE;
 726         offload->flow = flow;
 727         offload->priority = flowtable->priority;
 728         offload->flowtable = flowtable;
 729         flow->flags |= FLOW_OFFLOAD_HW;
 730
 731         flow_offload_queue_work(offload);
 732 }
 733
 734 void nf_flow_offload_del(struct nf_flowtable *flowtable,
 735                          struct flow_offload *flow)
 736 {
 737         struct flow_offload_work *offload;
 738
 739         offload = kzalloc(sizeof(struct flow_offload_work), GFP_ATOMIC);
 740         if (!offload)
 741                 return;
 742
 743         offload->cmd = FLOW_CLS_DESTROY;
 744         offload->flow = flow;
 745         offload->flow->flags |= FLOW_OFFLOAD_HW_DYING;
 746         offload->flowtable = flowtable;
 747
 748         flow_offload_queue_work(offload);
 749 }
 750
 751 void nf_flow_offload_stats(struct nf_flowtable *flowtable,
 752                            struct flow_offload *flow)
 753 {
 754         struct flow_offload_work *offload;
 755         s64 delta;
 756
 757         delta = flow->timeout - jiffies;
 758         if ((delta >= (9 * NF_FLOW_TIMEOUT) / 10) ||
 759             flow->flags & FLOW_OFFLOAD_HW_DYING)
 760                 return;
 761
 762         offload = kzalloc(sizeof(struct flow_offload_work), GFP_ATOMIC);
 763         if (!offload)
 764                 return;
 765
 766         offload->cmd = FLOW_CLS_STATS;
 767         offload->flow = flow;
 768         offload->flowtable = flowtable;
 769
 770         flow_offload_queue_work(offload);
 771 }
 772
 773 void nf_flow_table_offload_flush(struct nf_flowtable *flowtable)
 774 {
 775         if (flowtable->flags & NF_FLOWTABLE_HW_OFFLOAD)
 776                 flush_work(&nf_flow_offload_work);
 777 }
 778
 779 static int nf_flow_table_block_setup(struct nf_flowtable *flowtable,
 780                                      struct flow_block_offload *bo,
 781                                      enum flow_block_command cmd)
 782 {
 783         struct flow_block_cb *block_cb, *next;
 784         int err = 0;
 785
 786         switch (cmd) {
 787         case FLOW_BLOCK_BIND:
 788                 list_splice(&bo->cb_list, &flowtable->flow_block.cb_list);
 789                 break;
 790         case FLOW_BLOCK_UNBIND:
 791                 list_for_each_entry_safe(block_cb, next, &bo->cb_list, list) {
 792                         list_del(&block_cb->list);
 793                         flow_block_cb_free(block_cb);
 794                 }
 795                 break;
 796         default:
 797                 WARN_ON_ONCE(1);
 798                 err = -EOPNOTSUPP;
 799         }
 800
 801         return err;
 802 }
 803
 804 int nf_flow_table_offload_setup(struct nf_flowtable *flowtable,
 805                                 struct net_device *dev,
 806                                 enum flow_block_command cmd)
 807 {
 808         struct netlink_ext_ack extack = {};
 809         struct flow_block_offload bo = {};
 810         int err;
 811
 812         if (!(flowtable->flags & NF_FLOWTABLE_HW_OFFLOAD))
 813                 return 0;
 814
 815         if (!dev->netdev_ops->ndo_setup_tc)
 816                 return -EOPNOTSUPP;
 817
 818         bo.net          = dev_net(dev);
 819         bo.block        = &flowtable->flow_block;
 820         bo.command      = cmd;
 821         bo.binder_type  = FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS;
 822         bo.extack       = &extack;
 823         INIT_LIST_HEAD(&bo.cb_list);
 824
 825         err = dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_BLOCK, &bo);
 826         if (err < 0)
 827                 return err;
 828
 829         return nf_flow_table_block_setup(flowtable, &bo, cmd);
 830 }
 831 EXPORT_SYMBOL_GPL(nf_flow_table_offload_setup);
 832
 833 int nf_flow_table_offload_init(void)
 834 {
 835         INIT_WORK(&nf_flow_offload_work, flow_offload_work_handler);
 836
 837         return 0;
 838 }
 839
 840 void nf_flow_table_offload_exit(void)
 841 {
 842         struct flow_offload_work *offload, *next;
 843         LIST_HEAD(offload_pending_list);
 844
 845         cancel_work_sync(&nf_flow_offload_work);
 846
 847         list_for_each_entry_safe(offload, next, &offload_pending_list, list) {
 848                 list_del(&offload->list);
 849                 kfree(offload);
 850         }
 851 }