net/netfilter/nf_flow_table_ip.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 #include <linux/kernel.h>
   3 #include <linux/init.h>
   4 #include <linux/module.h>
   5 #include <linux/netfilter.h>
   6 #include <linux/rhashtable.h>
   7 #include <linux/ip.h>
   8 #include <linux/ipv6.h>
   9 #include <linux/netdevice.h>
  10 #include <net/ip.h>
  11 #include <net/ipv6.h>
  12 #include <net/ip6_route.h>
  13 #include <net/neighbour.h>
  14 #include <net/netfilter/nf_flow_table.h>
  15 /* For layer 4 checksum field offset. */
  16 #include <linux/tcp.h>
  17 #include <linux/udp.h>
  18
  19 static int nf_flow_state_check(struct flow_offload *flow, int proto,
  20                                struct sk_buff *skb, unsigned int thoff)
  21 {
  22         struct tcphdr *tcph;
  23
  24         if (proto != IPPROTO_TCP)
  25                 return 0;
  26
  27         if (!pskb_may_pull(skb, thoff + sizeof(*tcph)))
  28                 return -1;
  29
  30         tcph = (void *)(skb_network_header(skb) + thoff);
  31         if (unlikely(tcph->fin || tcph->rst)) {
  32                 flow_offload_teardown(flow);
  33                 return -1;
  34         }
  35
  36         return 0;
  37 }
  38
  39 static int nf_flow_nat_ip_tcp(struct sk_buff *skb, unsigned int thoff,
  40                               __be32 addr, __be32 new_addr)
  41 {
  42         struct tcphdr *tcph;
  43
  44         if (!pskb_may_pull(skb, thoff + sizeof(*tcph)) ||
  45             skb_try_make_writable(skb, thoff + sizeof(*tcph)))
  46                 return -1;
  47
  48         tcph = (void *)(skb_network_header(skb) + thoff);
  49         inet_proto_csum_replace4(&tcph->check, skb, addr, new_addr, true);
  50
  51         return 0;
  52 }
  53
  54 static int nf_flow_nat_ip_udp(struct sk_buff *skb, unsigned int thoff,
  55                               __be32 addr, __be32 new_addr)
  56 {
  57         struct udphdr *udph;
  58
  59         if (!pskb_may_pull(skb, thoff + sizeof(*udph)) ||
  60             skb_try_make_writable(skb, thoff + sizeof(*udph)))
  61                 return -1;
  62
  63         udph = (void *)(skb_network_header(skb) + thoff);
  64         if (udph->check || skb->ip_summed == CHECKSUM_PARTIAL) {
  65                 inet_proto_csum_replace4(&udph->check, skb, addr,
  66                                          new_addr, true);
  67                 if (!udph->check)
  68                         udph->check = CSUM_MANGLED_0;
  69         }
  70
  71         return 0;
  72 }
  73
  74 static int nf_flow_nat_ip_l4proto(struct sk_buff *skb, struct iphdr *iph,
  75                                   unsigned int thoff, __be32 addr,
  76                                   __be32 new_addr)
  77 {
  78         switch (iph->protocol) {
  79         case IPPROTO_TCP:
  80                 if (nf_flow_nat_ip_tcp(skb, thoff, addr, new_addr) < 0)
  81                         return NF_DROP;
  82                 break;
  83         case IPPROTO_UDP:
  84                 if (nf_flow_nat_ip_udp(skb, thoff, addr, new_addr) < 0)
  85                         return NF_DROP;
  86                 break;
  87         }
  88
  89         return 0;
  90 }
  91
  92 static int nf_flow_snat_ip(const struct flow_offload *flow, struct sk_buff *skb,
  93                            struct iphdr *iph, unsigned int thoff,
  94                            enum flow_offload_tuple_dir dir)
  95 {
  96         __be32 addr, new_addr;
  97
  98         switch (dir) {
  99         case FLOW_OFFLOAD_DIR_ORIGINAL:
 100                 addr = iph->saddr;
 101                 new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.dst_v4.s_addr;
 102                 iph->saddr = new_addr;
 103                 break;
 104         case FLOW_OFFLOAD_DIR_REPLY:
 105                 addr = iph->daddr;
 106                 new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.src_v4.s_addr;
 107                 iph->daddr = new_addr;
 108                 break;
 109         default:
 110                 return -1;
 111         }
 112         csum_replace4(&iph->check, addr, new_addr);
 113
 114         return nf_flow_nat_ip_l4proto(skb, iph, thoff, addr, new_addr);
 115 }
 116
 117 static int nf_flow_dnat_ip(const struct flow_offload *flow, struct sk_buff *skb,
 118                            struct iphdr *iph, unsigned int thoff,
 119                            enum flow_offload_tuple_dir dir)
 120 {
 121         __be32 addr, new_addr;
 122
 123         switch (dir) {
 124         case FLOW_OFFLOAD_DIR_ORIGINAL:
 125                 addr = iph->daddr;
 126                 new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.src_v4.s_addr;
 127                 iph->daddr = new_addr;
 128                 break;
 129         case FLOW_OFFLOAD_DIR_REPLY:
 130                 addr = iph->saddr;
 131                 new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.dst_v4.s_addr;
 132                 iph->saddr = new_addr;
 133                 break;
 134         default:
 135                 return -1;
 136         }
 137         csum_replace4(&iph->check, addr, new_addr);
 138
 139         return nf_flow_nat_ip_l4proto(skb, iph, thoff, addr, new_addr);
 140 }
 141
 142 static int nf_flow_nat_ip(const struct flow_offload *flow, struct sk_buff *skb,
 143                           unsigned int thoff, enum flow_offload_tuple_dir dir)
 144 {
 145         struct iphdr *iph = ip_hdr(skb);
 146
 147         if (test_bit(NF_FLOW_SNAT, &flow->flags) &&
 148             (nf_flow_snat_port(flow, skb, thoff, iph->protocol, dir) < 0 ||
 149              nf_flow_snat_ip(flow, skb, iph, thoff, dir) < 0))
 150                 return -1;
 151         if (test_bit(NF_FLOW_DNAT, &flow->flags) &&
 152             (nf_flow_dnat_port(flow, skb, thoff, iph->protocol, dir) < 0 ||
 153              nf_flow_dnat_ip(flow, skb, iph, thoff, dir) < 0))
 154                 return -1;
 155
 156         return 0;
 157 }
 158
 159 static bool ip_has_options(unsigned int thoff)
 160 {
 161         return thoff != sizeof(struct iphdr);
 162 }
 163
 164 static int nf_flow_tuple_ip(struct sk_buff *skb, const struct net_device *dev,
 165                             struct flow_offload_tuple *tuple)
 166 {
 167         struct flow_ports *ports;
 168         unsigned int thoff;
 169         struct iphdr *iph;
 170
 171         if (!pskb_may_pull(skb, sizeof(*iph)))
 172                 return -1;
 173
 174         iph = ip_hdr(skb);
 175         thoff = iph->ihl * 4;
 176
 177         if (ip_is_fragment(iph) ||
 178             unlikely(ip_has_options(thoff)))
 179                 return -1;
 180
 181         if (iph->protocol != IPPROTO_TCP &&
 182             iph->protocol != IPPROTO_UDP)
 183                 return -1;
 184
 185         if (iph->ttl <= 1)
 186                 return -1;
 187
 188         thoff = iph->ihl * 4;
 189         if (!pskb_may_pull(skb, thoff + sizeof(*ports)))
 190                 return -1;
 191
 192         ports = (struct flow_ports *)(skb_network_header(skb) + thoff);
 193
 194         tuple->src_v4.s_addr    = iph->saddr;
 195         tuple->dst_v4.s_addr    = iph->daddr;
 196         tuple->src_port         = ports->source;
 197         tuple->dst_port         = ports->dest;
 198         tuple->l3proto          = AF_INET;
 199         tuple->l4proto          = iph->protocol;
 200         tuple->iifidx           = dev->ifindex;
 201
 202         return 0;
 203 }
 204
 205 /* Based on ip_exceeds_mtu(). */
 206 static bool nf_flow_exceeds_mtu(const struct sk_buff *skb, unsigned int mtu)
 207 {
 208         if (skb->len <= mtu)
 209                 return false;
 210
 211         if (skb_is_gso(skb) && skb_gso_validate_network_len(skb, mtu))
 212                 return false;
 213
 214         return true;
 215 }
 216
 217 static int nf_flow_offload_dst_check(struct dst_entry *dst)
 218 {
 219         if (unlikely(dst_xfrm(dst)))
 220                 return dst_check(dst, 0) ? 0 : -1;
 221
 222         return 0;
 223 }
 224
 225 static unsigned int nf_flow_xmit_xfrm(struct sk_buff *skb,
 226                                       const struct nf_hook_state *state,
 227                                       struct dst_entry *dst)
 228 {
 229         skb_orphan(skb);
 230         skb_dst_set_noref(skb, dst);
 231         dst_output(state->net, state->sk, skb);
 232         return NF_STOLEN;
 233 }
 234
 235 static bool nf_flow_offload_refresh(struct nf_flowtable *flow_table,
 236                                     struct flow_offload *flow)
 237 {
 238         return nf_flowtable_hw_offload(flow_table) &&
 239                test_and_clear_bit(NF_FLOW_HW_REFRESH, &flow->flags);
 240 }
 241
 242 unsigned int
 243 nf_flow_offload_ip_hook(void *priv, struct sk_buff *skb,
 244                         const struct nf_hook_state *state)
 245 {
 246         struct flow_offload_tuple_rhash *tuplehash;
 247         struct nf_flowtable *flow_table = priv;
 248         struct flow_offload_tuple tuple = {};
 249         enum flow_offload_tuple_dir dir;
 250         struct flow_offload *flow;
 251         struct net_device *outdev;
 252         struct rtable *rt;
 253         unsigned int thoff;
 254         struct iphdr *iph;
 255         __be32 nexthop;
 256
 257         if (skb->protocol != htons(ETH_P_IP))
 258                 return NF_ACCEPT;
 259
 260         if (nf_flow_tuple_ip(skb, state->in, &tuple) < 0)
 261                 return NF_ACCEPT;
 262
 263         tuplehash = flow_offload_lookup(flow_table, &tuple);
 264         if (tuplehash == NULL)
 265                 return NF_ACCEPT;
 266
 267         dir = tuplehash->tuple.dir;
 268         flow = container_of(tuplehash, struct flow_offload, tuplehash[dir]);
 269         rt = (struct rtable *)flow->tuplehash[dir].tuple.dst_cache;
 270         outdev = rt->dst.dev;
 271
 272         if (unlikely(nf_flow_exceeds_mtu(skb, flow->tuplehash[dir].tuple.mtu)))
 273                 return NF_ACCEPT;
 274
 275         if (skb_try_make_writable(skb, sizeof(*iph)))
 276                 return NF_DROP;
 277
 278         thoff = ip_hdr(skb)->ihl * 4;
 279         if (nf_flow_state_check(flow, ip_hdr(skb)->protocol, skb, thoff))
 280                 return NF_ACCEPT;
 281
 282         if (unlikely(nf_flow_offload_refresh(flow_table, flow)))
 283                 nf_flow_offload_add(flow_table, flow);
 284
 285         if (nf_flow_offload_dst_check(&rt->dst)) {
 286                 flow_offload_teardown(flow);
 287                 return NF_ACCEPT;
 288         }
 289
 290         if (nf_flow_nat_ip(flow, skb, thoff, dir) < 0)
 291                 return NF_DROP;
 292
 293         flow->timeout = nf_flowtable_time_stamp + NF_FLOW_TIMEOUT;
 294         iph = ip_hdr(skb);
 295         ip_decrease_ttl(iph);
 296         skb->tstamp = 0;
 297
 298         if (unlikely(dst_xfrm(&rt->dst))) {
 299                 memset(skb->cb, 0, sizeof(struct inet_skb_parm));
 300                 IPCB(skb)->iif = skb->dev->ifindex;
 301                 IPCB(skb)->flags = IPSKB_FORWARDED;
 302                 return nf_flow_xmit_xfrm(skb, state, &rt->dst);
 303         }
 304
 305         skb->dev = outdev;
 306         nexthop = rt_nexthop(rt, flow->tuplehash[!dir].tuple.src_v4.s_addr);
 307         skb_dst_set_noref(skb, &rt->dst);
 308         neigh_xmit(NEIGH_ARP_TABLE, outdev, &nexthop, skb);
 309
 310         return NF_STOLEN;
 311 }
 312 EXPORT_SYMBOL_GPL(nf_flow_offload_ip_hook);
 313
 314 static int nf_flow_nat_ipv6_tcp(struct sk_buff *skb, unsigned int thoff,
 315                                 struct in6_addr *addr,
 316                                 struct in6_addr *new_addr)
 317 {
 318         struct tcphdr *tcph;
 319
 320         if (!pskb_may_pull(skb, thoff + sizeof(*tcph)) ||
 321             skb_try_make_writable(skb, thoff + sizeof(*tcph)))
 322                 return -1;
 323
 324         tcph = (void *)(skb_network_header(skb) + thoff);
 325         inet_proto_csum_replace16(&tcph->check, skb, addr->s6_addr32,
 326                                   new_addr->s6_addr32, true);
 327
 328         return 0;
 329 }
 330
 331 static int nf_flow_nat_ipv6_udp(struct sk_buff *skb, unsigned int thoff,
 332                                 struct in6_addr *addr,
 333                                 struct in6_addr *new_addr)
 334 {
 335         struct udphdr *udph;
 336
 337         if (!pskb_may_pull(skb, thoff + sizeof(*udph)) ||
 338             skb_try_make_writable(skb, thoff + sizeof(*udph)))
 339                 return -1;
 340
 341         udph = (void *)(skb_network_header(skb) + thoff);
 342         if (udph->check || skb->ip_summed == CHECKSUM_PARTIAL) {
 343                 inet_proto_csum_replace16(&udph->check, skb, addr->s6_addr32,
 344                                           new_addr->s6_addr32, true);
 345                 if (!udph->check)
 346                         udph->check = CSUM_MANGLED_0;
 347         }
 348
 349         return 0;
 350 }
 351
 352 static int nf_flow_nat_ipv6_l4proto(struct sk_buff *skb, struct ipv6hdr *ip6h,
 353                                     unsigned int thoff, struct in6_addr *addr,
 354                                     struct in6_addr *new_addr)
 355 {
 356         switch (ip6h->nexthdr) {
 357         case IPPROTO_TCP:
 358                 if (nf_flow_nat_ipv6_tcp(skb, thoff, addr, new_addr) < 0)
 359                         return NF_DROP;
 360                 break;
 361         case IPPROTO_UDP:
 362                 if (nf_flow_nat_ipv6_udp(skb, thoff, addr, new_addr) < 0)
 363                         return NF_DROP;
 364                 break;
 365         }
 366
 367         return 0;
 368 }
 369
 370 static int nf_flow_snat_ipv6(const struct flow_offload *flow,
 371                              struct sk_buff *skb, struct ipv6hdr *ip6h,
 372                              unsigned int thoff,
 373                              enum flow_offload_tuple_dir dir)
 374 {
 375         struct in6_addr addr, new_addr;
 376
 377         switch (dir) {
 378         case FLOW_OFFLOAD_DIR_ORIGINAL:
 379                 addr = ip6h->saddr;
 380                 new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.dst_v6;
 381                 ip6h->saddr = new_addr;
 382                 break;
 383         case FLOW_OFFLOAD_DIR_REPLY:
 384                 addr = ip6h->daddr;
 385                 new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.src_v6;
 386                 ip6h->daddr = new_addr;
 387                 break;
 388         default:
 389                 return -1;
 390         }
 391
 392         return nf_flow_nat_ipv6_l4proto(skb, ip6h, thoff, &addr, &new_addr);
 393 }
 394
 395 static int nf_flow_dnat_ipv6(const struct flow_offload *flow,
 396                              struct sk_buff *skb, struct ipv6hdr *ip6h,
 397                              unsigned int thoff,
 398                              enum flow_offload_tuple_dir dir)
 399 {
 400         struct in6_addr addr, new_addr;
 401
 402         switch (dir) {
 403         case FLOW_OFFLOAD_DIR_ORIGINAL:
 404                 addr = ip6h->daddr;
 405                 new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.src_v6;
 406                 ip6h->daddr = new_addr;
 407                 break;
 408         case FLOW_OFFLOAD_DIR_REPLY:
 409                 addr = ip6h->saddr;
 410                 new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.dst_v6;
 411                 ip6h->saddr = new_addr;
 412                 break;
 413         default:
 414                 return -1;
 415         }
 416
 417         return nf_flow_nat_ipv6_l4proto(skb, ip6h, thoff, &addr, &new_addr);
 418 }
 419
 420 static int nf_flow_nat_ipv6(const struct flow_offload *flow,
 421                             struct sk_buff *skb,
 422                             enum flow_offload_tuple_dir dir)
 423 {
 424         struct ipv6hdr *ip6h = ipv6_hdr(skb);
 425         unsigned int thoff = sizeof(*ip6h);
 426
 427         if (test_bit(NF_FLOW_SNAT, &flow->flags) &&
 428             (nf_flow_snat_port(flow, skb, thoff, ip6h->nexthdr, dir) < 0 ||
 429              nf_flow_snat_ipv6(flow, skb, ip6h, thoff, dir) < 0))
 430                 return -1;
 431         if (test_bit(NF_FLOW_DNAT, &flow->flags) &&
 432             (nf_flow_dnat_port(flow, skb, thoff, ip6h->nexthdr, dir) < 0 ||
 433              nf_flow_dnat_ipv6(flow, skb, ip6h, thoff, dir) < 0))
 434                 return -1;
 435
 436         return 0;
 437 }
 438
 439 static int nf_flow_tuple_ipv6(struct sk_buff *skb, const struct net_device *dev,
 440                               struct flow_offload_tuple *tuple)
 441 {
 442         struct flow_ports *ports;
 443         struct ipv6hdr *ip6h;
 444         unsigned int thoff;
 445
 446         if (!pskb_may_pull(skb, sizeof(*ip6h)))
 447                 return -1;
 448
 449         ip6h = ipv6_hdr(skb);
 450
 451         if (ip6h->nexthdr != IPPROTO_TCP &&
 452             ip6h->nexthdr != IPPROTO_UDP)
 453                 return -1;
 454
 455         if (ip6h->hop_limit <= 1)
 456                 return -1;
 457
 458         thoff = sizeof(*ip6h);
 459         if (!pskb_may_pull(skb, thoff + sizeof(*ports)))
 460                 return -1;
 461
 462         ports = (struct flow_ports *)(skb_network_header(skb) + thoff);
 463
 464         tuple->src_v6           = ip6h->saddr;
 465         tuple->dst_v6           = ip6h->daddr;
 466         tuple->src_port         = ports->source;
 467         tuple->dst_port         = ports->dest;
 468         tuple->l3proto          = AF_INET6;
 469         tuple->l4proto          = ip6h->nexthdr;
 470         tuple->iifidx           = dev->ifindex;
 471
 472         return 0;
 473 }
 474
 475 unsigned int
 476 nf_flow_offload_ipv6_hook(void *priv, struct sk_buff *skb,
 477                           const struct nf_hook_state *state)
 478 {
 479         struct flow_offload_tuple_rhash *tuplehash;
 480         struct nf_flowtable *flow_table = priv;
 481         struct flow_offload_tuple tuple = {};
 482         enum flow_offload_tuple_dir dir;
 483         const struct in6_addr *nexthop;
 484         struct flow_offload *flow;
 485         struct net_device *outdev;
 486         struct ipv6hdr *ip6h;
 487         struct rt6_info *rt;
 488
 489         if (skb->protocol != htons(ETH_P_IPV6))
 490                 return NF_ACCEPT;
 491
 492         if (nf_flow_tuple_ipv6(skb, state->in, &tuple) < 0)
 493                 return NF_ACCEPT;
 494
 495         tuplehash = flow_offload_lookup(flow_table, &tuple);
 496         if (tuplehash == NULL)
 497                 return NF_ACCEPT;
 498
 499         dir = tuplehash->tuple.dir;
 500         flow = container_of(tuplehash, struct flow_offload, tuplehash[dir]);
 501         rt = (struct rt6_info *)flow->tuplehash[dir].tuple.dst_cache;
 502         outdev = rt->dst.dev;
 503
 504         if (unlikely(nf_flow_exceeds_mtu(skb, flow->tuplehash[dir].tuple.mtu)))
 505                 return NF_ACCEPT;
 506
 507         if (nf_flow_state_check(flow, ipv6_hdr(skb)->nexthdr, skb,
 508                                 sizeof(*ip6h)))
 509                 return NF_ACCEPT;
 510
 511         if (unlikely(nf_flow_offload_refresh(flow_table, flow)))
 512                 nf_flow_offload_add(flow_table, flow);
 513
 514         if (nf_flow_offload_dst_check(&rt->dst)) {
 515                 flow_offload_teardown(flow);
 516                 return NF_ACCEPT;
 517         }
 518
 519         if (skb_try_make_writable(skb, sizeof(*ip6h)))
 520                 return NF_DROP;
 521
 522         if (nf_flow_nat_ipv6(flow, skb, dir) < 0)
 523                 return NF_DROP;
 524
 525         flow->timeout = nf_flowtable_time_stamp + NF_FLOW_TIMEOUT;
 526         ip6h = ipv6_hdr(skb);
 527         ip6h->hop_limit--;
 528         skb->tstamp = 0;
 529
 530         if (unlikely(dst_xfrm(&rt->dst))) {
 531                 memset(skb->cb, 0, sizeof(struct inet6_skb_parm));
 532                 IP6CB(skb)->iif = skb->dev->ifindex;
 533                 IP6CB(skb)->flags = IP6SKB_FORWARDED;
 534                 return nf_flow_xmit_xfrm(skb, state, &rt->dst);
 535         }
 536
 537         skb->dev = outdev;
 538         nexthop = rt6_nexthop(rt, &flow->tuplehash[!dir].tuple.src_v6);
 539         skb_dst_set_noref(skb, &rt->dst);
 540         neigh_xmit(NEIGH_ND_TABLE, outdev, nexthop, skb);
 541
 542         return NF_STOLEN;
 543 }
 544 EXPORT_SYMBOL_GPL(nf_flow_offload_ipv6_hook);