udp: avoid a cache miss on dequeue

[linux.git] / net / ipv4 / udp.c

diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c
index e7b6cfcca627a224914367c77be70181ec4a3f8d..d8b265f1a33be9aed413dedcd737193457bf71ee 100644 (file)
--- a/net/ipv4/udp.c
+++ b/net/ipv4/udp.c
@@ -1359,7 +1359,8 @@ void skb_consume_udp(struct sock *sk, struct sk_buff *skb, int len)
                sk_peek_offset_bwd(sk, len);
                unlock_sock_fast(sk, slow);
        }
-       consume_skb(skb);
+
+       consume_stateless_skb(skb);
 }
 EXPORT_SYMBOL_GPL(skb_consume_udp);
 
@@ -1727,7 +1728,7 @@ static void udp_v4_rehash(struct sock *sk)
        udp_lib_rehash(sk, new_hash);
 }
 
-int __udp_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
+static int __udp_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
 {
        int rc;
 
@@ -1739,6 +1740,9 @@ int __udp_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
                sk_mark_napi_id_once(sk, skb);
        }
 
+       /* clear all pending head states while they are hot in the cache */
+       skb_release_head_state(skb);
+
        rc = __udp_enqueue_schedule_skb(sk, skb);
        if (rc < 0) {
                int is_udplite = IS_UDPLITE(sk);
@@ -1772,7 +1776,7 @@ EXPORT_SYMBOL(udp_encap_enable);
  * Note that in the success and error cases, the skb is assumed to
  * have either been requeued or freed.
  */
-int udp_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
+static int udp_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
 {
        struct udp_sock *up = udp_sk(sk);
        int is_udplite = IS_UDPLITE(sk);