udp: avoid a cache miss on dequeue

[linux.git] / net / core / skbuff.c

diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index e508c1eae67fb433ae165875876450fd4307a7b2..304602784c3b78249c73609e8aad9cbc55e70e66 100644 (file)
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@ -643,12 +643,10 @@ static void kfree_skbmem(struct sk_buff *skb)
        kmem_cache_free(skbuff_fclone_cache, fclones);
 }
 
-static void skb_release_head_state(struct sk_buff *skb)
+void skb_release_head_state(struct sk_buff *skb)
 {
        skb_dst_drop(skb);
-#ifdef CONFIG_XFRM
-       secpath_put(skb->sp);
-#endif
+       secpath_reset(skb);
        if (skb->destructor) {
                WARN_ON(in_irq());
                skb->destructor(skb);
@@ -694,12 +692,9 @@ EXPORT_SYMBOL(__kfree_skb);
  */
 void kfree_skb(struct sk_buff *skb)
 {
-       if (unlikely(!skb))
-               return;
-       if (likely(atomic_read(&skb->users) == 1))
-               smp_rmb();
-       else if (likely(!atomic_dec_and_test(&skb->users)))
+       if (!skb_unref(skb))
                return;
+
        trace_kfree_skb(skb, __builtin_return_address(0));
        __kfree_skb(skb);
 }
@@ -746,17 +741,32 @@ EXPORT_SYMBOL(skb_tx_error);
  */
 void consume_skb(struct sk_buff *skb)
 {
-       if (unlikely(!skb))
-               return;
-       if (likely(atomic_read(&skb->users) == 1))
-               smp_rmb();
-       else if (likely(!atomic_dec_and_test(&skb->users)))
+       if (!skb_unref(skb))
                return;
+
        trace_consume_skb(skb);
        __kfree_skb(skb);
 }
 EXPORT_SYMBOL(consume_skb);
 
+/**
+ *     consume_stateless_skb - free an skbuff, assuming it is stateless
+ *     @skb: buffer to free
+ *
+ *     Works like consume_skb(), but this variant assumes that all the head
+ *     states have been already dropped.
+ */
+void consume_stateless_skb(struct sk_buff *skb)
+{
+       if (!skb_unref(skb))
+               return;
+
+       trace_consume_skb(skb);
+       if (likely(skb->head))
+               skb_release_data(skb);
+       kfree_skbmem(skb);
+}
+
 void __kfree_skb_flush(void)
 {
        struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);