Merge git://git.kernel.org/pub/scm/linux/kernel/git/davem/net

[linux.git] / include / net / sock.h

diff --git a/include/net/sock.h b/include/net/sock.h
index 341f8bafa0cf585fc72e5819054f1b2f15a8e338..bdd77bbce7d887dcc2dc90b754ae016628d559c7 100644 (file)
--- a/include/net/sock.h
+++ b/include/net/sock.h
@@ -368,6 +368,7 @@ struct sock {
        atomic_t                sk_drops;
        int                     sk_rcvlowat;
        struct sk_buff_head     sk_error_queue;
+       struct sk_buff          *sk_rx_skb_cache;
        struct sk_buff_head     sk_receive_queue;
        /*
         * The backlog queue is special, it is always used with
@@ -414,6 +415,7 @@ struct sock {
                struct sk_buff  *sk_send_head;
                struct rb_root  tcp_rtx_queue;
        };
+       struct sk_buff          *sk_tx_skb_cache;
        struct sk_buff_head     sk_write_queue;
        __s32                   sk_peek_off;
        int                     sk_write_pending;
@@ -966,7 +968,7 @@ static inline void sock_rps_record_flow_hash(__u32 hash)
 static inline void sock_rps_record_flow(const struct sock *sk)
 {
 #ifdef CONFIG_RPS
-       if (static_key_false(&rfs_needed)) {
+       if (static_branch_unlikely(&rfs_needed)) {
                /* Reading sk->sk_rxhash might incur an expensive cache line
                 * miss.
                 *
@@ -1466,6 +1468,11 @@ static inline void sk_wmem_free_skb(struct sock *sk, struct sk_buff *skb)
        sock_set_flag(sk, SOCK_QUEUE_SHRUNK);
        sk->sk_wmem_queued -= skb->truesize;
        sk_mem_uncharge(sk, skb->truesize);
+       if (!sk->sk_tx_skb_cache) {
+               skb_zcopy_clear(skb, true);
+               sk->sk_tx_skb_cache = skb;
+               return;
+       }
        __kfree_skb(skb);
 }
 
@@ -2427,6 +2434,15 @@ static inline void skb_setup_tx_timestamp(struct sk_buff *skb, __u16 tsflags)
 static inline void sk_eat_skb(struct sock *sk, struct sk_buff *skb)
 {
        __skb_unlink(skb, &sk->sk_receive_queue);
+       if (
+#ifdef CONFIG_RPS
+           !static_branch_unlikely(&rps_needed) &&
+#endif
+           !sk->sk_rx_skb_cache) {
+               sk->sk_rx_skb_cache = skb;
+               skb_orphan(skb);
+               return;
+       }
        __kfree_skb(skb);
 }