net: move skb->xmit_more hint to softnet data

[linux.git] / include / linux / netdevice.h

diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index 26f69cf763f43dd1e0d61c0692f649aab21baeea..eb9f05e0863dc5d6faea2274db2b3be5b5a99f33 100644 (file)
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -194,8 +194,8 @@ struct net_device_stats {
 
 #ifdef CONFIG_RPS
 #include <linux/static_key.h>
-extern struct static_key rps_needed;
-extern struct static_key rfs_needed;
+extern struct static_key_false rps_needed;
+extern struct static_key_false rfs_needed;
 #endif
 
 struct neighbour;
@@ -986,8 +986,7 @@ struct devlink;
  *     those the driver believes to be appropriate.
  *
  * u16 (*ndo_select_queue)(struct net_device *dev, struct sk_buff *skb,
- *                         struct net_device *sb_dev,
- *                         select_queue_fallback_t fallback);
+ *                         struct net_device *sb_dev);
  *     Called to decide which queue to use when device supports multiple
  *     transmit queues.
  *
@@ -1251,8 +1250,8 @@ struct devlink;
  *     that got dropped are freed/returned via xdp_return_frame().
  *     Returns negative number, means general error invoking ndo, meaning
  *     no frames were xmit'ed and core-caller will free all frames.
- * struct devlink *(*ndo_get_devlink)(struct net_device *dev);
- *     Get devlink instance associated with a given netdev.
+ * struct devlink_port *(*ndo_get_devlink_port)(struct net_device *dev);
+ *     Get devlink port instance associated with a given netdev.
  *     Called with a reference on the netdevice and devlink locks only,
  *     rtnl_lock is not held.
  */
@@ -1268,8 +1267,7 @@ struct net_device_ops {
                                                      netdev_features_t features);
        u16                     (*ndo_select_queue)(struct net_device *dev,
                                                    struct sk_buff *skb,
-                                                   struct net_device *sb_dev,
-                                                   select_queue_fallback_t fallback);
+                                                   struct net_device *sb_dev);
        void                    (*ndo_change_rx_flags)(struct net_device *dev,
                                                       int flags);
        void                    (*ndo_set_rx_mode)(struct net_device *dev);
@@ -1453,7 +1451,7 @@ struct net_device_ops {
                                                u32 flags);
        int                     (*ndo_xsk_async_xmit)(struct net_device *dev,
                                                      u32 queue_id);
-       struct devlink *        (*ndo_get_devlink)(struct net_device *dev);
+       struct devlink_port *   (*ndo_get_devlink_port)(struct net_device *dev);
 };
 
 /**
@@ -2152,9 +2150,11 @@ static inline void netdev_for_each_tx_queue(struct net_device *dev,
                                  &qdisc_xmit_lock_key);        \
 }
 
-struct netdev_queue *netdev_pick_tx(struct net_device *dev,
-                                   struct sk_buff *skb,
-                                   struct net_device *sb_dev);
+u16 netdev_pick_tx(struct net_device *dev, struct sk_buff *skb,
+                    struct net_device *sb_dev);
+struct netdev_queue *netdev_core_pick_tx(struct net_device *dev,
+                                        struct sk_buff *skb,
+                                        struct net_device *sb_dev);
 
 /* returns the headroom that the master device needs to take in account
  * when forwarding to this dev
@@ -2639,11 +2639,9 @@ void dev_close_many(struct list_head *head, bool unlink);
 void dev_disable_lro(struct net_device *dev);
 int dev_loopback_xmit(struct net *net, struct sock *sk, struct sk_buff *newskb);
 u16 dev_pick_tx_zero(struct net_device *dev, struct sk_buff *skb,
-                    struct net_device *sb_dev,
-                    select_queue_fallback_t fallback);
+                    struct net_device *sb_dev);
 u16 dev_pick_tx_cpu_id(struct net_device *dev, struct sk_buff *skb,
-                      struct net_device *sb_dev,
-                      select_queue_fallback_t fallback);
+                      struct net_device *sb_dev);
 int dev_queue_xmit(struct sk_buff *skb);
 int dev_queue_xmit_accel(struct sk_buff *skb, struct net_device *sb_dev);
 int dev_direct_xmit(struct sk_buff *skb, u16 queue_id);
@@ -2661,14 +2659,6 @@ void netdev_freemem(struct net_device *dev);
 void synchronize_net(void);
 int init_dummy_netdev(struct net_device *dev);
 
-DECLARE_PER_CPU(int, xmit_recursion);
-#define XMIT_RECURSION_LIMIT   10
-
-static inline int dev_recursion_level(void)
-{
-       return this_cpu_read(xmit_recursion);
-}
-
 struct net_device *dev_get_by_index(struct net *net, int ifindex);
 struct net_device *__dev_get_by_index(struct net *net, int ifindex);
 struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex);
@@ -3017,6 +3007,11 @@ struct softnet_data {
 #ifdef CONFIG_XFRM_OFFLOAD
        struct sk_buff_head     xfrm_backlog;
 #endif
+       /* written and read only by owning cpu: */
+       struct {
+               u16 recursion;
+               u8  more;
+       } xmit;
 #ifdef CONFIG_RPS
        /* input_queue_head should be written by cpu owning this struct,
         * and only read by other cpus. Worth using a cache line.
@@ -3052,6 +3047,28 @@ static inline void input_queue_tail_incr_save(struct softnet_data *sd,
 
 DECLARE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
 
+static inline int dev_recursion_level(void)
+{
+       return __this_cpu_read(softnet_data.xmit.recursion);
+}
+
+#define XMIT_RECURSION_LIMIT   10
+static inline bool dev_xmit_recursion(void)
+{
+       return unlikely(__this_cpu_read(softnet_data.xmit.recursion) >
+                       XMIT_RECURSION_LIMIT);
+}
+
+static inline void dev_xmit_recursion_inc(void)
+{
+       __this_cpu_inc(softnet_data.xmit.recursion);
+}
+
+static inline void dev_xmit_recursion_dec(void)
+{
+       __this_cpu_dec(softnet_data.xmit.recursion);
+}
+
 void __netif_schedule(struct Qdisc *q);
 void netif_schedule_queue(struct netdev_queue *txq);
 
@@ -4407,10 +4424,15 @@ static inline netdev_tx_t __netdev_start_xmit(const struct net_device_ops *ops,
                                              struct sk_buff *skb, struct net_device *dev,
                                              bool more)
 {
-       skb->xmit_more = more ? 1 : 0;
+       __this_cpu_write(softnet_data.xmit.more, more);
        return ops->ndo_start_xmit(skb, dev);
 }
 
+static inline bool netdev_xmit_more(void)
+{
+       return __this_cpu_read(softnet_data.xmit.more);
+}
+
 static inline netdev_tx_t netdev_start_xmit(struct sk_buff *skb, struct net_device *dev,
                                            struct netdev_queue *txq, bool more)
 {