net/mlx5e: Add AF_XDP need_wakeup support

This commit adds support for the new need_wakeup feature of AF_XDP. The
applications can opt-in by using the XDP_USE_NEED_WAKEUP bind() flag.
When this feature is enabled, some behavior changes:

RX side: If the Fill Ring is empty, instead of busy-polling, set the
flag to tell the application to kick the driver when it refills the Fill
Ring.

TX side: If there are pending completions or packets queued for
transmission, set the flag to tell the application that it can skip the
sendto() syscall and save time.

The performance testing was performed on a machine with the following
configuration:

- 24 cores of Intel Xeon E5-2620 v3 @ 2.40 GHz
- Mellanox ConnectX-5 Ex with 100 Gbit/s link

The results with retpoline disabled:

       | without need_wakeup  | with need_wakeup     |
       |----------------------|----------------------|
       | one core | two cores | one core | two cores |
-------|----------|-----------|----------|-----------|
txonly | 20.1     | 33.5      | 29.0     | 34.2      |
rxdrop | 0.065    | 14.1      | 12.0     | 14.1      |
l2fwd  | 0.032    | 7.3       | 6.6      | 7.2       |

"One core" means the application and NAPI run on the same core. "Two
cores" means they are pinned to different cores.

Signed-off-by: Maxim Mikityanskiy <maximmi@mellanox.com>
Reviewed-by: Tariq Toukan <tariqt@mellanox.com>
Reviewed-by: Saeed Mahameed <saeedm@mellanox.com>
Acked-by: Jonathan Lemon <jonathan.lemon@gmail.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.h b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.h
index 307b923a136139facb90fc44fb433fcb2fa1077a..cab0e93497ae6d2cf53197e86f5ec3ee3df8607d 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.h
@@ -5,6 +5,7 @@
 #define __MLX5_EN_XSK_RX_H__
 
 #include "en.h"
+#include <net/xdp_sock.h>
 
 /* RX data path */
 
@@ -24,4 +25,17 @@ struct sk_buff *mlx5e_xsk_skb_from_cqe_linear(struct mlx5e_rq *rq,
                                              struct mlx5e_wqe_frag_info *wi,
                                              u32 cqe_bcnt);
 
+static inline bool mlx5e_xsk_update_rx_wakeup(struct mlx5e_rq *rq, bool alloc_err)
+{
+       if (!xsk_umem_uses_need_wakeup(rq->umem))
+               return alloc_err;
+
+       if (unlikely(alloc_err))
+               xsk_set_rx_need_wakeup(rq->umem);
+       else
+               xsk_clear_rx_need_wakeup(rq->umem);
+
+       return false;
+}
+
 #endif /* __MLX5_EN_XSK_RX_H__ */

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.h b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.h
index 9c505158b975b5ac0063c70d511d6753dbf136b4..79b487d897570d1c7194d3b4e9b02de29078d246 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.h
@@ -5,6 +5,7 @@
 #define __MLX5_EN_XSK_TX_H__
 
 #include "en.h"
+#include <net/xdp_sock.h>
 
 /* TX data path */
 
@@ -12,4 +13,15 @@ int mlx5e_xsk_wakeup(struct net_device *dev, u32 qid, u32 flags);
 
 bool mlx5e_xsk_tx(struct mlx5e_xdpsq *sq, unsigned int budget);
 
+static inline void mlx5e_xsk_update_tx_wakeup(struct mlx5e_xdpsq *sq)
+{
+       if (!xsk_umem_uses_need_wakeup(sq->umem))
+               return;
+
+       if (sq->pc != sq->cc)
+               xsk_clear_tx_need_wakeup(sq->umem);
+       else
+               xsk_set_tx_need_wakeup(sq->umem);
+}
+
 #endif /* __MLX5_EN_XSK_TX_H__ */

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
index 60570b442fffe6d0ac29fa56a7f457f9475ff78e..fae0694d1cf80608dd5c50121610b2e9807c9a43 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
@@ -692,8 +692,11 @@ bool mlx5e_post_rx_mpwqes(struct mlx5e_rq *rq)
        rq->mpwqe.umr_in_progress += rq->mpwqe.umr_last_bulk;
        rq->mpwqe.actual_wq_head   = head;
 
-       /* If XSK Fill Ring doesn't have enough frames, busy poll by
-        * rescheduling the NAPI poll.
+       /* If XSK Fill Ring doesn't have enough frames, report the error, so
+        * that one of the actions can be performed:
+        * 1. If need_wakeup is used, signal that the application has to kick
+        * the driver when it refills the Fill Ring.
+        * 2. Otherwise, busy poll by rescheduling the NAPI poll.
         */
        if (unlikely(alloc_err == -ENOMEM && rq->umem))
                return true;

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c
index 6d16dee38edef2674237e79a0d6077a867955d3f..257a7c9f7a14d4cef416ec5992ed4e4c21832f43 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c
@@ -33,6 +33,7 @@
 #include <linux/irq.h>
 #include "en.h"
 #include "en/xdp.h"
+#include "en/xsk/rx.h"
 #include "en/xsk/tx.h"
 
 static inline bool mlx5e_channel_no_affinity_change(struct mlx5e_channel *c)
@@ -83,10 +84,23 @@ void mlx5e_trigger_irq(struct mlx5e_icosq *sq)
 
 static bool mlx5e_napi_xsk_post(struct mlx5e_xdpsq *xsksq, struct mlx5e_rq *xskrq)
 {
-       bool busy_xsk = false;
-
+       bool busy_xsk = false, xsk_rx_alloc_err;
+
+       /* Handle the race between the application querying need_wakeup and the
+        * driver setting it:
+        * 1. Update need_wakeup both before and after the TX. If it goes to
+        * "yes", it can only happen with the first update.
+        * 2. If the application queried need_wakeup before we set it, the
+        * packets will be transmitted anyway, even w/o a wakeup.
+        * 3. Give a chance to clear need_wakeup after new packets were queued
+        * for TX.
+        */
+       mlx5e_xsk_update_tx_wakeup(xsksq);
        busy_xsk |= mlx5e_xsk_tx(xsksq, MLX5E_TX_XSK_POLL_BUDGET);
-       busy_xsk |= xskrq->post_wqes(xskrq);
+       mlx5e_xsk_update_tx_wakeup(xsksq);
+
+       xsk_rx_alloc_err = xskrq->post_wqes(xskrq);
+       busy_xsk |= mlx5e_xsk_update_rx_wakeup(xskrq, xsk_rx_alloc_err);
 
        return busy_xsk;
 }