samples/bpf: add use of need_wakeup flag in xdpsock

This commit adds using the need_wakeup flag to the xdpsock sample
application. It is turned on by default as we think it is a feature
that seems to always produce a performance benefit, if the application
has been written taking advantage of it. It can be turned off in the
sample app by using the '-m' command line option.

The txpush and l2fwd sub applications have also been updated to
support poll() with multiple sockets.

Signed-off-by: Magnus Karlsson <magnus.karlsson@intel.com>
Acked-by: Jonathan Lemon <jonathan.lemon@gmail.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>

diff --git a/samples/bpf/xdpsock_user.c b/samples/bpf/xdpsock_user.c
index 93eaaf7239b293d366fedb57ac1e6e7df643385d..da84c760c0946b1a7ae3d0ffc692f8b42ffad501 100644 (file)
--- a/samples/bpf/xdpsock_user.c
+++ b/samples/bpf/xdpsock_user.c
@@ -67,8 +67,10 @@ static int opt_ifindex;
 static int opt_queue;
 static int opt_poll;
 static int opt_interval = 1;
-static u32 opt_xdp_bind_flags;
+static u32 opt_xdp_bind_flags = XDP_USE_NEED_WAKEUP;
 static int opt_xsk_frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE;
+static int opt_timeout = 1000;
+static bool opt_need_wakeup = true;
 static __u32 prog_id;
 
 struct xsk_umem_info {
@@ -352,6 +354,7 @@ static struct option long_options[] = {
        {"zero-copy", no_argument, 0, 'z'},
        {"copy", no_argument, 0, 'c'},
        {"frame-size", required_argument, 0, 'f'},
+       {"no-need-wakeup", no_argument, 0, 'm'},
        {0, 0, 0, 0}
 };
 
@@ -372,6 +375,7 @@ static void usage(const char *prog)
                "  -z, --zero-copy      Force zero-copy mode.\n"
                "  -c, --copy           Force copy mode.\n"
                "  -f, --frame-size=n   Set the frame size (must be a power of two, default is %d).\n"
+               "  -m, --no-need-wakeup Turn off use of driver need wakeup flag.\n"
                "\n";
        fprintf(stderr, str, prog, XSK_UMEM__DEFAULT_FRAME_SIZE);
        exit(EXIT_FAILURE);
@@ -384,8 +388,9 @@ static void parse_command_line(int argc, char **argv)
        opterr = 0;
 
        for (;;) {
-               c = getopt_long(argc, argv, "Frtli:q:psSNn:czf:", long_options,
-                               &option_index);
+
+               c = getopt_long(argc, argv, "Frtli:q:psSNn:czf:m",
+                               long_options, &option_index);
                if (c == -1)
                        break;
 
@@ -429,6 +434,9 @@ static void parse_command_line(int argc, char **argv)
                        break;
                case 'f':
                        opt_xsk_frame_size = atoi(optarg);
+               case 'm':
+                       opt_need_wakeup = false;
+                       opt_xdp_bind_flags &= ~XDP_USE_NEED_WAKEUP;
                        break;
                default:
                        usage(basename(argv[0]));
@@ -459,7 +467,8 @@ static void kick_tx(struct xsk_socket_info *xsk)
        exit_with_error(errno);
 }
 
-static inline void complete_tx_l2fwd(struct xsk_socket_info *xsk)
+static inline void complete_tx_l2fwd(struct xsk_socket_info *xsk,
+                                    struct pollfd *fds)
 {
        u32 idx_cq = 0, idx_fq = 0;
        unsigned int rcvd;
@@ -468,7 +477,9 @@ static inline void complete_tx_l2fwd(struct xsk_socket_info *xsk)
        if (!xsk->outstanding_tx)
                return;
 
-       kick_tx(xsk);
+       if (!opt_need_wakeup || xsk_ring_prod__needs_wakeup(&xsk->tx))
+               kick_tx(xsk);
+
        ndescs = (xsk->outstanding_tx > BATCH_SIZE) ? BATCH_SIZE :
                xsk->outstanding_tx;
 
@@ -482,6 +493,8 @@ static inline void complete_tx_l2fwd(struct xsk_socket_info *xsk)
                while (ret != rcvd) {
                        if (ret < 0)
                                exit_with_error(-ret);
+                       if (xsk_ring_prod__needs_wakeup(&xsk->umem->fq))
+                               ret = poll(fds, num_socks, opt_timeout);
                        ret = xsk_ring_prod__reserve(&xsk->umem->fq, rcvd,
                                                     &idx_fq);
                }
@@ -505,7 +518,8 @@ static inline void complete_tx_only(struct xsk_socket_info *xsk)
        if (!xsk->outstanding_tx)
                return;
 
-       kick_tx(xsk);
+       if (!opt_need_wakeup || xsk_ring_prod__needs_wakeup(&xsk->tx))
+               kick_tx(xsk);
 
        rcvd = xsk_ring_cons__peek(&xsk->umem->cq, BATCH_SIZE, &idx);
        if (rcvd > 0) {
@@ -515,20 +529,25 @@ static inline void complete_tx_only(struct xsk_socket_info *xsk)
        }
 }
 
-static void rx_drop(struct xsk_socket_info *xsk)
+static void rx_drop(struct xsk_socket_info *xsk, struct pollfd *fds)
 {
        unsigned int rcvd, i;
        u32 idx_rx = 0, idx_fq = 0;
        int ret;
 
        rcvd = xsk_ring_cons__peek(&xsk->rx, BATCH_SIZE, &idx_rx);
-       if (!rcvd)
+       if (!rcvd) {
+               if (xsk_ring_prod__needs_wakeup(&xsk->umem->fq))
+                       ret = poll(fds, num_socks, opt_timeout);
                return;
+       }
 
        ret = xsk_ring_prod__reserve(&xsk->umem->fq, rcvd, &idx_fq);
        while (ret != rcvd) {
                if (ret < 0)
                        exit_with_error(-ret);
+               if (xsk_ring_prod__needs_wakeup(&xsk->umem->fq))
+                       ret = poll(fds, num_socks, opt_timeout);
                ret = xsk_ring_prod__reserve(&xsk->umem->fq, rcvd, &idx_fq);
        }
 
@@ -549,42 +568,65 @@ static void rx_drop(struct xsk_socket_info *xsk)
 static void rx_drop_all(void)
 {
        struct pollfd fds[MAX_SOCKS + 1];
-       int i, ret, timeout, nfds = 1;
+       int i, ret;
 
        memset(fds, 0, sizeof(fds));
 
        for (i = 0; i < num_socks; i++) {
                fds[i].fd = xsk_socket__fd(xsks[i]->xsk);
                fds[i].events = POLLIN;
-               timeout = 1000; /* 1sn */
        }
 
        for (;;) {
                if (opt_poll) {
-                       ret = poll(fds, nfds, timeout);
+                       ret = poll(fds, num_socks, opt_timeout);
                        if (ret <= 0)
                                continue;
                }
 
                for (i = 0; i < num_socks; i++)
-                       rx_drop(xsks[i]);
+                       rx_drop(xsks[i], fds);
+       }
+}
+
+static void tx_only(struct xsk_socket_info *xsk, u32 frame_nb)
+{
+       u32 idx;
+
+       if (xsk_ring_prod__reserve(&xsk->tx, BATCH_SIZE, &idx) == BATCH_SIZE) {
+               unsigned int i;
+
+               for (i = 0; i < BATCH_SIZE; i++) {
+                       xsk_ring_prod__tx_desc(&xsk->tx, idx + i)->addr =
+                               (frame_nb + i) << XSK_UMEM__DEFAULT_FRAME_SHIFT;
+                       xsk_ring_prod__tx_desc(&xsk->tx, idx + i)->len =
+                               sizeof(pkt_data) - 1;
+               }
+
+               xsk_ring_prod__submit(&xsk->tx, BATCH_SIZE);
+               xsk->outstanding_tx += BATCH_SIZE;
+               frame_nb += BATCH_SIZE;
+               frame_nb %= NUM_FRAMES;
        }
+
+       complete_tx_only(xsk);
 }
 
-static void tx_only(struct xsk_socket_info *xsk)
+static void tx_only_all(void)
 {
-       int timeout, ret, nfds = 1;
-       struct pollfd fds[nfds + 1];
-       u32 idx, frame_nb = 0;
+       struct pollfd fds[MAX_SOCKS];
+       u32 frame_nb[MAX_SOCKS] = {};
+       int i, ret;
 
        memset(fds, 0, sizeof(fds));
-       fds[0].fd = xsk_socket__fd(xsk->xsk);
-       fds[0].events = POLLOUT;
-       timeout = 1000; /* 1sn */
+       for (i = 0; i < num_socks; i++) {
+               fds[0].fd = xsk_socket__fd(xsks[i]->xsk);
+               fds[0].events = POLLOUT;
+       }
 
        for (;;) {
                if (opt_poll) {
-                       ret = poll(fds, nfds, timeout);
+                       ret = poll(fds, num_socks, opt_timeout);
                        if (ret <= 0)
                                continue;
 
@@ -592,69 +634,75 @@ static void tx_only(struct xsk_socket_info *xsk)
                                continue;
                }
 
-               if (xsk_ring_prod__reserve(&xsk->tx, BATCH_SIZE, &idx) ==
-                   BATCH_SIZE) {
-                       unsigned int i;
-
-                       for (i = 0; i < BATCH_SIZE; i++) {
-                               xsk_ring_prod__tx_desc(&xsk->tx, idx + i)->addr
-                                       = (frame_nb + i) * opt_xsk_frame_size;
-                               xsk_ring_prod__tx_desc(&xsk->tx, idx + i)->len =
-                                       sizeof(pkt_data) - 1;
-                       }
-
-                       xsk_ring_prod__submit(&xsk->tx, BATCH_SIZE);
-                       xsk->outstanding_tx += BATCH_SIZE;
-                       frame_nb += BATCH_SIZE;
-                       frame_nb %= NUM_FRAMES;
-               }
-
-               complete_tx_only(xsk);
+               for (i = 0; i < num_socks; i++)
+                       tx_only(xsks[i], frame_nb[i]);
        }
 }
 
-static void l2fwd(struct xsk_socket_info *xsk)
+static void l2fwd(struct xsk_socket_info *xsk, struct pollfd *fds)
 {
-       for (;;) {
-               unsigned int rcvd, i;
-               u32 idx_rx = 0, idx_tx = 0;
-               int ret;
+       unsigned int rcvd, i;
+       u32 idx_rx = 0, idx_tx = 0;
+       int ret;
 
-               for (;;) {
-                       complete_tx_l2fwd(xsk);
+       complete_tx_l2fwd(xsk, fds);
 
-                       rcvd = xsk_ring_cons__peek(&xsk->rx, BATCH_SIZE,
-                                                  &idx_rx);
-                       if (rcvd > 0)
-                               break;
-               }
+       rcvd = xsk_ring_cons__peek(&xsk->rx, BATCH_SIZE, &idx_rx);
+       if (!rcvd) {
+               if (xsk_ring_prod__needs_wakeup(&xsk->umem->fq))
+                       ret = poll(fds, num_socks, opt_timeout);
+               return;
+       }
 
+       ret = xsk_ring_prod__reserve(&xsk->tx, rcvd, &idx_tx);
+       while (ret != rcvd) {
+               if (ret < 0)
+                       exit_with_error(-ret);
+               if (xsk_ring_prod__needs_wakeup(&xsk->tx))
+                       kick_tx(xsk);
                ret = xsk_ring_prod__reserve(&xsk->tx, rcvd, &idx_tx);
-               while (ret != rcvd) {
-                       if (ret < 0)
-                               exit_with_error(-ret);
-                       ret = xsk_ring_prod__reserve(&xsk->tx, rcvd, &idx_tx);
-               }
+       }
+
+       for (i = 0; i < rcvd; i++) {
+               u64 addr = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx)->addr;
+               u32 len = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx++)->len;
+               char *pkt = xsk_umem__get_data(xsk->umem->buffer, addr);
+
+               swap_mac_addresses(pkt);
 
-               for (i = 0; i < rcvd; i++) {
-                       u64 addr = xsk_ring_cons__rx_desc(&xsk->rx,
-                                                         idx_rx)->addr;
-                       u32 len = xsk_ring_cons__rx_desc(&xsk->rx,
-                                                        idx_rx++)->len;
-                       char *pkt = xsk_umem__get_data(xsk->umem->buffer, addr);
+               hex_dump(pkt, len, addr);
+               xsk_ring_prod__tx_desc(&xsk->tx, idx_tx)->addr = addr;
+               xsk_ring_prod__tx_desc(&xsk->tx, idx_tx++)->len = len;
+       }
 
-                       swap_mac_addresses(pkt);
+       xsk_ring_prod__submit(&xsk->tx, rcvd);
+       xsk_ring_cons__release(&xsk->rx, rcvd);
 
-                       hex_dump(pkt, len, addr);
-                       xsk_ring_prod__tx_desc(&xsk->tx, idx_tx)->addr = addr;
-                       xsk_ring_prod__tx_desc(&xsk->tx, idx_tx++)->len = len;
-               }
+       xsk->rx_npkts += rcvd;
+       xsk->outstanding_tx += rcvd;
+}
+
+static void l2fwd_all(void)
+{
+       struct pollfd fds[MAX_SOCKS];
+       int i, ret;
+
+       memset(fds, 0, sizeof(fds));
+
+       for (i = 0; i < num_socks; i++) {
+               fds[i].fd = xsk_socket__fd(xsks[i]->xsk);
+               fds[i].events = POLLOUT | POLLIN;
+       }
 
-               xsk_ring_prod__submit(&xsk->tx, rcvd);
-               xsk_ring_cons__release(&xsk->rx, rcvd);
+       for (;;) {
+               if (opt_poll) {
+                       ret = poll(fds, num_socks, opt_timeout);
+                       if (ret <= 0)
+                               continue;
+               }
 
-               xsk->rx_npkts += rcvd;
-               xsk->outstanding_tx += rcvd;
+               for (i = 0; i < num_socks; i++)
+                       l2fwd(xsks[i], fds);
        }
 }
 
@@ -705,9 +753,9 @@ int main(int argc, char **argv)
        if (opt_bench == BENCH_RXDROP)
                rx_drop_all();
        else if (opt_bench == BENCH_TXONLY)
-               tx_only(xsks[0]);
+               tx_only_all();
        else
-               l2fwd(xsks[0]);
+               l2fwd_all();
 
        return 0;
 }