xdp: implement convert_to_xdp_frame for MEM_TYPE_ZERO_COPY

This commit adds proper MEM_TYPE_ZERO_COPY support for
convert_to_xdp_frame. Converting a MEM_TYPE_ZERO_COPY xdp_buff to an
xdp_frame is done by transforming the MEM_TYPE_ZERO_COPY buffer into a
MEM_TYPE_PAGE_ORDER0 frame. This is costly, and in the future it might
make sense to implement a more sophisticated thread-safe alloc/free
scheme for MEM_TYPE_ZERO_COPY, so that no allocation and copy is
required in the fast-path.

Signed-off-by: Björn Töpel <bjorn.topel@intel.com>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

diff --git a/include/net/xdp.h b/include/net/xdp.h
index 76b95256c266495431a1c375f14cfc7ed76b0247..0d5c6fb4b2e266f84c706df2d9c3f6e10fe8a6bc 100644 (file)
--- a/include/net/xdp.h
+++ b/include/net/xdp.h
@@ -91,6 +91,8 @@ static inline void xdp_scrub_frame(struct xdp_frame *frame)
        frame->dev_rx = NULL;
 }
 
+struct xdp_frame *xdp_convert_zc_to_xdp_frame(struct xdp_buff *xdp);
+
 /* Convert xdp_buff to xdp_frame */
 static inline
 struct xdp_frame *convert_to_xdp_frame(struct xdp_buff *xdp)
@@ -99,9 +101,8 @@ struct xdp_frame *convert_to_xdp_frame(struct xdp_buff *xdp)
        int metasize;
        int headroom;
 
-       /* TODO: implement clone, copy, use "native" MEM_TYPE */
        if (xdp->rxq->mem.type == MEM_TYPE_ZERO_COPY)
-               return NULL;
+               return xdp_convert_zc_to_xdp_frame(xdp);
 
        /* Assure headroom is available for storing info */
        headroom = xdp->data - xdp->data_hard_start;

diff --git a/net/core/xdp.c b/net/core/xdp.c
index 89b6785cef2ad5db33557f9209c9038427fa5220..be6cb2f0e7222a254cd3d8c896038c2c18a8f739 100644 (file)
--- a/net/core/xdp.c
+++ b/net/core/xdp.c
@@ -398,3 +398,42 @@ void xdp_attachment_setup(struct xdp_attachment_info *info,
        info->flags = bpf->flags;
 }
 EXPORT_SYMBOL_GPL(xdp_attachment_setup);
+
+struct xdp_frame *xdp_convert_zc_to_xdp_frame(struct xdp_buff *xdp)
+{
+       unsigned int metasize, headroom, totsize;
+       void *addr, *data_to_copy;
+       struct xdp_frame *xdpf;
+       struct page *page;
+
+       /* Clone into a MEM_TYPE_PAGE_ORDER0 xdp_frame. */
+       metasize = xdp_data_meta_unsupported(xdp) ? 0 :
+                  xdp->data - xdp->data_meta;
+       headroom = xdp->data - xdp->data_hard_start;
+       totsize = xdp->data_end - xdp->data + metasize;
+
+       if (sizeof(*xdpf) + totsize > PAGE_SIZE)
+               return NULL;
+
+       page = dev_alloc_page();
+       if (!page)
+               return NULL;
+
+       addr = page_to_virt(page);
+       xdpf = addr;
+       memset(xdpf, 0, sizeof(*xdpf));
+
+       addr += sizeof(*xdpf);
+       data_to_copy = metasize ? xdp->data_meta : xdp->data;
+       memcpy(addr, data_to_copy, totsize);
+
+       xdpf->data = addr + metasize;
+       xdpf->len = totsize - metasize;
+       xdpf->headroom = 0;
+       xdpf->metasize = metasize;
+       xdpf->mem.type = MEM_TYPE_PAGE_ORDER0;
+
+       xdp_return_buff(xdp);
+       return xdpf;
+}
+EXPORT_SYMBOL_GPL(xdp_convert_zc_to_xdp_frame);