drivers/net/ethernet/intel/ice/ice_xsk.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright (c) 2019, Intel Corporation. */
   3
   4 #include <linux/bpf_trace.h>
   5 #include <net/xdp_sock.h>
   6 #include <net/xdp.h>
   7 #include "ice.h"
   8 #include "ice_base.h"
   9 #include "ice_type.h"
  10 #include "ice_xsk.h"
  11 #include "ice_txrx.h"
  12 #include "ice_txrx_lib.h"
  13 #include "ice_lib.h"
  14
  15 /**
  16  * ice_qp_reset_stats - Resets all stats for rings of given index
  17  * @vsi: VSI that contains rings of interest
  18  * @q_idx: ring index in array
  19  */
  20 static void ice_qp_reset_stats(struct ice_vsi *vsi, u16 q_idx)
  21 {
  22         memset(&vsi->rx_rings[q_idx]->rx_stats, 0,
  23                sizeof(vsi->rx_rings[q_idx]->rx_stats));
  24         memset(&vsi->tx_rings[q_idx]->stats, 0,
  25                sizeof(vsi->tx_rings[q_idx]->stats));
  26         if (ice_is_xdp_ena_vsi(vsi))
  27                 memset(&vsi->xdp_rings[q_idx]->stats, 0,
  28                        sizeof(vsi->xdp_rings[q_idx]->stats));
  29 }
  30
  31 /**
  32  * ice_qp_clean_rings - Cleans all the rings of a given index
  33  * @vsi: VSI that contains rings of interest
  34  * @q_idx: ring index in array
  35  */
  36 static void ice_qp_clean_rings(struct ice_vsi *vsi, u16 q_idx)
  37 {
  38         ice_clean_tx_ring(vsi->tx_rings[q_idx]);
  39         if (ice_is_xdp_ena_vsi(vsi))
  40                 ice_clean_tx_ring(vsi->xdp_rings[q_idx]);
  41         ice_clean_rx_ring(vsi->rx_rings[q_idx]);
  42 }
  43
  44 /**
  45  * ice_qvec_toggle_napi - Enables/disables NAPI for a given q_vector
  46  * @vsi: VSI that has netdev
  47  * @q_vector: q_vector that has NAPI context
  48  * @enable: true for enable, false for disable
  49  */
  50 static void
  51 ice_qvec_toggle_napi(struct ice_vsi *vsi, struct ice_q_vector *q_vector,
  52                      bool enable)
  53 {
  54         if (!vsi->netdev || !q_vector)
  55                 return;
  56
  57         if (enable)
  58                 napi_enable(&q_vector->napi);
  59         else
  60                 napi_disable(&q_vector->napi);
  61 }
  62
  63 /**
  64  * ice_qvec_dis_irq - Mask off queue interrupt generation on given ring
  65  * @vsi: the VSI that contains queue vector being un-configured
  66  * @rx_ring: Rx ring that will have its IRQ disabled
  67  * @q_vector: queue vector
  68  */
  69 static void
  70 ice_qvec_dis_irq(struct ice_vsi *vsi, struct ice_ring *rx_ring,
  71                  struct ice_q_vector *q_vector)
  72 {
  73         struct ice_pf *pf = vsi->back;
  74         struct ice_hw *hw = &pf->hw;
  75         int base = vsi->base_vector;
  76         u16 reg;
  77         u32 val;
  78
  79         /* QINT_TQCTL is being cleared in ice_vsi_stop_tx_ring, so handle
  80          * here only QINT_RQCTL
  81          */
  82         reg = rx_ring->reg_idx;
  83         val = rd32(hw, QINT_RQCTL(reg));
  84         val &= ~QINT_RQCTL_CAUSE_ENA_M;
  85         wr32(hw, QINT_RQCTL(reg), val);
  86
  87         if (q_vector) {
  88                 u16 v_idx = q_vector->v_idx;
  89
  90                 wr32(hw, GLINT_DYN_CTL(q_vector->reg_idx), 0);
  91                 ice_flush(hw);
  92                 synchronize_irq(pf->msix_entries[v_idx + base].vector);
  93         }
  94 }
  95
  96 /**
  97  * ice_qvec_cfg_msix - Enable IRQ for given queue vector
  98  * @vsi: the VSI that contains queue vector
  99  * @q_vector: queue vector
 100  */
 101 static void
 102 ice_qvec_cfg_msix(struct ice_vsi *vsi, struct ice_q_vector *q_vector)
 103 {
 104         u16 reg_idx = q_vector->reg_idx;
 105         struct ice_pf *pf = vsi->back;
 106         struct ice_hw *hw = &pf->hw;
 107         struct ice_ring *ring;
 108
 109         ice_cfg_itr(hw, q_vector);
 110
 111         wr32(hw, GLINT_RATE(reg_idx),
 112              ice_intrl_usec_to_reg(q_vector->intrl, hw->intrl_gran));
 113
 114         ice_for_each_ring(ring, q_vector->tx)
 115                 ice_cfg_txq_interrupt(vsi, ring->reg_idx, reg_idx,
 116                                       q_vector->tx.itr_idx);
 117
 118         ice_for_each_ring(ring, q_vector->rx)
 119                 ice_cfg_rxq_interrupt(vsi, ring->reg_idx, reg_idx,
 120                                       q_vector->rx.itr_idx);
 121
 122         ice_flush(hw);
 123 }
 124
 125 /**
 126  * ice_qvec_ena_irq - Enable IRQ for given queue vector
 127  * @vsi: the VSI that contains queue vector
 128  * @q_vector: queue vector
 129  */
 130 static void ice_qvec_ena_irq(struct ice_vsi *vsi, struct ice_q_vector *q_vector)
 131 {
 132         struct ice_pf *pf = vsi->back;
 133         struct ice_hw *hw = &pf->hw;
 134
 135         ice_irq_dynamic_ena(hw, vsi, q_vector);
 136
 137         ice_flush(hw);
 138 }
 139
 140 /**
 141  * ice_qp_dis - Disables a queue pair
 142  * @vsi: VSI of interest
 143  * @q_idx: ring index in array
 144  *
 145  * Returns 0 on success, negative on failure.
 146  */
 147 static int ice_qp_dis(struct ice_vsi *vsi, u16 q_idx)
 148 {
 149         struct ice_txq_meta txq_meta = { };
 150         struct ice_ring *tx_ring, *rx_ring;
 151         struct ice_q_vector *q_vector;
 152         int timeout = 50;
 153         int err;
 154
 155         if (q_idx >= vsi->num_rxq || q_idx >= vsi->num_txq)
 156                 return -EINVAL;
 157
 158         tx_ring = vsi->tx_rings[q_idx];
 159         rx_ring = vsi->rx_rings[q_idx];
 160         q_vector = rx_ring->q_vector;
 161
 162         while (test_and_set_bit(__ICE_CFG_BUSY, vsi->state)) {
 163                 timeout--;
 164                 if (!timeout)
 165                         return -EBUSY;
 166                 usleep_range(1000, 2000);
 167         }
 168         netif_tx_stop_queue(netdev_get_tx_queue(vsi->netdev, q_idx));
 169
 170         ice_qvec_dis_irq(vsi, rx_ring, q_vector);
 171
 172         ice_fill_txq_meta(vsi, tx_ring, &txq_meta);
 173         err = ice_vsi_stop_tx_ring(vsi, ICE_NO_RESET, 0, tx_ring, &txq_meta);
 174         if (err)
 175                 return err;
 176         if (ice_is_xdp_ena_vsi(vsi)) {
 177                 struct ice_ring *xdp_ring = vsi->xdp_rings[q_idx];
 178
 179                 memset(&txq_meta, 0, sizeof(txq_meta));
 180                 ice_fill_txq_meta(vsi, xdp_ring, &txq_meta);
 181                 err = ice_vsi_stop_tx_ring(vsi, ICE_NO_RESET, 0, xdp_ring,
 182                                            &txq_meta);
 183                 if (err)
 184                         return err;
 185         }
 186         err = ice_vsi_ctrl_rx_ring(vsi, false, q_idx);
 187         if (err)
 188                 return err;
 189
 190         ice_qvec_toggle_napi(vsi, q_vector, false);
 191         ice_qp_clean_rings(vsi, q_idx);
 192         ice_qp_reset_stats(vsi, q_idx);
 193
 194         return 0;
 195 }
 196
 197 /**
 198  * ice_qp_ena - Enables a queue pair
 199  * @vsi: VSI of interest
 200  * @q_idx: ring index in array
 201  *
 202  * Returns 0 on success, negative on failure.
 203  */
 204 static int ice_qp_ena(struct ice_vsi *vsi, u16 q_idx)
 205 {
 206         struct ice_aqc_add_tx_qgrp *qg_buf;
 207         struct ice_ring *tx_ring, *rx_ring;
 208         struct ice_q_vector *q_vector;
 209         int err;
 210
 211         if (q_idx >= vsi->num_rxq || q_idx >= vsi->num_txq)
 212                 return -EINVAL;
 213
 214         qg_buf = kzalloc(sizeof(*qg_buf), GFP_KERNEL);
 215         if (!qg_buf)
 216                 return -ENOMEM;
 217
 218         qg_buf->num_txqs = 1;
 219
 220         tx_ring = vsi->tx_rings[q_idx];
 221         rx_ring = vsi->rx_rings[q_idx];
 222         q_vector = rx_ring->q_vector;
 223
 224         err = ice_vsi_cfg_txq(vsi, tx_ring, qg_buf);
 225         if (err)
 226                 goto free_buf;
 227
 228         if (ice_is_xdp_ena_vsi(vsi)) {
 229                 struct ice_ring *xdp_ring = vsi->xdp_rings[q_idx];
 230
 231                 memset(qg_buf, 0, sizeof(*qg_buf));
 232                 qg_buf->num_txqs = 1;
 233                 err = ice_vsi_cfg_txq(vsi, xdp_ring, qg_buf);
 234                 if (err)
 235                         goto free_buf;
 236                 ice_set_ring_xdp(xdp_ring);
 237                 xdp_ring->xsk_umem = ice_xsk_umem(xdp_ring);
 238         }
 239
 240         err = ice_setup_rx_ctx(rx_ring);
 241         if (err)
 242                 goto free_buf;
 243
 244         ice_qvec_cfg_msix(vsi, q_vector);
 245
 246         err = ice_vsi_ctrl_rx_ring(vsi, true, q_idx);
 247         if (err)
 248                 goto free_buf;
 249
 250         clear_bit(__ICE_CFG_BUSY, vsi->state);
 251         ice_qvec_toggle_napi(vsi, q_vector, true);
 252         ice_qvec_ena_irq(vsi, q_vector);
 253
 254         netif_tx_start_queue(netdev_get_tx_queue(vsi->netdev, q_idx));
 255 free_buf:
 256         kfree(qg_buf);
 257         return err;
 258 }
 259
 260 /**
 261  * ice_xsk_alloc_umems - allocate a UMEM region for an XDP socket
 262  * @vsi: VSI to allocate the UMEM on
 263  *
 264  * Returns 0 on success, negative on error
 265  */
 266 static int ice_xsk_alloc_umems(struct ice_vsi *vsi)
 267 {
 268         if (vsi->xsk_umems)
 269                 return 0;
 270
 271         vsi->xsk_umems = kcalloc(vsi->num_xsk_umems, sizeof(*vsi->xsk_umems),
 272                                  GFP_KERNEL);
 273
 274         if (!vsi->xsk_umems) {
 275                 vsi->num_xsk_umems = 0;
 276                 return -ENOMEM;
 277         }
 278
 279         return 0;
 280 }
 281
 282 /**
 283  * ice_xsk_add_umem - add a UMEM region for XDP sockets
 284  * @vsi: VSI to which the UMEM will be added
 285  * @umem: pointer to a requested UMEM region
 286  * @qid: queue ID
 287  *
 288  * Returns 0 on success, negative on error
 289  */
 290 static int ice_xsk_add_umem(struct ice_vsi *vsi, struct xdp_umem *umem, u16 qid)
 291 {
 292         int err;
 293
 294         err = ice_xsk_alloc_umems(vsi);
 295         if (err)
 296                 return err;
 297
 298         vsi->xsk_umems[qid] = umem;
 299         vsi->num_xsk_umems_used++;
 300
 301         return 0;
 302 }
 303
 304 /**
 305  * ice_xsk_remove_umem - Remove an UMEM for a certain ring/qid
 306  * @vsi: VSI from which the VSI will be removed
 307  * @qid: Ring/qid associated with the UMEM
 308  */
 309 static void ice_xsk_remove_umem(struct ice_vsi *vsi, u16 qid)
 310 {
 311         vsi->xsk_umems[qid] = NULL;
 312         vsi->num_xsk_umems_used--;
 313
 314         if (vsi->num_xsk_umems_used == 0) {
 315                 kfree(vsi->xsk_umems);
 316                 vsi->xsk_umems = NULL;
 317                 vsi->num_xsk_umems = 0;
 318         }
 319 }
 320
 321 /**
 322  * ice_xsk_umem_dma_map - DMA map UMEM region for XDP sockets
 323  * @vsi: VSI to map the UMEM region
 324  * @umem: UMEM to map
 325  *
 326  * Returns 0 on success, negative on error
 327  */
 328 static int ice_xsk_umem_dma_map(struct ice_vsi *vsi, struct xdp_umem *umem)
 329 {
 330         struct ice_pf *pf = vsi->back;
 331         struct device *dev;
 332         unsigned int i;
 333
 334         dev = ice_pf_to_dev(pf);
 335         for (i = 0; i < umem->npgs; i++) {
 336                 dma_addr_t dma = dma_map_page_attrs(dev, umem->pgs[i], 0,
 337                                                     PAGE_SIZE,
 338                                                     DMA_BIDIRECTIONAL,
 339                                                     ICE_RX_DMA_ATTR);
 340                 if (dma_mapping_error(dev, dma)) {
 341                         dev_dbg(dev,
 342                                 "XSK UMEM DMA mapping error on page num %d", i);
 343                         goto out_unmap;
 344                 }
 345
 346                 umem->pages[i].dma = dma;
 347         }
 348
 349         return 0;
 350
 351 out_unmap:
 352         for (; i > 0; i--) {
 353                 dma_unmap_page_attrs(dev, umem->pages[i].dma, PAGE_SIZE,
 354                                      DMA_BIDIRECTIONAL, ICE_RX_DMA_ATTR);
 355                 umem->pages[i].dma = 0;
 356         }
 357
 358         return -EFAULT;
 359 }
 360
 361 /**
 362  * ice_xsk_umem_dma_unmap - DMA unmap UMEM region for XDP sockets
 363  * @vsi: VSI from which the UMEM will be unmapped
 364  * @umem: UMEM to unmap
 365  */
 366 static void ice_xsk_umem_dma_unmap(struct ice_vsi *vsi, struct xdp_umem *umem)
 367 {
 368         struct ice_pf *pf = vsi->back;
 369         struct device *dev;
 370         unsigned int i;
 371
 372         dev = ice_pf_to_dev(pf);
 373         for (i = 0; i < umem->npgs; i++) {
 374                 dma_unmap_page_attrs(dev, umem->pages[i].dma, PAGE_SIZE,
 375                                      DMA_BIDIRECTIONAL, ICE_RX_DMA_ATTR);
 376
 377                 umem->pages[i].dma = 0;
 378         }
 379 }
 380
 381 /**
 382  * ice_xsk_umem_disable - disable a UMEM region
 383  * @vsi: Current VSI
 384  * @qid: queue ID
 385  *
 386  * Returns 0 on success, negative on failure
 387  */
 388 static int ice_xsk_umem_disable(struct ice_vsi *vsi, u16 qid)
 389 {
 390         if (!vsi->xsk_umems || qid >= vsi->num_xsk_umems ||
 391             !vsi->xsk_umems[qid])
 392                 return -EINVAL;
 393
 394         ice_xsk_umem_dma_unmap(vsi, vsi->xsk_umems[qid]);
 395         ice_xsk_remove_umem(vsi, qid);
 396
 397         return 0;
 398 }
 399
 400 /**
 401  * ice_xsk_umem_enable - enable a UMEM region
 402  * @vsi: Current VSI
 403  * @umem: pointer to a requested UMEM region
 404  * @qid: queue ID
 405  *
 406  * Returns 0 on success, negative on failure
 407  */
 408 static int
 409 ice_xsk_umem_enable(struct ice_vsi *vsi, struct xdp_umem *umem, u16 qid)
 410 {
 411         struct xdp_umem_fq_reuse *reuseq;
 412         int err;
 413
 414         if (vsi->type != ICE_VSI_PF)
 415                 return -EINVAL;
 416
 417         vsi->num_xsk_umems = min_t(u16, vsi->num_rxq, vsi->num_txq);
 418         if (qid >= vsi->num_xsk_umems)
 419                 return -EINVAL;
 420
 421         if (vsi->xsk_umems && vsi->xsk_umems[qid])
 422                 return -EBUSY;
 423
 424         reuseq = xsk_reuseq_prepare(vsi->rx_rings[0]->count);
 425         if (!reuseq)
 426                 return -ENOMEM;
 427
 428         xsk_reuseq_free(xsk_reuseq_swap(umem, reuseq));
 429
 430         err = ice_xsk_umem_dma_map(vsi, umem);
 431         if (err)
 432                 return err;
 433
 434         err = ice_xsk_add_umem(vsi, umem, qid);
 435         if (err)
 436                 return err;
 437
 438         return 0;
 439 }
 440
 441 /**
 442  * ice_xsk_umem_setup - enable/disable a UMEM region depending on its state
 443  * @vsi: Current VSI
 444  * @umem: UMEM to enable/associate to a ring, NULL to disable
 445  * @qid: queue ID
 446  *
 447  * Returns 0 on success, negative on failure
 448  */
 449 int ice_xsk_umem_setup(struct ice_vsi *vsi, struct xdp_umem *umem, u16 qid)
 450 {
 451         bool if_running, umem_present = !!umem;
 452         int ret = 0, umem_failure = 0;
 453
 454         if_running = netif_running(vsi->netdev) && ice_is_xdp_ena_vsi(vsi);
 455
 456         if (if_running) {
 457                 ret = ice_qp_dis(vsi, qid);
 458                 if (ret) {
 459                         netdev_err(vsi->netdev, "ice_qp_dis error = %d", ret);
 460                         goto xsk_umem_if_up;
 461                 }
 462         }
 463
 464         umem_failure = umem_present ? ice_xsk_umem_enable(vsi, umem, qid) :
 465                                       ice_xsk_umem_disable(vsi, qid);
 466
 467 xsk_umem_if_up:
 468         if (if_running) {
 469                 ret = ice_qp_ena(vsi, qid);
 470                 if (!ret && umem_present)
 471                         napi_schedule(&vsi->xdp_rings[qid]->q_vector->napi);
 472                 else if (ret)
 473                         netdev_err(vsi->netdev, "ice_qp_ena error = %d", ret);
 474         }
 475
 476         if (umem_failure) {
 477                 netdev_err(vsi->netdev, "Could not %sable UMEM, error = %d",
 478                            umem_present ? "en" : "dis", umem_failure);
 479                 return umem_failure;
 480         }
 481
 482         return ret;
 483 }
 484
 485 /**
 486  * ice_zca_free - Callback for MEM_TYPE_ZERO_COPY allocations
 487  * @zca: zero-cpoy allocator
 488  * @handle: Buffer handle
 489  */
 490 void ice_zca_free(struct zero_copy_allocator *zca, unsigned long handle)
 491 {
 492         struct ice_rx_buf *rx_buf;
 493         struct ice_ring *rx_ring;
 494         struct xdp_umem *umem;
 495         u64 hr, mask;
 496         u16 nta;
 497
 498         rx_ring = container_of(zca, struct ice_ring, zca);
 499         umem = rx_ring->xsk_umem;
 500         hr = umem->headroom + XDP_PACKET_HEADROOM;
 501
 502         mask = umem->chunk_mask;
 503
 504         nta = rx_ring->next_to_alloc;
 505         rx_buf = &rx_ring->rx_buf[nta];
 506
 507         nta++;
 508         rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;
 509
 510         handle &= mask;
 511
 512         rx_buf->dma = xdp_umem_get_dma(umem, handle);
 513         rx_buf->dma += hr;
 514
 515         rx_buf->addr = xdp_umem_get_data(umem, handle);
 516         rx_buf->addr += hr;
 517
 518         rx_buf->handle = (u64)handle + umem->headroom;
 519 }
 520
 521 /**
 522  * ice_alloc_buf_fast_zc - Retrieve buffer address from XDP umem
 523  * @rx_ring: ring with an xdp_umem bound to it
 524  * @rx_buf: buffer to which xsk page address will be assigned
 525  *
 526  * This function allocates an Rx buffer in the hot path.
 527  * The buffer can come from fill queue or recycle queue.
 528  *
 529  * Returns true if an assignment was successful, false if not.
 530  */
 531 static __always_inline bool
 532 ice_alloc_buf_fast_zc(struct ice_ring *rx_ring, struct ice_rx_buf *rx_buf)
 533 {
 534         struct xdp_umem *umem = rx_ring->xsk_umem;
 535         void *addr = rx_buf->addr;
 536         u64 handle, hr;
 537
 538         if (addr) {
 539                 rx_ring->rx_stats.page_reuse_count++;
 540                 return true;
 541         }
 542
 543         if (!xsk_umem_peek_addr(umem, &handle)) {
 544                 rx_ring->rx_stats.alloc_page_failed++;
 545                 return false;
 546         }
 547
 548         hr = umem->headroom + XDP_PACKET_HEADROOM;
 549
 550         rx_buf->dma = xdp_umem_get_dma(umem, handle);
 551         rx_buf->dma += hr;
 552
 553         rx_buf->addr = xdp_umem_get_data(umem, handle);
 554         rx_buf->addr += hr;
 555
 556         rx_buf->handle = handle + umem->headroom;
 557
 558         xsk_umem_release_addr(umem);
 559         return true;
 560 }
 561
 562 /**
 563  * ice_alloc_buf_slow_zc - Retrieve buffer address from XDP umem
 564  * @rx_ring: ring with an xdp_umem bound to it
 565  * @rx_buf: buffer to which xsk page address will be assigned
 566  *
 567  * This function allocates an Rx buffer in the slow path.
 568  * The buffer can come from fill queue or recycle queue.
 569  *
 570  * Returns true if an assignment was successful, false if not.
 571  */
 572 static __always_inline bool
 573 ice_alloc_buf_slow_zc(struct ice_ring *rx_ring, struct ice_rx_buf *rx_buf)
 574 {
 575         struct xdp_umem *umem = rx_ring->xsk_umem;
 576         u64 handle, headroom;
 577
 578         if (!xsk_umem_peek_addr_rq(umem, &handle)) {
 579                 rx_ring->rx_stats.alloc_page_failed++;
 580                 return false;
 581         }
 582
 583         handle &= umem->chunk_mask;
 584         headroom = umem->headroom + XDP_PACKET_HEADROOM;
 585
 586         rx_buf->dma = xdp_umem_get_dma(umem, handle);
 587         rx_buf->dma += headroom;
 588
 589         rx_buf->addr = xdp_umem_get_data(umem, handle);
 590         rx_buf->addr += headroom;
 591
 592         rx_buf->handle = handle + umem->headroom;
 593
 594         xsk_umem_release_addr_rq(umem);
 595         return true;
 596 }
 597
 598 /**
 599  * ice_alloc_rx_bufs_zc - allocate a number of Rx buffers
 600  * @rx_ring: Rx ring
 601  * @count: The number of buffers to allocate
 602  * @alloc: the function pointer to call for allocation
 603  *
 604  * This function allocates a number of Rx buffers from the fill ring
 605  * or the internal recycle mechanism and places them on the Rx ring.
 606  *
 607  * Returns false if all allocations were successful, true if any fail.
 608  */
 609 static bool
 610 ice_alloc_rx_bufs_zc(struct ice_ring *rx_ring, int count,
 611                      bool alloc(struct ice_ring *, struct ice_rx_buf *))
 612 {
 613         union ice_32b_rx_flex_desc *rx_desc;
 614         u16 ntu = rx_ring->next_to_use;
 615         struct ice_rx_buf *rx_buf;
 616         bool ret = false;
 617
 618         if (!count)
 619                 return false;
 620
 621         rx_desc = ICE_RX_DESC(rx_ring, ntu);
 622         rx_buf = &rx_ring->rx_buf[ntu];
 623
 624         do {
 625                 if (!alloc(rx_ring, rx_buf)) {
 626                         ret = true;
 627                         break;
 628                 }
 629
 630                 dma_sync_single_range_for_device(rx_ring->dev, rx_buf->dma, 0,
 631                                                  rx_ring->rx_buf_len,
 632                                                  DMA_BIDIRECTIONAL);
 633
 634                 rx_desc->read.pkt_addr = cpu_to_le64(rx_buf->dma);
 635                 rx_desc->wb.status_error0 = 0;
 636
 637                 rx_desc++;
 638                 rx_buf++;
 639                 ntu++;
 640
 641                 if (unlikely(ntu == rx_ring->count)) {
 642                         rx_desc = ICE_RX_DESC(rx_ring, 0);
 643                         rx_buf = rx_ring->rx_buf;
 644                         ntu = 0;
 645                 }
 646         } while (--count);
 647
 648         if (rx_ring->next_to_use != ntu)
 649                 ice_release_rx_desc(rx_ring, ntu);
 650
 651         return ret;
 652 }
 653
 654 /**
 655  * ice_alloc_rx_bufs_fast_zc - allocate zero copy bufs in the hot path
 656  * @rx_ring: Rx ring
 657  * @count: number of bufs to allocate
 658  *
 659  * Returns false on success, true on failure.
 660  */
 661 static bool ice_alloc_rx_bufs_fast_zc(struct ice_ring *rx_ring, u16 count)
 662 {
 663         return ice_alloc_rx_bufs_zc(rx_ring, count,
 664                                     ice_alloc_buf_fast_zc);
 665 }
 666
 667 /**
 668  * ice_alloc_rx_bufs_slow_zc - allocate zero copy bufs in the slow path
 669  * @rx_ring: Rx ring
 670  * @count: number of bufs to allocate
 671  *
 672  * Returns false on success, true on failure.
 673  */
 674 bool ice_alloc_rx_bufs_slow_zc(struct ice_ring *rx_ring, u16 count)
 675 {
 676         return ice_alloc_rx_bufs_zc(rx_ring, count,
 677                                     ice_alloc_buf_slow_zc);
 678 }
 679
 680 /**
 681  * ice_bump_ntc - Bump the next_to_clean counter of an Rx ring
 682  * @rx_ring: Rx ring
 683  */
 684 static void ice_bump_ntc(struct ice_ring *rx_ring)
 685 {
 686         int ntc = rx_ring->next_to_clean + 1;
 687
 688         ntc = (ntc < rx_ring->count) ? ntc : 0;
 689         rx_ring->next_to_clean = ntc;
 690         prefetch(ICE_RX_DESC(rx_ring, ntc));
 691 }
 692
 693 /**
 694  * ice_get_rx_buf_zc - Fetch the current Rx buffer
 695  * @rx_ring: Rx ring
 696  * @size: size of a buffer
 697  *
 698  * This function returns the current, received Rx buffer and does
 699  * DMA synchronization.
 700  *
 701  * Returns a pointer to the received Rx buffer.
 702  */
 703 static struct ice_rx_buf *ice_get_rx_buf_zc(struct ice_ring *rx_ring, int size)
 704 {
 705         struct ice_rx_buf *rx_buf;
 706
 707         rx_buf = &rx_ring->rx_buf[rx_ring->next_to_clean];
 708
 709         dma_sync_single_range_for_cpu(rx_ring->dev, rx_buf->dma, 0,
 710                                       size, DMA_BIDIRECTIONAL);
 711
 712         return rx_buf;
 713 }
 714
 715 /**
 716  * ice_reuse_rx_buf_zc - reuse an Rx buffer
 717  * @rx_ring: Rx ring
 718  * @old_buf: The buffer to recycle
 719  *
 720  * This function recycles a finished Rx buffer, and places it on the recycle
 721  * queue (next_to_alloc).
 722  */
 723 static void
 724 ice_reuse_rx_buf_zc(struct ice_ring *rx_ring, struct ice_rx_buf *old_buf)
 725 {
 726         unsigned long mask = (unsigned long)rx_ring->xsk_umem->chunk_mask;
 727         u64 hr = rx_ring->xsk_umem->headroom + XDP_PACKET_HEADROOM;
 728         u16 nta = rx_ring->next_to_alloc;
 729         struct ice_rx_buf *new_buf;
 730
 731         new_buf = &rx_ring->rx_buf[nta++];
 732         rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;
 733
 734         new_buf->dma = old_buf->dma & mask;
 735         new_buf->dma += hr;
 736
 737         new_buf->addr = (void *)((unsigned long)old_buf->addr & mask);
 738         new_buf->addr += hr;
 739
 740         new_buf->handle = old_buf->handle & mask;
 741         new_buf->handle += rx_ring->xsk_umem->headroom;
 742
 743         old_buf->addr = NULL;
 744 }
 745
 746 /**
 747  * ice_construct_skb_zc - Create an sk_buff from zero-copy buffer
 748  * @rx_ring: Rx ring
 749  * @rx_buf: zero-copy Rx buffer
 750  * @xdp: XDP buffer
 751  *
 752  * This function allocates a new skb from a zero-copy Rx buffer.
 753  *
 754  * Returns the skb on success, NULL on failure.
 755  */
 756 static struct sk_buff *
 757 ice_construct_skb_zc(struct ice_ring *rx_ring, struct ice_rx_buf *rx_buf,
 758                      struct xdp_buff *xdp)
 759 {
 760         unsigned int metasize = xdp->data - xdp->data_meta;
 761         unsigned int datasize = xdp->data_end - xdp->data;
 762         unsigned int datasize_hard = xdp->data_end -
 763                                      xdp->data_hard_start;
 764         struct sk_buff *skb;
 765
 766         skb = __napi_alloc_skb(&rx_ring->q_vector->napi, datasize_hard,
 767                                GFP_ATOMIC | __GFP_NOWARN);
 768         if (unlikely(!skb))
 769                 return NULL;
 770
 771         skb_reserve(skb, xdp->data - xdp->data_hard_start);
 772         memcpy(__skb_put(skb, datasize), xdp->data, datasize);
 773         if (metasize)
 774                 skb_metadata_set(skb, metasize);
 775
 776         ice_reuse_rx_buf_zc(rx_ring, rx_buf);
 777
 778         return skb;
 779 }
 780
 781 /**
 782  * ice_run_xdp_zc - Executes an XDP program in zero-copy path
 783  * @rx_ring: Rx ring
 784  * @xdp: xdp_buff used as input to the XDP program
 785  *
 786  * Returns any of ICE_XDP_{PASS, CONSUMED, TX, REDIR}
 787  */
 788 static int
 789 ice_run_xdp_zc(struct ice_ring *rx_ring, struct xdp_buff *xdp)
 790 {
 791         int err, result = ICE_XDP_PASS;
 792         struct bpf_prog *xdp_prog;
 793         struct ice_ring *xdp_ring;
 794         u32 act;
 795
 796         rcu_read_lock();
 797         xdp_prog = READ_ONCE(rx_ring->xdp_prog);
 798         if (!xdp_prog) {
 799                 rcu_read_unlock();
 800                 return ICE_XDP_PASS;
 801         }
 802
 803         act = bpf_prog_run_xdp(xdp_prog, xdp);
 804         xdp->handle += xdp->data - xdp->data_hard_start;
 805         switch (act) {
 806         case XDP_PASS:
 807                 break;
 808         case XDP_TX:
 809                 xdp_ring = rx_ring->vsi->xdp_rings[rx_ring->q_index];
 810                 result = ice_xmit_xdp_buff(xdp, xdp_ring);
 811                 break;
 812         case XDP_REDIRECT:
 813                 err = xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog);
 814                 result = !err ? ICE_XDP_REDIR : ICE_XDP_CONSUMED;
 815                 break;
 816         default:
 817                 bpf_warn_invalid_xdp_action(act);
 818                 /* fallthrough -- not supported action */
 819         case XDP_ABORTED:
 820                 trace_xdp_exception(rx_ring->netdev, xdp_prog, act);
 821                 /* fallthrough -- handle aborts by dropping frame */
 822         case XDP_DROP:
 823                 result = ICE_XDP_CONSUMED;
 824                 break;
 825         }
 826
 827         rcu_read_unlock();
 828         return result;
 829 }
 830
 831 /**
 832  * ice_clean_rx_irq_zc - consumes packets from the hardware ring
 833  * @rx_ring: AF_XDP Rx ring
 834  * @budget: NAPI budget
 835  *
 836  * Returns number of processed packets on success, remaining budget on failure.
 837  */
 838 int ice_clean_rx_irq_zc(struct ice_ring *rx_ring, int budget)
 839 {
 840         unsigned int total_rx_bytes = 0, total_rx_packets = 0;
 841         u16 cleaned_count = ICE_DESC_UNUSED(rx_ring);
 842         unsigned int xdp_xmit = 0;
 843         struct xdp_buff xdp;
 844         bool failure = 0;
 845
 846         xdp.rxq = &rx_ring->xdp_rxq;
 847
 848         while (likely(total_rx_packets < (unsigned int)budget)) {
 849                 union ice_32b_rx_flex_desc *rx_desc;
 850                 unsigned int size, xdp_res = 0;
 851                 struct ice_rx_buf *rx_buf;
 852                 struct sk_buff *skb;
 853                 u16 stat_err_bits;
 854                 u16 vlan_tag = 0;
 855                 u8 rx_ptype;
 856
 857                 if (cleaned_count >= ICE_RX_BUF_WRITE) {
 858                         failure |= ice_alloc_rx_bufs_fast_zc(rx_ring,
 859                                                              cleaned_count);
 860                         cleaned_count = 0;
 861                 }
 862
 863                 rx_desc = ICE_RX_DESC(rx_ring, rx_ring->next_to_clean);
 864
 865                 stat_err_bits = BIT(ICE_RX_FLEX_DESC_STATUS0_DD_S);
 866                 if (!ice_test_staterr(rx_desc, stat_err_bits))
 867                         break;
 868
 869                 /* This memory barrier is needed to keep us from reading
 870                  * any other fields out of the rx_desc until we have
 871                  * verified the descriptor has been written back.
 872                  */
 873                 dma_rmb();
 874
 875                 size = le16_to_cpu(rx_desc->wb.pkt_len) &
 876                                    ICE_RX_FLX_DESC_PKT_LEN_M;
 877                 if (!size)
 878                         break;
 879
 880                 rx_buf = ice_get_rx_buf_zc(rx_ring, size);
 881                 if (!rx_buf->addr)
 882                         break;
 883
 884                 xdp.data = rx_buf->addr;
 885                 xdp.data_meta = xdp.data;
 886                 xdp.data_hard_start = xdp.data - XDP_PACKET_HEADROOM;
 887                 xdp.data_end = xdp.data + size;
 888                 xdp.handle = rx_buf->handle;
 889
 890                 xdp_res = ice_run_xdp_zc(rx_ring, &xdp);
 891                 if (xdp_res) {
 892                         if (xdp_res & (ICE_XDP_TX | ICE_XDP_REDIR)) {
 893                                 xdp_xmit |= xdp_res;
 894                                 rx_buf->addr = NULL;
 895                         } else {
 896                                 ice_reuse_rx_buf_zc(rx_ring, rx_buf);
 897                         }
 898
 899                         total_rx_bytes += size;
 900                         total_rx_packets++;
 901                         cleaned_count++;
 902
 903                         ice_bump_ntc(rx_ring);
 904                         continue;
 905                 }
 906
 907                 /* XDP_PASS path */
 908                 skb = ice_construct_skb_zc(rx_ring, rx_buf, &xdp);
 909                 if (!skb) {
 910                         rx_ring->rx_stats.alloc_buf_failed++;
 911                         break;
 912                 }
 913
 914                 cleaned_count++;
 915                 ice_bump_ntc(rx_ring);
 916
 917                 if (eth_skb_pad(skb)) {
 918                         skb = NULL;
 919                         continue;
 920                 }
 921
 922                 total_rx_bytes += skb->len;
 923                 total_rx_packets++;
 924
 925                 stat_err_bits = BIT(ICE_RX_FLEX_DESC_STATUS0_L2TAG1P_S);
 926                 if (ice_test_staterr(rx_desc, stat_err_bits))
 927                         vlan_tag = le16_to_cpu(rx_desc->wb.l2tag1);
 928
 929                 rx_ptype = le16_to_cpu(rx_desc->wb.ptype_flex_flags0) &
 930                                        ICE_RX_FLEX_DESC_PTYPE_M;
 931
 932                 ice_process_skb_fields(rx_ring, rx_desc, skb, rx_ptype);
 933                 ice_receive_skb(rx_ring, skb, vlan_tag);
 934         }
 935
 936         ice_finalize_xdp_rx(rx_ring, xdp_xmit);
 937         ice_update_rx_ring_stats(rx_ring, total_rx_packets, total_rx_bytes);
 938
 939         return failure ? budget : (int)total_rx_packets;
 940 }
 941
 942 /**
 943  * ice_xmit_zc - Completes AF_XDP entries, and cleans XDP entries
 944  * @xdp_ring: XDP Tx ring
 945  * @budget: max number of frames to xmit
 946  *
 947  * Returns true if cleanup/transmission is done.
 948  */
 949 static bool ice_xmit_zc(struct ice_ring *xdp_ring, int budget)
 950 {
 951         struct ice_tx_desc *tx_desc = NULL;
 952         bool work_done = true;
 953         struct xdp_desc desc;
 954         dma_addr_t dma;
 955
 956         while (likely(budget-- > 0)) {
 957                 struct ice_tx_buf *tx_buf;
 958
 959                 if (unlikely(!ICE_DESC_UNUSED(xdp_ring))) {
 960                         xdp_ring->tx_stats.tx_busy++;
 961                         work_done = false;
 962                         break;
 963                 }
 964
 965                 tx_buf = &xdp_ring->tx_buf[xdp_ring->next_to_use];
 966
 967                 if (!xsk_umem_consume_tx(xdp_ring->xsk_umem, &desc))
 968                         break;
 969
 970                 dma = xdp_umem_get_dma(xdp_ring->xsk_umem, desc.addr);
 971
 972                 dma_sync_single_for_device(xdp_ring->dev, dma, desc.len,
 973                                            DMA_BIDIRECTIONAL);
 974
 975                 tx_buf->bytecount = desc.len;
 976
 977                 tx_desc = ICE_TX_DESC(xdp_ring, xdp_ring->next_to_use);
 978                 tx_desc->buf_addr = cpu_to_le64(dma);
 979                 tx_desc->cmd_type_offset_bsz = build_ctob(ICE_TXD_LAST_DESC_CMD,
 980                                                           0, desc.len, 0);
 981
 982                 xdp_ring->next_to_use++;
 983                 if (xdp_ring->next_to_use == xdp_ring->count)
 984                         xdp_ring->next_to_use = 0;
 985         }
 986
 987         if (tx_desc) {
 988                 ice_xdp_ring_update_tail(xdp_ring);
 989                 xsk_umem_consume_tx_done(xdp_ring->xsk_umem);
 990         }
 991
 992         return budget > 0 && work_done;
 993 }
 994
 995 /**
 996  * ice_clean_xdp_tx_buf - Free and unmap XDP Tx buffer
 997  * @xdp_ring: XDP Tx ring
 998  * @tx_buf: Tx buffer to clean
 999  */
1000 static void
1001 ice_clean_xdp_tx_buf(struct ice_ring *xdp_ring, struct ice_tx_buf *tx_buf)
1002 {
1003         xdp_return_frame((struct xdp_frame *)tx_buf->raw_buf);
1004         dma_unmap_single(xdp_ring->dev, dma_unmap_addr(tx_buf, dma),
1005                          dma_unmap_len(tx_buf, len), DMA_TO_DEVICE);
1006         dma_unmap_len_set(tx_buf, len, 0);
1007 }
1008
1009 /**
1010  * ice_clean_tx_irq_zc - Completes AF_XDP entries, and cleans XDP entries
1011  * @xdp_ring: XDP Tx ring
1012  * @budget: NAPI budget
1013  *
1014  * Returns true if cleanup/tranmission is done.
1015  */
1016 bool ice_clean_tx_irq_zc(struct ice_ring *xdp_ring, int budget)
1017 {
1018         int total_packets = 0, total_bytes = 0;
1019         s16 ntc = xdp_ring->next_to_clean;
1020         struct ice_tx_desc *tx_desc;
1021         struct ice_tx_buf *tx_buf;
1022         bool xmit_done = true;
1023         u32 xsk_frames = 0;
1024
1025         tx_desc = ICE_TX_DESC(xdp_ring, ntc);
1026         tx_buf = &xdp_ring->tx_buf[ntc];
1027         ntc -= xdp_ring->count;
1028
1029         do {
1030                 if (!(tx_desc->cmd_type_offset_bsz &
1031                       cpu_to_le64(ICE_TX_DESC_DTYPE_DESC_DONE)))
1032                         break;
1033
1034                 total_bytes += tx_buf->bytecount;
1035                 total_packets++;
1036
1037                 if (tx_buf->raw_buf) {
1038                         ice_clean_xdp_tx_buf(xdp_ring, tx_buf);
1039                         tx_buf->raw_buf = NULL;
1040                 } else {
1041                         xsk_frames++;
1042                 }
1043
1044                 tx_desc->cmd_type_offset_bsz = 0;
1045                 tx_buf++;
1046                 tx_desc++;
1047                 ntc++;
1048
1049                 if (unlikely(!ntc)) {
1050                         ntc -= xdp_ring->count;
1051                         tx_buf = xdp_ring->tx_buf;
1052                         tx_desc = ICE_TX_DESC(xdp_ring, 0);
1053                 }
1054
1055                 prefetch(tx_desc);
1056
1057         } while (likely(--budget));
1058
1059         ntc += xdp_ring->count;
1060         xdp_ring->next_to_clean = ntc;
1061
1062         if (xsk_frames)
1063                 xsk_umem_complete_tx(xdp_ring->xsk_umem, xsk_frames);
1064
1065         ice_update_tx_ring_stats(xdp_ring, total_packets, total_bytes);
1066         xmit_done = ice_xmit_zc(xdp_ring, ICE_DFLT_IRQ_WORK);
1067
1068         return budget > 0 && xmit_done;
1069 }
1070
1071 /**
1072  * ice_xsk_wakeup - Implements ndo_xsk_wakeup
1073  * @netdev: net_device
1074  * @queue_id: queue to wake up
1075  * @flags: ignored in our case, since we have Rx and Tx in the same NAPI
1076  *
1077  * Returns negative on error, zero otherwise.
1078  */
1079 int
1080 ice_xsk_wakeup(struct net_device *netdev, u32 queue_id,
1081                u32 __always_unused flags)
1082 {
1083         struct ice_netdev_priv *np = netdev_priv(netdev);
1084         struct ice_q_vector *q_vector;
1085         struct ice_vsi *vsi = np->vsi;
1086         struct ice_ring *ring;
1087
1088         if (test_bit(__ICE_DOWN, vsi->state))
1089                 return -ENETDOWN;
1090
1091         if (!ice_is_xdp_ena_vsi(vsi))
1092                 return -ENXIO;
1093
1094         if (queue_id >= vsi->num_txq)
1095                 return -ENXIO;
1096
1097         if (!vsi->xdp_rings[queue_id]->xsk_umem)
1098                 return -ENXIO;
1099
1100         ring = vsi->xdp_rings[queue_id];
1101
1102         /* The idea here is that if NAPI is running, mark a miss, so
1103          * it will run again. If not, trigger an interrupt and
1104          * schedule the NAPI from interrupt context. If NAPI would be
1105          * scheduled here, the interrupt affinity would not be
1106          * honored.
1107          */
1108         q_vector = ring->q_vector;
1109         if (!napi_if_scheduled_mark_missed(&q_vector->napi))
1110                 ice_trigger_sw_intr(&vsi->back->hw, q_vector);
1111
1112         return 0;
1113 }
1114
1115 /**
1116  * ice_xsk_any_rx_ring_ena - Checks if Rx rings have AF_XDP UMEM attached
1117  * @vsi: VSI to be checked
1118  *
1119  * Returns true if any of the Rx rings has an AF_XDP UMEM attached
1120  */
1121 bool ice_xsk_any_rx_ring_ena(struct ice_vsi *vsi)
1122 {
1123         int i;
1124
1125         if (!vsi->xsk_umems)
1126                 return false;
1127
1128         for (i = 0; i < vsi->num_xsk_umems; i++) {
1129                 if (vsi->xsk_umems[i])
1130                         return true;
1131         }
1132
1133         return false;
1134 }
1135
1136 /**
1137  * ice_xsk_clean_rx_ring - clean UMEM queues connected to a given Rx ring
1138  * @rx_ring: ring to be cleaned
1139  */
1140 void ice_xsk_clean_rx_ring(struct ice_ring *rx_ring)
1141 {
1142         u16 i;
1143
1144         for (i = 0; i < rx_ring->count; i++) {
1145                 struct ice_rx_buf *rx_buf = &rx_ring->rx_buf[i];
1146
1147                 if (!rx_buf->addr)
1148                         continue;
1149
1150                 xsk_umem_fq_reuse(rx_ring->xsk_umem, rx_buf->handle);
1151                 rx_buf->addr = NULL;
1152         }
1153 }
1154
1155 /**
1156  * ice_xsk_clean_xdp_ring - Clean the XDP Tx ring and its UMEM queues
1157  * @xdp_ring: XDP_Tx ring
1158  */
1159 void ice_xsk_clean_xdp_ring(struct ice_ring *xdp_ring)
1160 {
1161         u16 ntc = xdp_ring->next_to_clean, ntu = xdp_ring->next_to_use;
1162         u32 xsk_frames = 0;
1163
1164         while (ntc != ntu) {
1165                 struct ice_tx_buf *tx_buf = &xdp_ring->tx_buf[ntc];
1166
1167                 if (tx_buf->raw_buf)
1168                         ice_clean_xdp_tx_buf(xdp_ring, tx_buf);
1169                 else
1170                         xsk_frames++;
1171
1172                 tx_buf->raw_buf = NULL;
1173
1174                 ntc++;
1175                 if (ntc >= xdp_ring->count)
1176                         ntc = 0;
1177         }
1178
1179         if (xsk_frames)
1180                 xsk_umem_complete_tx(xdp_ring->xsk_umem, xsk_frames);
1181 }