2 * Broadcom NetXtreme-E RoCE driver.
4 * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term
5 * Broadcom refers to Broadcom Limited and/or its subsidiaries.
7 * This software is available to you under a choice of one of two
8 * licenses. You may choose to be licensed under the terms of the GNU
9 * General Public License (GPL) Version 2, available from the file
10 * COPYING in the main directory of this source tree, or the
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
17 * 1. Redistributions of source code must retain the above copyright
18 * notice, this list of conditions and the following disclaimer.
19 * 2. Redistributions in binary form must reproduce the above copyright
20 * notice, this list of conditions and the following disclaimer in
21 * the documentation and/or other materials provided with the
24 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
25 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
26 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
27 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
28 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
31 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
32 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
33 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
34 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 * Description: Fast Path Operators
39 #include <linux/interrupt.h>
40 #include <linux/spinlock.h>
41 #include <linux/sched.h>
42 #include <linux/slab.h>
43 #include <linux/pci.h>
44 #include <linux/prefetch.h>
48 #include "qplib_res.h"
49 #include "qplib_rcfw.h"
53 static void bnxt_qplib_arm_cq_enable(struct bnxt_qplib_cq *cq);
55 static void bnxt_qplib_free_qp_hdr_buf(struct bnxt_qplib_res *res,
56 struct bnxt_qplib_qp *qp)
58 struct bnxt_qplib_q *rq = &qp->rq;
59 struct bnxt_qplib_q *sq = &qp->sq;
62 dma_free_coherent(&res->pdev->dev,
63 rq->hwq.max_elements * qp->rq_hdr_buf_size,
64 qp->rq_hdr_buf, qp->rq_hdr_buf_map);
66 dma_free_coherent(&res->pdev->dev,
67 sq->hwq.max_elements * qp->sq_hdr_buf_size,
68 qp->sq_hdr_buf, qp->sq_hdr_buf_map);
69 qp->rq_hdr_buf = NULL;
70 qp->sq_hdr_buf = NULL;
71 qp->rq_hdr_buf_map = 0;
72 qp->sq_hdr_buf_map = 0;
73 qp->sq_hdr_buf_size = 0;
74 qp->rq_hdr_buf_size = 0;
77 static int bnxt_qplib_alloc_qp_hdr_buf(struct bnxt_qplib_res *res,
78 struct bnxt_qplib_qp *qp)
80 struct bnxt_qplib_q *rq = &qp->rq;
81 struct bnxt_qplib_q *sq = &qp->rq;
84 if (qp->sq_hdr_buf_size && sq->hwq.max_elements) {
85 qp->sq_hdr_buf = dma_alloc_coherent(&res->pdev->dev,
86 sq->hwq.max_elements *
88 &qp->sq_hdr_buf_map, GFP_KERNEL);
89 if (!qp->sq_hdr_buf) {
91 dev_err(&res->pdev->dev,
92 "QPLIB: Failed to create sq_hdr_buf");
97 if (qp->rq_hdr_buf_size && rq->hwq.max_elements) {
98 qp->rq_hdr_buf = dma_alloc_coherent(&res->pdev->dev,
99 rq->hwq.max_elements *
103 if (!qp->rq_hdr_buf) {
105 dev_err(&res->pdev->dev,
106 "QPLIB: Failed to create rq_hdr_buf");
113 bnxt_qplib_free_qp_hdr_buf(res, qp);
117 static void bnxt_qplib_service_nq(unsigned long data)
119 struct bnxt_qplib_nq *nq = (struct bnxt_qplib_nq *)data;
120 struct bnxt_qplib_hwq *hwq = &nq->hwq;
121 struct nq_base *nqe, **nq_ptr;
122 int num_cqne_processed = 0;
123 u32 sw_cons, raw_cons;
125 int budget = nq->budget;
128 /* Service the NQ until empty */
129 raw_cons = hwq->cons;
131 sw_cons = HWQ_CMP(raw_cons, hwq);
132 nq_ptr = (struct nq_base **)hwq->pbl_ptr;
133 nqe = &nq_ptr[NQE_PG(sw_cons)][NQE_IDX(sw_cons)];
134 if (!NQE_CMP_VALID(nqe, raw_cons, hwq->max_elements))
137 type = le16_to_cpu(nqe->info10_type) & NQ_BASE_TYPE_MASK;
139 case NQ_BASE_TYPE_CQ_NOTIFICATION:
141 struct nq_cn *nqcne = (struct nq_cn *)nqe;
143 q_handle = le32_to_cpu(nqcne->cq_handle_low);
144 q_handle |= (u64)le32_to_cpu(nqcne->cq_handle_high)
146 bnxt_qplib_arm_cq_enable((struct bnxt_qplib_cq *)
147 ((unsigned long)q_handle));
148 if (!nq->cqn_handler(nq, (struct bnxt_qplib_cq *)
149 ((unsigned long)q_handle)))
150 num_cqne_processed++;
152 dev_warn(&nq->pdev->dev,
153 "QPLIB: cqn - type 0x%x not handled",
157 case NQ_BASE_TYPE_DBQ_EVENT:
160 dev_warn(&nq->pdev->dev,
161 "QPLIB: nqe with type = 0x%x not handled",
167 if (hwq->cons != raw_cons) {
168 hwq->cons = raw_cons;
169 NQ_DB_REARM(nq->bar_reg_iomem, hwq->cons, hwq->max_elements);
173 static irqreturn_t bnxt_qplib_nq_irq(int irq, void *dev_instance)
175 struct bnxt_qplib_nq *nq = dev_instance;
176 struct bnxt_qplib_hwq *hwq = &nq->hwq;
177 struct nq_base **nq_ptr;
180 /* Prefetch the NQ element */
181 sw_cons = HWQ_CMP(hwq->cons, hwq);
182 nq_ptr = (struct nq_base **)nq->hwq.pbl_ptr;
183 prefetch(&nq_ptr[NQE_PG(sw_cons)][NQE_IDX(sw_cons)]);
185 /* Fan out to CPU affinitized kthreads? */
186 tasklet_schedule(&nq->worker);
191 void bnxt_qplib_disable_nq(struct bnxt_qplib_nq *nq)
193 /* Make sure the HW is stopped! */
194 synchronize_irq(nq->vector);
195 tasklet_disable(&nq->worker);
196 tasklet_kill(&nq->worker);
199 free_irq(nq->vector, nq);
200 nq->requested = false;
202 if (nq->bar_reg_iomem)
203 iounmap(nq->bar_reg_iomem);
204 nq->bar_reg_iomem = NULL;
206 nq->cqn_handler = NULL;
207 nq->srqn_handler = NULL;
211 int bnxt_qplib_enable_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq,
212 int msix_vector, int bar_reg_offset,
213 int (*cqn_handler)(struct bnxt_qplib_nq *nq,
214 struct bnxt_qplib_cq *),
215 int (*srqn_handler)(struct bnxt_qplib_nq *nq,
218 resource_size_t nq_base;
222 nq->vector = msix_vector;
224 nq->cqn_handler = cqn_handler;
226 nq->srqn_handler = srqn_handler;
228 tasklet_init(&nq->worker, bnxt_qplib_service_nq, (unsigned long)nq);
230 nq->requested = false;
231 rc = request_irq(nq->vector, bnxt_qplib_nq_irq, 0, "bnxt_qplib_nq", nq);
233 dev_err(&nq->pdev->dev,
234 "Failed to request IRQ for NQ: %#x", rc);
235 bnxt_qplib_disable_nq(nq);
238 nq->requested = true;
239 nq->bar_reg = NQ_CONS_PCI_BAR_REGION;
240 nq->bar_reg_off = bar_reg_offset;
241 nq_base = pci_resource_start(pdev, nq->bar_reg);
246 nq->bar_reg_iomem = ioremap_nocache(nq_base + nq->bar_reg_off, 4);
247 if (!nq->bar_reg_iomem) {
251 NQ_DB_REARM(nq->bar_reg_iomem, nq->hwq.cons, nq->hwq.max_elements);
255 bnxt_qplib_disable_nq(nq);
259 void bnxt_qplib_free_nq(struct bnxt_qplib_nq *nq)
261 if (nq->hwq.max_elements)
262 bnxt_qplib_free_hwq(nq->pdev, &nq->hwq);
265 int bnxt_qplib_alloc_nq(struct pci_dev *pdev, struct bnxt_qplib_nq *nq)
268 if (!nq->hwq.max_elements ||
269 nq->hwq.max_elements > BNXT_QPLIB_NQE_MAX_CNT)
270 nq->hwq.max_elements = BNXT_QPLIB_NQE_MAX_CNT;
272 if (bnxt_qplib_alloc_init_hwq(nq->pdev, &nq->hwq, NULL, 0,
273 &nq->hwq.max_elements,
274 BNXT_QPLIB_MAX_NQE_ENTRY_SIZE, 0,
275 PAGE_SIZE, HWQ_TYPE_L2_CMPL))
283 int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
285 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
286 struct cmdq_create_qp1 req;
287 struct creq_create_qp1_resp resp;
288 struct bnxt_qplib_pbl *pbl;
289 struct bnxt_qplib_q *sq = &qp->sq;
290 struct bnxt_qplib_q *rq = &qp->rq;
295 RCFW_CMD_PREP(req, CREATE_QP1, cmd_flags);
299 req.dpi = cpu_to_le32(qp->dpi->dpi);
300 req.qp_handle = cpu_to_le64(qp->qp_handle);
303 sq->hwq.max_elements = sq->max_wqe;
304 rc = bnxt_qplib_alloc_init_hwq(res->pdev, &sq->hwq, NULL, 0,
305 &sq->hwq.max_elements,
306 BNXT_QPLIB_MAX_SQE_ENTRY_SIZE, 0,
307 PAGE_SIZE, HWQ_TYPE_QUEUE);
311 sq->swq = kcalloc(sq->hwq.max_elements, sizeof(*sq->swq), GFP_KERNEL);
316 pbl = &sq->hwq.pbl[PBL_LVL_0];
317 req.sq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
318 req.sq_pg_size_sq_lvl =
319 ((sq->hwq.level & CMDQ_CREATE_QP1_SQ_LVL_MASK)
320 << CMDQ_CREATE_QP1_SQ_LVL_SFT) |
321 (pbl->pg_size == ROCE_PG_SIZE_4K ?
322 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_4K :
323 pbl->pg_size == ROCE_PG_SIZE_8K ?
324 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8K :
325 pbl->pg_size == ROCE_PG_SIZE_64K ?
326 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_64K :
327 pbl->pg_size == ROCE_PG_SIZE_2M ?
328 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_2M :
329 pbl->pg_size == ROCE_PG_SIZE_8M ?
330 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_8M :
331 pbl->pg_size == ROCE_PG_SIZE_1G ?
332 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_1G :
333 CMDQ_CREATE_QP1_SQ_PG_SIZE_PG_4K);
336 req.scq_cid = cpu_to_le32(qp->scq->id);
338 qp_flags |= CMDQ_CREATE_QP1_QP_FLAGS_RESERVED_LKEY_ENABLE;
342 rq->hwq.max_elements = qp->rq.max_wqe;
343 rc = bnxt_qplib_alloc_init_hwq(res->pdev, &rq->hwq, NULL, 0,
344 &rq->hwq.max_elements,
345 BNXT_QPLIB_MAX_RQE_ENTRY_SIZE, 0,
346 PAGE_SIZE, HWQ_TYPE_QUEUE);
350 rq->swq = kcalloc(rq->hwq.max_elements, sizeof(*rq->swq),
356 pbl = &rq->hwq.pbl[PBL_LVL_0];
357 req.rq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
358 req.rq_pg_size_rq_lvl =
359 ((rq->hwq.level & CMDQ_CREATE_QP1_RQ_LVL_MASK) <<
360 CMDQ_CREATE_QP1_RQ_LVL_SFT) |
361 (pbl->pg_size == ROCE_PG_SIZE_4K ?
362 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_4K :
363 pbl->pg_size == ROCE_PG_SIZE_8K ?
364 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8K :
365 pbl->pg_size == ROCE_PG_SIZE_64K ?
366 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_64K :
367 pbl->pg_size == ROCE_PG_SIZE_2M ?
368 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_2M :
369 pbl->pg_size == ROCE_PG_SIZE_8M ?
370 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_8M :
371 pbl->pg_size == ROCE_PG_SIZE_1G ?
372 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_1G :
373 CMDQ_CREATE_QP1_RQ_PG_SIZE_PG_4K);
375 req.rcq_cid = cpu_to_le32(qp->rcq->id);
378 /* Header buffer - allow hdr_buf pass in */
379 rc = bnxt_qplib_alloc_qp_hdr_buf(res, qp);
384 req.qp_flags = cpu_to_le32(qp_flags);
385 req.sq_size = cpu_to_le32(sq->hwq.max_elements);
386 req.rq_size = cpu_to_le32(rq->hwq.max_elements);
389 cpu_to_le16((sq->max_sge & CMDQ_CREATE_QP1_SQ_SGE_MASK) <<
390 CMDQ_CREATE_QP1_SQ_SGE_SFT);
392 cpu_to_le16((rq->max_sge & CMDQ_CREATE_QP1_RQ_SGE_MASK) <<
393 CMDQ_CREATE_QP1_RQ_SGE_SFT);
395 req.pd_id = cpu_to_le32(qp->pd->id);
397 rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
398 (void *)&resp, NULL, 0);
402 qp->id = le32_to_cpu(resp.xid);
403 qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET;
404 sq->flush_in_progress = false;
405 rq->flush_in_progress = false;
410 bnxt_qplib_free_qp_hdr_buf(res, qp);
412 bnxt_qplib_free_hwq(res->pdev, &rq->hwq);
415 bnxt_qplib_free_hwq(res->pdev, &sq->hwq);
421 int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
423 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
424 struct sq_send *hw_sq_send_hdr, **hw_sq_send_ptr;
425 struct cmdq_create_qp req;
426 struct creq_create_qp_resp resp;
427 struct bnxt_qplib_pbl *pbl;
428 struct sq_psn_search **psn_search_ptr;
429 unsigned long int psn_search, poff = 0;
430 struct bnxt_qplib_q *sq = &qp->sq;
431 struct bnxt_qplib_q *rq = &qp->rq;
432 struct bnxt_qplib_hwq *xrrq;
433 int i, rc, req_size, psn_sz;
434 u16 cmd_flags = 0, max_ssge;
435 u32 sw_prod, qp_flags = 0;
437 RCFW_CMD_PREP(req, CREATE_QP, cmd_flags);
441 req.dpi = cpu_to_le32(qp->dpi->dpi);
442 req.qp_handle = cpu_to_le64(qp->qp_handle);
445 psn_sz = (qp->type == CMDQ_CREATE_QP_TYPE_RC) ?
446 sizeof(struct sq_psn_search) : 0;
447 sq->hwq.max_elements = sq->max_wqe;
448 rc = bnxt_qplib_alloc_init_hwq(res->pdev, &sq->hwq, sq->sglist,
449 sq->nmap, &sq->hwq.max_elements,
450 BNXT_QPLIB_MAX_SQE_ENTRY_SIZE,
452 PAGE_SIZE, HWQ_TYPE_QUEUE);
456 sq->swq = kcalloc(sq->hwq.max_elements, sizeof(*sq->swq), GFP_KERNEL);
461 hw_sq_send_ptr = (struct sq_send **)sq->hwq.pbl_ptr;
463 psn_search_ptr = (struct sq_psn_search **)
464 &hw_sq_send_ptr[get_sqe_pg
465 (sq->hwq.max_elements)];
466 psn_search = (unsigned long int)
467 &hw_sq_send_ptr[get_sqe_pg(sq->hwq.max_elements)]
468 [get_sqe_idx(sq->hwq.max_elements)];
469 if (psn_search & ~PAGE_MASK) {
470 /* If the psn_search does not start on a page boundary,
471 * then calculate the offset
473 poff = (psn_search & ~PAGE_MASK) /
474 BNXT_QPLIB_MAX_PSNE_ENTRY_SIZE;
476 for (i = 0; i < sq->hwq.max_elements; i++)
477 sq->swq[i].psn_search =
478 &psn_search_ptr[get_psne_pg(i + poff)]
479 [get_psne_idx(i + poff)];
481 pbl = &sq->hwq.pbl[PBL_LVL_0];
482 req.sq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
483 req.sq_pg_size_sq_lvl =
484 ((sq->hwq.level & CMDQ_CREATE_QP_SQ_LVL_MASK)
485 << CMDQ_CREATE_QP_SQ_LVL_SFT) |
486 (pbl->pg_size == ROCE_PG_SIZE_4K ?
487 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_4K :
488 pbl->pg_size == ROCE_PG_SIZE_8K ?
489 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8K :
490 pbl->pg_size == ROCE_PG_SIZE_64K ?
491 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_64K :
492 pbl->pg_size == ROCE_PG_SIZE_2M ?
493 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_2M :
494 pbl->pg_size == ROCE_PG_SIZE_8M ?
495 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_8M :
496 pbl->pg_size == ROCE_PG_SIZE_1G ?
497 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_1G :
498 CMDQ_CREATE_QP_SQ_PG_SIZE_PG_4K);
500 /* initialize all SQ WQEs to LOCAL_INVALID (sq prep for hw fetch) */
501 hw_sq_send_ptr = (struct sq_send **)sq->hwq.pbl_ptr;
502 for (sw_prod = 0; sw_prod < sq->hwq.max_elements; sw_prod++) {
503 hw_sq_send_hdr = &hw_sq_send_ptr[get_sqe_pg(sw_prod)]
504 [get_sqe_idx(sw_prod)];
505 hw_sq_send_hdr->wqe_type = SQ_BASE_WQE_TYPE_LOCAL_INVALID;
509 req.scq_cid = cpu_to_le32(qp->scq->id);
511 qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_RESERVED_LKEY_ENABLE;
512 qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FR_PMR_ENABLED;
514 qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FORCE_COMPLETION;
518 rq->hwq.max_elements = rq->max_wqe;
519 rc = bnxt_qplib_alloc_init_hwq(res->pdev, &rq->hwq, rq->sglist,
520 rq->nmap, &rq->hwq.max_elements,
521 BNXT_QPLIB_MAX_RQE_ENTRY_SIZE, 0,
522 PAGE_SIZE, HWQ_TYPE_QUEUE);
526 rq->swq = kcalloc(rq->hwq.max_elements, sizeof(*rq->swq),
532 pbl = &rq->hwq.pbl[PBL_LVL_0];
533 req.rq_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
534 req.rq_pg_size_rq_lvl =
535 ((rq->hwq.level & CMDQ_CREATE_QP_RQ_LVL_MASK) <<
536 CMDQ_CREATE_QP_RQ_LVL_SFT) |
537 (pbl->pg_size == ROCE_PG_SIZE_4K ?
538 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_4K :
539 pbl->pg_size == ROCE_PG_SIZE_8K ?
540 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8K :
541 pbl->pg_size == ROCE_PG_SIZE_64K ?
542 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_64K :
543 pbl->pg_size == ROCE_PG_SIZE_2M ?
544 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_2M :
545 pbl->pg_size == ROCE_PG_SIZE_8M ?
546 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_8M :
547 pbl->pg_size == ROCE_PG_SIZE_1G ?
548 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_1G :
549 CMDQ_CREATE_QP_RQ_PG_SIZE_PG_4K);
553 req.rcq_cid = cpu_to_le32(qp->rcq->id);
554 req.qp_flags = cpu_to_le32(qp_flags);
555 req.sq_size = cpu_to_le32(sq->hwq.max_elements);
556 req.rq_size = cpu_to_le32(rq->hwq.max_elements);
557 qp->sq_hdr_buf = NULL;
558 qp->rq_hdr_buf = NULL;
560 rc = bnxt_qplib_alloc_qp_hdr_buf(res, qp);
564 /* CTRL-22434: Irrespective of the requested SGE count on the SQ
565 * always create the QP with max send sges possible if the requested
566 * inline size is greater than 0.
568 max_ssge = qp->max_inline_data ? 6 : sq->max_sge;
569 req.sq_fwo_sq_sge = cpu_to_le16(
570 ((max_ssge & CMDQ_CREATE_QP_SQ_SGE_MASK)
571 << CMDQ_CREATE_QP_SQ_SGE_SFT) | 0);
572 req.rq_fwo_rq_sge = cpu_to_le16(
573 ((rq->max_sge & CMDQ_CREATE_QP_RQ_SGE_MASK)
574 << CMDQ_CREATE_QP_RQ_SGE_SFT) | 0);
579 ORD_LIMIT_TO_ORRQ_SLOTS(qp->max_rd_atomic);
580 req_size = xrrq->max_elements *
581 BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE + PAGE_SIZE - 1;
582 req_size &= ~(PAGE_SIZE - 1);
583 rc = bnxt_qplib_alloc_init_hwq(res->pdev, xrrq, NULL, 0,
585 BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE,
586 0, req_size, HWQ_TYPE_CTX);
589 pbl = &xrrq->pbl[PBL_LVL_0];
590 req.orrq_addr = cpu_to_le64(pbl->pg_map_arr[0]);
593 xrrq->max_elements = IRD_LIMIT_TO_IRRQ_SLOTS(
594 qp->max_dest_rd_atomic);
595 req_size = xrrq->max_elements *
596 BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE + PAGE_SIZE - 1;
597 req_size &= ~(PAGE_SIZE - 1);
599 rc = bnxt_qplib_alloc_init_hwq(res->pdev, xrrq, NULL, 0,
601 BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE,
602 0, req_size, HWQ_TYPE_CTX);
606 pbl = &xrrq->pbl[PBL_LVL_0];
607 req.irrq_addr = cpu_to_le64(pbl->pg_map_arr[0]);
609 req.pd_id = cpu_to_le32(qp->pd->id);
611 rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
612 (void *)&resp, NULL, 0);
616 qp->id = le32_to_cpu(resp.xid);
617 qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET;
618 sq->flush_in_progress = false;
619 rq->flush_in_progress = false;
624 if (qp->irrq.max_elements)
625 bnxt_qplib_free_hwq(res->pdev, &qp->irrq);
627 if (qp->orrq.max_elements)
628 bnxt_qplib_free_hwq(res->pdev, &qp->orrq);
630 bnxt_qplib_free_qp_hdr_buf(res, qp);
632 bnxt_qplib_free_hwq(res->pdev, &rq->hwq);
635 bnxt_qplib_free_hwq(res->pdev, &sq->hwq);
641 static void __modify_flags_from_init_state(struct bnxt_qplib_qp *qp)
644 case CMDQ_MODIFY_QP_NEW_STATE_RTR:
645 /* INIT->RTR, configure the path_mtu to the default
646 * 2048 if not being requested
648 if (!(qp->modify_flags &
649 CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU)) {
651 CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU;
653 CMDQ_MODIFY_QP_PATH_MTU_MTU_2048;
656 ~CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID;
657 /* Bono FW require the max_dest_rd_atomic to be >= 1 */
658 if (qp->max_dest_rd_atomic < 1)
659 qp->max_dest_rd_atomic = 1;
660 qp->modify_flags &= ~CMDQ_MODIFY_QP_MODIFY_MASK_SRC_MAC;
661 /* Bono FW 20.6.5 requires SGID_INDEX configuration */
662 if (!(qp->modify_flags &
663 CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX)) {
665 CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX;
666 qp->ah.sgid_index = 0;
674 static void __modify_flags_from_rtr_state(struct bnxt_qplib_qp *qp)
677 case CMDQ_MODIFY_QP_NEW_STATE_RTS:
678 /* Bono FW requires the max_rd_atomic to be >= 1 */
679 if (qp->max_rd_atomic < 1)
680 qp->max_rd_atomic = 1;
681 /* Bono FW does not allow PKEY_INDEX,
682 * DGID, FLOW_LABEL, SGID_INDEX, HOP_LIMIT,
683 * TRAFFIC_CLASS, DEST_MAC, PATH_MTU, RQ_PSN,
684 * MIN_RNR_TIMER, MAX_DEST_RD_ATOMIC, DEST_QP_ID
688 ~(CMDQ_MODIFY_QP_MODIFY_MASK_PKEY |
689 CMDQ_MODIFY_QP_MODIFY_MASK_DGID |
690 CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL |
691 CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX |
692 CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT |
693 CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS |
694 CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC |
695 CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU |
696 CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN |
697 CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER |
698 CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC |
699 CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID);
706 static void __filter_modify_flags(struct bnxt_qplib_qp *qp)
708 switch (qp->cur_qp_state) {
709 case CMDQ_MODIFY_QP_NEW_STATE_RESET:
711 case CMDQ_MODIFY_QP_NEW_STATE_INIT:
712 __modify_flags_from_init_state(qp);
714 case CMDQ_MODIFY_QP_NEW_STATE_RTR:
715 __modify_flags_from_rtr_state(qp);
717 case CMDQ_MODIFY_QP_NEW_STATE_RTS:
719 case CMDQ_MODIFY_QP_NEW_STATE_SQD:
721 case CMDQ_MODIFY_QP_NEW_STATE_SQE:
723 case CMDQ_MODIFY_QP_NEW_STATE_ERR:
730 int bnxt_qplib_modify_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
732 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
733 struct cmdq_modify_qp req;
734 struct creq_modify_qp_resp resp;
735 u16 cmd_flags = 0, pkey;
740 RCFW_CMD_PREP(req, MODIFY_QP, cmd_flags);
742 /* Filter out the qp_attr_mask based on the state->new transition */
743 __filter_modify_flags(qp);
744 bmask = qp->modify_flags;
745 req.modify_mask = cpu_to_le32(qp->modify_flags);
746 req.qp_cid = cpu_to_le32(qp->id);
747 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_STATE) {
748 req.network_type_en_sqd_async_notify_new_state =
749 (qp->state & CMDQ_MODIFY_QP_NEW_STATE_MASK) |
750 (qp->en_sqd_async_notify ?
751 CMDQ_MODIFY_QP_EN_SQD_ASYNC_NOTIFY : 0);
753 req.network_type_en_sqd_async_notify_new_state |= qp->nw_type;
755 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS)
756 req.access = qp->access;
758 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_PKEY) {
759 if (!bnxt_qplib_get_pkey(res, &res->pkey_tbl,
760 qp->pkey_index, &pkey))
761 req.pkey = cpu_to_le16(pkey);
763 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_QKEY)
764 req.qkey = cpu_to_le32(qp->qkey);
766 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DGID) {
767 memcpy(temp32, qp->ah.dgid.data, sizeof(struct bnxt_qplib_gid));
768 req.dgid[0] = cpu_to_le32(temp32[0]);
769 req.dgid[1] = cpu_to_le32(temp32[1]);
770 req.dgid[2] = cpu_to_le32(temp32[2]);
771 req.dgid[3] = cpu_to_le32(temp32[3]);
773 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL)
774 req.flow_label = cpu_to_le32(qp->ah.flow_label);
776 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX)
777 req.sgid_index = cpu_to_le16(res->sgid_tbl.hw_id
778 [qp->ah.sgid_index]);
780 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT)
781 req.hop_limit = qp->ah.hop_limit;
783 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS)
784 req.traffic_class = qp->ah.traffic_class;
786 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC)
787 memcpy(req.dest_mac, qp->ah.dmac, 6);
789 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU)
790 req.path_mtu = qp->path_mtu;
792 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT)
793 req.timeout = qp->timeout;
795 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT)
796 req.retry_cnt = qp->retry_cnt;
798 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY)
799 req.rnr_retry = qp->rnr_retry;
801 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER)
802 req.min_rnr_timer = qp->min_rnr_timer;
804 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN)
805 req.rq_psn = cpu_to_le32(qp->rq.psn);
807 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN)
808 req.sq_psn = cpu_to_le32(qp->sq.psn);
810 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC)
812 ORD_LIMIT_TO_ORRQ_SLOTS(qp->max_rd_atomic);
814 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC)
815 req.max_dest_rd_atomic =
816 IRD_LIMIT_TO_IRRQ_SLOTS(qp->max_dest_rd_atomic);
818 req.sq_size = cpu_to_le32(qp->sq.hwq.max_elements);
819 req.rq_size = cpu_to_le32(qp->rq.hwq.max_elements);
820 req.sq_sge = cpu_to_le16(qp->sq.max_sge);
821 req.rq_sge = cpu_to_le16(qp->rq.max_sge);
822 req.max_inline_data = cpu_to_le32(qp->max_inline_data);
823 if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID)
824 req.dest_qp_id = cpu_to_le32(qp->dest_qpn);
826 req.vlan_pcp_vlan_dei_vlan_id = cpu_to_le16(qp->vlan_id);
828 rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
829 (void *)&resp, NULL, 0);
832 qp->cur_qp_state = qp->state;
836 int bnxt_qplib_query_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
838 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
839 struct cmdq_query_qp req;
840 struct creq_query_qp_resp resp;
841 struct bnxt_qplib_rcfw_sbuf *sbuf;
842 struct creq_query_qp_resp_sb *sb;
847 RCFW_CMD_PREP(req, QUERY_QP, cmd_flags);
849 sbuf = bnxt_qplib_rcfw_alloc_sbuf(rcfw, sizeof(*sb));
854 req.qp_cid = cpu_to_le32(qp->id);
855 req.resp_size = sizeof(*sb) / BNXT_QPLIB_CMDQE_UNITS;
856 rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req, (void *)&resp,
860 /* Extract the context from the side buffer */
861 qp->state = sb->en_sqd_async_notify_state &
862 CREQ_QUERY_QP_RESP_SB_STATE_MASK;
863 qp->en_sqd_async_notify = sb->en_sqd_async_notify_state &
864 CREQ_QUERY_QP_RESP_SB_EN_SQD_ASYNC_NOTIFY ?
866 qp->access = sb->access;
867 qp->pkey_index = le16_to_cpu(sb->pkey);
868 qp->qkey = le32_to_cpu(sb->qkey);
870 temp32[0] = le32_to_cpu(sb->dgid[0]);
871 temp32[1] = le32_to_cpu(sb->dgid[1]);
872 temp32[2] = le32_to_cpu(sb->dgid[2]);
873 temp32[3] = le32_to_cpu(sb->dgid[3]);
874 memcpy(qp->ah.dgid.data, temp32, sizeof(qp->ah.dgid.data));
876 qp->ah.flow_label = le32_to_cpu(sb->flow_label);
878 qp->ah.sgid_index = 0;
879 for (i = 0; i < res->sgid_tbl.max; i++) {
880 if (res->sgid_tbl.hw_id[i] == le16_to_cpu(sb->sgid_index)) {
881 qp->ah.sgid_index = i;
885 if (i == res->sgid_tbl.max)
886 dev_warn(&res->pdev->dev, "QPLIB: SGID not found??");
888 qp->ah.hop_limit = sb->hop_limit;
889 qp->ah.traffic_class = sb->traffic_class;
890 memcpy(qp->ah.dmac, sb->dest_mac, 6);
891 qp->ah.vlan_id = (le16_to_cpu(sb->path_mtu_dest_vlan_id) &
892 CREQ_QUERY_QP_RESP_SB_VLAN_ID_MASK) >>
893 CREQ_QUERY_QP_RESP_SB_VLAN_ID_SFT;
894 qp->path_mtu = (le16_to_cpu(sb->path_mtu_dest_vlan_id) &
895 CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK) >>
896 CREQ_QUERY_QP_RESP_SB_PATH_MTU_SFT;
897 qp->timeout = sb->timeout;
898 qp->retry_cnt = sb->retry_cnt;
899 qp->rnr_retry = sb->rnr_retry;
900 qp->min_rnr_timer = sb->min_rnr_timer;
901 qp->rq.psn = le32_to_cpu(sb->rq_psn);
902 qp->max_rd_atomic = ORRQ_SLOTS_TO_ORD_LIMIT(sb->max_rd_atomic);
903 qp->sq.psn = le32_to_cpu(sb->sq_psn);
904 qp->max_dest_rd_atomic =
905 IRRQ_SLOTS_TO_IRD_LIMIT(sb->max_dest_rd_atomic);
906 qp->sq.max_wqe = qp->sq.hwq.max_elements;
907 qp->rq.max_wqe = qp->rq.hwq.max_elements;
908 qp->sq.max_sge = le16_to_cpu(sb->sq_sge);
909 qp->rq.max_sge = le16_to_cpu(sb->rq_sge);
910 qp->max_inline_data = le32_to_cpu(sb->max_inline_data);
911 qp->dest_qpn = le32_to_cpu(sb->dest_qp_id);
912 memcpy(qp->smac, sb->src_mac, 6);
913 qp->vlan_id = le16_to_cpu(sb->vlan_pcp_vlan_dei_vlan_id);
915 bnxt_qplib_rcfw_free_sbuf(rcfw, sbuf);
919 static void __clean_cq(struct bnxt_qplib_cq *cq, u64 qp)
921 struct bnxt_qplib_hwq *cq_hwq = &cq->hwq;
922 struct cq_base *hw_cqe, **hw_cqe_ptr;
925 for (i = 0; i < cq_hwq->max_elements; i++) {
926 hw_cqe_ptr = (struct cq_base **)cq_hwq->pbl_ptr;
927 hw_cqe = &hw_cqe_ptr[CQE_PG(i)][CQE_IDX(i)];
928 if (!CQE_CMP_VALID(hw_cqe, i, cq_hwq->max_elements))
930 switch (hw_cqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK) {
931 case CQ_BASE_CQE_TYPE_REQ:
932 case CQ_BASE_CQE_TYPE_TERMINAL:
934 struct cq_req *cqe = (struct cq_req *)hw_cqe;
936 if (qp == le64_to_cpu(cqe->qp_handle))
940 case CQ_BASE_CQE_TYPE_RES_RC:
941 case CQ_BASE_CQE_TYPE_RES_UD:
942 case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1:
944 struct cq_res_rc *cqe = (struct cq_res_rc *)hw_cqe;
946 if (qp == le64_to_cpu(cqe->qp_handle))
956 int bnxt_qplib_destroy_qp(struct bnxt_qplib_res *res,
957 struct bnxt_qplib_qp *qp)
959 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
960 struct cmdq_destroy_qp req;
961 struct creq_destroy_qp_resp resp;
966 RCFW_CMD_PREP(req, DESTROY_QP, cmd_flags);
968 req.qp_cid = cpu_to_le32(qp->id);
969 rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
970 (void *)&resp, NULL, 0);
974 /* Must walk the associated CQs to nullified the QP ptr */
975 spin_lock_irqsave(&qp->scq->hwq.lock, flags);
977 __clean_cq(qp->scq, (u64)(unsigned long)qp);
979 if (qp->rcq && qp->rcq != qp->scq) {
980 spin_lock(&qp->rcq->hwq.lock);
981 __clean_cq(qp->rcq, (u64)(unsigned long)qp);
982 spin_unlock(&qp->rcq->hwq.lock);
985 spin_unlock_irqrestore(&qp->scq->hwq.lock, flags);
987 bnxt_qplib_free_qp_hdr_buf(res, qp);
988 bnxt_qplib_free_hwq(res->pdev, &qp->sq.hwq);
991 bnxt_qplib_free_hwq(res->pdev, &qp->rq.hwq);
994 if (qp->irrq.max_elements)
995 bnxt_qplib_free_hwq(res->pdev, &qp->irrq);
996 if (qp->orrq.max_elements)
997 bnxt_qplib_free_hwq(res->pdev, &qp->orrq);
1002 void *bnxt_qplib_get_qp1_sq_buf(struct bnxt_qplib_qp *qp,
1003 struct bnxt_qplib_sge *sge)
1005 struct bnxt_qplib_q *sq = &qp->sq;
1008 memset(sge, 0, sizeof(*sge));
1010 if (qp->sq_hdr_buf) {
1011 sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);
1012 sge->addr = (dma_addr_t)(qp->sq_hdr_buf_map +
1013 sw_prod * qp->sq_hdr_buf_size);
1014 sge->lkey = 0xFFFFFFFF;
1015 sge->size = qp->sq_hdr_buf_size;
1016 return qp->sq_hdr_buf + sw_prod * sge->size;
1021 u32 bnxt_qplib_get_rq_prod_index(struct bnxt_qplib_qp *qp)
1023 struct bnxt_qplib_q *rq = &qp->rq;
1025 return HWQ_CMP(rq->hwq.prod, &rq->hwq);
1028 dma_addr_t bnxt_qplib_get_qp_buf_from_index(struct bnxt_qplib_qp *qp, u32 index)
1030 return (qp->rq_hdr_buf_map + index * qp->rq_hdr_buf_size);
1033 void *bnxt_qplib_get_qp1_rq_buf(struct bnxt_qplib_qp *qp,
1034 struct bnxt_qplib_sge *sge)
1036 struct bnxt_qplib_q *rq = &qp->rq;
1039 memset(sge, 0, sizeof(*sge));
1041 if (qp->rq_hdr_buf) {
1042 sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);
1043 sge->addr = (dma_addr_t)(qp->rq_hdr_buf_map +
1044 sw_prod * qp->rq_hdr_buf_size);
1045 sge->lkey = 0xFFFFFFFF;
1046 sge->size = qp->rq_hdr_buf_size;
1047 return qp->rq_hdr_buf + sw_prod * sge->size;
1052 void bnxt_qplib_post_send_db(struct bnxt_qplib_qp *qp)
1054 struct bnxt_qplib_q *sq = &qp->sq;
1055 struct dbr_dbr db_msg = { 0 };
1058 sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);
1060 db_msg.index = cpu_to_le32((sw_prod << DBR_DBR_INDEX_SFT) &
1061 DBR_DBR_INDEX_MASK);
1063 cpu_to_le32(((qp->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) |
1065 /* Flush all the WQE writes to HW */
1067 __iowrite64_copy(qp->dpi->dbr, &db_msg, sizeof(db_msg) / sizeof(u64));
1070 int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp,
1071 struct bnxt_qplib_swqe *wqe)
1073 struct bnxt_qplib_q *sq = &qp->sq;
1074 struct bnxt_qplib_swq *swq;
1075 struct sq_send *hw_sq_send_hdr, **hw_sq_send_ptr;
1076 struct sq_sge *hw_sge;
1079 int i, rc = 0, data_len = 0, pkt_num = 0;
1082 if (qp->state != CMDQ_MODIFY_QP_NEW_STATE_RTS) {
1087 if (bnxt_qplib_queue_full(sq)) {
1088 dev_err(&sq->hwq.pdev->dev,
1089 "QPLIB: prod = %#x cons = %#x qdepth = %#x delta = %#x",
1090 sq->hwq.prod, sq->hwq.cons, sq->hwq.max_elements,
1095 sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);
1096 swq = &sq->swq[sw_prod];
1097 swq->wr_id = wqe->wr_id;
1098 swq->type = wqe->type;
1099 swq->flags = wqe->flags;
1101 swq->flags |= SQ_SEND_FLAGS_SIGNAL_COMP;
1102 swq->start_psn = sq->psn & BTH_PSN_MASK;
1104 hw_sq_send_ptr = (struct sq_send **)sq->hwq.pbl_ptr;
1105 hw_sq_send_hdr = &hw_sq_send_ptr[get_sqe_pg(sw_prod)]
1106 [get_sqe_idx(sw_prod)];
1108 memset(hw_sq_send_hdr, 0, BNXT_QPLIB_MAX_SQE_ENTRY_SIZE);
1110 if (wqe->flags & BNXT_QPLIB_SWQE_FLAGS_INLINE) {
1111 /* Copy the inline data */
1112 if (wqe->inline_len > BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH) {
1113 dev_warn(&sq->hwq.pdev->dev,
1114 "QPLIB: Inline data length > 96 detected");
1115 data_len = BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH;
1117 data_len = wqe->inline_len;
1119 memcpy(hw_sq_send_hdr->data, wqe->inline_data, data_len);
1120 wqe_size16 = (data_len + 15) >> 4;
1122 for (i = 0, hw_sge = (struct sq_sge *)hw_sq_send_hdr->data;
1123 i < wqe->num_sge; i++, hw_sge++) {
1124 hw_sge->va_or_pa = cpu_to_le64(wqe->sg_list[i].addr);
1125 hw_sge->l_key = cpu_to_le32(wqe->sg_list[i].lkey);
1126 hw_sge->size = cpu_to_le32(wqe->sg_list[i].size);
1127 data_len += wqe->sg_list[i].size;
1129 /* Each SGE entry = 1 WQE size16 */
1130 wqe_size16 = wqe->num_sge;
1134 switch (wqe->type) {
1135 case BNXT_QPLIB_SWQE_TYPE_SEND:
1136 if (qp->type == CMDQ_CREATE_QP1_TYPE_GSI) {
1137 /* Assemble info for Raw Ethertype QPs */
1138 struct sq_send_raweth_qp1 *sqe =
1139 (struct sq_send_raweth_qp1 *)hw_sq_send_hdr;
1141 sqe->wqe_type = wqe->type;
1142 sqe->flags = wqe->flags;
1143 sqe->wqe_size = wqe_size16 +
1144 ((offsetof(typeof(*sqe), data) + 15) >> 4);
1145 sqe->cfa_action = cpu_to_le16(wqe->rawqp1.cfa_action);
1146 sqe->lflags = cpu_to_le16(wqe->rawqp1.lflags);
1147 sqe->length = cpu_to_le32(data_len);
1148 sqe->cfa_meta = cpu_to_le32((wqe->rawqp1.cfa_meta &
1149 SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_MASK) <<
1150 SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_SFT);
1154 /* else, just fall thru */
1155 case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM:
1156 case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV:
1158 struct sq_send *sqe = (struct sq_send *)hw_sq_send_hdr;
1160 sqe->wqe_type = wqe->type;
1161 sqe->flags = wqe->flags;
1162 sqe->wqe_size = wqe_size16 +
1163 ((offsetof(typeof(*sqe), data) + 15) >> 4);
1164 sqe->inv_key_or_imm_data = cpu_to_le32(
1166 if (qp->type == CMDQ_CREATE_QP_TYPE_UD) {
1167 sqe->q_key = cpu_to_le32(wqe->send.q_key);
1168 sqe->dst_qp = cpu_to_le32(
1169 wqe->send.dst_qp & SQ_SEND_DST_QP_MASK);
1170 sqe->length = cpu_to_le32(data_len);
1171 sqe->avid = cpu_to_le32(wqe->send.avid &
1173 sq->psn = (sq->psn + 1) & BTH_PSN_MASK;
1175 sqe->length = cpu_to_le32(data_len);
1179 pkt_num = (data_len + qp->mtu - 1) / qp->mtu;
1182 sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK;
1186 case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE:
1187 case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM:
1188 case BNXT_QPLIB_SWQE_TYPE_RDMA_READ:
1190 struct sq_rdma *sqe = (struct sq_rdma *)hw_sq_send_hdr;
1192 sqe->wqe_type = wqe->type;
1193 sqe->flags = wqe->flags;
1194 sqe->wqe_size = wqe_size16 +
1195 ((offsetof(typeof(*sqe), data) + 15) >> 4);
1196 sqe->imm_data = cpu_to_le32(wqe->rdma.inv_key);
1197 sqe->length = cpu_to_le32((u32)data_len);
1198 sqe->remote_va = cpu_to_le64(wqe->rdma.remote_va);
1199 sqe->remote_key = cpu_to_le32(wqe->rdma.r_key);
1201 pkt_num = (data_len + qp->mtu - 1) / qp->mtu;
1204 sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK;
1207 case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP:
1208 case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD:
1210 struct sq_atomic *sqe = (struct sq_atomic *)hw_sq_send_hdr;
1212 sqe->wqe_type = wqe->type;
1213 sqe->flags = wqe->flags;
1214 sqe->remote_key = cpu_to_le32(wqe->atomic.r_key);
1215 sqe->remote_va = cpu_to_le64(wqe->atomic.remote_va);
1216 sqe->swap_data = cpu_to_le64(wqe->atomic.swap_data);
1217 sqe->cmp_data = cpu_to_le64(wqe->atomic.cmp_data);
1219 pkt_num = (data_len + qp->mtu - 1) / qp->mtu;
1222 sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK;
1225 case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV:
1227 struct sq_localinvalidate *sqe =
1228 (struct sq_localinvalidate *)hw_sq_send_hdr;
1230 sqe->wqe_type = wqe->type;
1231 sqe->flags = wqe->flags;
1232 sqe->inv_l_key = cpu_to_le32(wqe->local_inv.inv_l_key);
1236 case BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR:
1238 struct sq_fr_pmr *sqe = (struct sq_fr_pmr *)hw_sq_send_hdr;
1240 sqe->wqe_type = wqe->type;
1241 sqe->flags = wqe->flags;
1242 sqe->access_cntl = wqe->frmr.access_cntl |
1243 SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE;
1244 sqe->zero_based_page_size_log =
1245 (wqe->frmr.pg_sz_log & SQ_FR_PMR_PAGE_SIZE_LOG_MASK) <<
1246 SQ_FR_PMR_PAGE_SIZE_LOG_SFT |
1247 (wqe->frmr.zero_based ? SQ_FR_PMR_ZERO_BASED : 0);
1248 sqe->l_key = cpu_to_le32(wqe->frmr.l_key);
1249 temp32 = cpu_to_le32(wqe->frmr.length);
1250 memcpy(sqe->length, &temp32, sizeof(wqe->frmr.length));
1251 sqe->numlevels_pbl_page_size_log =
1252 ((wqe->frmr.pbl_pg_sz_log <<
1253 SQ_FR_PMR_PBL_PAGE_SIZE_LOG_SFT) &
1254 SQ_FR_PMR_PBL_PAGE_SIZE_LOG_MASK) |
1255 ((wqe->frmr.levels << SQ_FR_PMR_NUMLEVELS_SFT) &
1256 SQ_FR_PMR_NUMLEVELS_MASK);
1258 for (i = 0; i < wqe->frmr.page_list_len; i++)
1259 wqe->frmr.pbl_ptr[i] = cpu_to_le64(
1260 wqe->frmr.page_list[i] |
1262 sqe->pblptr = cpu_to_le64(wqe->frmr.pbl_dma_ptr);
1263 sqe->va = cpu_to_le64(wqe->frmr.va);
1267 case BNXT_QPLIB_SWQE_TYPE_BIND_MW:
1269 struct sq_bind *sqe = (struct sq_bind *)hw_sq_send_hdr;
1271 sqe->wqe_type = wqe->type;
1272 sqe->flags = wqe->flags;
1273 sqe->access_cntl = wqe->bind.access_cntl;
1274 sqe->mw_type_zero_based = wqe->bind.mw_type |
1275 (wqe->bind.zero_based ? SQ_BIND_ZERO_BASED : 0);
1276 sqe->parent_l_key = cpu_to_le32(wqe->bind.parent_l_key);
1277 sqe->l_key = cpu_to_le32(wqe->bind.r_key);
1278 sqe->va = cpu_to_le64(wqe->bind.va);
1279 temp32 = cpu_to_le32(wqe->bind.length);
1280 memcpy(&sqe->length, &temp32, sizeof(wqe->bind.length));
1284 /* Bad wqe, return error */
1288 swq->next_psn = sq->psn & BTH_PSN_MASK;
1289 if (swq->psn_search) {
1290 swq->psn_search->opcode_start_psn = cpu_to_le32(
1291 ((swq->start_psn << SQ_PSN_SEARCH_START_PSN_SFT) &
1292 SQ_PSN_SEARCH_START_PSN_MASK) |
1293 ((wqe->type << SQ_PSN_SEARCH_OPCODE_SFT) &
1294 SQ_PSN_SEARCH_OPCODE_MASK));
1295 swq->psn_search->flags_next_psn = cpu_to_le32(
1296 ((swq->next_psn << SQ_PSN_SEARCH_NEXT_PSN_SFT) &
1297 SQ_PSN_SEARCH_NEXT_PSN_MASK));
1308 void bnxt_qplib_post_recv_db(struct bnxt_qplib_qp *qp)
1310 struct bnxt_qplib_q *rq = &qp->rq;
1311 struct dbr_dbr db_msg = { 0 };
1314 sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);
1315 db_msg.index = cpu_to_le32((sw_prod << DBR_DBR_INDEX_SFT) &
1316 DBR_DBR_INDEX_MASK);
1318 cpu_to_le32(((qp->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) |
1321 /* Flush the writes to HW Rx WQE before the ringing Rx DB */
1323 __iowrite64_copy(qp->dpi->dbr, &db_msg, sizeof(db_msg) / sizeof(u64));
1326 int bnxt_qplib_post_recv(struct bnxt_qplib_qp *qp,
1327 struct bnxt_qplib_swqe *wqe)
1329 struct bnxt_qplib_q *rq = &qp->rq;
1330 struct rq_wqe *rqe, **rqe_ptr;
1331 struct sq_sge *hw_sge;
1335 if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {
1336 dev_err(&rq->hwq.pdev->dev,
1337 "QPLIB: FP: QP (0x%x) is in the 0x%x state",
1342 if (bnxt_qplib_queue_full(rq)) {
1343 dev_err(&rq->hwq.pdev->dev,
1344 "QPLIB: FP: QP (0x%x) RQ is full!", qp->id);
1348 sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);
1349 rq->swq[sw_prod].wr_id = wqe->wr_id;
1351 rqe_ptr = (struct rq_wqe **)rq->hwq.pbl_ptr;
1352 rqe = &rqe_ptr[RQE_PG(sw_prod)][RQE_IDX(sw_prod)];
1354 memset(rqe, 0, BNXT_QPLIB_MAX_RQE_ENTRY_SIZE);
1356 /* Calculate wqe_size16 and data_len */
1357 for (i = 0, hw_sge = (struct sq_sge *)rqe->data;
1358 i < wqe->num_sge; i++, hw_sge++) {
1359 hw_sge->va_or_pa = cpu_to_le64(wqe->sg_list[i].addr);
1360 hw_sge->l_key = cpu_to_le32(wqe->sg_list[i].lkey);
1361 hw_sge->size = cpu_to_le32(wqe->sg_list[i].size);
1363 rqe->wqe_type = wqe->type;
1364 rqe->flags = wqe->flags;
1365 rqe->wqe_size = wqe->num_sge +
1366 ((offsetof(typeof(*rqe), data) + 15) >> 4);
1368 /* Supply the rqe->wr_id index to the wr_id_tbl for now */
1369 rqe->wr_id[0] = cpu_to_le32(sw_prod);
1378 /* Spinlock must be held */
1379 static void bnxt_qplib_arm_cq_enable(struct bnxt_qplib_cq *cq)
1381 struct dbr_dbr db_msg = { 0 };
1384 cpu_to_le32(((cq->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) |
1385 DBR_DBR_TYPE_CQ_ARMENA);
1386 /* Flush memory writes before enabling the CQ */
1388 __iowrite64_copy(cq->dbr_base, &db_msg, sizeof(db_msg) / sizeof(u64));
1391 static void bnxt_qplib_arm_cq(struct bnxt_qplib_cq *cq, u32 arm_type)
1393 struct bnxt_qplib_hwq *cq_hwq = &cq->hwq;
1394 struct dbr_dbr db_msg = { 0 };
1398 sw_cons = HWQ_CMP(cq_hwq->cons, cq_hwq);
1399 db_msg.index = cpu_to_le32((sw_cons << DBR_DBR_INDEX_SFT) &
1400 DBR_DBR_INDEX_MASK);
1402 cpu_to_le32(((cq->id << DBR_DBR_XID_SFT) & DBR_DBR_XID_MASK) |
1404 /* flush memory writes before arming the CQ */
1406 __iowrite64_copy(cq->dpi->dbr, &db_msg, sizeof(db_msg) / sizeof(u64));
1409 int bnxt_qplib_create_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq)
1411 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
1412 struct cmdq_create_cq req;
1413 struct creq_create_cq_resp resp;
1414 struct bnxt_qplib_pbl *pbl;
1418 cq->hwq.max_elements = cq->max_wqe;
1419 rc = bnxt_qplib_alloc_init_hwq(res->pdev, &cq->hwq, cq->sghead,
1420 cq->nmap, &cq->hwq.max_elements,
1421 BNXT_QPLIB_MAX_CQE_ENTRY_SIZE, 0,
1422 PAGE_SIZE, HWQ_TYPE_QUEUE);
1426 RCFW_CMD_PREP(req, CREATE_CQ, cmd_flags);
1429 dev_err(&rcfw->pdev->dev,
1430 "QPLIB: FP: CREATE_CQ failed due to NULL DPI");
1433 req.dpi = cpu_to_le32(cq->dpi->dpi);
1434 req.cq_handle = cpu_to_le64(cq->cq_handle);
1436 req.cq_size = cpu_to_le32(cq->hwq.max_elements);
1437 pbl = &cq->hwq.pbl[PBL_LVL_0];
1438 req.pg_size_lvl = cpu_to_le32(
1439 ((cq->hwq.level & CMDQ_CREATE_CQ_LVL_MASK) <<
1440 CMDQ_CREATE_CQ_LVL_SFT) |
1441 (pbl->pg_size == ROCE_PG_SIZE_4K ? CMDQ_CREATE_CQ_PG_SIZE_PG_4K :
1442 pbl->pg_size == ROCE_PG_SIZE_8K ? CMDQ_CREATE_CQ_PG_SIZE_PG_8K :
1443 pbl->pg_size == ROCE_PG_SIZE_64K ? CMDQ_CREATE_CQ_PG_SIZE_PG_64K :
1444 pbl->pg_size == ROCE_PG_SIZE_2M ? CMDQ_CREATE_CQ_PG_SIZE_PG_2M :
1445 pbl->pg_size == ROCE_PG_SIZE_8M ? CMDQ_CREATE_CQ_PG_SIZE_PG_8M :
1446 pbl->pg_size == ROCE_PG_SIZE_1G ? CMDQ_CREATE_CQ_PG_SIZE_PG_1G :
1447 CMDQ_CREATE_CQ_PG_SIZE_PG_4K));
1449 req.pbl = cpu_to_le64(pbl->pg_map_arr[0]);
1451 req.cq_fco_cnq_id = cpu_to_le32(
1452 (cq->cnq_hw_ring_id & CMDQ_CREATE_CQ_CNQ_ID_MASK) <<
1453 CMDQ_CREATE_CQ_CNQ_ID_SFT);
1455 rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
1456 (void *)&resp, NULL, 0);
1460 cq->id = le32_to_cpu(resp.xid);
1461 cq->dbr_base = res->dpi_tbl.dbr_bar_reg_iomem;
1462 cq->period = BNXT_QPLIB_QUEUE_START_PERIOD;
1463 init_waitqueue_head(&cq->waitq);
1465 bnxt_qplib_arm_cq_enable(cq);
1469 bnxt_qplib_free_hwq(res->pdev, &cq->hwq);
1474 int bnxt_qplib_destroy_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq)
1476 struct bnxt_qplib_rcfw *rcfw = res->rcfw;
1477 struct cmdq_destroy_cq req;
1478 struct creq_destroy_cq_resp resp;
1482 RCFW_CMD_PREP(req, DESTROY_CQ, cmd_flags);
1484 req.cq_cid = cpu_to_le32(cq->id);
1485 rc = bnxt_qplib_rcfw_send_message(rcfw, (void *)&req,
1486 (void *)&resp, NULL, 0);
1489 bnxt_qplib_free_hwq(res->pdev, &cq->hwq);
1493 static int __flush_sq(struct bnxt_qplib_q *sq, struct bnxt_qplib_qp *qp,
1494 struct bnxt_qplib_cqe **pcqe, int *budget)
1496 u32 sw_prod, sw_cons;
1497 struct bnxt_qplib_cqe *cqe;
1500 /* Now complete all outstanding SQEs with FLUSHED_ERR */
1501 sw_prod = HWQ_CMP(sq->hwq.prod, &sq->hwq);
1504 sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq);
1505 if (sw_cons == sw_prod) {
1506 sq->flush_in_progress = false;
1509 memset(cqe, 0, sizeof(*cqe));
1510 cqe->status = CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR;
1511 cqe->opcode = CQ_BASE_CQE_TYPE_REQ;
1512 cqe->qp_handle = (u64)(unsigned long)qp;
1513 cqe->wr_id = sq->swq[sw_cons].wr_id;
1514 cqe->src_qp = qp->id;
1515 cqe->type = sq->swq[sw_cons].type;
1521 if (!(*budget) && HWQ_CMP(sq->hwq.cons, &sq->hwq) != sw_prod)
1528 static int __flush_rq(struct bnxt_qplib_q *rq, struct bnxt_qplib_qp *qp,
1529 int opcode, struct bnxt_qplib_cqe **pcqe, int *budget)
1531 struct bnxt_qplib_cqe *cqe;
1532 u32 sw_prod, sw_cons;
1535 /* Flush the rest of the RQ */
1536 sw_prod = HWQ_CMP(rq->hwq.prod, &rq->hwq);
1539 sw_cons = HWQ_CMP(rq->hwq.cons, &rq->hwq);
1540 if (sw_cons == sw_prod)
1542 memset(cqe, 0, sizeof(*cqe));
1544 CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR;
1545 cqe->opcode = opcode;
1546 cqe->qp_handle = (unsigned long)qp;
1547 cqe->wr_id = rq->swq[sw_cons].wr_id;
1553 if (!*budget && HWQ_CMP(rq->hwq.cons, &rq->hwq) != sw_prod)
1560 /* Note: SQE is valid from sw_sq_cons up to cqe_sq_cons (exclusive)
1561 * CQE is track from sw_cq_cons to max_element but valid only if VALID=1
1563 static int do_wa9060(struct bnxt_qplib_qp *qp, struct bnxt_qplib_cq *cq,
1564 u32 cq_cons, u32 sw_sq_cons, u32 cqe_sq_cons)
1566 struct bnxt_qplib_q *sq = &qp->sq;
1567 struct bnxt_qplib_swq *swq;
1568 u32 peek_sw_cq_cons, peek_raw_cq_cons, peek_sq_cons_idx;
1569 struct cq_base *peek_hwcqe, **peek_hw_cqe_ptr;
1570 struct cq_req *peek_req_hwcqe;
1571 struct bnxt_qplib_qp *peek_qp;
1572 struct bnxt_qplib_q *peek_sq;
1576 /* Check for the psn_search marking before completing */
1577 swq = &sq->swq[sw_sq_cons];
1578 if (swq->psn_search &&
1579 le32_to_cpu(swq->psn_search->flags_next_psn) & 0x80000000) {
1581 swq->psn_search->flags_next_psn = cpu_to_le32
1582 (le32_to_cpu(swq->psn_search->flags_next_psn)
1584 dev_dbg(&cq->hwq.pdev->dev,
1585 "FP: Process Req cq_cons=0x%x qp=0x%x sq cons sw=0x%x cqe=0x%x marked!\n",
1586 cq_cons, qp->id, sw_sq_cons, cqe_sq_cons);
1587 sq->condition = true;
1588 sq->send_phantom = true;
1590 /* TODO: Only ARM if the previous SQE is ARMALL */
1591 bnxt_qplib_arm_cq(cq, DBR_DBR_TYPE_CQ_ARMALL);
1596 if (sq->condition) {
1597 /* Peek at the completions */
1598 peek_raw_cq_cons = cq->hwq.cons;
1599 peek_sw_cq_cons = cq_cons;
1600 i = cq->hwq.max_elements;
1602 peek_sw_cq_cons = HWQ_CMP((peek_sw_cq_cons), &cq->hwq);
1603 peek_hw_cqe_ptr = (struct cq_base **)cq->hwq.pbl_ptr;
1604 peek_hwcqe = &peek_hw_cqe_ptr[CQE_PG(peek_sw_cq_cons)]
1605 [CQE_IDX(peek_sw_cq_cons)];
1606 /* If the next hwcqe is VALID */
1607 if (CQE_CMP_VALID(peek_hwcqe, peek_raw_cq_cons,
1608 cq->hwq.max_elements)) {
1609 /* If the next hwcqe is a REQ */
1610 if ((peek_hwcqe->cqe_type_toggle &
1611 CQ_BASE_CQE_TYPE_MASK) ==
1612 CQ_BASE_CQE_TYPE_REQ) {
1613 peek_req_hwcqe = (struct cq_req *)
1615 peek_qp = (struct bnxt_qplib_qp *)
1618 (peek_req_hwcqe->qp_handle));
1619 peek_sq = &peek_qp->sq;
1620 peek_sq_cons_idx = HWQ_CMP(le16_to_cpu(
1621 peek_req_hwcqe->sq_cons_idx) - 1
1623 /* If the hwcqe's sq's wr_id matches */
1624 if (peek_sq == sq &&
1625 sq->swq[peek_sq_cons_idx].wr_id ==
1626 BNXT_QPLIB_FENCE_WRID) {
1628 * Unbreak only if the phantom
1631 dev_dbg(&cq->hwq.pdev->dev,
1632 "FP:Got Phantom CQE");
1633 sq->condition = false;
1639 /* Valid but not the phantom, so keep looping */
1641 /* Not valid yet, just exit and wait */
1648 dev_err(&cq->hwq.pdev->dev,
1649 "Should not have come here! cq_cons=0x%x qp=0x%x sq cons sw=0x%x hw=0x%x",
1650 cq_cons, qp->id, sw_sq_cons, cqe_sq_cons);
1657 static int bnxt_qplib_cq_process_req(struct bnxt_qplib_cq *cq,
1658 struct cq_req *hwcqe,
1659 struct bnxt_qplib_cqe **pcqe, int *budget,
1660 u32 cq_cons, struct bnxt_qplib_qp **lib_qp)
1662 struct bnxt_qplib_qp *qp;
1663 struct bnxt_qplib_q *sq;
1664 struct bnxt_qplib_cqe *cqe;
1665 u32 sw_sq_cons, cqe_sq_cons;
1666 struct bnxt_qplib_swq *swq;
1669 qp = (struct bnxt_qplib_qp *)((unsigned long)
1670 le64_to_cpu(hwcqe->qp_handle));
1672 dev_err(&cq->hwq.pdev->dev,
1673 "QPLIB: FP: Process Req qp is NULL");
1678 cqe_sq_cons = HWQ_CMP(le16_to_cpu(hwcqe->sq_cons_idx), &sq->hwq);
1679 if (cqe_sq_cons > sq->hwq.max_elements) {
1680 dev_err(&cq->hwq.pdev->dev,
1681 "QPLIB: FP: CQ Process req reported ");
1682 dev_err(&cq->hwq.pdev->dev,
1683 "QPLIB: sq_cons_idx 0x%x which exceeded max 0x%x",
1684 cqe_sq_cons, sq->hwq.max_elements);
1687 /* If we were in the middle of flushing the SQ, continue */
1688 if (sq->flush_in_progress)
1691 /* Require to walk the sq's swq to fabricate CQEs for all previously
1692 * signaled SWQEs due to CQE aggregation from the current sq cons
1693 * to the cqe_sq_cons
1697 sw_sq_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq);
1698 if (sw_sq_cons == cqe_sq_cons)
1702 swq = &sq->swq[sw_sq_cons];
1703 memset(cqe, 0, sizeof(*cqe));
1704 cqe->opcode = CQ_BASE_CQE_TYPE_REQ;
1705 cqe->qp_handle = (u64)(unsigned long)qp;
1706 cqe->src_qp = qp->id;
1707 cqe->wr_id = swq->wr_id;
1708 if (cqe->wr_id == BNXT_QPLIB_FENCE_WRID)
1710 cqe->type = swq->type;
1712 /* For the last CQE, check for status. For errors, regardless
1713 * of the request being signaled or not, it must complete with
1714 * the hwcqe error status
1716 if (HWQ_CMP((sw_sq_cons + 1), &sq->hwq) == cqe_sq_cons &&
1717 hwcqe->status != CQ_REQ_STATUS_OK) {
1718 cqe->status = hwcqe->status;
1719 dev_err(&cq->hwq.pdev->dev,
1720 "QPLIB: FP: CQ Processed Req ");
1721 dev_err(&cq->hwq.pdev->dev,
1722 "QPLIB: wr_id[%d] = 0x%llx with status 0x%x",
1723 sw_sq_cons, cqe->wr_id, cqe->status);
1726 sq->flush_in_progress = true;
1727 /* Must block new posting of SQ and RQ */
1728 qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR;
1729 sq->condition = false;
1732 if (swq->flags & SQ_SEND_FLAGS_SIGNAL_COMP) {
1733 /* Before we complete, do WA 9060 */
1734 if (do_wa9060(qp, cq, cq_cons, sw_sq_cons,
1739 cqe->status = CQ_REQ_STATUS_OK;
1751 if (HWQ_CMP(sq->hwq.cons, &sq->hwq) != cqe_sq_cons) {
1757 * Back to normal completion mode only after it has completed all of
1758 * the WC for this CQE
1761 if (!sq->flush_in_progress)
1764 /* Require to walk the sq's swq to fabricate CQEs for all
1765 * previously posted SWQEs due to the error CQE received
1767 rc = __flush_sq(sq, qp, pcqe, budget);
1769 sq->flush_in_progress = false;
1774 static int bnxt_qplib_cq_process_res_rc(struct bnxt_qplib_cq *cq,
1775 struct cq_res_rc *hwcqe,
1776 struct bnxt_qplib_cqe **pcqe,
1779 struct bnxt_qplib_qp *qp;
1780 struct bnxt_qplib_q *rq;
1781 struct bnxt_qplib_cqe *cqe;
1785 qp = (struct bnxt_qplib_qp *)((unsigned long)
1786 le64_to_cpu(hwcqe->qp_handle));
1788 dev_err(&cq->hwq.pdev->dev, "QPLIB: process_cq RC qp is NULL");
1792 cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK;
1793 cqe->length = le32_to_cpu(hwcqe->length);
1794 cqe->invrkey = le32_to_cpu(hwcqe->imm_data_or_inv_r_key);
1795 cqe->mr_handle = le64_to_cpu(hwcqe->mr_handle);
1796 cqe->flags = le16_to_cpu(hwcqe->flags);
1797 cqe->status = hwcqe->status;
1798 cqe->qp_handle = (u64)(unsigned long)qp;
1800 wr_id_idx = le32_to_cpu(hwcqe->srq_or_rq_wr_id) &
1801 CQ_RES_RC_SRQ_OR_RQ_WR_ID_MASK;
1803 if (wr_id_idx > rq->hwq.max_elements) {
1804 dev_err(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Process RC ");
1805 dev_err(&cq->hwq.pdev->dev,
1806 "QPLIB: wr_id idx 0x%x exceeded RQ max 0x%x",
1807 wr_id_idx, rq->hwq.max_elements);
1810 if (rq->flush_in_progress)
1813 cqe->wr_id = rq->swq[wr_id_idx].wr_id;
1819 if (hwcqe->status != CQ_RES_RC_STATUS_OK) {
1820 rq->flush_in_progress = true;
1822 rc = __flush_rq(rq, qp, CQ_BASE_CQE_TYPE_RES_RC, pcqe, budget);
1824 rq->flush_in_progress = false;
1829 static int bnxt_qplib_cq_process_res_ud(struct bnxt_qplib_cq *cq,
1830 struct cq_res_ud *hwcqe,
1831 struct bnxt_qplib_cqe **pcqe,
1834 struct bnxt_qplib_qp *qp;
1835 struct bnxt_qplib_q *rq;
1836 struct bnxt_qplib_cqe *cqe;
1840 qp = (struct bnxt_qplib_qp *)((unsigned long)
1841 le64_to_cpu(hwcqe->qp_handle));
1843 dev_err(&cq->hwq.pdev->dev, "QPLIB: process_cq UD qp is NULL");
1847 cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK;
1848 cqe->length = le32_to_cpu(hwcqe->length);
1849 cqe->invrkey = le32_to_cpu(hwcqe->imm_data);
1850 cqe->flags = le16_to_cpu(hwcqe->flags);
1851 cqe->status = hwcqe->status;
1852 cqe->qp_handle = (u64)(unsigned long)qp;
1853 memcpy(cqe->smac, hwcqe->src_mac, 6);
1854 wr_id_idx = le32_to_cpu(hwcqe->src_qp_high_srq_or_rq_wr_id)
1855 & CQ_RES_UD_SRQ_OR_RQ_WR_ID_MASK;
1856 cqe->src_qp = le16_to_cpu(hwcqe->src_qp_low) |
1858 hwcqe->src_qp_high_srq_or_rq_wr_id) &
1859 CQ_RES_UD_SRC_QP_HIGH_MASK) >> 8);
1862 if (wr_id_idx > rq->hwq.max_elements) {
1863 dev_err(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Process UD ");
1864 dev_err(&cq->hwq.pdev->dev,
1865 "QPLIB: wr_id idx %#x exceeded RQ max %#x",
1866 wr_id_idx, rq->hwq.max_elements);
1869 if (rq->flush_in_progress)
1872 cqe->wr_id = rq->swq[wr_id_idx].wr_id;
1878 if (hwcqe->status != CQ_RES_RC_STATUS_OK) {
1879 rq->flush_in_progress = true;
1881 rc = __flush_rq(rq, qp, CQ_BASE_CQE_TYPE_RES_UD, pcqe, budget);
1883 rq->flush_in_progress = false;
1888 static int bnxt_qplib_cq_process_res_raweth_qp1(struct bnxt_qplib_cq *cq,
1889 struct cq_res_raweth_qp1 *hwcqe,
1890 struct bnxt_qplib_cqe **pcqe,
1893 struct bnxt_qplib_qp *qp;
1894 struct bnxt_qplib_q *rq;
1895 struct bnxt_qplib_cqe *cqe;
1899 qp = (struct bnxt_qplib_qp *)((unsigned long)
1900 le64_to_cpu(hwcqe->qp_handle));
1902 dev_err(&cq->hwq.pdev->dev,
1903 "QPLIB: process_cq Raw/QP1 qp is NULL");
1907 cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK;
1908 cqe->flags = le16_to_cpu(hwcqe->flags);
1909 cqe->qp_handle = (u64)(unsigned long)qp;
1912 le32_to_cpu(hwcqe->raweth_qp1_payload_offset_srq_or_rq_wr_id)
1913 & CQ_RES_RAWETH_QP1_SRQ_OR_RQ_WR_ID_MASK;
1914 cqe->src_qp = qp->id;
1915 if (qp->id == 1 && !cqe->length) {
1916 /* Add workaround for the length misdetection */
1919 cqe->length = le16_to_cpu(hwcqe->length);
1921 cqe->pkey_index = qp->pkey_index;
1922 memcpy(cqe->smac, qp->smac, 6);
1924 cqe->raweth_qp1_flags = le16_to_cpu(hwcqe->raweth_qp1_flags);
1925 cqe->raweth_qp1_flags2 = le32_to_cpu(hwcqe->raweth_qp1_flags2);
1928 if (wr_id_idx > rq->hwq.max_elements) {
1929 dev_err(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Process Raw/QP1 RQ wr_id ");
1930 dev_err(&cq->hwq.pdev->dev, "QPLIB: ix 0x%x exceeded RQ max 0x%x",
1931 wr_id_idx, rq->hwq.max_elements);
1934 if (rq->flush_in_progress)
1937 cqe->wr_id = rq->swq[wr_id_idx].wr_id;
1943 if (hwcqe->status != CQ_RES_RC_STATUS_OK) {
1944 rq->flush_in_progress = true;
1946 rc = __flush_rq(rq, qp, CQ_BASE_CQE_TYPE_RES_RAWETH_QP1, pcqe,
1949 rq->flush_in_progress = false;
1954 static int bnxt_qplib_cq_process_terminal(struct bnxt_qplib_cq *cq,
1955 struct cq_terminal *hwcqe,
1956 struct bnxt_qplib_cqe **pcqe,
1959 struct bnxt_qplib_qp *qp;
1960 struct bnxt_qplib_q *sq, *rq;
1961 struct bnxt_qplib_cqe *cqe;
1962 u32 sw_cons = 0, cqe_cons;
1966 /* Check the Status */
1967 if (hwcqe->status != CQ_TERMINAL_STATUS_OK)
1968 dev_warn(&cq->hwq.pdev->dev,
1969 "QPLIB: FP: CQ Process Terminal Error status = 0x%x",
1972 qp = (struct bnxt_qplib_qp *)((unsigned long)
1973 le64_to_cpu(hwcqe->qp_handle));
1975 dev_err(&cq->hwq.pdev->dev,
1976 "QPLIB: FP: CQ Process terminal qp is NULL");
1979 /* Must block new posting of SQ and RQ */
1980 qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR;
1985 cqe_cons = le16_to_cpu(hwcqe->sq_cons_idx);
1986 if (cqe_cons == 0xFFFF)
1989 if (cqe_cons > sq->hwq.max_elements) {
1990 dev_err(&cq->hwq.pdev->dev,
1991 "QPLIB: FP: CQ Process terminal reported ");
1992 dev_err(&cq->hwq.pdev->dev,
1993 "QPLIB: sq_cons_idx 0x%x which exceeded max 0x%x",
1994 cqe_cons, sq->hwq.max_elements);
1997 /* If we were in the middle of flushing, continue */
1998 if (sq->flush_in_progress)
2001 /* Terminal CQE can also include aggregated successful CQEs prior.
2002 * So we must complete all CQEs from the current sq's cons to the
2003 * cq_cons with status OK
2007 sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq);
2008 if (sw_cons == cqe_cons)
2010 if (sq->swq[sw_cons].flags & SQ_SEND_FLAGS_SIGNAL_COMP) {
2011 memset(cqe, 0, sizeof(*cqe));
2012 cqe->status = CQ_REQ_STATUS_OK;
2013 cqe->opcode = CQ_BASE_CQE_TYPE_REQ;
2014 cqe->qp_handle = (u64)(unsigned long)qp;
2015 cqe->src_qp = qp->id;
2016 cqe->wr_id = sq->swq[sw_cons].wr_id;
2017 cqe->type = sq->swq[sw_cons].type;
2024 if (!(*budget) && sw_cons != cqe_cons) {
2029 sq->flush_in_progress = true;
2031 rc = __flush_sq(sq, qp, pcqe, budget);
2033 sq->flush_in_progress = false;
2038 cqe_cons = le16_to_cpu(hwcqe->rq_cons_idx);
2039 if (cqe_cons == 0xFFFF) {
2041 } else if (cqe_cons > rq->hwq.max_elements) {
2042 dev_err(&cq->hwq.pdev->dev,
2043 "QPLIB: FP: CQ Processed terminal ");
2044 dev_err(&cq->hwq.pdev->dev,
2045 "QPLIB: reported rq_cons_idx 0x%x exceeds max 0x%x",
2046 cqe_cons, rq->hwq.max_elements);
2049 /* Terminal CQE requires all posted RQEs to complete with FLUSHED_ERR
2050 * from the current rq->cons to the rq->prod regardless what the
2051 * rq->cons the terminal CQE indicates
2053 rq->flush_in_progress = true;
2055 case CMDQ_CREATE_QP1_TYPE_GSI:
2056 opcode = CQ_BASE_CQE_TYPE_RES_RAWETH_QP1;
2058 case CMDQ_CREATE_QP_TYPE_RC:
2059 opcode = CQ_BASE_CQE_TYPE_RES_RC;
2061 case CMDQ_CREATE_QP_TYPE_UD:
2062 opcode = CQ_BASE_CQE_TYPE_RES_UD;
2066 rc = __flush_rq(rq, qp, opcode, pcqe, budget);
2068 rq->flush_in_progress = false;
2073 static int bnxt_qplib_cq_process_cutoff(struct bnxt_qplib_cq *cq,
2074 struct cq_cutoff *hwcqe)
2076 /* Check the Status */
2077 if (hwcqe->status != CQ_CUTOFF_STATUS_OK) {
2078 dev_err(&cq->hwq.pdev->dev,
2079 "QPLIB: FP: CQ Process Cutoff Error status = 0x%x",
2083 clear_bit(CQ_FLAGS_RESIZE_IN_PROG, &cq->flags);
2084 wake_up_interruptible(&cq->waitq);
2089 int bnxt_qplib_poll_cq(struct bnxt_qplib_cq *cq, struct bnxt_qplib_cqe *cqe,
2090 int num_cqes, struct bnxt_qplib_qp **lib_qp)
2092 struct cq_base *hw_cqe, **hw_cqe_ptr;
2093 unsigned long flags;
2094 u32 sw_cons, raw_cons;
2097 spin_lock_irqsave(&cq->hwq.lock, flags);
2098 raw_cons = cq->hwq.cons;
2102 sw_cons = HWQ_CMP(raw_cons, &cq->hwq);
2103 hw_cqe_ptr = (struct cq_base **)cq->hwq.pbl_ptr;
2104 hw_cqe = &hw_cqe_ptr[CQE_PG(sw_cons)][CQE_IDX(sw_cons)];
2106 /* Check for Valid bit */
2107 if (!CQE_CMP_VALID(hw_cqe, raw_cons, cq->hwq.max_elements))
2110 /* From the device's respective CQE format to qplib_wc*/
2111 switch (hw_cqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK) {
2112 case CQ_BASE_CQE_TYPE_REQ:
2113 rc = bnxt_qplib_cq_process_req(cq,
2114 (struct cq_req *)hw_cqe,
2118 case CQ_BASE_CQE_TYPE_RES_RC:
2119 rc = bnxt_qplib_cq_process_res_rc(cq,
2120 (struct cq_res_rc *)
2124 case CQ_BASE_CQE_TYPE_RES_UD:
2125 rc = bnxt_qplib_cq_process_res_ud
2126 (cq, (struct cq_res_ud *)hw_cqe, &cqe,
2129 case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1:
2130 rc = bnxt_qplib_cq_process_res_raweth_qp1
2131 (cq, (struct cq_res_raweth_qp1 *)
2132 hw_cqe, &cqe, &budget);
2134 case CQ_BASE_CQE_TYPE_TERMINAL:
2135 rc = bnxt_qplib_cq_process_terminal
2136 (cq, (struct cq_terminal *)hw_cqe,
2139 case CQ_BASE_CQE_TYPE_CUT_OFF:
2140 bnxt_qplib_cq_process_cutoff
2141 (cq, (struct cq_cutoff *)hw_cqe);
2142 /* Done processing this CQ */
2145 dev_err(&cq->hwq.pdev->dev,
2146 "QPLIB: process_cq unknown type 0x%lx",
2147 hw_cqe->cqe_type_toggle &
2148 CQ_BASE_CQE_TYPE_MASK);
2155 /* Error while processing the CQE, just skip to the
2158 dev_err(&cq->hwq.pdev->dev,
2159 "QPLIB: process_cqe error rc = 0x%x", rc);
2163 if (cq->hwq.cons != raw_cons) {
2164 cq->hwq.cons = raw_cons;
2165 bnxt_qplib_arm_cq(cq, DBR_DBR_TYPE_CQ);
2168 spin_unlock_irqrestore(&cq->hwq.lock, flags);
2169 return num_cqes - budget;
2172 void bnxt_qplib_req_notify_cq(struct bnxt_qplib_cq *cq, u32 arm_type)
2174 unsigned long flags;
2176 spin_lock_irqsave(&cq->hwq.lock, flags);
2178 bnxt_qplib_arm_cq(cq, arm_type);
2180 spin_unlock_irqrestore(&cq->hwq.lock, flags);