1 /* QLogic qedr NIC Driver
2 * Copyright (c) 2015-2016 QLogic Corporation
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and /or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 #include <linux/dma-mapping.h>
33 #include <linux/crc32.h>
37 #include <linux/iommu.h>
39 #include <rdma/ib_verbs.h>
40 #include <rdma/ib_user_verbs.h>
41 #include <rdma/iw_cm.h>
42 #include <rdma/ib_umem.h>
43 #include <rdma/ib_addr.h>
44 #include <rdma/ib_cache.h>
46 #include <linux/qed/common_hsi.h>
47 #include "qedr_hsi_rdma.h"
48 #include <linux/qed/qed_if.h>
51 #include <rdma/qedr-abi.h>
52 #include "qedr_roce_cm.h"
54 #define QEDR_SRQ_WQE_ELEM_SIZE sizeof(union rdma_srq_elm)
55 #define RDMA_MAX_SGE_PER_SRQ (4)
56 #define RDMA_MAX_SRQ_WQE_SIZE (RDMA_MAX_SGE_PER_SRQ + 1)
58 #define DB_ADDR_SHIFT(addr) ((addr) << DB_PWM_ADDR_OFFSET_SHIFT)
60 static inline int qedr_ib_copy_to_udata(struct ib_udata *udata, void *src,
63 size_t min_len = min_t(size_t, len, udata->outlen);
65 return ib_copy_to_udata(udata, src, min_len);
68 int qedr_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey)
70 if (index >= QEDR_ROCE_PKEY_TABLE_LEN)
73 *pkey = QEDR_ROCE_PKEY_DEFAULT;
77 int qedr_iw_query_gid(struct ib_device *ibdev, u8 port,
78 int index, union ib_gid *sgid)
80 struct qedr_dev *dev = get_qedr_dev(ibdev);
82 memset(sgid->raw, 0, sizeof(sgid->raw));
83 ether_addr_copy(sgid->raw, dev->ndev->dev_addr);
85 DP_DEBUG(dev, QEDR_MSG_INIT, "QUERY sgid[%d]=%llx:%llx\n", index,
86 sgid->global.interface_id, sgid->global.subnet_prefix);
91 int qedr_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr)
93 struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
94 struct qedr_device_attr *qattr = &dev->attr;
95 struct qedr_srq *srq = get_qedr_srq(ibsrq);
97 srq_attr->srq_limit = srq->srq_limit;
98 srq_attr->max_wr = qattr->max_srq_wr;
99 srq_attr->max_sge = qattr->max_sge;
104 int qedr_query_device(struct ib_device *ibdev,
105 struct ib_device_attr *attr, struct ib_udata *udata)
107 struct qedr_dev *dev = get_qedr_dev(ibdev);
108 struct qedr_device_attr *qattr = &dev->attr;
110 if (!dev->rdma_ctx) {
112 "qedr_query_device called with invalid params rdma_ctx=%p\n",
117 memset(attr, 0, sizeof(*attr));
119 attr->fw_ver = qattr->fw_ver;
120 attr->sys_image_guid = qattr->sys_image_guid;
121 attr->max_mr_size = qattr->max_mr_size;
122 attr->page_size_cap = qattr->page_size_caps;
123 attr->vendor_id = qattr->vendor_id;
124 attr->vendor_part_id = qattr->vendor_part_id;
125 attr->hw_ver = qattr->hw_ver;
126 attr->max_qp = qattr->max_qp;
127 attr->max_qp_wr = max_t(u32, qattr->max_sqe, qattr->max_rqe);
128 attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD |
129 IB_DEVICE_RC_RNR_NAK_GEN |
130 IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_MGT_EXTENSIONS;
132 attr->max_send_sge = qattr->max_sge;
133 attr->max_recv_sge = qattr->max_sge;
134 attr->max_sge_rd = qattr->max_sge;
135 attr->max_cq = qattr->max_cq;
136 attr->max_cqe = qattr->max_cqe;
137 attr->max_mr = qattr->max_mr;
138 attr->max_mw = qattr->max_mw;
139 attr->max_pd = qattr->max_pd;
140 attr->atomic_cap = dev->atomic_cap;
141 attr->max_fmr = qattr->max_fmr;
142 attr->max_map_per_fmr = 16;
143 attr->max_qp_init_rd_atom =
144 1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1);
145 attr->max_qp_rd_atom =
146 min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1),
147 attr->max_qp_init_rd_atom);
149 attr->max_srq = qattr->max_srq;
150 attr->max_srq_sge = qattr->max_srq_sge;
151 attr->max_srq_wr = qattr->max_srq_wr;
153 attr->local_ca_ack_delay = qattr->dev_ack_delay;
154 attr->max_fast_reg_page_list_len = qattr->max_mr / 8;
155 attr->max_pkeys = QEDR_ROCE_PKEY_MAX;
156 attr->max_ah = qattr->max_ah;
161 #define QEDR_SPEED_SDR (1)
162 #define QEDR_SPEED_DDR (2)
163 #define QEDR_SPEED_QDR (4)
164 #define QEDR_SPEED_FDR10 (8)
165 #define QEDR_SPEED_FDR (16)
166 #define QEDR_SPEED_EDR (32)
168 static inline void get_link_speed_and_width(int speed, u8 *ib_speed,
173 *ib_speed = QEDR_SPEED_SDR;
174 *ib_width = IB_WIDTH_1X;
177 *ib_speed = QEDR_SPEED_QDR;
178 *ib_width = IB_WIDTH_1X;
182 *ib_speed = QEDR_SPEED_DDR;
183 *ib_width = IB_WIDTH_4X;
187 *ib_speed = QEDR_SPEED_EDR;
188 *ib_width = IB_WIDTH_1X;
192 *ib_speed = QEDR_SPEED_QDR;
193 *ib_width = IB_WIDTH_4X;
197 *ib_speed = QEDR_SPEED_QDR;
198 *ib_width = IB_WIDTH_4X;
202 *ib_speed = QEDR_SPEED_EDR;
203 *ib_width = IB_WIDTH_4X;
208 *ib_speed = QEDR_SPEED_SDR;
209 *ib_width = IB_WIDTH_1X;
213 int qedr_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *attr)
215 struct qedr_dev *dev;
216 struct qed_rdma_port *rdma_port;
218 dev = get_qedr_dev(ibdev);
220 if (!dev->rdma_ctx) {
221 DP_ERR(dev, "rdma_ctx is NULL\n");
225 rdma_port = dev->ops->rdma_query_port(dev->rdma_ctx);
227 /* *attr being zeroed by the caller, avoid zeroing it here */
228 if (rdma_port->port_state == QED_RDMA_PORT_UP) {
229 attr->state = IB_PORT_ACTIVE;
230 attr->phys_state = 5;
232 attr->state = IB_PORT_DOWN;
233 attr->phys_state = 3;
235 attr->max_mtu = IB_MTU_4096;
236 attr->active_mtu = iboe_get_mtu(dev->ndev->mtu);
241 attr->ip_gids = true;
242 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
243 attr->gid_tbl_len = 1;
244 attr->pkey_tbl_len = 1;
246 attr->gid_tbl_len = QEDR_MAX_SGID;
247 attr->pkey_tbl_len = QEDR_ROCE_PKEY_TABLE_LEN;
249 attr->bad_pkey_cntr = rdma_port->pkey_bad_counter;
250 attr->qkey_viol_cntr = 0;
251 get_link_speed_and_width(rdma_port->link_speed,
252 &attr->active_speed, &attr->active_width);
253 attr->max_msg_sz = rdma_port->max_msg_size;
254 attr->max_vl_num = 4;
259 int qedr_modify_port(struct ib_device *ibdev, u8 port, int mask,
260 struct ib_port_modify *props)
265 static int qedr_add_mmap(struct qedr_ucontext *uctx, u64 phy_addr,
270 mm = kzalloc(sizeof(*mm), GFP_KERNEL);
274 mm->key.phy_addr = phy_addr;
275 /* This function might be called with a length which is not a multiple
276 * of PAGE_SIZE, while the mapping is PAGE_SIZE grained and the kernel
277 * forces this granularity by increasing the requested size if needed.
278 * When qedr_mmap is called, it will search the list with the updated
279 * length as a key. To prevent search failures, the length is rounded up
280 * in advance to PAGE_SIZE.
282 mm->key.len = roundup(len, PAGE_SIZE);
283 INIT_LIST_HEAD(&mm->entry);
285 mutex_lock(&uctx->mm_list_lock);
286 list_add(&mm->entry, &uctx->mm_head);
287 mutex_unlock(&uctx->mm_list_lock);
289 DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
290 "added (addr=0x%llx,len=0x%lx) for ctx=%p\n",
291 (unsigned long long)mm->key.phy_addr,
292 (unsigned long)mm->key.len, uctx);
297 static bool qedr_search_mmap(struct qedr_ucontext *uctx, u64 phy_addr,
303 mutex_lock(&uctx->mm_list_lock);
304 list_for_each_entry(mm, &uctx->mm_head, entry) {
305 if (len != mm->key.len || phy_addr != mm->key.phy_addr)
311 mutex_unlock(&uctx->mm_list_lock);
312 DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
313 "searched for (addr=0x%llx,len=0x%lx) for ctx=%p, result=%d\n",
314 mm->key.phy_addr, mm->key.len, uctx, found);
319 int qedr_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata)
321 struct ib_device *ibdev = uctx->device;
323 struct qedr_ucontext *ctx = get_qedr_ucontext(uctx);
324 struct qedr_alloc_ucontext_resp uresp = {};
325 struct qedr_dev *dev = get_qedr_dev(ibdev);
326 struct qed_rdma_add_user_out_params oparams;
331 rc = dev->ops->rdma_add_user(dev->rdma_ctx, &oparams);
334 "failed to allocate a DPI for a new RoCE application, rc=%d. To overcome this consider to increase the number of DPIs, increase the doorbell BAR size or just close unnecessary RoCE applications. In order to increase the number of DPIs consult the qedr readme\n",
339 ctx->dpi = oparams.dpi;
340 ctx->dpi_addr = oparams.dpi_addr;
341 ctx->dpi_phys_addr = oparams.dpi_phys_addr;
342 ctx->dpi_size = oparams.dpi_size;
343 INIT_LIST_HEAD(&ctx->mm_head);
344 mutex_init(&ctx->mm_list_lock);
346 uresp.dpm_enabled = dev->user_dpm_enabled;
347 uresp.wids_enabled = 1;
348 uresp.wid_count = oparams.wid_count;
349 uresp.db_pa = ctx->dpi_phys_addr;
350 uresp.db_size = ctx->dpi_size;
351 uresp.max_send_wr = dev->attr.max_sqe;
352 uresp.max_recv_wr = dev->attr.max_rqe;
353 uresp.max_srq_wr = dev->attr.max_srq_wr;
354 uresp.sges_per_send_wr = QEDR_MAX_SQE_ELEMENTS_PER_SQE;
355 uresp.sges_per_recv_wr = QEDR_MAX_RQE_ELEMENTS_PER_RQE;
356 uresp.sges_per_srq_wr = dev->attr.max_srq_sge;
357 uresp.max_cqes = QEDR_MAX_CQES;
359 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
365 rc = qedr_add_mmap(ctx, ctx->dpi_phys_addr, ctx->dpi_size);
369 DP_DEBUG(dev, QEDR_MSG_INIT, "Allocating user context %p\n",
374 void qedr_dealloc_ucontext(struct ib_ucontext *ibctx)
376 struct qedr_ucontext *uctx = get_qedr_ucontext(ibctx);
377 struct qedr_mm *mm, *tmp;
379 DP_DEBUG(uctx->dev, QEDR_MSG_INIT, "Deallocating user context %p\n",
381 uctx->dev->ops->rdma_remove_user(uctx->dev->rdma_ctx, uctx->dpi);
383 list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) {
384 DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
385 "deleted (addr=0x%llx,len=0x%lx) for ctx=%p\n",
386 mm->key.phy_addr, mm->key.len, uctx);
387 list_del(&mm->entry);
392 int qedr_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
394 struct qedr_ucontext *ucontext = get_qedr_ucontext(context);
395 struct qedr_dev *dev = get_qedr_dev(context->device);
396 unsigned long phys_addr = vma->vm_pgoff << PAGE_SHIFT;
397 unsigned long len = (vma->vm_end - vma->vm_start);
398 unsigned long dpi_start;
400 dpi_start = dev->db_phys_addr + (ucontext->dpi * ucontext->dpi_size);
402 DP_DEBUG(dev, QEDR_MSG_INIT,
403 "mmap invoked with vm_start=0x%pK, vm_end=0x%pK,vm_pgoff=0x%pK; dpi_start=0x%pK dpi_size=0x%x\n",
404 (void *)vma->vm_start, (void *)vma->vm_end,
405 (void *)vma->vm_pgoff, (void *)dpi_start, ucontext->dpi_size);
407 if ((vma->vm_start & (PAGE_SIZE - 1)) || (len & (PAGE_SIZE - 1))) {
409 "failed mmap, addresses must be page aligned: start=0x%pK, end=0x%pK\n",
410 (void *)vma->vm_start, (void *)vma->vm_end);
414 if (!qedr_search_mmap(ucontext, phys_addr, len)) {
415 DP_ERR(dev, "failed mmap, vm_pgoff=0x%lx is not authorized\n",
420 if (phys_addr < dpi_start ||
421 ((phys_addr + len) > (dpi_start + ucontext->dpi_size))) {
423 "failed mmap, pages are outside of dpi; page address=0x%pK, dpi_start=0x%pK, dpi_size=0x%x\n",
424 (void *)phys_addr, (void *)dpi_start,
429 if (vma->vm_flags & VM_READ) {
430 DP_ERR(dev, "failed mmap, cannot map doorbell bar for read\n");
434 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
435 return io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, len,
439 int qedr_alloc_pd(struct ib_pd *ibpd, struct ib_ucontext *context,
440 struct ib_udata *udata)
442 struct ib_device *ibdev = ibpd->device;
443 struct qedr_dev *dev = get_qedr_dev(ibdev);
444 struct qedr_pd *pd = get_qedr_pd(ibpd);
448 DP_DEBUG(dev, QEDR_MSG_INIT, "Function called from: %s\n",
449 (udata && context) ? "User Lib" : "Kernel");
451 if (!dev->rdma_ctx) {
452 DP_ERR(dev, "invalid RDMA context\n");
456 rc = dev->ops->rdma_alloc_pd(dev->rdma_ctx, &pd_id);
462 if (udata && context) {
463 struct qedr_alloc_pd_uresp uresp = {
467 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
469 DP_ERR(dev, "copy error pd_id=0x%x.\n", pd_id);
470 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd_id);
474 pd->uctx = get_qedr_ucontext(context);
481 void qedr_dealloc_pd(struct ib_pd *ibpd)
483 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
484 struct qedr_pd *pd = get_qedr_pd(ibpd);
486 DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id);
487 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id);
490 static void qedr_free_pbl(struct qedr_dev *dev,
491 struct qedr_pbl_info *pbl_info, struct qedr_pbl *pbl)
493 struct pci_dev *pdev = dev->pdev;
496 for (i = 0; i < pbl_info->num_pbls; i++) {
499 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
500 pbl[i].va, pbl[i].pa);
506 #define MIN_FW_PBL_PAGE_SIZE (4 * 1024)
507 #define MAX_FW_PBL_PAGE_SIZE (64 * 1024)
509 #define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64))
510 #define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE)
511 #define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE * MAX_PBES_ON_PAGE)
513 static struct qedr_pbl *qedr_alloc_pbl_tbl(struct qedr_dev *dev,
514 struct qedr_pbl_info *pbl_info,
517 struct pci_dev *pdev = dev->pdev;
518 struct qedr_pbl *pbl_table;
519 dma_addr_t *pbl_main_tbl;
524 pbl_table = kcalloc(pbl_info->num_pbls, sizeof(*pbl_table), flags);
526 return ERR_PTR(-ENOMEM);
528 for (i = 0; i < pbl_info->num_pbls; i++) {
529 va = dma_alloc_coherent(&pdev->dev, pbl_info->pbl_size, &pa,
534 pbl_table[i].va = va;
535 pbl_table[i].pa = pa;
538 /* Two-Layer PBLs, if we have more than one pbl we need to initialize
539 * the first one with physical pointers to all of the rest
541 pbl_main_tbl = (dma_addr_t *)pbl_table[0].va;
542 for (i = 0; i < pbl_info->num_pbls - 1; i++)
543 pbl_main_tbl[i] = pbl_table[i + 1].pa;
548 for (i--; i >= 0; i--)
549 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
550 pbl_table[i].va, pbl_table[i].pa);
552 qedr_free_pbl(dev, pbl_info, pbl_table);
554 return ERR_PTR(-ENOMEM);
557 static int qedr_prepare_pbl_tbl(struct qedr_dev *dev,
558 struct qedr_pbl_info *pbl_info,
559 u32 num_pbes, int two_layer_capable)
565 if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) {
566 if (num_pbes > MAX_PBES_TWO_LAYER) {
567 DP_ERR(dev, "prepare pbl table: too many pages %d\n",
572 /* calculate required pbl page size */
573 pbl_size = MIN_FW_PBL_PAGE_SIZE;
574 pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) *
575 NUM_PBES_ON_PAGE(pbl_size);
577 while (pbl_capacity < num_pbes) {
579 pbl_capacity = pbl_size / sizeof(u64);
580 pbl_capacity = pbl_capacity * pbl_capacity;
583 num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size));
584 num_pbls++; /* One for the layer0 ( points to the pbls) */
585 pbl_info->two_layered = true;
587 /* One layered PBL */
589 pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE,
590 roundup_pow_of_two((num_pbes * sizeof(u64))));
591 pbl_info->two_layered = false;
594 pbl_info->num_pbls = num_pbls;
595 pbl_info->pbl_size = pbl_size;
596 pbl_info->num_pbes = num_pbes;
598 DP_DEBUG(dev, QEDR_MSG_MR,
599 "prepare pbl table: num_pbes=%d, num_pbls=%d, pbl_size=%d\n",
600 pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size);
605 static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem,
606 struct qedr_pbl *pbl,
607 struct qedr_pbl_info *pbl_info, u32 pg_shift)
609 int pbe_cnt, total_num_pbes = 0;
610 u32 fw_pg_cnt, fw_pg_per_umem_pg;
611 struct qedr_pbl *pbl_tbl;
612 struct sg_dma_page_iter sg_iter;
616 if (!pbl_info->num_pbes)
619 /* If we have a two layered pbl, the first pbl points to the rest
620 * of the pbls and the first entry lays on the second pbl in the table
622 if (pbl_info->two_layered)
627 pbe = (struct regpair *)pbl_tbl->va;
629 DP_ERR(dev, "cannot populate PBL due to a NULL PBE\n");
635 fw_pg_per_umem_pg = BIT(PAGE_SHIFT - pg_shift);
637 for_each_sg_dma_page (umem->sg_head.sgl, &sg_iter, umem->nmap, 0) {
638 pg_addr = sg_page_iter_dma_address(&sg_iter);
639 for (fw_pg_cnt = 0; fw_pg_cnt < fw_pg_per_umem_pg;) {
640 pbe->lo = cpu_to_le32(pg_addr);
641 pbe->hi = cpu_to_le32(upper_32_bits(pg_addr));
643 pg_addr += BIT(pg_shift);
648 if (total_num_pbes == pbl_info->num_pbes)
651 /* If the given pbl is full storing the pbes,
654 if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) {
656 pbe = (struct regpair *)pbl_tbl->va;
665 static int qedr_copy_cq_uresp(struct qedr_dev *dev,
666 struct qedr_cq *cq, struct ib_udata *udata)
668 struct qedr_create_cq_uresp uresp;
671 memset(&uresp, 0, sizeof(uresp));
673 uresp.db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
674 uresp.icid = cq->icid;
676 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
678 DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid);
683 static void consume_cqe(struct qedr_cq *cq)
685 if (cq->latest_cqe == cq->toggle_cqe)
686 cq->pbl_toggle ^= RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
688 cq->latest_cqe = qed_chain_consume(&cq->pbl);
691 static inline int qedr_align_cq_entries(int entries)
693 u64 size, aligned_size;
695 /* We allocate an extra entry that we don't report to the FW. */
696 size = (entries + 1) * QEDR_CQE_SIZE;
697 aligned_size = ALIGN(size, PAGE_SIZE);
699 return aligned_size / QEDR_CQE_SIZE;
702 static inline int qedr_init_user_queue(struct ib_udata *udata,
703 struct qedr_dev *dev,
704 struct qedr_userq *q, u64 buf_addr,
705 size_t buf_len, int access, int dmasync,
711 q->buf_addr = buf_addr;
712 q->buf_len = buf_len;
713 q->umem = ib_umem_get(udata, q->buf_addr, q->buf_len, access, dmasync);
714 if (IS_ERR(q->umem)) {
715 DP_ERR(dev, "create user queue: failed ib_umem_get, got %ld\n",
717 return PTR_ERR(q->umem);
720 fw_pages = ib_umem_page_count(q->umem) <<
721 (PAGE_SHIFT - FW_PAGE_SHIFT);
723 rc = qedr_prepare_pbl_tbl(dev, &q->pbl_info, fw_pages, 0);
727 if (alloc_and_init) {
728 q->pbl_tbl = qedr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL);
729 if (IS_ERR(q->pbl_tbl)) {
730 rc = PTR_ERR(q->pbl_tbl);
733 qedr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info,
736 q->pbl_tbl = kzalloc(sizeof(*q->pbl_tbl), GFP_KERNEL);
746 ib_umem_release(q->umem);
752 static inline void qedr_init_cq_params(struct qedr_cq *cq,
753 struct qedr_ucontext *ctx,
754 struct qedr_dev *dev, int vector,
755 int chain_entries, int page_cnt,
757 struct qed_rdma_create_cq_in_params
760 memset(params, 0, sizeof(*params));
761 params->cq_handle_hi = upper_32_bits((uintptr_t)cq);
762 params->cq_handle_lo = lower_32_bits((uintptr_t)cq);
763 params->cnq_id = vector;
764 params->cq_size = chain_entries - 1;
765 params->dpi = (ctx) ? ctx->dpi : dev->dpi;
766 params->pbl_num_pages = page_cnt;
767 params->pbl_ptr = pbl_ptr;
768 params->pbl_two_level = 0;
771 static void doorbell_cq(struct qedr_cq *cq, u32 cons, u8 flags)
773 cq->db.data.agg_flags = flags;
774 cq->db.data.value = cpu_to_le32(cons);
775 writeq(cq->db.raw, cq->db_addr);
777 /* Make sure write would stick */
781 int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
783 struct qedr_cq *cq = get_qedr_cq(ibcq);
784 unsigned long sflags;
785 struct qedr_dev *dev;
787 dev = get_qedr_dev(ibcq->device);
791 "warning: arm was invoked after destroy for cq %p (icid=%d)\n",
797 if (cq->cq_type == QEDR_CQ_TYPE_GSI)
800 spin_lock_irqsave(&cq->cq_lock, sflags);
804 if (flags & IB_CQ_SOLICITED)
805 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD;
807 if (flags & IB_CQ_NEXT_COMP)
808 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD;
810 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
812 spin_unlock_irqrestore(&cq->cq_lock, sflags);
817 struct ib_cq *qedr_create_cq(struct ib_device *ibdev,
818 const struct ib_cq_init_attr *attr,
819 struct ib_ucontext *ib_ctx, struct ib_udata *udata)
821 struct qedr_ucontext *ctx = get_qedr_ucontext(ib_ctx);
822 struct qed_rdma_destroy_cq_out_params destroy_oparams;
823 struct qed_rdma_destroy_cq_in_params destroy_iparams;
824 struct qedr_dev *dev = get_qedr_dev(ibdev);
825 struct qed_rdma_create_cq_in_params params;
826 struct qedr_create_cq_ureq ureq;
827 int vector = attr->comp_vector;
828 int entries = attr->cqe;
836 DP_DEBUG(dev, QEDR_MSG_INIT,
837 "create_cq: called from %s. entries=%d, vector=%d\n",
838 udata ? "User Lib" : "Kernel", entries, vector);
840 if (entries > QEDR_MAX_CQES) {
842 "create cq: the number of entries %d is too high. Must be equal or below %d.\n",
843 entries, QEDR_MAX_CQES);
844 return ERR_PTR(-EINVAL);
847 chain_entries = qedr_align_cq_entries(entries);
848 chain_entries = min_t(int, chain_entries, QEDR_MAX_CQES);
850 cq = kzalloc(sizeof(*cq), GFP_KERNEL);
852 return ERR_PTR(-ENOMEM);
855 memset(&ureq, 0, sizeof(ureq));
856 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) {
858 "create cq: problem copying data from user space\n");
864 "create cq: cannot create a cq with 0 entries\n");
868 cq->cq_type = QEDR_CQ_TYPE_USER;
870 rc = qedr_init_user_queue(udata, dev, &cq->q, ureq.addr,
871 ureq.len, IB_ACCESS_LOCAL_WRITE, 1,
876 pbl_ptr = cq->q.pbl_tbl->pa;
877 page_cnt = cq->q.pbl_info.num_pbes;
879 cq->ibcq.cqe = chain_entries;
881 cq->cq_type = QEDR_CQ_TYPE_KERNEL;
883 rc = dev->ops->common->chain_alloc(dev->cdev,
884 QED_CHAIN_USE_TO_CONSUME,
886 QED_CHAIN_CNT_TYPE_U32,
888 sizeof(union rdma_cqe),
893 page_cnt = qed_chain_get_page_cnt(&cq->pbl);
894 pbl_ptr = qed_chain_get_pbl_phys(&cq->pbl);
895 cq->ibcq.cqe = cq->pbl.capacity;
898 qedr_init_cq_params(cq, ctx, dev, vector, chain_entries, page_cnt,
901 rc = dev->ops->rdma_create_cq(dev->rdma_ctx, ¶ms, &icid);
906 cq->sig = QEDR_CQ_MAGIC_NUMBER;
907 spin_lock_init(&cq->cq_lock);
910 rc = qedr_copy_cq_uresp(dev, cq, udata);
914 /* Generate doorbell address. */
915 cq->db_addr = dev->db_addr +
916 DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
917 cq->db.data.icid = cq->icid;
918 cq->db.data.params = DB_AGG_CMD_SET <<
919 RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT;
921 /* point to the very last element, passing it we will toggle */
922 cq->toggle_cqe = qed_chain_get_last_elem(&cq->pbl);
923 cq->pbl_toggle = RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
924 cq->latest_cqe = NULL;
926 cq->cq_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
929 DP_DEBUG(dev, QEDR_MSG_CQ,
930 "create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n",
931 cq->icid, cq, params.cq_size);
936 destroy_iparams.icid = cq->icid;
937 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams,
941 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
943 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
946 ib_umem_release(cq->q.umem);
949 return ERR_PTR(-EINVAL);
952 int qedr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata)
954 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
955 struct qedr_cq *cq = get_qedr_cq(ibcq);
957 DP_ERR(dev, "cq %p RESIZE NOT SUPPORTED\n", cq);
962 #define QEDR_DESTROY_CQ_MAX_ITERATIONS (10)
963 #define QEDR_DESTROY_CQ_ITER_DURATION (10)
965 int qedr_destroy_cq(struct ib_cq *ibcq)
967 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
968 struct qed_rdma_destroy_cq_out_params oparams;
969 struct qed_rdma_destroy_cq_in_params iparams;
970 struct qedr_cq *cq = get_qedr_cq(ibcq);
974 DP_DEBUG(dev, QEDR_MSG_CQ, "destroy cq %p (icid=%d)\n", cq, cq->icid);
978 /* GSIs CQs are handled by driver, so they don't exist in the FW */
979 if (cq->cq_type == QEDR_CQ_TYPE_GSI)
982 iparams.icid = cq->icid;
983 rc = dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams);
987 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
989 if (ibcq->uobject && ibcq->uobject->context) {
990 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
991 ib_umem_release(cq->q.umem);
994 /* We don't want the IRQ handler to handle a non-existing CQ so we
995 * wait until all CNQ interrupts, if any, are received. This will always
996 * happen and will always happen very fast. If not, then a serious error
997 * has occured. That is why we can use a long delay.
998 * We spin for a short time so we don’t lose time on context switching
999 * in case all the completions are handled in that span. Otherwise
1000 * we sleep for a while and check again. Since the CNQ may be
1001 * associated with (only) the current CPU we use msleep to allow the
1002 * current CPU to be freed.
1003 * The CNQ notification is increased in qedr_irq_handler().
1005 iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1006 while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1007 udelay(QEDR_DESTROY_CQ_ITER_DURATION);
1011 iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1012 while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1013 msleep(QEDR_DESTROY_CQ_ITER_DURATION);
1017 if (oparams.num_cq_notif != cq->cnq_notif)
1020 /* Note that we don't need to have explicit code to wait for the
1021 * completion of the event handler because it is invoked from the EQ.
1022 * Since the destroy CQ ramrod has also been received on the EQ we can
1023 * be certain that there's no event handler in process.
1034 "CQ %p (icid=%d) not freed, expecting %d ints but got %d ints\n",
1035 cq, cq->icid, oparams.num_cq_notif, cq->cnq_notif);
1040 static inline int get_gid_info_from_table(struct ib_qp *ibqp,
1041 struct ib_qp_attr *attr,
1043 struct qed_rdma_modify_qp_in_params
1046 const struct ib_gid_attr *gid_attr;
1047 enum rdma_network_type nw_type;
1048 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
1052 gid_attr = grh->sgid_attr;
1053 qp_params->vlan_id = rdma_vlan_dev_vlan_id(gid_attr->ndev);
1055 nw_type = rdma_gid_attr_network_type(gid_attr);
1057 case RDMA_NETWORK_IPV6:
1058 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0],
1059 sizeof(qp_params->sgid));
1060 memcpy(&qp_params->dgid.bytes[0],
1062 sizeof(qp_params->dgid));
1063 qp_params->roce_mode = ROCE_V2_IPV6;
1064 SET_FIELD(qp_params->modify_flags,
1065 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1067 case RDMA_NETWORK_IB:
1068 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0],
1069 sizeof(qp_params->sgid));
1070 memcpy(&qp_params->dgid.bytes[0],
1072 sizeof(qp_params->dgid));
1073 qp_params->roce_mode = ROCE_V1;
1075 case RDMA_NETWORK_IPV4:
1076 memset(&qp_params->sgid, 0, sizeof(qp_params->sgid));
1077 memset(&qp_params->dgid, 0, sizeof(qp_params->dgid));
1078 ipv4_addr = qedr_get_ipv4_from_gid(gid_attr->gid.raw);
1079 qp_params->sgid.ipv4_addr = ipv4_addr;
1081 qedr_get_ipv4_from_gid(grh->dgid.raw);
1082 qp_params->dgid.ipv4_addr = ipv4_addr;
1083 SET_FIELD(qp_params->modify_flags,
1084 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1085 qp_params->roce_mode = ROCE_V2_IPV4;
1089 for (i = 0; i < 4; i++) {
1090 qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]);
1091 qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]);
1094 if (qp_params->vlan_id >= VLAN_CFI_MASK)
1095 qp_params->vlan_id = 0;
1100 static int qedr_check_qp_attrs(struct ib_pd *ibpd, struct qedr_dev *dev,
1101 struct ib_qp_init_attr *attrs,
1102 struct ib_udata *udata)
1104 struct qedr_device_attr *qattr = &dev->attr;
1106 /* QP0... attrs->qp_type == IB_QPT_GSI */
1107 if (attrs->qp_type != IB_QPT_RC && attrs->qp_type != IB_QPT_GSI) {
1108 DP_DEBUG(dev, QEDR_MSG_QP,
1109 "create qp: unsupported qp type=0x%x requested\n",
1114 if (attrs->cap.max_send_wr > qattr->max_sqe) {
1116 "create qp: cannot create a SQ with %d elements (max_send_wr=0x%x)\n",
1117 attrs->cap.max_send_wr, qattr->max_sqe);
1121 if (attrs->cap.max_inline_data > qattr->max_inline) {
1123 "create qp: unsupported inline data size=0x%x requested (max_inline=0x%x)\n",
1124 attrs->cap.max_inline_data, qattr->max_inline);
1128 if (attrs->cap.max_send_sge > qattr->max_sge) {
1130 "create qp: unsupported send_sge=0x%x requested (max_send_sge=0x%x)\n",
1131 attrs->cap.max_send_sge, qattr->max_sge);
1135 if (attrs->cap.max_recv_sge > qattr->max_sge) {
1137 "create qp: unsupported recv_sge=0x%x requested (max_recv_sge=0x%x)\n",
1138 attrs->cap.max_recv_sge, qattr->max_sge);
1142 /* Unprivileged user space cannot create special QP */
1143 if (udata && attrs->qp_type == IB_QPT_GSI) {
1145 "create qp: userspace can't create special QPs of type=0x%x\n",
1153 static int qedr_copy_srq_uresp(struct qedr_dev *dev,
1154 struct qedr_srq *srq, struct ib_udata *udata)
1156 struct qedr_create_srq_uresp uresp = {};
1159 uresp.srq_id = srq->srq_id;
1161 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1163 DP_ERR(dev, "create srq: problem copying data to user space\n");
1168 static void qedr_copy_rq_uresp(struct qedr_dev *dev,
1169 struct qedr_create_qp_uresp *uresp,
1172 /* iWARP requires two doorbells per RQ. */
1173 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1174 uresp->rq_db_offset =
1175 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
1176 uresp->rq_db2_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
1178 uresp->rq_db_offset =
1179 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1182 uresp->rq_icid = qp->icid;
1185 static void qedr_copy_sq_uresp(struct qedr_dev *dev,
1186 struct qedr_create_qp_uresp *uresp,
1189 uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1191 /* iWARP uses the same cid for rq and sq */
1192 if (rdma_protocol_iwarp(&dev->ibdev, 1))
1193 uresp->sq_icid = qp->icid;
1195 uresp->sq_icid = qp->icid + 1;
1198 static int qedr_copy_qp_uresp(struct qedr_dev *dev,
1199 struct qedr_qp *qp, struct ib_udata *udata)
1201 struct qedr_create_qp_uresp uresp;
1204 memset(&uresp, 0, sizeof(uresp));
1205 qedr_copy_sq_uresp(dev, &uresp, qp);
1206 qedr_copy_rq_uresp(dev, &uresp, qp);
1208 uresp.atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE;
1209 uresp.qp_id = qp->qp_id;
1211 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1214 "create qp: failed a copy to user space with qp icid=0x%x.\n",
1220 static void qedr_set_common_qp_params(struct qedr_dev *dev,
1223 struct ib_qp_init_attr *attrs)
1225 spin_lock_init(&qp->q_lock);
1226 atomic_set(&qp->refcnt, 1);
1228 qp->qp_type = attrs->qp_type;
1229 qp->max_inline_data = attrs->cap.max_inline_data;
1230 qp->sq.max_sges = attrs->cap.max_send_sge;
1231 qp->state = QED_ROCE_QP_STATE_RESET;
1232 qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false;
1233 qp->sq_cq = get_qedr_cq(attrs->send_cq);
1237 qp->srq = get_qedr_srq(attrs->srq);
1239 qp->rq_cq = get_qedr_cq(attrs->recv_cq);
1240 qp->rq.max_sges = attrs->cap.max_recv_sge;
1241 DP_DEBUG(dev, QEDR_MSG_QP,
1242 "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n",
1243 qp->rq.max_sges, qp->rq_cq->icid);
1246 DP_DEBUG(dev, QEDR_MSG_QP,
1247 "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d, state = %d, signaled = %d, use_srq=%d\n",
1248 pd->pd_id, qp->qp_type, qp->max_inline_data,
1249 qp->state, qp->signaled, (attrs->srq) ? 1 : 0);
1250 DP_DEBUG(dev, QEDR_MSG_QP,
1251 "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n",
1252 qp->sq.max_sges, qp->sq_cq->icid);
1255 static void qedr_set_roce_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
1257 qp->sq.db = dev->db_addr +
1258 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1259 qp->sq.db_data.data.icid = qp->icid + 1;
1261 qp->rq.db = dev->db_addr +
1262 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1263 qp->rq.db_data.data.icid = qp->icid;
1267 static int qedr_check_srq_params(struct ib_pd *ibpd, struct qedr_dev *dev,
1268 struct ib_srq_init_attr *attrs,
1269 struct ib_udata *udata)
1271 struct qedr_device_attr *qattr = &dev->attr;
1273 if (attrs->attr.max_wr > qattr->max_srq_wr) {
1275 "create srq: unsupported srq_wr=0x%x requested (max_srq_wr=0x%x)\n",
1276 attrs->attr.max_wr, qattr->max_srq_wr);
1280 if (attrs->attr.max_sge > qattr->max_sge) {
1282 "create srq: unsupported sge=0x%x requested (max_srq_sge=0x%x)\n",
1283 attrs->attr.max_sge, qattr->max_sge);
1290 static void qedr_free_srq_user_params(struct qedr_srq *srq)
1292 qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
1293 ib_umem_release(srq->usrq.umem);
1294 ib_umem_release(srq->prod_umem);
1297 static void qedr_free_srq_kernel_params(struct qedr_srq *srq)
1299 struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq;
1300 struct qedr_dev *dev = srq->dev;
1302 dev->ops->common->chain_free(dev->cdev, &hw_srq->pbl);
1304 dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers),
1305 hw_srq->virt_prod_pair_addr,
1306 hw_srq->phy_prod_pair_addr);
1309 static int qedr_init_srq_user_params(struct ib_udata *udata,
1310 struct qedr_srq *srq,
1311 struct qedr_create_srq_ureq *ureq,
1312 int access, int dmasync)
1314 struct scatterlist *sg;
1317 rc = qedr_init_user_queue(udata, srq->dev, &srq->usrq, ureq->srq_addr,
1318 ureq->srq_len, access, dmasync, 1);
1323 ib_umem_get(udata, ureq->prod_pair_addr,
1324 sizeof(struct rdma_srq_producers), access, dmasync);
1325 if (IS_ERR(srq->prod_umem)) {
1326 qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
1327 ib_umem_release(srq->usrq.umem);
1329 "create srq: failed ib_umem_get for producer, got %ld\n",
1330 PTR_ERR(srq->prod_umem));
1331 return PTR_ERR(srq->prod_umem);
1334 sg = srq->prod_umem->sg_head.sgl;
1335 srq->hw_srq.phy_prod_pair_addr = sg_dma_address(sg);
1340 static int qedr_alloc_srq_kernel_params(struct qedr_srq *srq,
1341 struct qedr_dev *dev,
1342 struct ib_srq_init_attr *init_attr)
1344 struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq;
1345 dma_addr_t phy_prod_pair_addr;
1350 va = dma_alloc_coherent(&dev->pdev->dev,
1351 sizeof(struct rdma_srq_producers),
1352 &phy_prod_pair_addr, GFP_KERNEL);
1355 "create srq: failed to allocate dma memory for producer\n");
1359 hw_srq->phy_prod_pair_addr = phy_prod_pair_addr;
1360 hw_srq->virt_prod_pair_addr = va;
1362 num_elems = init_attr->attr.max_wr * RDMA_MAX_SRQ_WQE_SIZE;
1363 rc = dev->ops->common->chain_alloc(dev->cdev,
1364 QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1366 QED_CHAIN_CNT_TYPE_U32,
1368 QEDR_SRQ_WQE_ELEM_SIZE,
1369 &hw_srq->pbl, NULL);
1373 hw_srq->num_elems = num_elems;
1378 dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers),
1379 va, phy_prod_pair_addr);
1383 static int qedr_idr_add(struct qedr_dev *dev, struct qedr_idr *qidr,
1385 static void qedr_idr_remove(struct qedr_dev *dev,
1386 struct qedr_idr *qidr, u32 id);
1388 struct ib_srq *qedr_create_srq(struct ib_pd *ibpd,
1389 struct ib_srq_init_attr *init_attr,
1390 struct ib_udata *udata)
1392 struct qed_rdma_destroy_srq_in_params destroy_in_params;
1393 struct qed_rdma_create_srq_in_params in_params = {};
1394 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
1395 struct qed_rdma_create_srq_out_params out_params;
1396 struct qedr_pd *pd = get_qedr_pd(ibpd);
1397 struct qedr_create_srq_ureq ureq = {};
1398 u64 pbl_base_addr, phy_prod_pair_addr;
1399 struct qedr_srq_hwq_info *hw_srq;
1400 u32 page_cnt, page_size;
1401 struct qedr_srq *srq;
1404 DP_DEBUG(dev, QEDR_MSG_QP,
1405 "create SRQ called from %s (pd %p)\n",
1406 (udata) ? "User lib" : "kernel", pd);
1408 rc = qedr_check_srq_params(ibpd, dev, init_attr, udata);
1410 return ERR_PTR(-EINVAL);
1412 srq = kzalloc(sizeof(*srq), GFP_KERNEL);
1414 return ERR_PTR(-ENOMEM);
1417 hw_srq = &srq->hw_srq;
1418 spin_lock_init(&srq->lock);
1420 hw_srq->max_wr = init_attr->attr.max_wr;
1421 hw_srq->max_sges = init_attr->attr.max_sge;
1424 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) {
1426 "create srq: problem copying data from user space\n");
1430 rc = qedr_init_srq_user_params(udata, srq, &ureq, 0, 0);
1434 page_cnt = srq->usrq.pbl_info.num_pbes;
1435 pbl_base_addr = srq->usrq.pbl_tbl->pa;
1436 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
1437 page_size = PAGE_SIZE;
1439 struct qed_chain *pbl;
1441 rc = qedr_alloc_srq_kernel_params(srq, dev, init_attr);
1446 page_cnt = qed_chain_get_page_cnt(pbl);
1447 pbl_base_addr = qed_chain_get_pbl_phys(pbl);
1448 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
1449 page_size = QED_CHAIN_PAGE_SIZE;
1452 in_params.pd_id = pd->pd_id;
1453 in_params.pbl_base_addr = pbl_base_addr;
1454 in_params.prod_pair_addr = phy_prod_pair_addr;
1455 in_params.num_pages = page_cnt;
1456 in_params.page_size = page_size;
1458 rc = dev->ops->rdma_create_srq(dev->rdma_ctx, &in_params, &out_params);
1462 srq->srq_id = out_params.srq_id;
1465 rc = qedr_copy_srq_uresp(dev, srq, udata);
1470 rc = qedr_idr_add(dev, &dev->srqidr, srq, srq->srq_id);
1474 DP_DEBUG(dev, QEDR_MSG_SRQ,
1475 "create srq: created srq with srq_id=0x%0x\n", srq->srq_id);
1479 destroy_in_params.srq_id = srq->srq_id;
1481 dev->ops->rdma_destroy_srq(dev->rdma_ctx, &destroy_in_params);
1484 qedr_free_srq_user_params(srq);
1486 qedr_free_srq_kernel_params(srq);
1490 return ERR_PTR(-EFAULT);
1493 int qedr_destroy_srq(struct ib_srq *ibsrq)
1495 struct qed_rdma_destroy_srq_in_params in_params = {};
1496 struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1497 struct qedr_srq *srq = get_qedr_srq(ibsrq);
1499 qedr_idr_remove(dev, &dev->srqidr, srq->srq_id);
1500 in_params.srq_id = srq->srq_id;
1501 dev->ops->rdma_destroy_srq(dev->rdma_ctx, &in_params);
1504 qedr_free_srq_user_params(srq);
1506 qedr_free_srq_kernel_params(srq);
1508 DP_DEBUG(dev, QEDR_MSG_SRQ,
1509 "destroy srq: destroyed srq with srq_id=0x%0x\n",
1516 int qedr_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
1517 enum ib_srq_attr_mask attr_mask, struct ib_udata *udata)
1519 struct qed_rdma_modify_srq_in_params in_params = {};
1520 struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1521 struct qedr_srq *srq = get_qedr_srq(ibsrq);
1524 if (attr_mask & IB_SRQ_MAX_WR) {
1526 "modify srq: invalid attribute mask=0x%x specified for %p\n",
1531 if (attr_mask & IB_SRQ_LIMIT) {
1532 if (attr->srq_limit >= srq->hw_srq.max_wr) {
1534 "modify srq: invalid srq_limit=0x%x (max_srq_limit=0x%x)\n",
1535 attr->srq_limit, srq->hw_srq.max_wr);
1539 in_params.srq_id = srq->srq_id;
1540 in_params.wqe_limit = attr->srq_limit;
1541 rc = dev->ops->rdma_modify_srq(dev->rdma_ctx, &in_params);
1546 srq->srq_limit = attr->srq_limit;
1548 DP_DEBUG(dev, QEDR_MSG_SRQ,
1549 "modify srq: modified srq with srq_id=0x%0x\n", srq->srq_id);
1555 qedr_init_common_qp_in_params(struct qedr_dev *dev,
1558 struct ib_qp_init_attr *attrs,
1559 bool fmr_and_reserved_lkey,
1560 struct qed_rdma_create_qp_in_params *params)
1562 /* QP handle to be written in an async event */
1563 params->qp_handle_async_lo = lower_32_bits((uintptr_t) qp);
1564 params->qp_handle_async_hi = upper_32_bits((uintptr_t) qp);
1566 params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR);
1567 params->fmr_and_reserved_lkey = fmr_and_reserved_lkey;
1568 params->pd = pd->pd_id;
1569 params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi;
1570 params->sq_cq_id = get_qedr_cq(attrs->send_cq)->icid;
1571 params->stats_queue = 0;
1573 params->use_srq = false;
1576 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1579 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1580 params->srq_id = qp->srq->srq_id;
1581 params->use_srq = true;
1585 static inline void qedr_qp_user_print(struct qedr_dev *dev, struct qedr_qp *qp)
1587 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: successfully created user QP. "
1596 qp->usq.buf_len, qp->urq.buf_addr, qp->urq.buf_len);
1599 static int qedr_idr_add(struct qedr_dev *dev, struct qedr_idr *qidr,
1604 idr_preload(GFP_KERNEL);
1605 spin_lock_irq(&qidr->idr_lock);
1607 rc = idr_alloc(&qidr->idr, ptr, id, id + 1, GFP_ATOMIC);
1609 spin_unlock_irq(&qidr->idr_lock);
1612 return rc < 0 ? rc : 0;
1615 static void qedr_idr_remove(struct qedr_dev *dev, struct qedr_idr *qidr, u32 id)
1617 spin_lock_irq(&qidr->idr_lock);
1618 idr_remove(&qidr->idr, id);
1619 spin_unlock_irq(&qidr->idr_lock);
1623 qedr_iwarp_populate_user_qp(struct qedr_dev *dev,
1625 struct qed_rdma_create_qp_out_params *out_params)
1627 qp->usq.pbl_tbl->va = out_params->sq_pbl_virt;
1628 qp->usq.pbl_tbl->pa = out_params->sq_pbl_phys;
1630 qedr_populate_pbls(dev, qp->usq.umem, qp->usq.pbl_tbl,
1631 &qp->usq.pbl_info, FW_PAGE_SHIFT);
1633 qp->urq.pbl_tbl->va = out_params->rq_pbl_virt;
1634 qp->urq.pbl_tbl->pa = out_params->rq_pbl_phys;
1637 qedr_populate_pbls(dev, qp->urq.umem, qp->urq.pbl_tbl,
1638 &qp->urq.pbl_info, FW_PAGE_SHIFT);
1641 static void qedr_cleanup_user(struct qedr_dev *dev, struct qedr_qp *qp)
1644 ib_umem_release(qp->usq.umem);
1645 qp->usq.umem = NULL;
1648 ib_umem_release(qp->urq.umem);
1649 qp->urq.umem = NULL;
1652 static int qedr_create_user_qp(struct qedr_dev *dev,
1655 struct ib_udata *udata,
1656 struct ib_qp_init_attr *attrs)
1658 struct qed_rdma_create_qp_in_params in_params;
1659 struct qed_rdma_create_qp_out_params out_params;
1660 struct qedr_pd *pd = get_qedr_pd(ibpd);
1661 struct qedr_create_qp_ureq ureq;
1662 int alloc_and_init = rdma_protocol_roce(&dev->ibdev, 1);
1665 memset(&ureq, 0, sizeof(ureq));
1666 rc = ib_copy_from_udata(&ureq, udata, sizeof(ureq));
1668 DP_ERR(dev, "Problem copying data from user space\n");
1672 /* SQ - read access only (0), dma sync not required (0) */
1673 rc = qedr_init_user_queue(udata, dev, &qp->usq, ureq.sq_addr,
1674 ureq.sq_len, 0, 0, alloc_and_init);
1679 /* RQ - read access only (0), dma sync not required (0) */
1680 rc = qedr_init_user_queue(udata, dev, &qp->urq, ureq.rq_addr,
1681 ureq.rq_len, 0, 0, alloc_and_init);
1686 memset(&in_params, 0, sizeof(in_params));
1687 qedr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params);
1688 in_params.qp_handle_lo = ureq.qp_handle_lo;
1689 in_params.qp_handle_hi = ureq.qp_handle_hi;
1690 in_params.sq_num_pages = qp->usq.pbl_info.num_pbes;
1691 in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa;
1693 in_params.rq_num_pages = qp->urq.pbl_info.num_pbes;
1694 in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa;
1697 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1698 &in_params, &out_params);
1705 if (rdma_protocol_iwarp(&dev->ibdev, 1))
1706 qedr_iwarp_populate_user_qp(dev, qp, &out_params);
1708 qp->qp_id = out_params.qp_id;
1709 qp->icid = out_params.icid;
1711 rc = qedr_copy_qp_uresp(dev, qp, udata);
1715 qedr_qp_user_print(dev, qp);
1719 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
1721 DP_ERR(dev, "create qp: fatal fault. rc=%d", rc);
1724 qedr_cleanup_user(dev, qp);
1728 static void qedr_set_iwarp_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
1730 qp->sq.db = dev->db_addr +
1731 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1732 qp->sq.db_data.data.icid = qp->icid;
1734 qp->rq.db = dev->db_addr +
1735 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
1736 qp->rq.db_data.data.icid = qp->icid;
1737 qp->rq.iwarp_db2 = dev->db_addr +
1738 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
1739 qp->rq.iwarp_db2_data.data.icid = qp->icid;
1740 qp->rq.iwarp_db2_data.data.value = DQ_TCM_IWARP_POST_RQ_CF_CMD;
1744 qedr_roce_create_kernel_qp(struct qedr_dev *dev,
1746 struct qed_rdma_create_qp_in_params *in_params,
1747 u32 n_sq_elems, u32 n_rq_elems)
1749 struct qed_rdma_create_qp_out_params out_params;
1752 rc = dev->ops->common->chain_alloc(dev->cdev,
1753 QED_CHAIN_USE_TO_PRODUCE,
1755 QED_CHAIN_CNT_TYPE_U32,
1757 QEDR_SQE_ELEMENT_SIZE,
1763 in_params->sq_num_pages = qed_chain_get_page_cnt(&qp->sq.pbl);
1764 in_params->sq_pbl_ptr = qed_chain_get_pbl_phys(&qp->sq.pbl);
1766 rc = dev->ops->common->chain_alloc(dev->cdev,
1767 QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1769 QED_CHAIN_CNT_TYPE_U32,
1771 QEDR_RQE_ELEMENT_SIZE,
1776 in_params->rq_num_pages = qed_chain_get_page_cnt(&qp->rq.pbl);
1777 in_params->rq_pbl_ptr = qed_chain_get_pbl_phys(&qp->rq.pbl);
1779 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1780 in_params, &out_params);
1785 qp->qp_id = out_params.qp_id;
1786 qp->icid = out_params.icid;
1788 qedr_set_roce_db_info(dev, qp);
1793 qedr_iwarp_create_kernel_qp(struct qedr_dev *dev,
1795 struct qed_rdma_create_qp_in_params *in_params,
1796 u32 n_sq_elems, u32 n_rq_elems)
1798 struct qed_rdma_create_qp_out_params out_params;
1799 struct qed_chain_ext_pbl ext_pbl;
1802 in_params->sq_num_pages = QED_CHAIN_PAGE_CNT(n_sq_elems,
1803 QEDR_SQE_ELEMENT_SIZE,
1804 QED_CHAIN_MODE_PBL);
1805 in_params->rq_num_pages = QED_CHAIN_PAGE_CNT(n_rq_elems,
1806 QEDR_RQE_ELEMENT_SIZE,
1807 QED_CHAIN_MODE_PBL);
1809 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1810 in_params, &out_params);
1815 /* Now we allocate the chain */
1816 ext_pbl.p_pbl_virt = out_params.sq_pbl_virt;
1817 ext_pbl.p_pbl_phys = out_params.sq_pbl_phys;
1819 rc = dev->ops->common->chain_alloc(dev->cdev,
1820 QED_CHAIN_USE_TO_PRODUCE,
1822 QED_CHAIN_CNT_TYPE_U32,
1824 QEDR_SQE_ELEMENT_SIZE,
1825 &qp->sq.pbl, &ext_pbl);
1830 ext_pbl.p_pbl_virt = out_params.rq_pbl_virt;
1831 ext_pbl.p_pbl_phys = out_params.rq_pbl_phys;
1833 rc = dev->ops->common->chain_alloc(dev->cdev,
1834 QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1836 QED_CHAIN_CNT_TYPE_U32,
1838 QEDR_RQE_ELEMENT_SIZE,
1839 &qp->rq.pbl, &ext_pbl);
1844 qp->qp_id = out_params.qp_id;
1845 qp->icid = out_params.icid;
1847 qedr_set_iwarp_db_info(dev, qp);
1851 dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
1856 static void qedr_cleanup_kernel(struct qedr_dev *dev, struct qedr_qp *qp)
1858 dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl);
1859 kfree(qp->wqe_wr_id);
1861 dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl);
1862 kfree(qp->rqe_wr_id);
1865 static int qedr_create_kernel_qp(struct qedr_dev *dev,
1868 struct ib_qp_init_attr *attrs)
1870 struct qed_rdma_create_qp_in_params in_params;
1871 struct qedr_pd *pd = get_qedr_pd(ibpd);
1877 memset(&in_params, 0, sizeof(in_params));
1879 /* A single work request may take up to QEDR_MAX_SQ_WQE_SIZE elements in
1880 * the ring. The ring should allow at least a single WR, even if the
1881 * user requested none, due to allocation issues.
1882 * We should add an extra WR since the prod and cons indices of
1883 * wqe_wr_id are managed in such a way that the WQ is considered full
1884 * when (prod+1)%max_wr==cons. We currently don't do that because we
1885 * double the number of entries due an iSER issue that pushes far more
1886 * WRs than indicated. If we decline its ib_post_send() then we get
1887 * error prints in the dmesg we'd like to avoid.
1889 qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier,
1892 qp->wqe_wr_id = kcalloc(qp->sq.max_wr, sizeof(*qp->wqe_wr_id),
1894 if (!qp->wqe_wr_id) {
1895 DP_ERR(dev, "create qp: failed SQ shadow memory allocation\n");
1899 /* QP handle to be written in CQE */
1900 in_params.qp_handle_lo = lower_32_bits((uintptr_t) qp);
1901 in_params.qp_handle_hi = upper_32_bits((uintptr_t) qp);
1903 /* A single work request may take up to QEDR_MAX_RQ_WQE_SIZE elements in
1904 * the ring. There ring should allow at least a single WR, even if the
1905 * user requested none, due to allocation issues.
1907 qp->rq.max_wr = (u16) max_t(u32, attrs->cap.max_recv_wr, 1);
1909 /* Allocate driver internal RQ array */
1910 qp->rqe_wr_id = kcalloc(qp->rq.max_wr, sizeof(*qp->rqe_wr_id),
1912 if (!qp->rqe_wr_id) {
1914 "create qp: failed RQ shadow memory allocation\n");
1915 kfree(qp->wqe_wr_id);
1919 qedr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params);
1921 n_sq_entries = attrs->cap.max_send_wr;
1922 n_sq_entries = min_t(u32, n_sq_entries, dev->attr.max_sqe);
1923 n_sq_entries = max_t(u32, n_sq_entries, 1);
1924 n_sq_elems = n_sq_entries * QEDR_MAX_SQE_ELEMENTS_PER_SQE;
1926 n_rq_elems = qp->rq.max_wr * QEDR_MAX_RQE_ELEMENTS_PER_RQE;
1928 if (rdma_protocol_iwarp(&dev->ibdev, 1))
1929 rc = qedr_iwarp_create_kernel_qp(dev, qp, &in_params,
1930 n_sq_elems, n_rq_elems);
1932 rc = qedr_roce_create_kernel_qp(dev, qp, &in_params,
1933 n_sq_elems, n_rq_elems);
1935 qedr_cleanup_kernel(dev, qp);
1940 struct ib_qp *qedr_create_qp(struct ib_pd *ibpd,
1941 struct ib_qp_init_attr *attrs,
1942 struct ib_udata *udata)
1944 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
1945 struct qedr_pd *pd = get_qedr_pd(ibpd);
1950 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: called from %s, pd=%p\n",
1951 udata ? "user library" : "kernel", pd);
1953 rc = qedr_check_qp_attrs(ibpd, dev, attrs, udata);
1957 DP_DEBUG(dev, QEDR_MSG_QP,
1958 "create qp: called from %s, event_handler=%p, eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n",
1959 udata ? "user library" : "kernel", attrs->event_handler, pd,
1960 get_qedr_cq(attrs->send_cq),
1961 get_qedr_cq(attrs->send_cq)->icid,
1962 get_qedr_cq(attrs->recv_cq),
1963 attrs->recv_cq ? get_qedr_cq(attrs->recv_cq)->icid : 0);
1965 qp = kzalloc(sizeof(*qp), GFP_KERNEL);
1967 DP_ERR(dev, "create qp: failed allocating memory\n");
1968 return ERR_PTR(-ENOMEM);
1971 qedr_set_common_qp_params(dev, qp, pd, attrs);
1973 if (attrs->qp_type == IB_QPT_GSI) {
1974 ibqp = qedr_create_gsi_qp(dev, attrs, qp);
1981 rc = qedr_create_user_qp(dev, qp, ibpd, udata, attrs);
1983 rc = qedr_create_kernel_qp(dev, qp, ibpd, attrs);
1988 qp->ibqp.qp_num = qp->qp_id;
1990 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1991 rc = qedr_idr_add(dev, &dev->qpidr, qp, qp->qp_id);
2001 return ERR_PTR(-EFAULT);
2004 static enum ib_qp_state qedr_get_ibqp_state(enum qed_roce_qp_state qp_state)
2007 case QED_ROCE_QP_STATE_RESET:
2008 return IB_QPS_RESET;
2009 case QED_ROCE_QP_STATE_INIT:
2011 case QED_ROCE_QP_STATE_RTR:
2013 case QED_ROCE_QP_STATE_RTS:
2015 case QED_ROCE_QP_STATE_SQD:
2017 case QED_ROCE_QP_STATE_ERR:
2019 case QED_ROCE_QP_STATE_SQE:
2025 static enum qed_roce_qp_state qedr_get_state_from_ibqp(
2026 enum ib_qp_state qp_state)
2030 return QED_ROCE_QP_STATE_RESET;
2032 return QED_ROCE_QP_STATE_INIT;
2034 return QED_ROCE_QP_STATE_RTR;
2036 return QED_ROCE_QP_STATE_RTS;
2038 return QED_ROCE_QP_STATE_SQD;
2040 return QED_ROCE_QP_STATE_ERR;
2042 return QED_ROCE_QP_STATE_ERR;
2046 static void qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info *qph)
2048 qed_chain_reset(&qph->pbl);
2052 qph->db_data.data.value = cpu_to_le16(0);
2055 static int qedr_update_qp_state(struct qedr_dev *dev,
2057 enum qed_roce_qp_state cur_state,
2058 enum qed_roce_qp_state new_state)
2062 if (new_state == cur_state)
2065 switch (cur_state) {
2066 case QED_ROCE_QP_STATE_RESET:
2067 switch (new_state) {
2068 case QED_ROCE_QP_STATE_INIT:
2069 qp->prev_wqe_size = 0;
2070 qedr_reset_qp_hwq_info(&qp->sq);
2071 qedr_reset_qp_hwq_info(&qp->rq);
2078 case QED_ROCE_QP_STATE_INIT:
2079 switch (new_state) {
2080 case QED_ROCE_QP_STATE_RTR:
2081 /* Update doorbell (in case post_recv was
2082 * done before move to RTR)
2085 if (rdma_protocol_roce(&dev->ibdev, 1)) {
2086 writel(qp->rq.db_data.raw, qp->rq.db);
2087 /* Make sure write takes effect */
2091 case QED_ROCE_QP_STATE_ERR:
2094 /* Invalid state change. */
2099 case QED_ROCE_QP_STATE_RTR:
2101 switch (new_state) {
2102 case QED_ROCE_QP_STATE_RTS:
2104 case QED_ROCE_QP_STATE_ERR:
2107 /* Invalid state change. */
2112 case QED_ROCE_QP_STATE_RTS:
2114 switch (new_state) {
2115 case QED_ROCE_QP_STATE_SQD:
2117 case QED_ROCE_QP_STATE_ERR:
2120 /* Invalid state change. */
2125 case QED_ROCE_QP_STATE_SQD:
2127 switch (new_state) {
2128 case QED_ROCE_QP_STATE_RTS:
2129 case QED_ROCE_QP_STATE_ERR:
2132 /* Invalid state change. */
2137 case QED_ROCE_QP_STATE_ERR:
2139 switch (new_state) {
2140 case QED_ROCE_QP_STATE_RESET:
2141 if ((qp->rq.prod != qp->rq.cons) ||
2142 (qp->sq.prod != qp->sq.cons)) {
2144 "Error->Reset with rq/sq not empty rq.prod=%x rq.cons=%x sq.prod=%x sq.cons=%x\n",
2145 qp->rq.prod, qp->rq.cons, qp->sq.prod,
2163 int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
2164 int attr_mask, struct ib_udata *udata)
2166 struct qedr_qp *qp = get_qedr_qp(ibqp);
2167 struct qed_rdma_modify_qp_in_params qp_params = { 0 };
2168 struct qedr_dev *dev = get_qedr_dev(&qp->dev->ibdev);
2169 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
2170 enum ib_qp_state old_qp_state, new_qp_state;
2171 enum qed_roce_qp_state cur_state;
2174 DP_DEBUG(dev, QEDR_MSG_QP,
2175 "modify qp: qp %p attr_mask=0x%x, state=%d", qp, attr_mask,
2178 old_qp_state = qedr_get_ibqp_state(qp->state);
2179 if (attr_mask & IB_QP_STATE)
2180 new_qp_state = attr->qp_state;
2182 new_qp_state = old_qp_state;
2184 if (rdma_protocol_roce(&dev->ibdev, 1)) {
2185 if (!ib_modify_qp_is_ok(old_qp_state, new_qp_state,
2186 ibqp->qp_type, attr_mask)) {
2188 "modify qp: invalid attribute mask=0x%x specified for\n"
2189 "qpn=0x%x of type=0x%x old_qp_state=0x%x, new_qp_state=0x%x\n",
2190 attr_mask, qp->qp_id, ibqp->qp_type,
2191 old_qp_state, new_qp_state);
2197 /* Translate the masks... */
2198 if (attr_mask & IB_QP_STATE) {
2199 SET_FIELD(qp_params.modify_flags,
2200 QED_RDMA_MODIFY_QP_VALID_NEW_STATE, 1);
2201 qp_params.new_state = qedr_get_state_from_ibqp(attr->qp_state);
2204 if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY)
2205 qp_params.sqd_async = true;
2207 if (attr_mask & IB_QP_PKEY_INDEX) {
2208 SET_FIELD(qp_params.modify_flags,
2209 QED_ROCE_MODIFY_QP_VALID_PKEY, 1);
2210 if (attr->pkey_index >= QEDR_ROCE_PKEY_TABLE_LEN) {
2215 qp_params.pkey = QEDR_ROCE_PKEY_DEFAULT;
2218 if (attr_mask & IB_QP_QKEY)
2219 qp->qkey = attr->qkey;
2221 if (attr_mask & IB_QP_ACCESS_FLAGS) {
2222 SET_FIELD(qp_params.modify_flags,
2223 QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1);
2224 qp_params.incoming_rdma_read_en = attr->qp_access_flags &
2225 IB_ACCESS_REMOTE_READ;
2226 qp_params.incoming_rdma_write_en = attr->qp_access_flags &
2227 IB_ACCESS_REMOTE_WRITE;
2228 qp_params.incoming_atomic_en = attr->qp_access_flags &
2229 IB_ACCESS_REMOTE_ATOMIC;
2232 if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) {
2233 if (rdma_protocol_iwarp(&dev->ibdev, 1))
2236 if (attr_mask & IB_QP_PATH_MTU) {
2237 if (attr->path_mtu < IB_MTU_256 ||
2238 attr->path_mtu > IB_MTU_4096) {
2239 pr_err("error: Only MTU sizes of 256, 512, 1024, 2048 and 4096 are supported by RoCE\n");
2243 qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu),
2244 ib_mtu_enum_to_int(iboe_get_mtu
2250 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
2251 pr_err("Fixing zeroed MTU to qp->mtu = %d\n", qp->mtu);
2254 SET_FIELD(qp_params.modify_flags,
2255 QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, 1);
2257 qp_params.traffic_class_tos = grh->traffic_class;
2258 qp_params.flow_label = grh->flow_label;
2259 qp_params.hop_limit_ttl = grh->hop_limit;
2261 qp->sgid_idx = grh->sgid_index;
2263 rc = get_gid_info_from_table(ibqp, attr, attr_mask, &qp_params);
2266 "modify qp: problems with GID index %d (rc=%d)\n",
2267 grh->sgid_index, rc);
2271 rc = qedr_get_dmac(dev, &attr->ah_attr,
2272 qp_params.remote_mac_addr);
2276 qp_params.use_local_mac = true;
2277 ether_addr_copy(qp_params.local_mac_addr, dev->ndev->dev_addr);
2279 DP_DEBUG(dev, QEDR_MSG_QP, "dgid=%x:%x:%x:%x\n",
2280 qp_params.dgid.dwords[0], qp_params.dgid.dwords[1],
2281 qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]);
2282 DP_DEBUG(dev, QEDR_MSG_QP, "sgid=%x:%x:%x:%x\n",
2283 qp_params.sgid.dwords[0], qp_params.sgid.dwords[1],
2284 qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]);
2285 DP_DEBUG(dev, QEDR_MSG_QP, "remote_mac=[%pM]\n",
2286 qp_params.remote_mac_addr);
2288 qp_params.mtu = qp->mtu;
2289 qp_params.lb_indication = false;
2292 if (!qp_params.mtu) {
2293 /* Stay with current MTU */
2295 qp_params.mtu = qp->mtu;
2298 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
2301 if (attr_mask & IB_QP_TIMEOUT) {
2302 SET_FIELD(qp_params.modify_flags,
2303 QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1);
2305 /* The received timeout value is an exponent used like this:
2306 * "12.7.34 LOCAL ACK TIMEOUT
2307 * Value representing the transport (ACK) timeout for use by
2308 * the remote, expressed as: 4.096 * 2^timeout [usec]"
2309 * The FW expects timeout in msec so we need to divide the usec
2310 * result by 1000. We'll approximate 1000~2^10, and 4.096 ~ 2^2,
2311 * so we get: 2^2 * 2^timeout / 2^10 = 2^(timeout - 8).
2312 * The value of zero means infinite so we use a 'max_t' to make
2313 * sure that sub 1 msec values will be configured as 1 msec.
2316 qp_params.ack_timeout =
2317 1 << max_t(int, attr->timeout - 8, 0);
2319 qp_params.ack_timeout = 0;
2322 if (attr_mask & IB_QP_RETRY_CNT) {
2323 SET_FIELD(qp_params.modify_flags,
2324 QED_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1);
2325 qp_params.retry_cnt = attr->retry_cnt;
2328 if (attr_mask & IB_QP_RNR_RETRY) {
2329 SET_FIELD(qp_params.modify_flags,
2330 QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, 1);
2331 qp_params.rnr_retry_cnt = attr->rnr_retry;
2334 if (attr_mask & IB_QP_RQ_PSN) {
2335 SET_FIELD(qp_params.modify_flags,
2336 QED_ROCE_MODIFY_QP_VALID_RQ_PSN, 1);
2337 qp_params.rq_psn = attr->rq_psn;
2338 qp->rq_psn = attr->rq_psn;
2341 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
2342 if (attr->max_rd_atomic > dev->attr.max_qp_req_rd_atomic_resc) {
2345 "unsupported max_rd_atomic=%d, supported=%d\n",
2346 attr->max_rd_atomic,
2347 dev->attr.max_qp_req_rd_atomic_resc);
2351 SET_FIELD(qp_params.modify_flags,
2352 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, 1);
2353 qp_params.max_rd_atomic_req = attr->max_rd_atomic;
2356 if (attr_mask & IB_QP_MIN_RNR_TIMER) {
2357 SET_FIELD(qp_params.modify_flags,
2358 QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, 1);
2359 qp_params.min_rnr_nak_timer = attr->min_rnr_timer;
2362 if (attr_mask & IB_QP_SQ_PSN) {
2363 SET_FIELD(qp_params.modify_flags,
2364 QED_ROCE_MODIFY_QP_VALID_SQ_PSN, 1);
2365 qp_params.sq_psn = attr->sq_psn;
2366 qp->sq_psn = attr->sq_psn;
2369 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
2370 if (attr->max_dest_rd_atomic >
2371 dev->attr.max_qp_resp_rd_atomic_resc) {
2373 "unsupported max_dest_rd_atomic=%d, supported=%d\n",
2374 attr->max_dest_rd_atomic,
2375 dev->attr.max_qp_resp_rd_atomic_resc);
2381 SET_FIELD(qp_params.modify_flags,
2382 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, 1);
2383 qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic;
2386 if (attr_mask & IB_QP_DEST_QPN) {
2387 SET_FIELD(qp_params.modify_flags,
2388 QED_ROCE_MODIFY_QP_VALID_DEST_QP, 1);
2390 qp_params.dest_qp = attr->dest_qp_num;
2391 qp->dest_qp_num = attr->dest_qp_num;
2394 cur_state = qp->state;
2396 /* Update the QP state before the actual ramrod to prevent a race with
2397 * fast path. Modifying the QP state to error will cause the device to
2398 * flush the CQEs and while polling the flushed CQEs will considered as
2399 * a potential issue if the QP isn't in error state.
2401 if ((attr_mask & IB_QP_STATE) && qp->qp_type != IB_QPT_GSI &&
2402 !udata && qp_params.new_state == QED_ROCE_QP_STATE_ERR)
2403 qp->state = QED_ROCE_QP_STATE_ERR;
2405 if (qp->qp_type != IB_QPT_GSI)
2406 rc = dev->ops->rdma_modify_qp(dev->rdma_ctx,
2407 qp->qed_qp, &qp_params);
2409 if (attr_mask & IB_QP_STATE) {
2410 if ((qp->qp_type != IB_QPT_GSI) && (!udata))
2411 rc = qedr_update_qp_state(dev, qp, cur_state,
2412 qp_params.new_state);
2413 qp->state = qp_params.new_state;
2420 static int qedr_to_ib_qp_acc_flags(struct qed_rdma_query_qp_out_params *params)
2422 int ib_qp_acc_flags = 0;
2424 if (params->incoming_rdma_write_en)
2425 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE;
2426 if (params->incoming_rdma_read_en)
2427 ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ;
2428 if (params->incoming_atomic_en)
2429 ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC;
2430 ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE;
2431 return ib_qp_acc_flags;
2434 int qedr_query_qp(struct ib_qp *ibqp,
2435 struct ib_qp_attr *qp_attr,
2436 int attr_mask, struct ib_qp_init_attr *qp_init_attr)
2438 struct qed_rdma_query_qp_out_params params;
2439 struct qedr_qp *qp = get_qedr_qp(ibqp);
2440 struct qedr_dev *dev = qp->dev;
2443 memset(¶ms, 0, sizeof(params));
2445 rc = dev->ops->rdma_query_qp(dev->rdma_ctx, qp->qed_qp, ¶ms);
2449 memset(qp_attr, 0, sizeof(*qp_attr));
2450 memset(qp_init_attr, 0, sizeof(*qp_init_attr));
2452 qp_attr->qp_state = qedr_get_ibqp_state(params.state);
2453 qp_attr->cur_qp_state = qedr_get_ibqp_state(params.state);
2454 qp_attr->path_mtu = ib_mtu_int_to_enum(params.mtu);
2455 qp_attr->path_mig_state = IB_MIG_MIGRATED;
2456 qp_attr->rq_psn = params.rq_psn;
2457 qp_attr->sq_psn = params.sq_psn;
2458 qp_attr->dest_qp_num = params.dest_qp;
2460 qp_attr->qp_access_flags = qedr_to_ib_qp_acc_flags(¶ms);
2462 qp_attr->cap.max_send_wr = qp->sq.max_wr;
2463 qp_attr->cap.max_recv_wr = qp->rq.max_wr;
2464 qp_attr->cap.max_send_sge = qp->sq.max_sges;
2465 qp_attr->cap.max_recv_sge = qp->rq.max_sges;
2466 qp_attr->cap.max_inline_data = ROCE_REQ_MAX_INLINE_DATA_SIZE;
2467 qp_init_attr->cap = qp_attr->cap;
2469 qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
2470 rdma_ah_set_grh(&qp_attr->ah_attr, NULL,
2471 params.flow_label, qp->sgid_idx,
2472 params.hop_limit_ttl, params.traffic_class_tos);
2473 rdma_ah_set_dgid_raw(&qp_attr->ah_attr, ¶ms.dgid.bytes[0]);
2474 rdma_ah_set_port_num(&qp_attr->ah_attr, 1);
2475 rdma_ah_set_sl(&qp_attr->ah_attr, 0);
2476 qp_attr->timeout = params.timeout;
2477 qp_attr->rnr_retry = params.rnr_retry;
2478 qp_attr->retry_cnt = params.retry_cnt;
2479 qp_attr->min_rnr_timer = params.min_rnr_nak_timer;
2480 qp_attr->pkey_index = params.pkey_index;
2481 qp_attr->port_num = 1;
2482 rdma_ah_set_path_bits(&qp_attr->ah_attr, 0);
2483 rdma_ah_set_static_rate(&qp_attr->ah_attr, 0);
2484 qp_attr->alt_pkey_index = 0;
2485 qp_attr->alt_port_num = 0;
2486 qp_attr->alt_timeout = 0;
2487 memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr));
2489 qp_attr->sq_draining = (params.state == QED_ROCE_QP_STATE_SQD) ? 1 : 0;
2490 qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic;
2491 qp_attr->max_rd_atomic = params.max_rd_atomic;
2492 qp_attr->en_sqd_async_notify = (params.sqd_async) ? 1 : 0;
2494 DP_DEBUG(dev, QEDR_MSG_QP, "QEDR_QUERY_QP: max_inline_data=%d\n",
2495 qp_attr->cap.max_inline_data);
2501 static int qedr_free_qp_resources(struct qedr_dev *dev, struct qedr_qp *qp)
2505 if (qp->qp_type != IB_QPT_GSI) {
2506 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2511 if (qp->ibqp.uobject && qp->ibqp.uobject->context)
2512 qedr_cleanup_user(dev, qp);
2514 qedr_cleanup_kernel(dev, qp);
2519 int qedr_destroy_qp(struct ib_qp *ibqp)
2521 struct qedr_qp *qp = get_qedr_qp(ibqp);
2522 struct qedr_dev *dev = qp->dev;
2523 struct ib_qp_attr attr;
2527 DP_DEBUG(dev, QEDR_MSG_QP, "destroy qp: destroying %p, qp type=%d\n",
2530 if (rdma_protocol_roce(&dev->ibdev, 1)) {
2531 if ((qp->state != QED_ROCE_QP_STATE_RESET) &&
2532 (qp->state != QED_ROCE_QP_STATE_ERR) &&
2533 (qp->state != QED_ROCE_QP_STATE_INIT)) {
2535 attr.qp_state = IB_QPS_ERR;
2536 attr_mask |= IB_QP_STATE;
2538 /* Change the QP state to ERROR */
2539 qedr_modify_qp(ibqp, &attr, attr_mask, NULL);
2542 /* Wait for the connect/accept to complete */
2546 while (qp->ep->during_connect) {
2547 DP_DEBUG(dev, QEDR_MSG_QP,
2548 "Still in during connect/accept\n");
2551 if (wait_count++ > 200) {
2553 "during connect timeout\n");
2560 if (qp->qp_type == IB_QPT_GSI)
2561 qedr_destroy_gsi_qp(dev);
2563 qedr_free_qp_resources(dev, qp);
2565 if (atomic_dec_and_test(&qp->refcnt) &&
2566 rdma_protocol_iwarp(&dev->ibdev, 1)) {
2567 qedr_idr_remove(dev, &dev->qpidr, qp->qp_id);
2573 struct ib_ah *qedr_create_ah(struct ib_pd *ibpd, struct rdma_ah_attr *attr,
2574 u32 flags, struct ib_udata *udata)
2578 ah = kzalloc(sizeof(*ah), GFP_ATOMIC);
2580 return ERR_PTR(-ENOMEM);
2582 rdma_copy_ah_attr(&ah->attr, attr);
2587 int qedr_destroy_ah(struct ib_ah *ibah, u32 flags)
2589 struct qedr_ah *ah = get_qedr_ah(ibah);
2591 rdma_destroy_ah_attr(&ah->attr);
2596 static void free_mr_info(struct qedr_dev *dev, struct mr_info *info)
2598 struct qedr_pbl *pbl, *tmp;
2600 if (info->pbl_table)
2601 list_add_tail(&info->pbl_table->list_entry,
2602 &info->free_pbl_list);
2604 if (!list_empty(&info->inuse_pbl_list))
2605 list_splice(&info->inuse_pbl_list, &info->free_pbl_list);
2607 list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) {
2608 list_del(&pbl->list_entry);
2609 qedr_free_pbl(dev, &info->pbl_info, pbl);
2613 static int init_mr_info(struct qedr_dev *dev, struct mr_info *info,
2614 size_t page_list_len, bool two_layered)
2616 struct qedr_pbl *tmp;
2619 INIT_LIST_HEAD(&info->free_pbl_list);
2620 INIT_LIST_HEAD(&info->inuse_pbl_list);
2622 rc = qedr_prepare_pbl_tbl(dev, &info->pbl_info,
2623 page_list_len, two_layered);
2627 info->pbl_table = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2628 if (IS_ERR(info->pbl_table)) {
2629 rc = PTR_ERR(info->pbl_table);
2633 DP_DEBUG(dev, QEDR_MSG_MR, "pbl_table_pa = %pa\n",
2634 &info->pbl_table->pa);
2636 /* in usual case we use 2 PBLs, so we add one to free
2637 * list and allocating another one
2639 tmp = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2641 DP_DEBUG(dev, QEDR_MSG_MR, "Extra PBL is not allocated\n");
2645 list_add_tail(&tmp->list_entry, &info->free_pbl_list);
2647 DP_DEBUG(dev, QEDR_MSG_MR, "extra pbl_table_pa = %pa\n", &tmp->pa);
2651 free_mr_info(dev, info);
2656 struct ib_mr *qedr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
2657 u64 usr_addr, int acc, struct ib_udata *udata)
2659 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2664 pd = get_qedr_pd(ibpd);
2665 DP_DEBUG(dev, QEDR_MSG_MR,
2666 "qedr_register user mr pd = %d start = %lld, len = %lld, usr_addr = %lld, acc = %d\n",
2667 pd->pd_id, start, len, usr_addr, acc);
2669 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE))
2670 return ERR_PTR(-EINVAL);
2672 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2676 mr->type = QEDR_MR_USER;
2678 mr->umem = ib_umem_get(udata, start, len, acc, 0);
2679 if (IS_ERR(mr->umem)) {
2684 rc = init_mr_info(dev, &mr->info, ib_umem_page_count(mr->umem), 1);
2688 qedr_populate_pbls(dev, mr->umem, mr->info.pbl_table,
2689 &mr->info.pbl_info, PAGE_SHIFT);
2691 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2693 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2697 /* Index only, 18 bit long, lkey = itid << 8 | key */
2698 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
2700 mr->hw_mr.pd = pd->pd_id;
2701 mr->hw_mr.local_read = 1;
2702 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
2703 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
2704 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
2705 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
2706 mr->hw_mr.mw_bind = false;
2707 mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa;
2708 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
2709 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
2710 mr->hw_mr.page_size_log = PAGE_SHIFT;
2711 mr->hw_mr.fbo = ib_umem_offset(mr->umem);
2712 mr->hw_mr.length = len;
2713 mr->hw_mr.vaddr = usr_addr;
2714 mr->hw_mr.zbva = false;
2715 mr->hw_mr.phy_mr = false;
2716 mr->hw_mr.dma_mr = false;
2718 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2720 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2724 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2725 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
2726 mr->hw_mr.remote_atomic)
2727 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2729 DP_DEBUG(dev, QEDR_MSG_MR, "register user mr lkey: %x\n",
2734 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2736 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
2742 int qedr_dereg_mr(struct ib_mr *ib_mr)
2744 struct qedr_mr *mr = get_qedr_mr(ib_mr);
2745 struct qedr_dev *dev = get_qedr_dev(ib_mr->device);
2748 rc = dev->ops->rdma_deregister_tid(dev->rdma_ctx, mr->hw_mr.itid);
2752 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2754 if ((mr->type != QEDR_MR_DMA) && (mr->type != QEDR_MR_FRMR))
2755 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
2757 /* it could be user registered memory. */
2759 ib_umem_release(mr->umem);
2766 static struct qedr_mr *__qedr_alloc_mr(struct ib_pd *ibpd,
2767 int max_page_list_len)
2769 struct qedr_pd *pd = get_qedr_pd(ibpd);
2770 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2774 DP_DEBUG(dev, QEDR_MSG_MR,
2775 "qedr_alloc_frmr pd = %d max_page_list_len= %d\n", pd->pd_id,
2778 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2783 mr->type = QEDR_MR_FRMR;
2785 rc = init_mr_info(dev, &mr->info, max_page_list_len, 1);
2789 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2791 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2795 /* Index only, 18 bit long, lkey = itid << 8 | key */
2796 mr->hw_mr.tid_type = QED_RDMA_TID_FMR;
2798 mr->hw_mr.pd = pd->pd_id;
2799 mr->hw_mr.local_read = 1;
2800 mr->hw_mr.local_write = 0;
2801 mr->hw_mr.remote_read = 0;
2802 mr->hw_mr.remote_write = 0;
2803 mr->hw_mr.remote_atomic = 0;
2804 mr->hw_mr.mw_bind = false;
2805 mr->hw_mr.pbl_ptr = 0;
2806 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
2807 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
2809 mr->hw_mr.length = 0;
2810 mr->hw_mr.vaddr = 0;
2811 mr->hw_mr.zbva = false;
2812 mr->hw_mr.phy_mr = true;
2813 mr->hw_mr.dma_mr = false;
2815 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2817 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2821 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2822 mr->ibmr.rkey = mr->ibmr.lkey;
2824 DP_DEBUG(dev, QEDR_MSG_MR, "alloc frmr: %x\n", mr->ibmr.lkey);
2828 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2834 struct ib_mr *qedr_alloc_mr(struct ib_pd *ibpd,
2835 enum ib_mr_type mr_type, u32 max_num_sg)
2839 if (mr_type != IB_MR_TYPE_MEM_REG)
2840 return ERR_PTR(-EINVAL);
2842 mr = __qedr_alloc_mr(ibpd, max_num_sg);
2845 return ERR_PTR(-EINVAL);
2850 static int qedr_set_page(struct ib_mr *ibmr, u64 addr)
2852 struct qedr_mr *mr = get_qedr_mr(ibmr);
2853 struct qedr_pbl *pbl_table;
2854 struct regpair *pbe;
2857 if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) {
2858 DP_ERR(mr->dev, "qedr_set_page fails when %d\n", mr->npages);
2862 DP_DEBUG(mr->dev, QEDR_MSG_MR, "qedr_set_page pages[%d] = 0x%llx\n",
2865 pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64);
2866 pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page);
2867 pbe = (struct regpair *)pbl_table->va;
2868 pbe += mr->npages % pbes_in_page;
2869 pbe->lo = cpu_to_le32((u32)addr);
2870 pbe->hi = cpu_to_le32((u32)upper_32_bits(addr));
2877 static void handle_completed_mrs(struct qedr_dev *dev, struct mr_info *info)
2879 int work = info->completed - info->completed_handled - 1;
2881 DP_DEBUG(dev, QEDR_MSG_MR, "Special FMR work = %d\n", work);
2882 while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) {
2883 struct qedr_pbl *pbl;
2885 /* Free all the page list that are possible to be freed
2886 * (all the ones that were invalidated), under the assumption
2887 * that if an FMR was completed successfully that means that
2888 * if there was an invalidate operation before it also ended
2890 pbl = list_first_entry(&info->inuse_pbl_list,
2891 struct qedr_pbl, list_entry);
2892 list_move_tail(&pbl->list_entry, &info->free_pbl_list);
2893 info->completed_handled++;
2897 int qedr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
2898 int sg_nents, unsigned int *sg_offset)
2900 struct qedr_mr *mr = get_qedr_mr(ibmr);
2904 handle_completed_mrs(mr->dev, &mr->info);
2905 return ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qedr_set_page);
2908 struct ib_mr *qedr_get_dma_mr(struct ib_pd *ibpd, int acc)
2910 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2911 struct qedr_pd *pd = get_qedr_pd(ibpd);
2915 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2917 return ERR_PTR(-ENOMEM);
2919 mr->type = QEDR_MR_DMA;
2921 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2923 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2927 /* index only, 18 bit long, lkey = itid << 8 | key */
2928 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
2929 mr->hw_mr.pd = pd->pd_id;
2930 mr->hw_mr.local_read = 1;
2931 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
2932 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
2933 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
2934 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
2935 mr->hw_mr.dma_mr = true;
2937 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2939 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2943 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2944 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
2945 mr->hw_mr.remote_atomic)
2946 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2948 DP_DEBUG(dev, QEDR_MSG_MR, "get dma mr: lkey = %x\n", mr->ibmr.lkey);
2952 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2958 static inline int qedr_wq_is_full(struct qedr_qp_hwq_info *wq)
2960 return (((wq->prod + 1) % wq->max_wr) == wq->cons);
2963 static int sge_data_len(struct ib_sge *sg_list, int num_sge)
2967 for (i = 0; i < num_sge; i++)
2968 len += sg_list[i].length;
2973 static void swap_wqe_data64(u64 *p)
2977 for (i = 0; i < QEDR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++)
2978 *p = cpu_to_be64(cpu_to_le64(*p));
2981 static u32 qedr_prepare_sq_inline_data(struct qedr_dev *dev,
2982 struct qedr_qp *qp, u8 *wqe_size,
2983 const struct ib_send_wr *wr,
2984 const struct ib_send_wr **bad_wr,
2987 u32 data_size = sge_data_len(wr->sg_list, wr->num_sge);
2988 char *seg_prt, *wqe;
2991 if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) {
2992 DP_ERR(dev, "Too much inline data in WR: %d\n", data_size);
3006 /* Copy data inline */
3007 for (i = 0; i < wr->num_sge; i++) {
3008 u32 len = wr->sg_list[i].length;
3009 void *src = (void *)(uintptr_t)wr->sg_list[i].addr;
3014 /* New segment required */
3016 wqe = (char *)qed_chain_produce(&qp->sq.pbl);
3018 seg_siz = sizeof(struct rdma_sq_common_wqe);
3022 /* Calculate currently allowed length */
3023 cur = min_t(u32, len, seg_siz);
3024 memcpy(seg_prt, src, cur);
3026 /* Update segment variables */
3030 /* Update sge variables */
3034 /* Swap fully-completed segments */
3036 swap_wqe_data64((u64 *)wqe);
3040 /* swap last not completed segment */
3042 swap_wqe_data64((u64 *)wqe);
3047 #define RQ_SGE_SET(sge, vaddr, vlength, vflags) \
3049 DMA_REGPAIR_LE(sge->addr, vaddr); \
3050 (sge)->length = cpu_to_le32(vlength); \
3051 (sge)->flags = cpu_to_le32(vflags); \
3054 #define SRQ_HDR_SET(hdr, vwr_id, num_sge) \
3056 DMA_REGPAIR_LE(hdr->wr_id, vwr_id); \
3057 (hdr)->num_sges = num_sge; \
3060 #define SRQ_SGE_SET(sge, vaddr, vlength, vlkey) \
3062 DMA_REGPAIR_LE(sge->addr, vaddr); \
3063 (sge)->length = cpu_to_le32(vlength); \
3064 (sge)->l_key = cpu_to_le32(vlkey); \
3067 static u32 qedr_prepare_sq_sges(struct qedr_qp *qp, u8 *wqe_size,
3068 const struct ib_send_wr *wr)
3073 for (i = 0; i < wr->num_sge; i++) {
3074 struct rdma_sq_sge *sge = qed_chain_produce(&qp->sq.pbl);
3076 DMA_REGPAIR_LE(sge->addr, wr->sg_list[i].addr);
3077 sge->l_key = cpu_to_le32(wr->sg_list[i].lkey);
3078 sge->length = cpu_to_le32(wr->sg_list[i].length);
3079 data_size += wr->sg_list[i].length;
3083 *wqe_size += wr->num_sge;
3088 static u32 qedr_prepare_sq_rdma_data(struct qedr_dev *dev,
3090 struct rdma_sq_rdma_wqe_1st *rwqe,
3091 struct rdma_sq_rdma_wqe_2nd *rwqe2,
3092 const struct ib_send_wr *wr,
3093 const struct ib_send_wr **bad_wr)
3095 rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey);
3096 DMA_REGPAIR_LE(rwqe2->remote_va, rdma_wr(wr)->remote_addr);
3098 if (wr->send_flags & IB_SEND_INLINE &&
3099 (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM ||
3100 wr->opcode == IB_WR_RDMA_WRITE)) {
3103 SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1);
3104 return qedr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, wr,
3105 bad_wr, &rwqe->flags, flags);
3108 return qedr_prepare_sq_sges(qp, &rwqe->wqe_size, wr);
3111 static u32 qedr_prepare_sq_send_data(struct qedr_dev *dev,
3113 struct rdma_sq_send_wqe_1st *swqe,
3114 struct rdma_sq_send_wqe_2st *swqe2,
3115 const struct ib_send_wr *wr,
3116 const struct ib_send_wr **bad_wr)
3118 memset(swqe2, 0, sizeof(*swqe2));
3119 if (wr->send_flags & IB_SEND_INLINE) {
3122 SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1);
3123 return qedr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, wr,
3124 bad_wr, &swqe->flags, flags);
3127 return qedr_prepare_sq_sges(qp, &swqe->wqe_size, wr);
3130 static int qedr_prepare_reg(struct qedr_qp *qp,
3131 struct rdma_sq_fmr_wqe_1st *fwqe1,
3132 const struct ib_reg_wr *wr)
3134 struct qedr_mr *mr = get_qedr_mr(wr->mr);
3135 struct rdma_sq_fmr_wqe_2nd *fwqe2;
3137 fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)qed_chain_produce(&qp->sq.pbl);
3138 fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova);
3139 fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova);
3140 fwqe1->l_key = wr->key;
3142 fwqe2->access_ctrl = 0;
3144 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ,
3145 !!(wr->access & IB_ACCESS_REMOTE_READ));
3146 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE,
3147 !!(wr->access & IB_ACCESS_REMOTE_WRITE));
3148 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC,
3149 !!(wr->access & IB_ACCESS_REMOTE_ATOMIC));
3150 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1);
3151 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE,
3152 !!(wr->access & IB_ACCESS_LOCAL_WRITE));
3153 fwqe2->fmr_ctrl = 0;
3155 SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG,
3156 ilog2(mr->ibmr.page_size) - 12);
3158 fwqe2->length_hi = 0;
3159 fwqe2->length_lo = mr->ibmr.length;
3160 fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa);
3161 fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa);
3163 qp->wqe_wr_id[qp->sq.prod].mr = mr;
3168 static enum ib_wc_opcode qedr_ib_to_wc_opcode(enum ib_wr_opcode opcode)
3171 case IB_WR_RDMA_WRITE:
3172 case IB_WR_RDMA_WRITE_WITH_IMM:
3173 return IB_WC_RDMA_WRITE;
3174 case IB_WR_SEND_WITH_IMM:
3176 case IB_WR_SEND_WITH_INV:
3178 case IB_WR_RDMA_READ:
3179 case IB_WR_RDMA_READ_WITH_INV:
3180 return IB_WC_RDMA_READ;
3181 case IB_WR_ATOMIC_CMP_AND_SWP:
3182 return IB_WC_COMP_SWAP;
3183 case IB_WR_ATOMIC_FETCH_AND_ADD:
3184 return IB_WC_FETCH_ADD;
3186 return IB_WC_REG_MR;
3187 case IB_WR_LOCAL_INV:
3188 return IB_WC_LOCAL_INV;
3194 static inline bool qedr_can_post_send(struct qedr_qp *qp,
3195 const struct ib_send_wr *wr)
3197 int wq_is_full, err_wr, pbl_is_full;
3198 struct qedr_dev *dev = qp->dev;
3200 /* prevent SQ overflow and/or processing of a bad WR */
3201 err_wr = wr->num_sge > qp->sq.max_sges;
3202 wq_is_full = qedr_wq_is_full(&qp->sq);
3203 pbl_is_full = qed_chain_get_elem_left_u32(&qp->sq.pbl) <
3204 QEDR_MAX_SQE_ELEMENTS_PER_SQE;
3205 if (wq_is_full || err_wr || pbl_is_full) {
3206 if (wq_is_full && !(qp->err_bitmap & QEDR_QP_ERR_SQ_FULL)) {
3208 "error: WQ is full. Post send on QP %p failed (this error appears only once)\n",
3210 qp->err_bitmap |= QEDR_QP_ERR_SQ_FULL;
3213 if (err_wr && !(qp->err_bitmap & QEDR_QP_ERR_BAD_SR)) {
3215 "error: WR is bad. Post send on QP %p failed (this error appears only once)\n",
3217 qp->err_bitmap |= QEDR_QP_ERR_BAD_SR;
3221 !(qp->err_bitmap & QEDR_QP_ERR_SQ_PBL_FULL)) {
3223 "error: WQ PBL is full. Post send on QP %p failed (this error appears only once)\n",
3225 qp->err_bitmap |= QEDR_QP_ERR_SQ_PBL_FULL;
3232 static int __qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
3233 const struct ib_send_wr **bad_wr)
3235 struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3236 struct qedr_qp *qp = get_qedr_qp(ibqp);
3237 struct rdma_sq_atomic_wqe_1st *awqe1;
3238 struct rdma_sq_atomic_wqe_2nd *awqe2;
3239 struct rdma_sq_atomic_wqe_3rd *awqe3;
3240 struct rdma_sq_send_wqe_2st *swqe2;
3241 struct rdma_sq_local_inv_wqe *iwqe;
3242 struct rdma_sq_rdma_wqe_2nd *rwqe2;
3243 struct rdma_sq_send_wqe_1st *swqe;
3244 struct rdma_sq_rdma_wqe_1st *rwqe;
3245 struct rdma_sq_fmr_wqe_1st *fwqe1;
3246 struct rdma_sq_common_wqe *wqe;
3251 if (!qedr_can_post_send(qp, wr)) {
3256 wqe = qed_chain_produce(&qp->sq.pbl);
3257 qp->wqe_wr_id[qp->sq.prod].signaled =
3258 !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled;
3261 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG,
3262 !!(wr->send_flags & IB_SEND_SOLICITED));
3263 comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || qp->signaled;
3264 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp);
3265 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG,
3266 !!(wr->send_flags & IB_SEND_FENCE));
3267 wqe->prev_wqe_size = qp->prev_wqe_size;
3269 qp->wqe_wr_id[qp->sq.prod].opcode = qedr_ib_to_wc_opcode(wr->opcode);
3271 switch (wr->opcode) {
3272 case IB_WR_SEND_WITH_IMM:
3273 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
3278 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM;
3279 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3281 swqe2 = qed_chain_produce(&qp->sq.pbl);
3283 swqe->inv_key_or_imm_data = cpu_to_le32(be32_to_cpu(wr->ex.imm_data));
3284 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3286 swqe->length = cpu_to_le32(length);
3287 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3288 qp->prev_wqe_size = swqe->wqe_size;
3289 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3292 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND;
3293 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3296 swqe2 = qed_chain_produce(&qp->sq.pbl);
3297 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3299 swqe->length = cpu_to_le32(length);
3300 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3301 qp->prev_wqe_size = swqe->wqe_size;
3302 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3304 case IB_WR_SEND_WITH_INV:
3305 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE;
3306 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3307 swqe2 = qed_chain_produce(&qp->sq.pbl);
3309 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.invalidate_rkey);
3310 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3312 swqe->length = cpu_to_le32(length);
3313 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3314 qp->prev_wqe_size = swqe->wqe_size;
3315 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3318 case IB_WR_RDMA_WRITE_WITH_IMM:
3319 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
3324 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM;
3325 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3328 rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data));
3329 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3330 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3332 rwqe->length = cpu_to_le32(length);
3333 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3334 qp->prev_wqe_size = rwqe->wqe_size;
3335 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3337 case IB_WR_RDMA_WRITE:
3338 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR;
3339 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3342 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3343 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3345 rwqe->length = cpu_to_le32(length);
3346 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3347 qp->prev_wqe_size = rwqe->wqe_size;
3348 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3350 case IB_WR_RDMA_READ_WITH_INV:
3351 SET_FIELD2(wqe->flags, RDMA_SQ_RDMA_WQE_1ST_READ_INV_FLG, 1);
3352 /* fallthrough -- same is identical to RDMA READ */
3354 case IB_WR_RDMA_READ:
3355 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD;
3356 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3359 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3360 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3362 rwqe->length = cpu_to_le32(length);
3363 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3364 qp->prev_wqe_size = rwqe->wqe_size;
3365 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3368 case IB_WR_ATOMIC_CMP_AND_SWP:
3369 case IB_WR_ATOMIC_FETCH_AND_ADD:
3370 awqe1 = (struct rdma_sq_atomic_wqe_1st *)wqe;
3371 awqe1->wqe_size = 4;
3373 awqe2 = qed_chain_produce(&qp->sq.pbl);
3374 DMA_REGPAIR_LE(awqe2->remote_va, atomic_wr(wr)->remote_addr);
3375 awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey);
3377 awqe3 = qed_chain_produce(&qp->sq.pbl);
3379 if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
3380 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD;
3381 DMA_REGPAIR_LE(awqe3->swap_data,
3382 atomic_wr(wr)->compare_add);
3384 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP;
3385 DMA_REGPAIR_LE(awqe3->swap_data,
3386 atomic_wr(wr)->swap);
3387 DMA_REGPAIR_LE(awqe3->cmp_data,
3388 atomic_wr(wr)->compare_add);
3391 qedr_prepare_sq_sges(qp, NULL, wr);
3393 qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->wqe_size;
3394 qp->prev_wqe_size = awqe1->wqe_size;
3397 case IB_WR_LOCAL_INV:
3398 iwqe = (struct rdma_sq_local_inv_wqe *)wqe;
3401 iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE;
3402 iwqe->inv_l_key = wr->ex.invalidate_rkey;
3403 qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->wqe_size;
3404 qp->prev_wqe_size = iwqe->wqe_size;
3407 DP_DEBUG(dev, QEDR_MSG_CQ, "REG_MR\n");
3408 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR;
3409 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe;
3410 fwqe1->wqe_size = 2;
3412 rc = qedr_prepare_reg(qp, fwqe1, reg_wr(wr));
3414 DP_ERR(dev, "IB_REG_MR failed rc=%d\n", rc);
3419 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size;
3420 qp->prev_wqe_size = fwqe1->wqe_size;
3423 DP_ERR(dev, "invalid opcode 0x%x!\n", wr->opcode);
3432 /* Restore prod to its position before
3433 * this WR was processed
3435 value = le16_to_cpu(qp->sq.db_data.data.value);
3436 qed_chain_set_prod(&qp->sq.pbl, value, wqe);
3438 /* Restore prev_wqe_size */
3439 qp->prev_wqe_size = wqe->prev_wqe_size;
3441 DP_ERR(dev, "POST SEND FAILED\n");
3447 int qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
3448 const struct ib_send_wr **bad_wr)
3450 struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3451 struct qedr_qp *qp = get_qedr_qp(ibqp);
3452 unsigned long flags;
3457 if (qp->qp_type == IB_QPT_GSI)
3458 return qedr_gsi_post_send(ibqp, wr, bad_wr);
3460 spin_lock_irqsave(&qp->q_lock, flags);
3462 if (rdma_protocol_roce(&dev->ibdev, 1)) {
3463 if ((qp->state != QED_ROCE_QP_STATE_RTS) &&
3464 (qp->state != QED_ROCE_QP_STATE_ERR) &&
3465 (qp->state != QED_ROCE_QP_STATE_SQD)) {
3466 spin_unlock_irqrestore(&qp->q_lock, flags);
3468 DP_DEBUG(dev, QEDR_MSG_CQ,
3469 "QP in wrong state! QP icid=0x%x state %d\n",
3470 qp->icid, qp->state);
3476 rc = __qedr_post_send(ibqp, wr, bad_wr);
3480 qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id;
3482 qedr_inc_sw_prod(&qp->sq);
3484 qp->sq.db_data.data.value++;
3490 * If there was a failure in the first WR then it will be triggered in
3491 * vane. However this is not harmful (as long as the producer value is
3492 * unchanged). For performance reasons we avoid checking for this
3493 * redundant doorbell.
3495 * qp->wqe_wr_id is accessed during qedr_poll_cq, as
3496 * soon as we give the doorbell, we could get a completion
3497 * for this wr, therefore we need to make sure that the
3498 * memory is updated before giving the doorbell.
3499 * During qedr_poll_cq, rmb is called before accessing the
3500 * cqe. This covers for the smp_rmb as well.
3503 writel(qp->sq.db_data.raw, qp->sq.db);
3505 /* Make sure write sticks */
3508 spin_unlock_irqrestore(&qp->q_lock, flags);
3513 static u32 qedr_srq_elem_left(struct qedr_srq_hwq_info *hw_srq)
3517 /* Calculate number of elements used based on producer
3518 * count and consumer count and subtract it from max
3519 * work request supported so that we get elements left.
3521 used = hw_srq->wr_prod_cnt - hw_srq->wr_cons_cnt;
3523 return hw_srq->max_wr - used;
3526 int qedr_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,
3527 const struct ib_recv_wr **bad_wr)
3529 struct qedr_srq *srq = get_qedr_srq(ibsrq);
3530 struct qedr_srq_hwq_info *hw_srq;
3531 struct qedr_dev *dev = srq->dev;
3532 struct qed_chain *pbl;
3533 unsigned long flags;
3538 spin_lock_irqsave(&srq->lock, flags);
3540 hw_srq = &srq->hw_srq;
3541 pbl = &srq->hw_srq.pbl;
3543 struct rdma_srq_wqe_header *hdr;
3546 if (!qedr_srq_elem_left(hw_srq) ||
3547 wr->num_sge > srq->hw_srq.max_sges) {
3548 DP_ERR(dev, "Can't post WR (%d,%d) || (%d > %d)\n",
3549 hw_srq->wr_prod_cnt, hw_srq->wr_cons_cnt,
3550 wr->num_sge, srq->hw_srq.max_sges);
3556 hdr = qed_chain_produce(pbl);
3557 num_sge = wr->num_sge;
3558 /* Set number of sge and work request id in header */
3559 SRQ_HDR_SET(hdr, wr->wr_id, num_sge);
3561 srq->hw_srq.wr_prod_cnt++;
3565 DP_DEBUG(dev, QEDR_MSG_SRQ,
3566 "SRQ WR: SGEs: %d with wr_id[%d] = %llx\n",
3567 wr->num_sge, hw_srq->wqe_prod, wr->wr_id);
3569 for (i = 0; i < wr->num_sge; i++) {
3570 struct rdma_srq_sge *srq_sge = qed_chain_produce(pbl);
3572 /* Set SGE length, lkey and address */
3573 SRQ_SGE_SET(srq_sge, wr->sg_list[i].addr,
3574 wr->sg_list[i].length, wr->sg_list[i].lkey);
3576 DP_DEBUG(dev, QEDR_MSG_SRQ,
3577 "[%d]: len %d key %x addr %x:%x\n",
3578 i, srq_sge->length, srq_sge->l_key,
3579 srq_sge->addr.hi, srq_sge->addr.lo);
3583 /* Flush WQE and SGE information before
3584 * updating producer.
3588 /* SRQ producer is 8 bytes. Need to update SGE producer index
3589 * in first 4 bytes and need to update WQE producer in
3592 *srq->hw_srq.virt_prod_pair_addr = hw_srq->sge_prod;
3593 offset = offsetof(struct rdma_srq_producers, wqe_prod);
3594 *((u8 *)srq->hw_srq.virt_prod_pair_addr + offset) =
3597 /* Flush producer after updating it. */
3602 DP_DEBUG(dev, QEDR_MSG_SRQ, "POST: Elements in S-RQ: %d\n",
3603 qed_chain_get_elem_left(pbl));
3604 spin_unlock_irqrestore(&srq->lock, flags);
3609 int qedr_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
3610 const struct ib_recv_wr **bad_wr)
3612 struct qedr_qp *qp = get_qedr_qp(ibqp);
3613 struct qedr_dev *dev = qp->dev;
3614 unsigned long flags;
3617 if (qp->qp_type == IB_QPT_GSI)
3618 return qedr_gsi_post_recv(ibqp, wr, bad_wr);
3620 spin_lock_irqsave(&qp->q_lock, flags);
3622 if (qp->state == QED_ROCE_QP_STATE_RESET) {
3623 spin_unlock_irqrestore(&qp->q_lock, flags);
3631 if (qed_chain_get_elem_left_u32(&qp->rq.pbl) <
3632 QEDR_MAX_RQE_ELEMENTS_PER_RQE ||
3633 wr->num_sge > qp->rq.max_sges) {
3634 DP_ERR(dev, "Can't post WR (%d < %d) || (%d > %d)\n",
3635 qed_chain_get_elem_left_u32(&qp->rq.pbl),
3636 QEDR_MAX_RQE_ELEMENTS_PER_RQE, wr->num_sge,
3642 for (i = 0; i < wr->num_sge; i++) {
3644 struct rdma_rq_sge *rqe =
3645 qed_chain_produce(&qp->rq.pbl);
3647 /* First one must include the number
3648 * of SGE in the list
3651 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES,
3654 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO,
3655 wr->sg_list[i].lkey);
3657 RQ_SGE_SET(rqe, wr->sg_list[i].addr,
3658 wr->sg_list[i].length, flags);
3661 /* Special case of no sges. FW requires between 1-4 sges...
3662 * in this case we need to post 1 sge with length zero. this is
3663 * because rdma write with immediate consumes an RQ.
3667 struct rdma_rq_sge *rqe =
3668 qed_chain_produce(&qp->rq.pbl);
3670 /* First one must include the number
3671 * of SGE in the list
3673 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO, 0);
3674 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1);
3676 RQ_SGE_SET(rqe, 0, 0, flags);
3680 qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id;
3681 qp->rqe_wr_id[qp->rq.prod].wqe_size = i;
3683 qedr_inc_sw_prod(&qp->rq);
3685 /* qp->rqe_wr_id is accessed during qedr_poll_cq, as
3686 * soon as we give the doorbell, we could get a completion
3687 * for this wr, therefore we need to make sure that the
3688 * memory is update before giving the doorbell.
3689 * During qedr_poll_cq, rmb is called before accessing the
3690 * cqe. This covers for the smp_rmb as well.
3694 qp->rq.db_data.data.value++;
3696 writel(qp->rq.db_data.raw, qp->rq.db);
3698 /* Make sure write sticks */
3701 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
3702 writel(qp->rq.iwarp_db2_data.raw, qp->rq.iwarp_db2);
3703 mmiowb(); /* for second doorbell */
3709 spin_unlock_irqrestore(&qp->q_lock, flags);
3714 static int is_valid_cqe(struct qedr_cq *cq, union rdma_cqe *cqe)
3716 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3718 return (resp_cqe->flags & RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK) ==
3722 static struct qedr_qp *cqe_get_qp(union rdma_cqe *cqe)
3724 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3727 qp = (struct qedr_qp *)(uintptr_t)HILO_GEN(resp_cqe->qp_handle.hi,
3728 resp_cqe->qp_handle.lo,
3733 static enum rdma_cqe_type cqe_get_type(union rdma_cqe *cqe)
3735 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3737 return GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE);
3740 /* Return latest CQE (needs processing) */
3741 static union rdma_cqe *get_cqe(struct qedr_cq *cq)
3743 return cq->latest_cqe;
3746 /* In fmr we need to increase the number of fmr completed counter for the fmr
3747 * algorithm determining whether we can free a pbl or not.
3748 * we need to perform this whether the work request was signaled or not. for
3749 * this purpose we call this function from the condition that checks if a wr
3750 * should be skipped, to make sure we don't miss it ( possibly this fmr
3751 * operation was not signalted)
3753 static inline void qedr_chk_if_fmr(struct qedr_qp *qp)
3755 if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR)
3756 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
3759 static int process_req(struct qedr_dev *dev, struct qedr_qp *qp,
3760 struct qedr_cq *cq, int num_entries,
3761 struct ib_wc *wc, u16 hw_cons, enum ib_wc_status status,
3766 while (num_entries && qp->sq.wqe_cons != hw_cons) {
3767 if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) {
3768 qedr_chk_if_fmr(qp);
3774 wc->status = status;
3777 wc->src_qp = qp->id;
3780 wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id;
3781 wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode;
3783 switch (wc->opcode) {
3784 case IB_WC_RDMA_WRITE:
3785 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
3787 case IB_WC_COMP_SWAP:
3788 case IB_WC_FETCH_ADD:
3792 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
3794 case IB_WC_RDMA_READ:
3796 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
3806 while (qp->wqe_wr_id[qp->sq.cons].wqe_size--)
3807 qed_chain_consume(&qp->sq.pbl);
3808 qedr_inc_sw_cons(&qp->sq);
3814 static int qedr_poll_cq_req(struct qedr_dev *dev,
3815 struct qedr_qp *qp, struct qedr_cq *cq,
3816 int num_entries, struct ib_wc *wc,
3817 struct rdma_cqe_requester *req)
3821 switch (req->status) {
3822 case RDMA_CQE_REQ_STS_OK:
3823 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
3826 case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR:
3827 if (qp->state != QED_ROCE_QP_STATE_ERR)
3828 DP_DEBUG(dev, QEDR_MSG_CQ,
3829 "Error: POLL CQ with RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3830 cq->icid, qp->icid);
3831 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
3832 IB_WC_WR_FLUSH_ERR, 1);
3835 /* process all WQE before the cosumer */
3836 qp->state = QED_ROCE_QP_STATE_ERR;
3837 cnt = process_req(dev, qp, cq, num_entries, wc,
3838 req->sq_cons - 1, IB_WC_SUCCESS, 0);
3840 /* if we have extra WC fill it with actual error info */
3841 if (cnt < num_entries) {
3842 enum ib_wc_status wc_status;
3844 switch (req->status) {
3845 case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR:
3847 "Error: POLL CQ with RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3848 cq->icid, qp->icid);
3849 wc_status = IB_WC_BAD_RESP_ERR;
3851 case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR:
3853 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3854 cq->icid, qp->icid);
3855 wc_status = IB_WC_LOC_LEN_ERR;
3857 case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR:
3859 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3860 cq->icid, qp->icid);
3861 wc_status = IB_WC_LOC_QP_OP_ERR;
3863 case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR:
3865 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3866 cq->icid, qp->icid);
3867 wc_status = IB_WC_LOC_PROT_ERR;
3869 case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR:
3871 "Error: POLL CQ with RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3872 cq->icid, qp->icid);
3873 wc_status = IB_WC_MW_BIND_ERR;
3875 case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR:
3877 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3878 cq->icid, qp->icid);
3879 wc_status = IB_WC_REM_INV_REQ_ERR;
3881 case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR:
3883 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3884 cq->icid, qp->icid);
3885 wc_status = IB_WC_REM_ACCESS_ERR;
3887 case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR:
3889 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3890 cq->icid, qp->icid);
3891 wc_status = IB_WC_REM_OP_ERR;
3893 case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR:
3895 "Error: POLL CQ with RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3896 cq->icid, qp->icid);
3897 wc_status = IB_WC_RNR_RETRY_EXC_ERR;
3899 case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR:
3901 "Error: POLL CQ with ROCE_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3902 cq->icid, qp->icid);
3903 wc_status = IB_WC_RETRY_EXC_ERR;
3907 "Error: POLL CQ with IB_WC_GENERAL_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3908 cq->icid, qp->icid);
3909 wc_status = IB_WC_GENERAL_ERR;
3911 cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons,
3919 static inline int qedr_cqe_resp_status_to_ib(u8 status)
3922 case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR:
3923 return IB_WC_LOC_ACCESS_ERR;
3924 case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR:
3925 return IB_WC_LOC_LEN_ERR;
3926 case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR:
3927 return IB_WC_LOC_QP_OP_ERR;
3928 case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR:
3929 return IB_WC_LOC_PROT_ERR;
3930 case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR:
3931 return IB_WC_MW_BIND_ERR;
3932 case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR:
3933 return IB_WC_REM_INV_RD_REQ_ERR;
3934 case RDMA_CQE_RESP_STS_OK:
3935 return IB_WC_SUCCESS;
3937 return IB_WC_GENERAL_ERR;
3941 static inline int qedr_set_ok_cqe_resp_wc(struct rdma_cqe_responder *resp,
3944 wc->status = IB_WC_SUCCESS;
3945 wc->byte_len = le32_to_cpu(resp->length);
3947 if (resp->flags & QEDR_RESP_IMM) {
3948 wc->ex.imm_data = cpu_to_be32(le32_to_cpu(resp->imm_data_or_inv_r_Key));
3949 wc->wc_flags |= IB_WC_WITH_IMM;
3951 if (resp->flags & QEDR_RESP_RDMA)
3952 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
3954 if (resp->flags & QEDR_RESP_INV)
3957 } else if (resp->flags & QEDR_RESP_INV) {
3958 wc->ex.imm_data = le32_to_cpu(resp->imm_data_or_inv_r_Key);
3959 wc->wc_flags |= IB_WC_WITH_INVALIDATE;
3961 if (resp->flags & QEDR_RESP_RDMA)
3964 } else if (resp->flags & QEDR_RESP_RDMA) {
3971 static void __process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
3972 struct qedr_cq *cq, struct ib_wc *wc,
3973 struct rdma_cqe_responder *resp, u64 wr_id)
3975 /* Must fill fields before qedr_set_ok_cqe_resp_wc() */
3976 wc->opcode = IB_WC_RECV;
3979 if (likely(resp->status == RDMA_CQE_RESP_STS_OK)) {
3980 if (qedr_set_ok_cqe_resp_wc(resp, wc))
3982 "CQ %p (icid=%d) has invalid CQE responder flags=0x%x\n",
3983 cq, cq->icid, resp->flags);
3986 wc->status = qedr_cqe_resp_status_to_ib(resp->status);
3987 if (wc->status == IB_WC_GENERAL_ERR)
3989 "CQ %p (icid=%d) contains an invalid CQE status %d\n",
3990 cq, cq->icid, resp->status);
3993 /* Fill the rest of the WC */
3995 wc->src_qp = qp->id;
4000 static int process_resp_one_srq(struct qedr_dev *dev, struct qedr_qp *qp,
4001 struct qedr_cq *cq, struct ib_wc *wc,
4002 struct rdma_cqe_responder *resp)
4004 struct qedr_srq *srq = qp->srq;
4007 wr_id = HILO_GEN(le32_to_cpu(resp->srq_wr_id.hi),
4008 le32_to_cpu(resp->srq_wr_id.lo), u64);
4010 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
4011 wc->status = IB_WC_WR_FLUSH_ERR;
4015 wc->src_qp = qp->id;
4019 __process_resp_one(dev, qp, cq, wc, resp, wr_id);
4021 srq->hw_srq.wr_cons_cnt++;
4025 static int process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
4026 struct qedr_cq *cq, struct ib_wc *wc,
4027 struct rdma_cqe_responder *resp)
4029 u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
4031 __process_resp_one(dev, qp, cq, wc, resp, wr_id);
4033 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
4034 qed_chain_consume(&qp->rq.pbl);
4035 qedr_inc_sw_cons(&qp->rq);
4040 static int process_resp_flush(struct qedr_qp *qp, struct qedr_cq *cq,
4041 int num_entries, struct ib_wc *wc, u16 hw_cons)
4045 while (num_entries && qp->rq.wqe_cons != hw_cons) {
4047 wc->status = IB_WC_WR_FLUSH_ERR;
4050 wc->src_qp = qp->id;
4052 wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
4057 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
4058 qed_chain_consume(&qp->rq.pbl);
4059 qedr_inc_sw_cons(&qp->rq);
4065 static void try_consume_resp_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
4066 struct rdma_cqe_responder *resp, int *update)
4068 if (le16_to_cpu(resp->rq_cons_or_srq_id) == qp->rq.wqe_cons) {
4074 static int qedr_poll_cq_resp_srq(struct qedr_dev *dev, struct qedr_qp *qp,
4075 struct qedr_cq *cq, int num_entries,
4077 struct rdma_cqe_responder *resp)
4081 cnt = process_resp_one_srq(dev, qp, cq, wc, resp);
4087 static int qedr_poll_cq_resp(struct qedr_dev *dev, struct qedr_qp *qp,
4088 struct qedr_cq *cq, int num_entries,
4089 struct ib_wc *wc, struct rdma_cqe_responder *resp,
4094 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
4095 cnt = process_resp_flush(qp, cq, num_entries, wc,
4096 resp->rq_cons_or_srq_id);
4097 try_consume_resp_cqe(cq, qp, resp, update);
4099 cnt = process_resp_one(dev, qp, cq, wc, resp);
4107 static void try_consume_req_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
4108 struct rdma_cqe_requester *req, int *update)
4110 if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) {
4116 int qedr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
4118 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
4119 struct qedr_cq *cq = get_qedr_cq(ibcq);
4120 union rdma_cqe *cqe;
4121 u32 old_cons, new_cons;
4122 unsigned long flags;
4126 if (cq->destroyed) {
4128 "warning: poll was invoked after destroy for cq %p (icid=%d)\n",
4133 if (cq->cq_type == QEDR_CQ_TYPE_GSI)
4134 return qedr_gsi_poll_cq(ibcq, num_entries, wc);
4136 spin_lock_irqsave(&cq->cq_lock, flags);
4137 cqe = cq->latest_cqe;
4138 old_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
4139 while (num_entries && is_valid_cqe(cq, cqe)) {
4143 /* prevent speculative reads of any field of CQE */
4146 qp = cqe_get_qp(cqe);
4148 WARN(1, "Error: CQE QP pointer is NULL. CQE=%p\n", cqe);
4154 switch (cqe_get_type(cqe)) {
4155 case RDMA_CQE_TYPE_REQUESTER:
4156 cnt = qedr_poll_cq_req(dev, qp, cq, num_entries, wc,
4158 try_consume_req_cqe(cq, qp, &cqe->req, &update);
4160 case RDMA_CQE_TYPE_RESPONDER_RQ:
4161 cnt = qedr_poll_cq_resp(dev, qp, cq, num_entries, wc,
4162 &cqe->resp, &update);
4164 case RDMA_CQE_TYPE_RESPONDER_SRQ:
4165 cnt = qedr_poll_cq_resp_srq(dev, qp, cq, num_entries,
4169 case RDMA_CQE_TYPE_INVALID:
4171 DP_ERR(dev, "Error: invalid CQE type = %d\n",
4180 new_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
4182 cq->cq_cons += new_cons - old_cons;
4185 /* doorbell notifies abount latest VALID entry,
4186 * but chain already point to the next INVALID one
4188 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
4190 spin_unlock_irqrestore(&cq->cq_lock, flags);
4194 int qedr_process_mad(struct ib_device *ibdev, int process_mad_flags,
4196 const struct ib_wc *in_wc,
4197 const struct ib_grh *in_grh,
4198 const struct ib_mad_hdr *mad_hdr,
4199 size_t in_mad_size, struct ib_mad_hdr *out_mad,
4200 size_t *out_mad_size, u16 *out_mad_pkey_index)
4202 struct qedr_dev *dev = get_qedr_dev(ibdev);
4204 DP_DEBUG(dev, QEDR_MSG_GSI,
4205 "QEDR_PROCESS_MAD in_mad %x %x %x %x %x %x %x %x\n",
4206 mad_hdr->attr_id, mad_hdr->base_version, mad_hdr->attr_mod,
4207 mad_hdr->class_specific, mad_hdr->class_version,
4208 mad_hdr->method, mad_hdr->mgmt_class, mad_hdr->status);
4209 return IB_MAD_RESULT_SUCCESS;