1 /* QLogic qedr NIC Driver
2 * Copyright (c) 2015-2016 QLogic Corporation
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and /or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 #include <linux/dma-mapping.h>
33 #include <linux/crc32.h>
37 #include <linux/iommu.h>
39 #include <rdma/ib_verbs.h>
40 #include <rdma/ib_user_verbs.h>
41 #include <rdma/iw_cm.h>
42 #include <rdma/ib_umem.h>
43 #include <rdma/ib_addr.h>
44 #include <rdma/ib_cache.h>
46 #include <linux/qed/common_hsi.h>
47 #include "qedr_hsi_rdma.h"
48 #include <linux/qed/qed_if.h>
51 #include <rdma/qedr-abi.h>
54 #define DB_ADDR_SHIFT(addr) ((addr) << DB_PWM_ADDR_OFFSET_SHIFT)
56 static inline int qedr_ib_copy_to_udata(struct ib_udata *udata, void *src,
59 size_t min_len = min_t(size_t, len, udata->outlen);
61 return ib_copy_to_udata(udata, src, min_len);
64 int qedr_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey)
66 if (index > QEDR_ROCE_PKEY_TABLE_LEN)
69 *pkey = QEDR_ROCE_PKEY_DEFAULT;
73 int qedr_query_gid(struct ib_device *ibdev, u8 port, int index,
76 struct qedr_dev *dev = get_qedr_dev(ibdev);
79 if (!rdma_cap_roce_gid_table(ibdev, port))
82 rc = ib_get_cached_gid(ibdev, port, index, sgid, NULL);
84 memcpy(sgid, &zgid, sizeof(*sgid));
88 DP_DEBUG(dev, QEDR_MSG_INIT, "query gid: index=%d %llx:%llx\n", index,
89 sgid->global.interface_id, sgid->global.subnet_prefix);
94 int qedr_add_gid(struct ib_device *device, u8 port_num,
95 unsigned int index, const union ib_gid *gid,
96 const struct ib_gid_attr *attr, void **context)
98 if (!rdma_cap_roce_gid_table(device, port_num))
101 if (port_num > QEDR_MAX_PORT)
110 int qedr_del_gid(struct ib_device *device, u8 port_num,
111 unsigned int index, void **context)
113 if (!rdma_cap_roce_gid_table(device, port_num))
116 if (port_num > QEDR_MAX_PORT)
125 int qedr_query_device(struct ib_device *ibdev,
126 struct ib_device_attr *attr, struct ib_udata *udata)
128 struct qedr_dev *dev = get_qedr_dev(ibdev);
129 struct qedr_device_attr *qattr = &dev->attr;
131 if (!dev->rdma_ctx) {
133 "qedr_query_device called with invalid params rdma_ctx=%p\n",
138 memset(attr, 0, sizeof(*attr));
140 attr->fw_ver = qattr->fw_ver;
141 attr->sys_image_guid = qattr->sys_image_guid;
142 attr->max_mr_size = qattr->max_mr_size;
143 attr->page_size_cap = qattr->page_size_caps;
144 attr->vendor_id = qattr->vendor_id;
145 attr->vendor_part_id = qattr->vendor_part_id;
146 attr->hw_ver = qattr->hw_ver;
147 attr->max_qp = qattr->max_qp;
148 attr->max_qp_wr = max_t(u32, qattr->max_sqe, qattr->max_rqe);
149 attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD |
150 IB_DEVICE_RC_RNR_NAK_GEN |
151 IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_MGT_EXTENSIONS;
153 attr->max_sge = qattr->max_sge;
154 attr->max_sge_rd = qattr->max_sge;
155 attr->max_cq = qattr->max_cq;
156 attr->max_cqe = qattr->max_cqe;
157 attr->max_mr = qattr->max_mr;
158 attr->max_mw = qattr->max_mw;
159 attr->max_pd = qattr->max_pd;
160 attr->atomic_cap = dev->atomic_cap;
161 attr->max_fmr = qattr->max_fmr;
162 attr->max_map_per_fmr = 16;
163 attr->max_qp_init_rd_atom =
164 1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1);
165 attr->max_qp_rd_atom =
166 min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1),
167 attr->max_qp_init_rd_atom);
169 attr->max_srq = qattr->max_srq;
170 attr->max_srq_sge = qattr->max_srq_sge;
171 attr->max_srq_wr = qattr->max_srq_wr;
173 attr->local_ca_ack_delay = qattr->dev_ack_delay;
174 attr->max_fast_reg_page_list_len = qattr->max_mr / 8;
175 attr->max_pkeys = QEDR_ROCE_PKEY_MAX;
176 attr->max_ah = qattr->max_ah;
181 #define QEDR_SPEED_SDR (1)
182 #define QEDR_SPEED_DDR (2)
183 #define QEDR_SPEED_QDR (4)
184 #define QEDR_SPEED_FDR10 (8)
185 #define QEDR_SPEED_FDR (16)
186 #define QEDR_SPEED_EDR (32)
188 static inline void get_link_speed_and_width(int speed, u8 *ib_speed,
193 *ib_speed = QEDR_SPEED_SDR;
194 *ib_width = IB_WIDTH_1X;
197 *ib_speed = QEDR_SPEED_QDR;
198 *ib_width = IB_WIDTH_1X;
202 *ib_speed = QEDR_SPEED_DDR;
203 *ib_width = IB_WIDTH_4X;
207 *ib_speed = QEDR_SPEED_EDR;
208 *ib_width = IB_WIDTH_1X;
212 *ib_speed = QEDR_SPEED_QDR;
213 *ib_width = IB_WIDTH_4X;
217 *ib_speed = QEDR_SPEED_QDR;
218 *ib_width = IB_WIDTH_4X;
222 *ib_speed = QEDR_SPEED_EDR;
223 *ib_width = IB_WIDTH_4X;
228 *ib_speed = QEDR_SPEED_SDR;
229 *ib_width = IB_WIDTH_1X;
233 int qedr_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *attr)
235 struct qedr_dev *dev;
236 struct qed_rdma_port *rdma_port;
238 dev = get_qedr_dev(ibdev);
240 DP_ERR(dev, "invalid_port=0x%x\n", port);
244 if (!dev->rdma_ctx) {
245 DP_ERR(dev, "rdma_ctx is NULL\n");
249 rdma_port = dev->ops->rdma_query_port(dev->rdma_ctx);
251 /* *attr being zeroed by the caller, avoid zeroing it here */
252 if (rdma_port->port_state == QED_RDMA_PORT_UP) {
253 attr->state = IB_PORT_ACTIVE;
254 attr->phys_state = 5;
256 attr->state = IB_PORT_DOWN;
257 attr->phys_state = 3;
259 attr->max_mtu = IB_MTU_4096;
260 attr->active_mtu = iboe_get_mtu(dev->ndev->mtu);
265 attr->port_cap_flags = IB_PORT_IP_BASED_GIDS;
266 attr->gid_tbl_len = QEDR_MAX_SGID;
267 attr->pkey_tbl_len = QEDR_ROCE_PKEY_TABLE_LEN;
268 attr->bad_pkey_cntr = rdma_port->pkey_bad_counter;
269 attr->qkey_viol_cntr = 0;
270 get_link_speed_and_width(rdma_port->link_speed,
271 &attr->active_speed, &attr->active_width);
272 attr->max_msg_sz = rdma_port->max_msg_size;
273 attr->max_vl_num = 4;
278 int qedr_modify_port(struct ib_device *ibdev, u8 port, int mask,
279 struct ib_port_modify *props)
281 struct qedr_dev *dev;
283 dev = get_qedr_dev(ibdev);
285 DP_ERR(dev, "invalid_port=0x%x\n", port);
292 static int qedr_add_mmap(struct qedr_ucontext *uctx, u64 phy_addr,
297 mm = kzalloc(sizeof(*mm), GFP_KERNEL);
301 mm->key.phy_addr = phy_addr;
302 /* This function might be called with a length which is not a multiple
303 * of PAGE_SIZE, while the mapping is PAGE_SIZE grained and the kernel
304 * forces this granularity by increasing the requested size if needed.
305 * When qedr_mmap is called, it will search the list with the updated
306 * length as a key. To prevent search failures, the length is rounded up
307 * in advance to PAGE_SIZE.
309 mm->key.len = roundup(len, PAGE_SIZE);
310 INIT_LIST_HEAD(&mm->entry);
312 mutex_lock(&uctx->mm_list_lock);
313 list_add(&mm->entry, &uctx->mm_head);
314 mutex_unlock(&uctx->mm_list_lock);
316 DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
317 "added (addr=0x%llx,len=0x%lx) for ctx=%p\n",
318 (unsigned long long)mm->key.phy_addr,
319 (unsigned long)mm->key.len, uctx);
324 static bool qedr_search_mmap(struct qedr_ucontext *uctx, u64 phy_addr,
330 mutex_lock(&uctx->mm_list_lock);
331 list_for_each_entry(mm, &uctx->mm_head, entry) {
332 if (len != mm->key.len || phy_addr != mm->key.phy_addr)
338 mutex_unlock(&uctx->mm_list_lock);
339 DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
340 "searched for (addr=0x%llx,len=0x%lx) for ctx=%p, result=%d\n",
341 mm->key.phy_addr, mm->key.len, uctx, found);
346 struct ib_ucontext *qedr_alloc_ucontext(struct ib_device *ibdev,
347 struct ib_udata *udata)
350 struct qedr_ucontext *ctx;
351 struct qedr_alloc_ucontext_resp uresp;
352 struct qedr_dev *dev = get_qedr_dev(ibdev);
353 struct qed_rdma_add_user_out_params oparams;
356 return ERR_PTR(-EFAULT);
358 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
360 return ERR_PTR(-ENOMEM);
362 rc = dev->ops->rdma_add_user(dev->rdma_ctx, &oparams);
365 "failed to allocate a DPI for a new RoCE application, rc=%d. To overcome this consider to increase the number of DPIs, increase the doorbell BAR size or just close unnecessary RoCE applications. In order to increase the number of DPIs consult the qedr readme\n",
370 ctx->dpi = oparams.dpi;
371 ctx->dpi_addr = oparams.dpi_addr;
372 ctx->dpi_phys_addr = oparams.dpi_phys_addr;
373 ctx->dpi_size = oparams.dpi_size;
374 INIT_LIST_HEAD(&ctx->mm_head);
375 mutex_init(&ctx->mm_list_lock);
377 memset(&uresp, 0, sizeof(uresp));
379 uresp.db_pa = ctx->dpi_phys_addr;
380 uresp.db_size = ctx->dpi_size;
381 uresp.max_send_wr = dev->attr.max_sqe;
382 uresp.max_recv_wr = dev->attr.max_rqe;
383 uresp.max_srq_wr = dev->attr.max_srq_wr;
384 uresp.sges_per_send_wr = QEDR_MAX_SQE_ELEMENTS_PER_SQE;
385 uresp.sges_per_recv_wr = QEDR_MAX_RQE_ELEMENTS_PER_RQE;
386 uresp.sges_per_srq_wr = dev->attr.max_srq_sge;
387 uresp.max_cqes = QEDR_MAX_CQES;
389 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
395 rc = qedr_add_mmap(ctx, ctx->dpi_phys_addr, ctx->dpi_size);
399 DP_DEBUG(dev, QEDR_MSG_INIT, "Allocating user context %p\n",
401 return &ctx->ibucontext;
408 int qedr_dealloc_ucontext(struct ib_ucontext *ibctx)
410 struct qedr_ucontext *uctx = get_qedr_ucontext(ibctx);
411 struct qedr_mm *mm, *tmp;
414 DP_DEBUG(uctx->dev, QEDR_MSG_INIT, "Deallocating user context %p\n",
416 uctx->dev->ops->rdma_remove_user(uctx->dev->rdma_ctx, uctx->dpi);
418 list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) {
419 DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
420 "deleted (addr=0x%llx,len=0x%lx) for ctx=%p\n",
421 mm->key.phy_addr, mm->key.len, uctx);
422 list_del(&mm->entry);
430 int qedr_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
432 struct qedr_ucontext *ucontext = get_qedr_ucontext(context);
433 struct qedr_dev *dev = get_qedr_dev(context->device);
434 unsigned long vm_page = vma->vm_pgoff << PAGE_SHIFT;
435 u64 unmapped_db = dev->db_phys_addr;
436 unsigned long len = (vma->vm_end - vma->vm_start);
440 DP_DEBUG(dev, QEDR_MSG_INIT,
441 "qedr_mmap called vm_page=0x%lx vm_pgoff=0x%lx unmapped_db=0x%llx db_size=%x, len=%lx\n",
442 vm_page, vma->vm_pgoff, unmapped_db, dev->db_size, len);
443 if (vma->vm_start & (PAGE_SIZE - 1)) {
444 DP_ERR(dev, "Vma_start not page aligned = %ld\n",
449 found = qedr_search_mmap(ucontext, vm_page, len);
451 DP_ERR(dev, "Vma_pgoff not found in mapped array = %ld\n",
456 DP_DEBUG(dev, QEDR_MSG_INIT, "Mapping doorbell bar\n");
458 if ((vm_page >= unmapped_db) && (vm_page <= (unmapped_db +
460 DP_DEBUG(dev, QEDR_MSG_INIT, "Mapping doorbell bar\n");
461 if (vma->vm_flags & VM_READ) {
462 DP_ERR(dev, "Trying to map doorbell bar for read\n");
466 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
468 rc = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
469 PAGE_SIZE, vma->vm_page_prot);
471 DP_DEBUG(dev, QEDR_MSG_INIT, "Mapping chains\n");
472 rc = remap_pfn_range(vma, vma->vm_start,
473 vma->vm_pgoff, len, vma->vm_page_prot);
475 DP_DEBUG(dev, QEDR_MSG_INIT, "qedr_mmap return code: %d\n", rc);
479 struct ib_pd *qedr_alloc_pd(struct ib_device *ibdev,
480 struct ib_ucontext *context, struct ib_udata *udata)
482 struct qedr_dev *dev = get_qedr_dev(ibdev);
487 DP_DEBUG(dev, QEDR_MSG_INIT, "Function called from: %s\n",
488 (udata && context) ? "User Lib" : "Kernel");
490 if (!dev->rdma_ctx) {
491 DP_ERR(dev, "invlaid RDMA context\n");
492 return ERR_PTR(-EINVAL);
495 pd = kzalloc(sizeof(*pd), GFP_KERNEL);
497 return ERR_PTR(-ENOMEM);
499 rc = dev->ops->rdma_alloc_pd(dev->rdma_ctx, &pd_id);
505 if (udata && context) {
506 struct qedr_alloc_pd_uresp uresp;
510 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
512 DP_ERR(dev, "copy error pd_id=0x%x.\n", pd_id);
513 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd_id);
517 pd->uctx = get_qedr_ucontext(context);
528 int qedr_dealloc_pd(struct ib_pd *ibpd)
530 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
531 struct qedr_pd *pd = get_qedr_pd(ibpd);
534 pr_err("Invalid PD received in dealloc_pd\n");
538 DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id);
539 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id);
546 static void qedr_free_pbl(struct qedr_dev *dev,
547 struct qedr_pbl_info *pbl_info, struct qedr_pbl *pbl)
549 struct pci_dev *pdev = dev->pdev;
552 for (i = 0; i < pbl_info->num_pbls; i++) {
555 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
556 pbl[i].va, pbl[i].pa);
562 #define MIN_FW_PBL_PAGE_SIZE (4 * 1024)
563 #define MAX_FW_PBL_PAGE_SIZE (64 * 1024)
565 #define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64))
566 #define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE)
567 #define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE * MAX_PBES_ON_PAGE)
569 static struct qedr_pbl *qedr_alloc_pbl_tbl(struct qedr_dev *dev,
570 struct qedr_pbl_info *pbl_info,
573 struct pci_dev *pdev = dev->pdev;
574 struct qedr_pbl *pbl_table;
575 dma_addr_t *pbl_main_tbl;
580 pbl_table = kcalloc(pbl_info->num_pbls, sizeof(*pbl_table), flags);
582 return ERR_PTR(-ENOMEM);
584 for (i = 0; i < pbl_info->num_pbls; i++) {
585 va = dma_alloc_coherent(&pdev->dev, pbl_info->pbl_size,
590 memset(va, 0, pbl_info->pbl_size);
591 pbl_table[i].va = va;
592 pbl_table[i].pa = pa;
595 /* Two-Layer PBLs, if we have more than one pbl we need to initialize
596 * the first one with physical pointers to all of the rest
598 pbl_main_tbl = (dma_addr_t *)pbl_table[0].va;
599 for (i = 0; i < pbl_info->num_pbls - 1; i++)
600 pbl_main_tbl[i] = pbl_table[i + 1].pa;
605 for (i--; i >= 0; i--)
606 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
607 pbl_table[i].va, pbl_table[i].pa);
609 qedr_free_pbl(dev, pbl_info, pbl_table);
611 return ERR_PTR(-ENOMEM);
614 static int qedr_prepare_pbl_tbl(struct qedr_dev *dev,
615 struct qedr_pbl_info *pbl_info,
616 u32 num_pbes, int two_layer_capable)
622 if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) {
623 if (num_pbes > MAX_PBES_TWO_LAYER) {
624 DP_ERR(dev, "prepare pbl table: too many pages %d\n",
629 /* calculate required pbl page size */
630 pbl_size = MIN_FW_PBL_PAGE_SIZE;
631 pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) *
632 NUM_PBES_ON_PAGE(pbl_size);
634 while (pbl_capacity < num_pbes) {
636 pbl_capacity = pbl_size / sizeof(u64);
637 pbl_capacity = pbl_capacity * pbl_capacity;
640 num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size));
641 num_pbls++; /* One for the layer0 ( points to the pbls) */
642 pbl_info->two_layered = true;
644 /* One layered PBL */
646 pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE,
647 roundup_pow_of_two((num_pbes * sizeof(u64))));
648 pbl_info->two_layered = false;
651 pbl_info->num_pbls = num_pbls;
652 pbl_info->pbl_size = pbl_size;
653 pbl_info->num_pbes = num_pbes;
655 DP_DEBUG(dev, QEDR_MSG_MR,
656 "prepare pbl table: num_pbes=%d, num_pbls=%d, pbl_size=%d\n",
657 pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size);
662 static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem,
663 struct qedr_pbl *pbl,
664 struct qedr_pbl_info *pbl_info, u32 pg_shift)
666 int shift, pg_cnt, pages, pbe_cnt, total_num_pbes = 0;
667 u32 fw_pg_cnt, fw_pg_per_umem_pg;
668 struct qedr_pbl *pbl_tbl;
669 struct scatterlist *sg;
674 if (!pbl_info->num_pbes)
677 /* If we have a two layered pbl, the first pbl points to the rest
678 * of the pbls and the first entry lays on the second pbl in the table
680 if (pbl_info->two_layered)
685 pbe = (struct regpair *)pbl_tbl->va;
687 DP_ERR(dev, "cannot populate PBL due to a NULL PBE\n");
693 shift = umem->page_shift;
695 fw_pg_per_umem_pg = BIT(umem->page_shift - pg_shift);
697 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
698 pages = sg_dma_len(sg) >> shift;
699 pg_addr = sg_dma_address(sg);
700 for (pg_cnt = 0; pg_cnt < pages; pg_cnt++) {
701 for (fw_pg_cnt = 0; fw_pg_cnt < fw_pg_per_umem_pg;) {
702 pbe->lo = cpu_to_le32(pg_addr);
703 pbe->hi = cpu_to_le32(upper_32_bits(pg_addr));
705 pg_addr += BIT(pg_shift);
710 if (total_num_pbes == pbl_info->num_pbes)
713 /* If the given pbl is full storing the pbes,
717 (pbl_info->pbl_size / sizeof(u64))) {
719 pbe = (struct regpair *)pbl_tbl->va;
729 static int qedr_copy_cq_uresp(struct qedr_dev *dev,
730 struct qedr_cq *cq, struct ib_udata *udata)
732 struct qedr_create_cq_uresp uresp;
735 memset(&uresp, 0, sizeof(uresp));
737 uresp.db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
738 uresp.icid = cq->icid;
740 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
742 DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid);
747 static void consume_cqe(struct qedr_cq *cq)
749 if (cq->latest_cqe == cq->toggle_cqe)
750 cq->pbl_toggle ^= RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
752 cq->latest_cqe = qed_chain_consume(&cq->pbl);
755 static inline int qedr_align_cq_entries(int entries)
757 u64 size, aligned_size;
759 /* We allocate an extra entry that we don't report to the FW. */
760 size = (entries + 1) * QEDR_CQE_SIZE;
761 aligned_size = ALIGN(size, PAGE_SIZE);
763 return aligned_size / QEDR_CQE_SIZE;
766 static inline int qedr_init_user_queue(struct ib_ucontext *ib_ctx,
767 struct qedr_dev *dev,
768 struct qedr_userq *q,
769 u64 buf_addr, size_t buf_len,
770 int access, int dmasync)
775 q->buf_addr = buf_addr;
776 q->buf_len = buf_len;
777 q->umem = ib_umem_get(ib_ctx, q->buf_addr, q->buf_len, access, dmasync);
778 if (IS_ERR(q->umem)) {
779 DP_ERR(dev, "create user queue: failed ib_umem_get, got %ld\n",
781 return PTR_ERR(q->umem);
784 fw_pages = ib_umem_page_count(q->umem) <<
785 (q->umem->page_shift - FW_PAGE_SHIFT);
787 rc = qedr_prepare_pbl_tbl(dev, &q->pbl_info, fw_pages, 0);
791 q->pbl_tbl = qedr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL);
792 if (IS_ERR(q->pbl_tbl)) {
793 rc = PTR_ERR(q->pbl_tbl);
797 qedr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info,
803 ib_umem_release(q->umem);
808 static inline void qedr_init_cq_params(struct qedr_cq *cq,
809 struct qedr_ucontext *ctx,
810 struct qedr_dev *dev, int vector,
811 int chain_entries, int page_cnt,
813 struct qed_rdma_create_cq_in_params
816 memset(params, 0, sizeof(*params));
817 params->cq_handle_hi = upper_32_bits((uintptr_t)cq);
818 params->cq_handle_lo = lower_32_bits((uintptr_t)cq);
819 params->cnq_id = vector;
820 params->cq_size = chain_entries - 1;
821 params->dpi = (ctx) ? ctx->dpi : dev->dpi;
822 params->pbl_num_pages = page_cnt;
823 params->pbl_ptr = pbl_ptr;
824 params->pbl_two_level = 0;
827 static void doorbell_cq(struct qedr_cq *cq, u32 cons, u8 flags)
829 /* Flush data before signalling doorbell */
831 cq->db.data.agg_flags = flags;
832 cq->db.data.value = cpu_to_le32(cons);
833 writeq(cq->db.raw, cq->db_addr);
835 /* Make sure write would stick */
839 int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
841 struct qedr_cq *cq = get_qedr_cq(ibcq);
842 unsigned long sflags;
843 struct qedr_dev *dev;
845 dev = get_qedr_dev(ibcq->device);
849 "warning: arm was invoked after destroy for cq %p (icid=%d)\n",
855 if (cq->cq_type == QEDR_CQ_TYPE_GSI)
858 spin_lock_irqsave(&cq->cq_lock, sflags);
862 if (flags & IB_CQ_SOLICITED)
863 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD;
865 if (flags & IB_CQ_NEXT_COMP)
866 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD;
868 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
870 spin_unlock_irqrestore(&cq->cq_lock, sflags);
875 struct ib_cq *qedr_create_cq(struct ib_device *ibdev,
876 const struct ib_cq_init_attr *attr,
877 struct ib_ucontext *ib_ctx, struct ib_udata *udata)
879 struct qedr_ucontext *ctx = get_qedr_ucontext(ib_ctx);
880 struct qed_rdma_destroy_cq_out_params destroy_oparams;
881 struct qed_rdma_destroy_cq_in_params destroy_iparams;
882 struct qedr_dev *dev = get_qedr_dev(ibdev);
883 struct qed_rdma_create_cq_in_params params;
884 struct qedr_create_cq_ureq ureq;
885 int vector = attr->comp_vector;
886 int entries = attr->cqe;
894 DP_DEBUG(dev, QEDR_MSG_INIT,
895 "create_cq: called from %s. entries=%d, vector=%d\n",
896 udata ? "User Lib" : "Kernel", entries, vector);
898 if (entries > QEDR_MAX_CQES) {
900 "create cq: the number of entries %d is too high. Must be equal or below %d.\n",
901 entries, QEDR_MAX_CQES);
902 return ERR_PTR(-EINVAL);
905 chain_entries = qedr_align_cq_entries(entries);
906 chain_entries = min_t(int, chain_entries, QEDR_MAX_CQES);
908 cq = kzalloc(sizeof(*cq), GFP_KERNEL);
910 return ERR_PTR(-ENOMEM);
913 memset(&ureq, 0, sizeof(ureq));
914 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) {
916 "create cq: problem copying data from user space\n");
922 "create cq: cannot create a cq with 0 entries\n");
926 cq->cq_type = QEDR_CQ_TYPE_USER;
928 rc = qedr_init_user_queue(ib_ctx, dev, &cq->q, ureq.addr,
929 ureq.len, IB_ACCESS_LOCAL_WRITE, 1);
933 pbl_ptr = cq->q.pbl_tbl->pa;
934 page_cnt = cq->q.pbl_info.num_pbes;
936 cq->ibcq.cqe = chain_entries;
938 cq->cq_type = QEDR_CQ_TYPE_KERNEL;
940 rc = dev->ops->common->chain_alloc(dev->cdev,
941 QED_CHAIN_USE_TO_CONSUME,
943 QED_CHAIN_CNT_TYPE_U32,
945 sizeof(union rdma_cqe),
950 page_cnt = qed_chain_get_page_cnt(&cq->pbl);
951 pbl_ptr = qed_chain_get_pbl_phys(&cq->pbl);
952 cq->ibcq.cqe = cq->pbl.capacity;
955 qedr_init_cq_params(cq, ctx, dev, vector, chain_entries, page_cnt,
958 rc = dev->ops->rdma_create_cq(dev->rdma_ctx, ¶ms, &icid);
963 cq->sig = QEDR_CQ_MAGIC_NUMBER;
964 spin_lock_init(&cq->cq_lock);
967 rc = qedr_copy_cq_uresp(dev, cq, udata);
971 /* Generate doorbell address. */
972 cq->db_addr = dev->db_addr +
973 DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
974 cq->db.data.icid = cq->icid;
975 cq->db.data.params = DB_AGG_CMD_SET <<
976 RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT;
978 /* point to the very last element, passing it we will toggle */
979 cq->toggle_cqe = qed_chain_get_last_elem(&cq->pbl);
980 cq->pbl_toggle = RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
981 cq->latest_cqe = NULL;
983 cq->cq_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
986 DP_DEBUG(dev, QEDR_MSG_CQ,
987 "create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n",
988 cq->icid, cq, params.cq_size);
993 destroy_iparams.icid = cq->icid;
994 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams,
998 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
1000 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
1003 ib_umem_release(cq->q.umem);
1006 return ERR_PTR(-EINVAL);
1009 int qedr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata)
1011 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
1012 struct qedr_cq *cq = get_qedr_cq(ibcq);
1014 DP_ERR(dev, "cq %p RESIZE NOT SUPPORTED\n", cq);
1019 #define QEDR_DESTROY_CQ_MAX_ITERATIONS (10)
1020 #define QEDR_DESTROY_CQ_ITER_DURATION (10)
1022 int qedr_destroy_cq(struct ib_cq *ibcq)
1024 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
1025 struct qed_rdma_destroy_cq_out_params oparams;
1026 struct qed_rdma_destroy_cq_in_params iparams;
1027 struct qedr_cq *cq = get_qedr_cq(ibcq);
1031 DP_DEBUG(dev, QEDR_MSG_CQ, "destroy cq %p (icid=%d)\n", cq, cq->icid);
1035 /* GSIs CQs are handled by driver, so they don't exist in the FW */
1036 if (cq->cq_type == QEDR_CQ_TYPE_GSI)
1039 iparams.icid = cq->icid;
1040 rc = dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams);
1044 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
1046 if (ibcq->uobject && ibcq->uobject->context) {
1047 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
1048 ib_umem_release(cq->q.umem);
1051 /* We don't want the IRQ handler to handle a non-existing CQ so we
1052 * wait until all CNQ interrupts, if any, are received. This will always
1053 * happen and will always happen very fast. If not, then a serious error
1054 * has occured. That is why we can use a long delay.
1055 * We spin for a short time so we don’t lose time on context switching
1056 * in case all the completions are handled in that span. Otherwise
1057 * we sleep for a while and check again. Since the CNQ may be
1058 * associated with (only) the current CPU we use msleep to allow the
1059 * current CPU to be freed.
1060 * The CNQ notification is increased in qedr_irq_handler().
1062 iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1063 while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1064 udelay(QEDR_DESTROY_CQ_ITER_DURATION);
1068 iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1069 while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1070 msleep(QEDR_DESTROY_CQ_ITER_DURATION);
1074 if (oparams.num_cq_notif != cq->cnq_notif)
1077 /* Note that we don't need to have explicit code to wait for the
1078 * completion of the event handler because it is invoked from the EQ.
1079 * Since the destroy CQ ramrod has also been received on the EQ we can
1080 * be certain that there's no event handler in process.
1091 "CQ %p (icid=%d) not freed, expecting %d ints but got %d ints\n",
1092 cq, cq->icid, oparams.num_cq_notif, cq->cnq_notif);
1097 static inline int get_gid_info_from_table(struct ib_qp *ibqp,
1098 struct ib_qp_attr *attr,
1100 struct qed_rdma_modify_qp_in_params
1103 enum rdma_network_type nw_type;
1104 struct ib_gid_attr gid_attr;
1105 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
1111 rc = ib_get_cached_gid(ibqp->device,
1112 rdma_ah_get_port_num(&attr->ah_attr),
1113 grh->sgid_index, &gid, &gid_attr);
1117 if (!memcmp(&gid, &zgid, sizeof(gid)))
1120 if (gid_attr.ndev) {
1121 qp_params->vlan_id = rdma_vlan_dev_vlan_id(gid_attr.ndev);
1123 dev_put(gid_attr.ndev);
1124 nw_type = ib_gid_to_network_type(gid_attr.gid_type, &gid);
1126 case RDMA_NETWORK_IPV6:
1127 memcpy(&qp_params->sgid.bytes[0], &gid.raw[0],
1128 sizeof(qp_params->sgid));
1129 memcpy(&qp_params->dgid.bytes[0],
1131 sizeof(qp_params->dgid));
1132 qp_params->roce_mode = ROCE_V2_IPV6;
1133 SET_FIELD(qp_params->modify_flags,
1134 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1136 case RDMA_NETWORK_IB:
1137 memcpy(&qp_params->sgid.bytes[0], &gid.raw[0],
1138 sizeof(qp_params->sgid));
1139 memcpy(&qp_params->dgid.bytes[0],
1141 sizeof(qp_params->dgid));
1142 qp_params->roce_mode = ROCE_V1;
1144 case RDMA_NETWORK_IPV4:
1145 memset(&qp_params->sgid, 0, sizeof(qp_params->sgid));
1146 memset(&qp_params->dgid, 0, sizeof(qp_params->dgid));
1147 ipv4_addr = qedr_get_ipv4_from_gid(gid.raw);
1148 qp_params->sgid.ipv4_addr = ipv4_addr;
1150 qedr_get_ipv4_from_gid(grh->dgid.raw);
1151 qp_params->dgid.ipv4_addr = ipv4_addr;
1152 SET_FIELD(qp_params->modify_flags,
1153 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1154 qp_params->roce_mode = ROCE_V2_IPV4;
1159 for (i = 0; i < 4; i++) {
1160 qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]);
1161 qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]);
1164 if (qp_params->vlan_id >= VLAN_CFI_MASK)
1165 qp_params->vlan_id = 0;
1170 static int qedr_check_qp_attrs(struct ib_pd *ibpd, struct qedr_dev *dev,
1171 struct ib_qp_init_attr *attrs)
1173 struct qedr_device_attr *qattr = &dev->attr;
1175 /* QP0... attrs->qp_type == IB_QPT_GSI */
1176 if (attrs->qp_type != IB_QPT_RC && attrs->qp_type != IB_QPT_GSI) {
1177 DP_DEBUG(dev, QEDR_MSG_QP,
1178 "create qp: unsupported qp type=0x%x requested\n",
1183 if (attrs->cap.max_send_wr > qattr->max_sqe) {
1185 "create qp: cannot create a SQ with %d elements (max_send_wr=0x%x)\n",
1186 attrs->cap.max_send_wr, qattr->max_sqe);
1190 if (attrs->cap.max_inline_data > qattr->max_inline) {
1192 "create qp: unsupported inline data size=0x%x requested (max_inline=0x%x)\n",
1193 attrs->cap.max_inline_data, qattr->max_inline);
1197 if (attrs->cap.max_send_sge > qattr->max_sge) {
1199 "create qp: unsupported send_sge=0x%x requested (max_send_sge=0x%x)\n",
1200 attrs->cap.max_send_sge, qattr->max_sge);
1204 if (attrs->cap.max_recv_sge > qattr->max_sge) {
1206 "create qp: unsupported recv_sge=0x%x requested (max_recv_sge=0x%x)\n",
1207 attrs->cap.max_recv_sge, qattr->max_sge);
1211 /* Unprivileged user space cannot create special QP */
1212 if (ibpd->uobject && attrs->qp_type == IB_QPT_GSI) {
1214 "create qp: userspace can't create special QPs of type=0x%x\n",
1222 static void qedr_copy_rq_uresp(struct qedr_create_qp_uresp *uresp,
1225 uresp->rq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1226 uresp->rq_icid = qp->icid;
1229 static void qedr_copy_sq_uresp(struct qedr_create_qp_uresp *uresp,
1232 uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1233 uresp->sq_icid = qp->icid + 1;
1236 static int qedr_copy_qp_uresp(struct qedr_dev *dev,
1237 struct qedr_qp *qp, struct ib_udata *udata)
1239 struct qedr_create_qp_uresp uresp;
1242 memset(&uresp, 0, sizeof(uresp));
1243 qedr_copy_sq_uresp(&uresp, qp);
1244 qedr_copy_rq_uresp(&uresp, qp);
1246 uresp.atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE;
1247 uresp.qp_id = qp->qp_id;
1249 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1252 "create qp: failed a copy to user space with qp icid=0x%x.\n",
1258 static void qedr_set_common_qp_params(struct qedr_dev *dev,
1261 struct ib_qp_init_attr *attrs)
1263 spin_lock_init(&qp->q_lock);
1265 qp->qp_type = attrs->qp_type;
1266 qp->max_inline_data = attrs->cap.max_inline_data;
1267 qp->sq.max_sges = attrs->cap.max_send_sge;
1268 qp->state = QED_ROCE_QP_STATE_RESET;
1269 qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false;
1270 qp->sq_cq = get_qedr_cq(attrs->send_cq);
1271 qp->rq_cq = get_qedr_cq(attrs->recv_cq);
1273 qp->rq.max_sges = attrs->cap.max_recv_sge;
1275 DP_DEBUG(dev, QEDR_MSG_QP,
1276 "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n",
1277 qp->rq.max_sges, qp->rq_cq->icid);
1278 DP_DEBUG(dev, QEDR_MSG_QP,
1279 "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d, state = %d, signaled = %d, use_srq=%d\n",
1280 pd->pd_id, qp->qp_type, qp->max_inline_data,
1281 qp->state, qp->signaled, (attrs->srq) ? 1 : 0);
1282 DP_DEBUG(dev, QEDR_MSG_QP,
1283 "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n",
1284 qp->sq.max_sges, qp->sq_cq->icid);
1287 static void qedr_set_roce_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
1289 qp->sq.db = dev->db_addr +
1290 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1291 qp->sq.db_data.data.icid = qp->icid + 1;
1292 qp->rq.db = dev->db_addr +
1293 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1294 qp->rq.db_data.data.icid = qp->icid;
1298 qedr_init_common_qp_in_params(struct qedr_dev *dev,
1301 struct ib_qp_init_attr *attrs,
1302 bool fmr_and_reserved_lkey,
1303 struct qed_rdma_create_qp_in_params *params)
1305 /* QP handle to be written in an async event */
1306 params->qp_handle_async_lo = lower_32_bits((uintptr_t) qp);
1307 params->qp_handle_async_hi = upper_32_bits((uintptr_t) qp);
1309 params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR);
1310 params->fmr_and_reserved_lkey = fmr_and_reserved_lkey;
1311 params->pd = pd->pd_id;
1312 params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi;
1313 params->sq_cq_id = get_qedr_cq(attrs->send_cq)->icid;
1314 params->stats_queue = 0;
1315 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1317 params->use_srq = false;
1320 static inline void qedr_qp_user_print(struct qedr_dev *dev, struct qedr_qp *qp)
1322 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: successfully created user QP. "
1331 qp->usq.buf_len, qp->urq.buf_addr, qp->urq.buf_len);
1334 static void qedr_cleanup_user(struct qedr_dev *dev, struct qedr_qp *qp)
1337 ib_umem_release(qp->usq.umem);
1338 qp->usq.umem = NULL;
1341 ib_umem_release(qp->urq.umem);
1342 qp->urq.umem = NULL;
1345 static int qedr_create_user_qp(struct qedr_dev *dev,
1348 struct ib_udata *udata,
1349 struct ib_qp_init_attr *attrs)
1351 struct qed_rdma_create_qp_in_params in_params;
1352 struct qed_rdma_create_qp_out_params out_params;
1353 struct qedr_pd *pd = get_qedr_pd(ibpd);
1354 struct ib_ucontext *ib_ctx = NULL;
1355 struct qedr_ucontext *ctx = NULL;
1356 struct qedr_create_qp_ureq ureq;
1359 ib_ctx = ibpd->uobject->context;
1360 ctx = get_qedr_ucontext(ib_ctx);
1362 memset(&ureq, 0, sizeof(ureq));
1363 rc = ib_copy_from_udata(&ureq, udata, sizeof(ureq));
1365 DP_ERR(dev, "Problem copying data from user space\n");
1369 /* SQ - read access only (0), dma sync not required (0) */
1370 rc = qedr_init_user_queue(ib_ctx, dev, &qp->usq, ureq.sq_addr,
1375 /* RQ - read access only (0), dma sync not required (0) */
1376 rc = qedr_init_user_queue(ib_ctx, dev, &qp->urq, ureq.rq_addr,
1382 memset(&in_params, 0, sizeof(in_params));
1383 qedr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params);
1384 in_params.qp_handle_lo = ureq.qp_handle_lo;
1385 in_params.qp_handle_hi = ureq.qp_handle_hi;
1386 in_params.sq_num_pages = qp->usq.pbl_info.num_pbes;
1387 in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa;
1388 in_params.rq_num_pages = qp->urq.pbl_info.num_pbes;
1389 in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa;
1391 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1392 &in_params, &out_params);
1399 qp->qp_id = out_params.qp_id;
1400 qp->icid = out_params.icid;
1402 rc = qedr_copy_qp_uresp(dev, qp, udata);
1406 qedr_qp_user_print(dev, qp);
1410 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
1412 DP_ERR(dev, "create qp: fatal fault. rc=%d", rc);
1415 qedr_cleanup_user(dev, qp);
1420 qedr_roce_create_kernel_qp(struct qedr_dev *dev,
1422 struct qed_rdma_create_qp_in_params *in_params,
1423 u32 n_sq_elems, u32 n_rq_elems)
1425 struct qed_rdma_create_qp_out_params out_params;
1428 rc = dev->ops->common->chain_alloc(dev->cdev,
1429 QED_CHAIN_USE_TO_PRODUCE,
1431 QED_CHAIN_CNT_TYPE_U32,
1433 QEDR_SQE_ELEMENT_SIZE,
1439 in_params->sq_num_pages = qed_chain_get_page_cnt(&qp->sq.pbl);
1440 in_params->sq_pbl_ptr = qed_chain_get_pbl_phys(&qp->sq.pbl);
1442 rc = dev->ops->common->chain_alloc(dev->cdev,
1443 QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1445 QED_CHAIN_CNT_TYPE_U32,
1447 QEDR_RQE_ELEMENT_SIZE,
1452 in_params->rq_num_pages = qed_chain_get_page_cnt(&qp->rq.pbl);
1453 in_params->rq_pbl_ptr = qed_chain_get_pbl_phys(&qp->rq.pbl);
1455 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1456 in_params, &out_params);
1461 qp->qp_id = out_params.qp_id;
1462 qp->icid = out_params.icid;
1464 qedr_set_roce_db_info(dev, qp);
1469 static void qedr_cleanup_kernel(struct qedr_dev *dev, struct qedr_qp *qp)
1471 dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl);
1472 kfree(qp->wqe_wr_id);
1474 dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl);
1475 kfree(qp->rqe_wr_id);
1478 static int qedr_create_kernel_qp(struct qedr_dev *dev,
1481 struct ib_qp_init_attr *attrs)
1483 struct qed_rdma_create_qp_in_params in_params;
1484 struct qedr_pd *pd = get_qedr_pd(ibpd);
1490 memset(&in_params, 0, sizeof(in_params));
1492 /* A single work request may take up to QEDR_MAX_SQ_WQE_SIZE elements in
1493 * the ring. The ring should allow at least a single WR, even if the
1494 * user requested none, due to allocation issues.
1495 * We should add an extra WR since the prod and cons indices of
1496 * wqe_wr_id are managed in such a way that the WQ is considered full
1497 * when (prod+1)%max_wr==cons. We currently don't do that because we
1498 * double the number of entries due an iSER issue that pushes far more
1499 * WRs than indicated. If we decline its ib_post_send() then we get
1500 * error prints in the dmesg we'd like to avoid.
1502 qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier,
1505 qp->wqe_wr_id = kzalloc(qp->sq.max_wr * sizeof(*qp->wqe_wr_id),
1507 if (!qp->wqe_wr_id) {
1508 DP_ERR(dev, "create qp: failed SQ shadow memory allocation\n");
1512 /* QP handle to be written in CQE */
1513 in_params.qp_handle_lo = lower_32_bits((uintptr_t) qp);
1514 in_params.qp_handle_hi = upper_32_bits((uintptr_t) qp);
1516 /* A single work request may take up to QEDR_MAX_RQ_WQE_SIZE elements in
1517 * the ring. There ring should allow at least a single WR, even if the
1518 * user requested none, due to allocation issues.
1520 qp->rq.max_wr = (u16) max_t(u32, attrs->cap.max_recv_wr, 1);
1522 /* Allocate driver internal RQ array */
1523 qp->rqe_wr_id = kzalloc(qp->rq.max_wr * sizeof(*qp->rqe_wr_id),
1525 if (!qp->rqe_wr_id) {
1527 "create qp: failed RQ shadow memory allocation\n");
1528 kfree(qp->wqe_wr_id);
1532 qedr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params);
1534 n_sq_entries = attrs->cap.max_send_wr;
1535 n_sq_entries = min_t(u32, n_sq_entries, dev->attr.max_sqe);
1536 n_sq_entries = max_t(u32, n_sq_entries, 1);
1537 n_sq_elems = n_sq_entries * QEDR_MAX_SQE_ELEMENTS_PER_SQE;
1539 n_rq_elems = qp->rq.max_wr * QEDR_MAX_RQE_ELEMENTS_PER_RQE;
1541 rc = qedr_roce_create_kernel_qp(dev, qp, &in_params,
1542 n_sq_elems, n_rq_elems);
1544 qedr_cleanup_kernel(dev, qp);
1549 struct ib_qp *qedr_create_qp(struct ib_pd *ibpd,
1550 struct ib_qp_init_attr *attrs,
1551 struct ib_udata *udata)
1553 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
1554 struct qedr_pd *pd = get_qedr_pd(ibpd);
1559 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: called from %s, pd=%p\n",
1560 udata ? "user library" : "kernel", pd);
1562 rc = qedr_check_qp_attrs(ibpd, dev, attrs);
1567 return ERR_PTR(-EINVAL);
1569 DP_DEBUG(dev, QEDR_MSG_QP,
1570 "create qp: called from %s, event_handler=%p, eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n",
1571 udata ? "user library" : "kernel", attrs->event_handler, pd,
1572 get_qedr_cq(attrs->send_cq),
1573 get_qedr_cq(attrs->send_cq)->icid,
1574 get_qedr_cq(attrs->recv_cq),
1575 get_qedr_cq(attrs->recv_cq)->icid);
1577 qp = kzalloc(sizeof(*qp), GFP_KERNEL);
1579 DP_ERR(dev, "create qp: failed allocating memory\n");
1580 return ERR_PTR(-ENOMEM);
1583 qedr_set_common_qp_params(dev, qp, pd, attrs);
1585 if (attrs->qp_type == IB_QPT_GSI) {
1586 ibqp = qedr_create_gsi_qp(dev, attrs, qp);
1593 rc = qedr_create_user_qp(dev, qp, ibpd, udata, attrs);
1595 rc = qedr_create_kernel_qp(dev, qp, ibpd, attrs);
1600 qp->ibqp.qp_num = qp->qp_id;
1607 return ERR_PTR(-EFAULT);
1610 static enum ib_qp_state qedr_get_ibqp_state(enum qed_roce_qp_state qp_state)
1613 case QED_ROCE_QP_STATE_RESET:
1614 return IB_QPS_RESET;
1615 case QED_ROCE_QP_STATE_INIT:
1617 case QED_ROCE_QP_STATE_RTR:
1619 case QED_ROCE_QP_STATE_RTS:
1621 case QED_ROCE_QP_STATE_SQD:
1623 case QED_ROCE_QP_STATE_ERR:
1625 case QED_ROCE_QP_STATE_SQE:
1631 static enum qed_roce_qp_state qedr_get_state_from_ibqp(
1632 enum ib_qp_state qp_state)
1636 return QED_ROCE_QP_STATE_RESET;
1638 return QED_ROCE_QP_STATE_INIT;
1640 return QED_ROCE_QP_STATE_RTR;
1642 return QED_ROCE_QP_STATE_RTS;
1644 return QED_ROCE_QP_STATE_SQD;
1646 return QED_ROCE_QP_STATE_ERR;
1648 return QED_ROCE_QP_STATE_ERR;
1652 static void qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info *qph)
1654 qed_chain_reset(&qph->pbl);
1658 qph->db_data.data.value = cpu_to_le16(0);
1661 static int qedr_update_qp_state(struct qedr_dev *dev,
1663 enum qed_roce_qp_state new_state)
1667 if (new_state == qp->state)
1670 switch (qp->state) {
1671 case QED_ROCE_QP_STATE_RESET:
1672 switch (new_state) {
1673 case QED_ROCE_QP_STATE_INIT:
1674 qp->prev_wqe_size = 0;
1675 qedr_reset_qp_hwq_info(&qp->sq);
1676 qedr_reset_qp_hwq_info(&qp->rq);
1683 case QED_ROCE_QP_STATE_INIT:
1684 switch (new_state) {
1685 case QED_ROCE_QP_STATE_RTR:
1686 /* Update doorbell (in case post_recv was
1687 * done before move to RTR)
1690 writel(qp->rq.db_data.raw, qp->rq.db);
1691 /* Make sure write takes effect */
1694 case QED_ROCE_QP_STATE_ERR:
1697 /* Invalid state change. */
1702 case QED_ROCE_QP_STATE_RTR:
1704 switch (new_state) {
1705 case QED_ROCE_QP_STATE_RTS:
1707 case QED_ROCE_QP_STATE_ERR:
1710 /* Invalid state change. */
1715 case QED_ROCE_QP_STATE_RTS:
1717 switch (new_state) {
1718 case QED_ROCE_QP_STATE_SQD:
1720 case QED_ROCE_QP_STATE_ERR:
1723 /* Invalid state change. */
1728 case QED_ROCE_QP_STATE_SQD:
1730 switch (new_state) {
1731 case QED_ROCE_QP_STATE_RTS:
1732 case QED_ROCE_QP_STATE_ERR:
1735 /* Invalid state change. */
1740 case QED_ROCE_QP_STATE_ERR:
1742 switch (new_state) {
1743 case QED_ROCE_QP_STATE_RESET:
1744 if ((qp->rq.prod != qp->rq.cons) ||
1745 (qp->sq.prod != qp->sq.cons)) {
1747 "Error->Reset with rq/sq not empty rq.prod=%x rq.cons=%x sq.prod=%x sq.cons=%x\n",
1748 qp->rq.prod, qp->rq.cons, qp->sq.prod,
1766 int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
1767 int attr_mask, struct ib_udata *udata)
1769 struct qedr_qp *qp = get_qedr_qp(ibqp);
1770 struct qed_rdma_modify_qp_in_params qp_params = { 0 };
1771 struct qedr_dev *dev = get_qedr_dev(&qp->dev->ibdev);
1772 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
1773 enum ib_qp_state old_qp_state, new_qp_state;
1776 DP_DEBUG(dev, QEDR_MSG_QP,
1777 "modify qp: qp %p attr_mask=0x%x, state=%d", qp, attr_mask,
1780 old_qp_state = qedr_get_ibqp_state(qp->state);
1781 if (attr_mask & IB_QP_STATE)
1782 new_qp_state = attr->qp_state;
1784 new_qp_state = old_qp_state;
1786 if (!ib_modify_qp_is_ok
1787 (old_qp_state, new_qp_state, ibqp->qp_type, attr_mask,
1788 IB_LINK_LAYER_ETHERNET)) {
1790 "modify qp: invalid attribute mask=0x%x specified for\n"
1791 "qpn=0x%x of type=0x%x old_qp_state=0x%x, new_qp_state=0x%x\n",
1792 attr_mask, qp->qp_id, ibqp->qp_type, old_qp_state,
1798 /* Translate the masks... */
1799 if (attr_mask & IB_QP_STATE) {
1800 SET_FIELD(qp_params.modify_flags,
1801 QED_RDMA_MODIFY_QP_VALID_NEW_STATE, 1);
1802 qp_params.new_state = qedr_get_state_from_ibqp(attr->qp_state);
1805 if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY)
1806 qp_params.sqd_async = true;
1808 if (attr_mask & IB_QP_PKEY_INDEX) {
1809 SET_FIELD(qp_params.modify_flags,
1810 QED_ROCE_MODIFY_QP_VALID_PKEY, 1);
1811 if (attr->pkey_index >= QEDR_ROCE_PKEY_TABLE_LEN) {
1816 qp_params.pkey = QEDR_ROCE_PKEY_DEFAULT;
1819 if (attr_mask & IB_QP_QKEY)
1820 qp->qkey = attr->qkey;
1822 if (attr_mask & IB_QP_ACCESS_FLAGS) {
1823 SET_FIELD(qp_params.modify_flags,
1824 QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1);
1825 qp_params.incoming_rdma_read_en = attr->qp_access_flags &
1826 IB_ACCESS_REMOTE_READ;
1827 qp_params.incoming_rdma_write_en = attr->qp_access_flags &
1828 IB_ACCESS_REMOTE_WRITE;
1829 qp_params.incoming_atomic_en = attr->qp_access_flags &
1830 IB_ACCESS_REMOTE_ATOMIC;
1833 if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) {
1834 if (attr_mask & IB_QP_PATH_MTU) {
1835 if (attr->path_mtu < IB_MTU_256 ||
1836 attr->path_mtu > IB_MTU_4096) {
1837 pr_err("error: Only MTU sizes of 256, 512, 1024, 2048 and 4096 are supported by RoCE\n");
1841 qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu),
1842 ib_mtu_enum_to_int(iboe_get_mtu
1848 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
1849 pr_err("Fixing zeroed MTU to qp->mtu = %d\n", qp->mtu);
1852 SET_FIELD(qp_params.modify_flags,
1853 QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, 1);
1855 qp_params.traffic_class_tos = grh->traffic_class;
1856 qp_params.flow_label = grh->flow_label;
1857 qp_params.hop_limit_ttl = grh->hop_limit;
1859 qp->sgid_idx = grh->sgid_index;
1861 rc = get_gid_info_from_table(ibqp, attr, attr_mask, &qp_params);
1864 "modify qp: problems with GID index %d (rc=%d)\n",
1865 grh->sgid_index, rc);
1869 rc = qedr_get_dmac(dev, &attr->ah_attr,
1870 qp_params.remote_mac_addr);
1874 qp_params.use_local_mac = true;
1875 ether_addr_copy(qp_params.local_mac_addr, dev->ndev->dev_addr);
1877 DP_DEBUG(dev, QEDR_MSG_QP, "dgid=%x:%x:%x:%x\n",
1878 qp_params.dgid.dwords[0], qp_params.dgid.dwords[1],
1879 qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]);
1880 DP_DEBUG(dev, QEDR_MSG_QP, "sgid=%x:%x:%x:%x\n",
1881 qp_params.sgid.dwords[0], qp_params.sgid.dwords[1],
1882 qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]);
1883 DP_DEBUG(dev, QEDR_MSG_QP, "remote_mac=[%pM]\n",
1884 qp_params.remote_mac_addr);
1886 qp_params.mtu = qp->mtu;
1887 qp_params.lb_indication = false;
1890 if (!qp_params.mtu) {
1891 /* Stay with current MTU */
1893 qp_params.mtu = qp->mtu;
1896 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
1899 if (attr_mask & IB_QP_TIMEOUT) {
1900 SET_FIELD(qp_params.modify_flags,
1901 QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1);
1903 qp_params.ack_timeout = attr->timeout;
1904 if (attr->timeout) {
1907 temp = 4096 * (1UL << attr->timeout) / 1000 / 1000;
1908 /* FW requires [msec] */
1909 qp_params.ack_timeout = temp;
1912 qp_params.ack_timeout = 0;
1915 if (attr_mask & IB_QP_RETRY_CNT) {
1916 SET_FIELD(qp_params.modify_flags,
1917 QED_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1);
1918 qp_params.retry_cnt = attr->retry_cnt;
1921 if (attr_mask & IB_QP_RNR_RETRY) {
1922 SET_FIELD(qp_params.modify_flags,
1923 QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, 1);
1924 qp_params.rnr_retry_cnt = attr->rnr_retry;
1927 if (attr_mask & IB_QP_RQ_PSN) {
1928 SET_FIELD(qp_params.modify_flags,
1929 QED_ROCE_MODIFY_QP_VALID_RQ_PSN, 1);
1930 qp_params.rq_psn = attr->rq_psn;
1931 qp->rq_psn = attr->rq_psn;
1934 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
1935 if (attr->max_rd_atomic > dev->attr.max_qp_req_rd_atomic_resc) {
1938 "unsupported max_rd_atomic=%d, supported=%d\n",
1939 attr->max_rd_atomic,
1940 dev->attr.max_qp_req_rd_atomic_resc);
1944 SET_FIELD(qp_params.modify_flags,
1945 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, 1);
1946 qp_params.max_rd_atomic_req = attr->max_rd_atomic;
1949 if (attr_mask & IB_QP_MIN_RNR_TIMER) {
1950 SET_FIELD(qp_params.modify_flags,
1951 QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, 1);
1952 qp_params.min_rnr_nak_timer = attr->min_rnr_timer;
1955 if (attr_mask & IB_QP_SQ_PSN) {
1956 SET_FIELD(qp_params.modify_flags,
1957 QED_ROCE_MODIFY_QP_VALID_SQ_PSN, 1);
1958 qp_params.sq_psn = attr->sq_psn;
1959 qp->sq_psn = attr->sq_psn;
1962 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
1963 if (attr->max_dest_rd_atomic >
1964 dev->attr.max_qp_resp_rd_atomic_resc) {
1966 "unsupported max_dest_rd_atomic=%d, supported=%d\n",
1967 attr->max_dest_rd_atomic,
1968 dev->attr.max_qp_resp_rd_atomic_resc);
1974 SET_FIELD(qp_params.modify_flags,
1975 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, 1);
1976 qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic;
1979 if (attr_mask & IB_QP_DEST_QPN) {
1980 SET_FIELD(qp_params.modify_flags,
1981 QED_ROCE_MODIFY_QP_VALID_DEST_QP, 1);
1983 qp_params.dest_qp = attr->dest_qp_num;
1984 qp->dest_qp_num = attr->dest_qp_num;
1987 if (qp->qp_type != IB_QPT_GSI)
1988 rc = dev->ops->rdma_modify_qp(dev->rdma_ctx,
1989 qp->qed_qp, &qp_params);
1991 if (attr_mask & IB_QP_STATE) {
1992 if ((qp->qp_type != IB_QPT_GSI) && (!udata))
1993 rc = qedr_update_qp_state(dev, qp, qp_params.new_state);
1994 qp->state = qp_params.new_state;
2001 static int qedr_to_ib_qp_acc_flags(struct qed_rdma_query_qp_out_params *params)
2003 int ib_qp_acc_flags = 0;
2005 if (params->incoming_rdma_write_en)
2006 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE;
2007 if (params->incoming_rdma_read_en)
2008 ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ;
2009 if (params->incoming_atomic_en)
2010 ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC;
2011 ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE;
2012 return ib_qp_acc_flags;
2015 int qedr_query_qp(struct ib_qp *ibqp,
2016 struct ib_qp_attr *qp_attr,
2017 int attr_mask, struct ib_qp_init_attr *qp_init_attr)
2019 struct qed_rdma_query_qp_out_params params;
2020 struct qedr_qp *qp = get_qedr_qp(ibqp);
2021 struct qedr_dev *dev = qp->dev;
2024 memset(¶ms, 0, sizeof(params));
2026 rc = dev->ops->rdma_query_qp(dev->rdma_ctx, qp->qed_qp, ¶ms);
2030 memset(qp_attr, 0, sizeof(*qp_attr));
2031 memset(qp_init_attr, 0, sizeof(*qp_init_attr));
2033 qp_attr->qp_state = qedr_get_ibqp_state(params.state);
2034 qp_attr->cur_qp_state = qedr_get_ibqp_state(params.state);
2035 qp_attr->path_mtu = ib_mtu_int_to_enum(params.mtu);
2036 qp_attr->path_mig_state = IB_MIG_MIGRATED;
2037 qp_attr->rq_psn = params.rq_psn;
2038 qp_attr->sq_psn = params.sq_psn;
2039 qp_attr->dest_qp_num = params.dest_qp;
2041 qp_attr->qp_access_flags = qedr_to_ib_qp_acc_flags(¶ms);
2043 qp_attr->cap.max_send_wr = qp->sq.max_wr;
2044 qp_attr->cap.max_recv_wr = qp->rq.max_wr;
2045 qp_attr->cap.max_send_sge = qp->sq.max_sges;
2046 qp_attr->cap.max_recv_sge = qp->rq.max_sges;
2047 qp_attr->cap.max_inline_data = ROCE_REQ_MAX_INLINE_DATA_SIZE;
2048 qp_init_attr->cap = qp_attr->cap;
2050 qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
2051 rdma_ah_set_grh(&qp_attr->ah_attr, NULL,
2052 params.flow_label, qp->sgid_idx,
2053 params.hop_limit_ttl, params.traffic_class_tos);
2054 rdma_ah_set_dgid_raw(&qp_attr->ah_attr, ¶ms.dgid.bytes[0]);
2055 rdma_ah_set_port_num(&qp_attr->ah_attr, 1);
2056 rdma_ah_set_sl(&qp_attr->ah_attr, 0);
2057 qp_attr->timeout = params.timeout;
2058 qp_attr->rnr_retry = params.rnr_retry;
2059 qp_attr->retry_cnt = params.retry_cnt;
2060 qp_attr->min_rnr_timer = params.min_rnr_nak_timer;
2061 qp_attr->pkey_index = params.pkey_index;
2062 qp_attr->port_num = 1;
2063 rdma_ah_set_path_bits(&qp_attr->ah_attr, 0);
2064 rdma_ah_set_static_rate(&qp_attr->ah_attr, 0);
2065 qp_attr->alt_pkey_index = 0;
2066 qp_attr->alt_port_num = 0;
2067 qp_attr->alt_timeout = 0;
2068 memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr));
2070 qp_attr->sq_draining = (params.state == QED_ROCE_QP_STATE_SQD) ? 1 : 0;
2071 qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic;
2072 qp_attr->max_rd_atomic = params.max_rd_atomic;
2073 qp_attr->en_sqd_async_notify = (params.sqd_async) ? 1 : 0;
2075 DP_DEBUG(dev, QEDR_MSG_QP, "QEDR_QUERY_QP: max_inline_data=%d\n",
2076 qp_attr->cap.max_inline_data);
2082 int qedr_free_qp_resources(struct qedr_dev *dev, struct qedr_qp *qp)
2086 if (qp->qp_type != IB_QPT_GSI) {
2087 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2092 if (qp->ibqp.uobject && qp->ibqp.uobject->context)
2093 qedr_cleanup_user(dev, qp);
2095 qedr_cleanup_kernel(dev, qp);
2100 int qedr_destroy_qp(struct ib_qp *ibqp)
2102 struct qedr_qp *qp = get_qedr_qp(ibqp);
2103 struct qedr_dev *dev = qp->dev;
2104 struct ib_qp_attr attr;
2108 DP_DEBUG(dev, QEDR_MSG_QP, "destroy qp: destroying %p, qp type=%d\n",
2111 if ((qp->state != QED_ROCE_QP_STATE_RESET) &&
2112 (qp->state != QED_ROCE_QP_STATE_ERR) &&
2113 (qp->state != QED_ROCE_QP_STATE_INIT)) {
2115 attr.qp_state = IB_QPS_ERR;
2116 attr_mask |= IB_QP_STATE;
2118 /* Change the QP state to ERROR */
2119 qedr_modify_qp(ibqp, &attr, attr_mask, NULL);
2122 if (qp->qp_type == IB_QPT_GSI)
2123 qedr_destroy_gsi_qp(dev);
2125 qedr_free_qp_resources(dev, qp);
2132 struct ib_ah *qedr_create_ah(struct ib_pd *ibpd, struct rdma_ah_attr *attr,
2133 struct ib_udata *udata)
2137 ah = kzalloc(sizeof(*ah), GFP_ATOMIC);
2139 return ERR_PTR(-ENOMEM);
2146 int qedr_destroy_ah(struct ib_ah *ibah)
2148 struct qedr_ah *ah = get_qedr_ah(ibah);
2154 static void free_mr_info(struct qedr_dev *dev, struct mr_info *info)
2156 struct qedr_pbl *pbl, *tmp;
2158 if (info->pbl_table)
2159 list_add_tail(&info->pbl_table->list_entry,
2160 &info->free_pbl_list);
2162 if (!list_empty(&info->inuse_pbl_list))
2163 list_splice(&info->inuse_pbl_list, &info->free_pbl_list);
2165 list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) {
2166 list_del(&pbl->list_entry);
2167 qedr_free_pbl(dev, &info->pbl_info, pbl);
2171 static int init_mr_info(struct qedr_dev *dev, struct mr_info *info,
2172 size_t page_list_len, bool two_layered)
2174 struct qedr_pbl *tmp;
2177 INIT_LIST_HEAD(&info->free_pbl_list);
2178 INIT_LIST_HEAD(&info->inuse_pbl_list);
2180 rc = qedr_prepare_pbl_tbl(dev, &info->pbl_info,
2181 page_list_len, two_layered);
2185 info->pbl_table = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2186 if (IS_ERR(info->pbl_table)) {
2187 rc = PTR_ERR(info->pbl_table);
2191 DP_DEBUG(dev, QEDR_MSG_MR, "pbl_table_pa = %pa\n",
2192 &info->pbl_table->pa);
2194 /* in usual case we use 2 PBLs, so we add one to free
2195 * list and allocating another one
2197 tmp = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2199 DP_DEBUG(dev, QEDR_MSG_MR, "Extra PBL is not allocated\n");
2203 list_add_tail(&tmp->list_entry, &info->free_pbl_list);
2205 DP_DEBUG(dev, QEDR_MSG_MR, "extra pbl_table_pa = %pa\n", &tmp->pa);
2209 free_mr_info(dev, info);
2214 struct ib_mr *qedr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
2215 u64 usr_addr, int acc, struct ib_udata *udata)
2217 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2222 pd = get_qedr_pd(ibpd);
2223 DP_DEBUG(dev, QEDR_MSG_MR,
2224 "qedr_register user mr pd = %d start = %lld, len = %lld, usr_addr = %lld, acc = %d\n",
2225 pd->pd_id, start, len, usr_addr, acc);
2227 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE))
2228 return ERR_PTR(-EINVAL);
2230 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2234 mr->type = QEDR_MR_USER;
2236 mr->umem = ib_umem_get(ibpd->uobject->context, start, len, acc, 0);
2237 if (IS_ERR(mr->umem)) {
2242 rc = init_mr_info(dev, &mr->info, ib_umem_page_count(mr->umem), 1);
2246 qedr_populate_pbls(dev, mr->umem, mr->info.pbl_table,
2247 &mr->info.pbl_info, mr->umem->page_shift);
2249 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2251 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2255 /* Index only, 18 bit long, lkey = itid << 8 | key */
2256 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
2258 mr->hw_mr.pd = pd->pd_id;
2259 mr->hw_mr.local_read = 1;
2260 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
2261 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
2262 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
2263 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
2264 mr->hw_mr.mw_bind = false;
2265 mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa;
2266 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
2267 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
2268 mr->hw_mr.page_size_log = mr->umem->page_shift;
2269 mr->hw_mr.fbo = ib_umem_offset(mr->umem);
2270 mr->hw_mr.length = len;
2271 mr->hw_mr.vaddr = usr_addr;
2272 mr->hw_mr.zbva = false;
2273 mr->hw_mr.phy_mr = false;
2274 mr->hw_mr.dma_mr = false;
2276 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2278 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2282 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2283 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
2284 mr->hw_mr.remote_atomic)
2285 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2287 DP_DEBUG(dev, QEDR_MSG_MR, "register user mr lkey: %x\n",
2292 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2294 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
2300 int qedr_dereg_mr(struct ib_mr *ib_mr)
2302 struct qedr_mr *mr = get_qedr_mr(ib_mr);
2303 struct qedr_dev *dev = get_qedr_dev(ib_mr->device);
2306 rc = dev->ops->rdma_deregister_tid(dev->rdma_ctx, mr->hw_mr.itid);
2310 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2312 if ((mr->type != QEDR_MR_DMA) && (mr->type != QEDR_MR_FRMR))
2313 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
2315 /* it could be user registered memory. */
2317 ib_umem_release(mr->umem);
2324 static struct qedr_mr *__qedr_alloc_mr(struct ib_pd *ibpd,
2325 int max_page_list_len)
2327 struct qedr_pd *pd = get_qedr_pd(ibpd);
2328 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2332 DP_DEBUG(dev, QEDR_MSG_MR,
2333 "qedr_alloc_frmr pd = %d max_page_list_len= %d\n", pd->pd_id,
2336 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2341 mr->type = QEDR_MR_FRMR;
2343 rc = init_mr_info(dev, &mr->info, max_page_list_len, 1);
2347 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2349 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2353 /* Index only, 18 bit long, lkey = itid << 8 | key */
2354 mr->hw_mr.tid_type = QED_RDMA_TID_FMR;
2356 mr->hw_mr.pd = pd->pd_id;
2357 mr->hw_mr.local_read = 1;
2358 mr->hw_mr.local_write = 0;
2359 mr->hw_mr.remote_read = 0;
2360 mr->hw_mr.remote_write = 0;
2361 mr->hw_mr.remote_atomic = 0;
2362 mr->hw_mr.mw_bind = false;
2363 mr->hw_mr.pbl_ptr = 0;
2364 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
2365 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
2367 mr->hw_mr.length = 0;
2368 mr->hw_mr.vaddr = 0;
2369 mr->hw_mr.zbva = false;
2370 mr->hw_mr.phy_mr = true;
2371 mr->hw_mr.dma_mr = false;
2373 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2375 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2379 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2380 mr->ibmr.rkey = mr->ibmr.lkey;
2382 DP_DEBUG(dev, QEDR_MSG_MR, "alloc frmr: %x\n", mr->ibmr.lkey);
2386 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2392 struct ib_mr *qedr_alloc_mr(struct ib_pd *ibpd,
2393 enum ib_mr_type mr_type, u32 max_num_sg)
2395 struct qedr_dev *dev;
2398 if (mr_type != IB_MR_TYPE_MEM_REG)
2399 return ERR_PTR(-EINVAL);
2401 mr = __qedr_alloc_mr(ibpd, max_num_sg);
2404 return ERR_PTR(-EINVAL);
2411 static int qedr_set_page(struct ib_mr *ibmr, u64 addr)
2413 struct qedr_mr *mr = get_qedr_mr(ibmr);
2414 struct qedr_pbl *pbl_table;
2415 struct regpair *pbe;
2418 if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) {
2419 DP_ERR(mr->dev, "qedr_set_page failes when %d\n", mr->npages);
2423 DP_DEBUG(mr->dev, QEDR_MSG_MR, "qedr_set_page pages[%d] = 0x%llx\n",
2426 pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64);
2427 pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page);
2428 pbe = (struct regpair *)pbl_table->va;
2429 pbe += mr->npages % pbes_in_page;
2430 pbe->lo = cpu_to_le32((u32)addr);
2431 pbe->hi = cpu_to_le32((u32)upper_32_bits(addr));
2438 static void handle_completed_mrs(struct qedr_dev *dev, struct mr_info *info)
2440 int work = info->completed - info->completed_handled - 1;
2442 DP_DEBUG(dev, QEDR_MSG_MR, "Special FMR work = %d\n", work);
2443 while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) {
2444 struct qedr_pbl *pbl;
2446 /* Free all the page list that are possible to be freed
2447 * (all the ones that were invalidated), under the assumption
2448 * that if an FMR was completed successfully that means that
2449 * if there was an invalidate operation before it also ended
2451 pbl = list_first_entry(&info->inuse_pbl_list,
2452 struct qedr_pbl, list_entry);
2453 list_move_tail(&pbl->list_entry, &info->free_pbl_list);
2454 info->completed_handled++;
2458 int qedr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
2459 int sg_nents, unsigned int *sg_offset)
2461 struct qedr_mr *mr = get_qedr_mr(ibmr);
2465 handle_completed_mrs(mr->dev, &mr->info);
2466 return ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qedr_set_page);
2469 struct ib_mr *qedr_get_dma_mr(struct ib_pd *ibpd, int acc)
2471 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2472 struct qedr_pd *pd = get_qedr_pd(ibpd);
2476 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2478 return ERR_PTR(-ENOMEM);
2480 mr->type = QEDR_MR_DMA;
2482 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2484 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2488 /* index only, 18 bit long, lkey = itid << 8 | key */
2489 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
2490 mr->hw_mr.pd = pd->pd_id;
2491 mr->hw_mr.local_read = 1;
2492 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
2493 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
2494 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
2495 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
2496 mr->hw_mr.dma_mr = true;
2498 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2500 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2504 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2505 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
2506 mr->hw_mr.remote_atomic)
2507 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2509 DP_DEBUG(dev, QEDR_MSG_MR, "get dma mr: lkey = %x\n", mr->ibmr.lkey);
2513 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2519 static inline int qedr_wq_is_full(struct qedr_qp_hwq_info *wq)
2521 return (((wq->prod + 1) % wq->max_wr) == wq->cons);
2524 static int sge_data_len(struct ib_sge *sg_list, int num_sge)
2528 for (i = 0; i < num_sge; i++)
2529 len += sg_list[i].length;
2534 static void swap_wqe_data64(u64 *p)
2538 for (i = 0; i < QEDR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++)
2539 *p = cpu_to_be64(cpu_to_le64(*p));
2542 static u32 qedr_prepare_sq_inline_data(struct qedr_dev *dev,
2543 struct qedr_qp *qp, u8 *wqe_size,
2544 struct ib_send_wr *wr,
2545 struct ib_send_wr **bad_wr, u8 *bits,
2548 u32 data_size = sge_data_len(wr->sg_list, wr->num_sge);
2549 char *seg_prt, *wqe;
2552 if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) {
2553 DP_ERR(dev, "Too much inline data in WR: %d\n", data_size);
2567 /* Copy data inline */
2568 for (i = 0; i < wr->num_sge; i++) {
2569 u32 len = wr->sg_list[i].length;
2570 void *src = (void *)(uintptr_t)wr->sg_list[i].addr;
2575 /* New segment required */
2577 wqe = (char *)qed_chain_produce(&qp->sq.pbl);
2579 seg_siz = sizeof(struct rdma_sq_common_wqe);
2583 /* Calculate currently allowed length */
2584 cur = min_t(u32, len, seg_siz);
2585 memcpy(seg_prt, src, cur);
2587 /* Update segment variables */
2591 /* Update sge variables */
2595 /* Swap fully-completed segments */
2597 swap_wqe_data64((u64 *)wqe);
2601 /* swap last not completed segment */
2603 swap_wqe_data64((u64 *)wqe);
2608 #define RQ_SGE_SET(sge, vaddr, vlength, vflags) \
2610 DMA_REGPAIR_LE(sge->addr, vaddr); \
2611 (sge)->length = cpu_to_le32(vlength); \
2612 (sge)->flags = cpu_to_le32(vflags); \
2615 #define SRQ_HDR_SET(hdr, vwr_id, num_sge) \
2617 DMA_REGPAIR_LE(hdr->wr_id, vwr_id); \
2618 (hdr)->num_sges = num_sge; \
2621 #define SRQ_SGE_SET(sge, vaddr, vlength, vlkey) \
2623 DMA_REGPAIR_LE(sge->addr, vaddr); \
2624 (sge)->length = cpu_to_le32(vlength); \
2625 (sge)->l_key = cpu_to_le32(vlkey); \
2628 static u32 qedr_prepare_sq_sges(struct qedr_qp *qp, u8 *wqe_size,
2629 struct ib_send_wr *wr)
2634 for (i = 0; i < wr->num_sge; i++) {
2635 struct rdma_sq_sge *sge = qed_chain_produce(&qp->sq.pbl);
2637 DMA_REGPAIR_LE(sge->addr, wr->sg_list[i].addr);
2638 sge->l_key = cpu_to_le32(wr->sg_list[i].lkey);
2639 sge->length = cpu_to_le32(wr->sg_list[i].length);
2640 data_size += wr->sg_list[i].length;
2644 *wqe_size += wr->num_sge;
2649 static u32 qedr_prepare_sq_rdma_data(struct qedr_dev *dev,
2651 struct rdma_sq_rdma_wqe_1st *rwqe,
2652 struct rdma_sq_rdma_wqe_2nd *rwqe2,
2653 struct ib_send_wr *wr,
2654 struct ib_send_wr **bad_wr)
2656 rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey);
2657 DMA_REGPAIR_LE(rwqe2->remote_va, rdma_wr(wr)->remote_addr);
2659 if (wr->send_flags & IB_SEND_INLINE &&
2660 (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM ||
2661 wr->opcode == IB_WR_RDMA_WRITE)) {
2664 SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1);
2665 return qedr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, wr,
2666 bad_wr, &rwqe->flags, flags);
2669 return qedr_prepare_sq_sges(qp, &rwqe->wqe_size, wr);
2672 static u32 qedr_prepare_sq_send_data(struct qedr_dev *dev,
2674 struct rdma_sq_send_wqe_1st *swqe,
2675 struct rdma_sq_send_wqe_2st *swqe2,
2676 struct ib_send_wr *wr,
2677 struct ib_send_wr **bad_wr)
2679 memset(swqe2, 0, sizeof(*swqe2));
2680 if (wr->send_flags & IB_SEND_INLINE) {
2683 SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1);
2684 return qedr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, wr,
2685 bad_wr, &swqe->flags, flags);
2688 return qedr_prepare_sq_sges(qp, &swqe->wqe_size, wr);
2691 static int qedr_prepare_reg(struct qedr_qp *qp,
2692 struct rdma_sq_fmr_wqe_1st *fwqe1,
2693 struct ib_reg_wr *wr)
2695 struct qedr_mr *mr = get_qedr_mr(wr->mr);
2696 struct rdma_sq_fmr_wqe_2nd *fwqe2;
2698 fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)qed_chain_produce(&qp->sq.pbl);
2699 fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova);
2700 fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova);
2701 fwqe1->l_key = wr->key;
2703 fwqe2->access_ctrl = 0;
2705 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ,
2706 !!(wr->access & IB_ACCESS_REMOTE_READ));
2707 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE,
2708 !!(wr->access & IB_ACCESS_REMOTE_WRITE));
2709 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC,
2710 !!(wr->access & IB_ACCESS_REMOTE_ATOMIC));
2711 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1);
2712 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE,
2713 !!(wr->access & IB_ACCESS_LOCAL_WRITE));
2714 fwqe2->fmr_ctrl = 0;
2716 SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG,
2717 ilog2(mr->ibmr.page_size) - 12);
2719 fwqe2->length_hi = 0;
2720 fwqe2->length_lo = mr->ibmr.length;
2721 fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa);
2722 fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa);
2724 qp->wqe_wr_id[qp->sq.prod].mr = mr;
2729 static enum ib_wc_opcode qedr_ib_to_wc_opcode(enum ib_wr_opcode opcode)
2732 case IB_WR_RDMA_WRITE:
2733 case IB_WR_RDMA_WRITE_WITH_IMM:
2734 return IB_WC_RDMA_WRITE;
2735 case IB_WR_SEND_WITH_IMM:
2737 case IB_WR_SEND_WITH_INV:
2739 case IB_WR_RDMA_READ:
2740 return IB_WC_RDMA_READ;
2741 case IB_WR_ATOMIC_CMP_AND_SWP:
2742 return IB_WC_COMP_SWAP;
2743 case IB_WR_ATOMIC_FETCH_AND_ADD:
2744 return IB_WC_FETCH_ADD;
2746 return IB_WC_REG_MR;
2747 case IB_WR_LOCAL_INV:
2748 return IB_WC_LOCAL_INV;
2754 static inline bool qedr_can_post_send(struct qedr_qp *qp, struct ib_send_wr *wr)
2756 int wq_is_full, err_wr, pbl_is_full;
2757 struct qedr_dev *dev = qp->dev;
2759 /* prevent SQ overflow and/or processing of a bad WR */
2760 err_wr = wr->num_sge > qp->sq.max_sges;
2761 wq_is_full = qedr_wq_is_full(&qp->sq);
2762 pbl_is_full = qed_chain_get_elem_left_u32(&qp->sq.pbl) <
2763 QEDR_MAX_SQE_ELEMENTS_PER_SQE;
2764 if (wq_is_full || err_wr || pbl_is_full) {
2765 if (wq_is_full && !(qp->err_bitmap & QEDR_QP_ERR_SQ_FULL)) {
2767 "error: WQ is full. Post send on QP %p failed (this error appears only once)\n",
2769 qp->err_bitmap |= QEDR_QP_ERR_SQ_FULL;
2772 if (err_wr && !(qp->err_bitmap & QEDR_QP_ERR_BAD_SR)) {
2774 "error: WR is bad. Post send on QP %p failed (this error appears only once)\n",
2776 qp->err_bitmap |= QEDR_QP_ERR_BAD_SR;
2780 !(qp->err_bitmap & QEDR_QP_ERR_SQ_PBL_FULL)) {
2782 "error: WQ PBL is full. Post send on QP %p failed (this error appears only once)\n",
2784 qp->err_bitmap |= QEDR_QP_ERR_SQ_PBL_FULL;
2791 static int __qedr_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
2792 struct ib_send_wr **bad_wr)
2794 struct qedr_dev *dev = get_qedr_dev(ibqp->device);
2795 struct qedr_qp *qp = get_qedr_qp(ibqp);
2796 struct rdma_sq_atomic_wqe_1st *awqe1;
2797 struct rdma_sq_atomic_wqe_2nd *awqe2;
2798 struct rdma_sq_atomic_wqe_3rd *awqe3;
2799 struct rdma_sq_send_wqe_2st *swqe2;
2800 struct rdma_sq_local_inv_wqe *iwqe;
2801 struct rdma_sq_rdma_wqe_2nd *rwqe2;
2802 struct rdma_sq_send_wqe_1st *swqe;
2803 struct rdma_sq_rdma_wqe_1st *rwqe;
2804 struct rdma_sq_fmr_wqe_1st *fwqe1;
2805 struct rdma_sq_common_wqe *wqe;
2810 if (!qedr_can_post_send(qp, wr)) {
2815 wqe = qed_chain_produce(&qp->sq.pbl);
2816 qp->wqe_wr_id[qp->sq.prod].signaled =
2817 !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled;
2820 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG,
2821 !!(wr->send_flags & IB_SEND_SOLICITED));
2822 comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || qp->signaled;
2823 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp);
2824 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG,
2825 !!(wr->send_flags & IB_SEND_FENCE));
2826 wqe->prev_wqe_size = qp->prev_wqe_size;
2828 qp->wqe_wr_id[qp->sq.prod].opcode = qedr_ib_to_wc_opcode(wr->opcode);
2830 switch (wr->opcode) {
2831 case IB_WR_SEND_WITH_IMM:
2832 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM;
2833 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
2835 swqe2 = qed_chain_produce(&qp->sq.pbl);
2837 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.imm_data);
2838 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
2840 swqe->length = cpu_to_le32(length);
2841 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
2842 qp->prev_wqe_size = swqe->wqe_size;
2843 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
2846 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND;
2847 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
2850 swqe2 = qed_chain_produce(&qp->sq.pbl);
2851 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
2853 swqe->length = cpu_to_le32(length);
2854 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
2855 qp->prev_wqe_size = swqe->wqe_size;
2856 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
2858 case IB_WR_SEND_WITH_INV:
2859 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE;
2860 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
2861 swqe2 = qed_chain_produce(&qp->sq.pbl);
2863 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.invalidate_rkey);
2864 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
2866 swqe->length = cpu_to_le32(length);
2867 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
2868 qp->prev_wqe_size = swqe->wqe_size;
2869 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
2872 case IB_WR_RDMA_WRITE_WITH_IMM:
2873 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM;
2874 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
2877 rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data));
2878 rwqe2 = qed_chain_produce(&qp->sq.pbl);
2879 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
2881 rwqe->length = cpu_to_le32(length);
2882 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
2883 qp->prev_wqe_size = rwqe->wqe_size;
2884 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
2886 case IB_WR_RDMA_WRITE:
2887 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR;
2888 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
2891 rwqe2 = qed_chain_produce(&qp->sq.pbl);
2892 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
2894 rwqe->length = cpu_to_le32(length);
2895 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
2896 qp->prev_wqe_size = rwqe->wqe_size;
2897 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
2899 case IB_WR_RDMA_READ_WITH_INV:
2901 "RDMA READ WITH INVALIDATE not supported\n");
2906 case IB_WR_RDMA_READ:
2907 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD;
2908 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
2911 rwqe2 = qed_chain_produce(&qp->sq.pbl);
2912 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
2914 rwqe->length = cpu_to_le32(length);
2915 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
2916 qp->prev_wqe_size = rwqe->wqe_size;
2917 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
2920 case IB_WR_ATOMIC_CMP_AND_SWP:
2921 case IB_WR_ATOMIC_FETCH_AND_ADD:
2922 awqe1 = (struct rdma_sq_atomic_wqe_1st *)wqe;
2923 awqe1->wqe_size = 4;
2925 awqe2 = qed_chain_produce(&qp->sq.pbl);
2926 DMA_REGPAIR_LE(awqe2->remote_va, atomic_wr(wr)->remote_addr);
2927 awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey);
2929 awqe3 = qed_chain_produce(&qp->sq.pbl);
2931 if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
2932 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD;
2933 DMA_REGPAIR_LE(awqe3->swap_data,
2934 atomic_wr(wr)->compare_add);
2936 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP;
2937 DMA_REGPAIR_LE(awqe3->swap_data,
2938 atomic_wr(wr)->swap);
2939 DMA_REGPAIR_LE(awqe3->cmp_data,
2940 atomic_wr(wr)->compare_add);
2943 qedr_prepare_sq_sges(qp, NULL, wr);
2945 qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->wqe_size;
2946 qp->prev_wqe_size = awqe1->wqe_size;
2949 case IB_WR_LOCAL_INV:
2950 iwqe = (struct rdma_sq_local_inv_wqe *)wqe;
2953 iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE;
2954 iwqe->inv_l_key = wr->ex.invalidate_rkey;
2955 qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->wqe_size;
2956 qp->prev_wqe_size = iwqe->wqe_size;
2959 DP_DEBUG(dev, QEDR_MSG_CQ, "REG_MR\n");
2960 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR;
2961 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe;
2962 fwqe1->wqe_size = 2;
2964 rc = qedr_prepare_reg(qp, fwqe1, reg_wr(wr));
2966 DP_ERR(dev, "IB_REG_MR failed rc=%d\n", rc);
2971 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size;
2972 qp->prev_wqe_size = fwqe1->wqe_size;
2975 DP_ERR(dev, "invalid opcode 0x%x!\n", wr->opcode);
2984 /* Restore prod to its position before
2985 * this WR was processed
2987 value = le16_to_cpu(qp->sq.db_data.data.value);
2988 qed_chain_set_prod(&qp->sq.pbl, value, wqe);
2990 /* Restore prev_wqe_size */
2991 qp->prev_wqe_size = wqe->prev_wqe_size;
2993 DP_ERR(dev, "POST SEND FAILED\n");
2999 int qedr_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
3000 struct ib_send_wr **bad_wr)
3002 struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3003 struct qedr_qp *qp = get_qedr_qp(ibqp);
3004 unsigned long flags;
3009 if (qp->qp_type == IB_QPT_GSI)
3010 return qedr_gsi_post_send(ibqp, wr, bad_wr);
3012 spin_lock_irqsave(&qp->q_lock, flags);
3014 if ((qp->state != QED_ROCE_QP_STATE_RTS) &&
3015 (qp->state != QED_ROCE_QP_STATE_ERR) &&
3016 (qp->state != QED_ROCE_QP_STATE_SQD)) {
3017 spin_unlock_irqrestore(&qp->q_lock, flags);
3019 DP_DEBUG(dev, QEDR_MSG_CQ,
3020 "QP in wrong state! QP icid=0x%x state %d\n",
3021 qp->icid, qp->state);
3026 rc = __qedr_post_send(ibqp, wr, bad_wr);
3030 qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id;
3032 qedr_inc_sw_prod(&qp->sq);
3034 qp->sq.db_data.data.value++;
3040 * If there was a failure in the first WR then it will be triggered in
3041 * vane. However this is not harmful (as long as the producer value is
3042 * unchanged). For performance reasons we avoid checking for this
3043 * redundant doorbell.
3046 writel(qp->sq.db_data.raw, qp->sq.db);
3048 /* Make sure write sticks */
3051 spin_unlock_irqrestore(&qp->q_lock, flags);
3056 int qedr_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr,
3057 struct ib_recv_wr **bad_wr)
3059 struct qedr_qp *qp = get_qedr_qp(ibqp);
3060 struct qedr_dev *dev = qp->dev;
3061 unsigned long flags;
3064 if (qp->qp_type == IB_QPT_GSI)
3065 return qedr_gsi_post_recv(ibqp, wr, bad_wr);
3067 spin_lock_irqsave(&qp->q_lock, flags);
3069 if (qp->state == QED_ROCE_QP_STATE_RESET) {
3070 spin_unlock_irqrestore(&qp->q_lock, flags);
3078 if (qed_chain_get_elem_left_u32(&qp->rq.pbl) <
3079 QEDR_MAX_RQE_ELEMENTS_PER_RQE ||
3080 wr->num_sge > qp->rq.max_sges) {
3081 DP_ERR(dev, "Can't post WR (%d < %d) || (%d > %d)\n",
3082 qed_chain_get_elem_left_u32(&qp->rq.pbl),
3083 QEDR_MAX_RQE_ELEMENTS_PER_RQE, wr->num_sge,
3089 for (i = 0; i < wr->num_sge; i++) {
3091 struct rdma_rq_sge *rqe =
3092 qed_chain_produce(&qp->rq.pbl);
3094 /* First one must include the number
3095 * of SGE in the list
3098 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES,
3101 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY,
3102 wr->sg_list[i].lkey);
3104 RQ_SGE_SET(rqe, wr->sg_list[i].addr,
3105 wr->sg_list[i].length, flags);
3108 /* Special case of no sges. FW requires between 1-4 sges...
3109 * in this case we need to post 1 sge with length zero. this is
3110 * because rdma write with immediate consumes an RQ.
3114 struct rdma_rq_sge *rqe =
3115 qed_chain_produce(&qp->rq.pbl);
3117 /* First one must include the number
3118 * of SGE in the list
3120 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY, 0);
3121 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1);
3123 RQ_SGE_SET(rqe, 0, 0, flags);
3127 qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id;
3128 qp->rqe_wr_id[qp->rq.prod].wqe_size = i;
3130 qedr_inc_sw_prod(&qp->rq);
3132 /* Flush all the writes before signalling doorbell */
3135 qp->rq.db_data.data.value++;
3137 writel(qp->rq.db_data.raw, qp->rq.db);
3139 /* Make sure write sticks */
3145 spin_unlock_irqrestore(&qp->q_lock, flags);
3150 static int is_valid_cqe(struct qedr_cq *cq, union rdma_cqe *cqe)
3152 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3154 return (resp_cqe->flags & RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK) ==
3158 static struct qedr_qp *cqe_get_qp(union rdma_cqe *cqe)
3160 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3163 qp = (struct qedr_qp *)(uintptr_t)HILO_GEN(resp_cqe->qp_handle.hi,
3164 resp_cqe->qp_handle.lo,
3169 static enum rdma_cqe_type cqe_get_type(union rdma_cqe *cqe)
3171 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3173 return GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE);
3176 /* Return latest CQE (needs processing) */
3177 static union rdma_cqe *get_cqe(struct qedr_cq *cq)
3179 return cq->latest_cqe;
3182 /* In fmr we need to increase the number of fmr completed counter for the fmr
3183 * algorithm determining whether we can free a pbl or not.
3184 * we need to perform this whether the work request was signaled or not. for
3185 * this purpose we call this function from the condition that checks if a wr
3186 * should be skipped, to make sure we don't miss it ( possibly this fmr
3187 * operation was not signalted)
3189 static inline void qedr_chk_if_fmr(struct qedr_qp *qp)
3191 if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR)
3192 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
3195 static int process_req(struct qedr_dev *dev, struct qedr_qp *qp,
3196 struct qedr_cq *cq, int num_entries,
3197 struct ib_wc *wc, u16 hw_cons, enum ib_wc_status status,
3202 while (num_entries && qp->sq.wqe_cons != hw_cons) {
3203 if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) {
3204 qedr_chk_if_fmr(qp);
3210 wc->status = status;
3213 wc->src_qp = qp->id;
3216 wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id;
3217 wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode;
3219 switch (wc->opcode) {
3220 case IB_WC_RDMA_WRITE:
3221 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
3223 case IB_WC_COMP_SWAP:
3224 case IB_WC_FETCH_ADD:
3228 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
3230 case IB_WC_RDMA_READ:
3232 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
3242 while (qp->wqe_wr_id[qp->sq.cons].wqe_size--)
3243 qed_chain_consume(&qp->sq.pbl);
3244 qedr_inc_sw_cons(&qp->sq);
3250 static int qedr_poll_cq_req(struct qedr_dev *dev,
3251 struct qedr_qp *qp, struct qedr_cq *cq,
3252 int num_entries, struct ib_wc *wc,
3253 struct rdma_cqe_requester *req)
3257 switch (req->status) {
3258 case RDMA_CQE_REQ_STS_OK:
3259 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
3262 case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR:
3263 if (qp->state != QED_ROCE_QP_STATE_ERR)
3265 "Error: POLL CQ with RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3266 cq->icid, qp->icid);
3267 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
3268 IB_WC_WR_FLUSH_ERR, 1);
3271 /* process all WQE before the cosumer */
3272 qp->state = QED_ROCE_QP_STATE_ERR;
3273 cnt = process_req(dev, qp, cq, num_entries, wc,
3274 req->sq_cons - 1, IB_WC_SUCCESS, 0);
3276 /* if we have extra WC fill it with actual error info */
3277 if (cnt < num_entries) {
3278 enum ib_wc_status wc_status;
3280 switch (req->status) {
3281 case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR:
3283 "Error: POLL CQ with RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3284 cq->icid, qp->icid);
3285 wc_status = IB_WC_BAD_RESP_ERR;
3287 case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR:
3289 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3290 cq->icid, qp->icid);
3291 wc_status = IB_WC_LOC_LEN_ERR;
3293 case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR:
3295 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3296 cq->icid, qp->icid);
3297 wc_status = IB_WC_LOC_QP_OP_ERR;
3299 case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR:
3301 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3302 cq->icid, qp->icid);
3303 wc_status = IB_WC_LOC_PROT_ERR;
3305 case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR:
3307 "Error: POLL CQ with RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3308 cq->icid, qp->icid);
3309 wc_status = IB_WC_MW_BIND_ERR;
3311 case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR:
3313 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3314 cq->icid, qp->icid);
3315 wc_status = IB_WC_REM_INV_REQ_ERR;
3317 case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR:
3319 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3320 cq->icid, qp->icid);
3321 wc_status = IB_WC_REM_ACCESS_ERR;
3323 case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR:
3325 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3326 cq->icid, qp->icid);
3327 wc_status = IB_WC_REM_OP_ERR;
3329 case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR:
3331 "Error: POLL CQ with RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3332 cq->icid, qp->icid);
3333 wc_status = IB_WC_RNR_RETRY_EXC_ERR;
3335 case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR:
3337 "Error: POLL CQ with ROCE_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3338 cq->icid, qp->icid);
3339 wc_status = IB_WC_RETRY_EXC_ERR;
3343 "Error: POLL CQ with IB_WC_GENERAL_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3344 cq->icid, qp->icid);
3345 wc_status = IB_WC_GENERAL_ERR;
3347 cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons,
3355 static inline int qedr_cqe_resp_status_to_ib(u8 status)
3358 case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR:
3359 return IB_WC_LOC_ACCESS_ERR;
3360 case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR:
3361 return IB_WC_LOC_LEN_ERR;
3362 case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR:
3363 return IB_WC_LOC_QP_OP_ERR;
3364 case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR:
3365 return IB_WC_LOC_PROT_ERR;
3366 case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR:
3367 return IB_WC_MW_BIND_ERR;
3368 case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR:
3369 return IB_WC_REM_INV_RD_REQ_ERR;
3370 case RDMA_CQE_RESP_STS_OK:
3371 return IB_WC_SUCCESS;
3373 return IB_WC_GENERAL_ERR;
3377 static inline int qedr_set_ok_cqe_resp_wc(struct rdma_cqe_responder *resp,
3380 wc->status = IB_WC_SUCCESS;
3381 wc->byte_len = le32_to_cpu(resp->length);
3383 if (resp->flags & QEDR_RESP_IMM) {
3384 wc->ex.imm_data = le32_to_cpu(resp->imm_data_or_inv_r_Key);
3385 wc->wc_flags |= IB_WC_WITH_IMM;
3387 if (resp->flags & QEDR_RESP_RDMA)
3388 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
3390 if (resp->flags & QEDR_RESP_INV)
3393 } else if (resp->flags & QEDR_RESP_INV) {
3394 wc->ex.imm_data = le32_to_cpu(resp->imm_data_or_inv_r_Key);
3395 wc->wc_flags |= IB_WC_WITH_INVALIDATE;
3397 if (resp->flags & QEDR_RESP_RDMA)
3400 } else if (resp->flags & QEDR_RESP_RDMA) {
3407 static void __process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
3408 struct qedr_cq *cq, struct ib_wc *wc,
3409 struct rdma_cqe_responder *resp, u64 wr_id)
3411 /* Must fill fields before qedr_set_ok_cqe_resp_wc() */
3412 wc->opcode = IB_WC_RECV;
3415 if (likely(resp->status == RDMA_CQE_RESP_STS_OK)) {
3416 if (qedr_set_ok_cqe_resp_wc(resp, wc))
3418 "CQ %p (icid=%d) has invalid CQE responder flags=0x%x\n",
3419 cq, cq->icid, resp->flags);
3422 wc->status = qedr_cqe_resp_status_to_ib(resp->status);
3423 if (wc->status == IB_WC_GENERAL_ERR)
3425 "CQ %p (icid=%d) contains an invalid CQE status %d\n",
3426 cq, cq->icid, resp->status);
3429 /* Fill the rest of the WC */
3431 wc->src_qp = qp->id;
3436 static int process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
3437 struct qedr_cq *cq, struct ib_wc *wc,
3438 struct rdma_cqe_responder *resp)
3440 u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
3442 __process_resp_one(dev, qp, cq, wc, resp, wr_id);
3444 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
3445 qed_chain_consume(&qp->rq.pbl);
3446 qedr_inc_sw_cons(&qp->rq);
3451 static int process_resp_flush(struct qedr_qp *qp, struct qedr_cq *cq,
3452 int num_entries, struct ib_wc *wc, u16 hw_cons)
3456 while (num_entries && qp->rq.wqe_cons != hw_cons) {
3458 wc->status = IB_WC_WR_FLUSH_ERR;
3461 wc->src_qp = qp->id;
3463 wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
3468 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
3469 qed_chain_consume(&qp->rq.pbl);
3470 qedr_inc_sw_cons(&qp->rq);
3476 static void try_consume_resp_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
3477 struct rdma_cqe_responder *resp, int *update)
3479 if (le16_to_cpu(resp->rq_cons) == qp->rq.wqe_cons) {
3485 static int qedr_poll_cq_resp(struct qedr_dev *dev, struct qedr_qp *qp,
3486 struct qedr_cq *cq, int num_entries,
3487 struct ib_wc *wc, struct rdma_cqe_responder *resp,
3492 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
3493 cnt = process_resp_flush(qp, cq, num_entries, wc,
3495 try_consume_resp_cqe(cq, qp, resp, update);
3497 cnt = process_resp_one(dev, qp, cq, wc, resp);
3505 static void try_consume_req_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
3506 struct rdma_cqe_requester *req, int *update)
3508 if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) {
3514 int qedr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
3516 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
3517 struct qedr_cq *cq = get_qedr_cq(ibcq);
3518 union rdma_cqe *cqe = cq->latest_cqe;
3519 u32 old_cons, new_cons;
3520 unsigned long flags;
3524 if (cq->destroyed) {
3526 "warning: poll was invoked after destroy for cq %p (icid=%d)\n",
3531 if (cq->cq_type == QEDR_CQ_TYPE_GSI)
3532 return qedr_gsi_poll_cq(ibcq, num_entries, wc);
3534 spin_lock_irqsave(&cq->cq_lock, flags);
3535 old_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
3536 while (num_entries && is_valid_cqe(cq, cqe)) {
3540 /* prevent speculative reads of any field of CQE */
3543 qp = cqe_get_qp(cqe);
3545 WARN(1, "Error: CQE QP pointer is NULL. CQE=%p\n", cqe);
3551 switch (cqe_get_type(cqe)) {
3552 case RDMA_CQE_TYPE_REQUESTER:
3553 cnt = qedr_poll_cq_req(dev, qp, cq, num_entries, wc,
3555 try_consume_req_cqe(cq, qp, &cqe->req, &update);
3557 case RDMA_CQE_TYPE_RESPONDER_RQ:
3558 cnt = qedr_poll_cq_resp(dev, qp, cq, num_entries, wc,
3559 &cqe->resp, &update);
3561 case RDMA_CQE_TYPE_INVALID:
3563 DP_ERR(dev, "Error: invalid CQE type = %d\n",
3572 new_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
3574 cq->cq_cons += new_cons - old_cons;
3577 /* doorbell notifies abount latest VALID entry,
3578 * but chain already point to the next INVALID one
3580 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
3582 spin_unlock_irqrestore(&cq->cq_lock, flags);
3586 int qedr_process_mad(struct ib_device *ibdev, int process_mad_flags,
3588 const struct ib_wc *in_wc,
3589 const struct ib_grh *in_grh,
3590 const struct ib_mad_hdr *mad_hdr,
3591 size_t in_mad_size, struct ib_mad_hdr *out_mad,
3592 size_t *out_mad_size, u16 *out_mad_pkey_index)
3594 struct qedr_dev *dev = get_qedr_dev(ibdev);
3596 DP_DEBUG(dev, QEDR_MSG_GSI,
3597 "QEDR_PROCESS_MAD in_mad %x %x %x %x %x %x %x %x\n",
3598 mad_hdr->attr_id, mad_hdr->base_version, mad_hdr->attr_mod,
3599 mad_hdr->class_specific, mad_hdr->class_version,
3600 mad_hdr->method, mad_hdr->mgmt_class, mad_hdr->status);
3601 return IB_MAD_RESULT_SUCCESS;
3604 int qedr_port_immutable(struct ib_device *ibdev, u8 port_num,
3605 struct ib_port_immutable *immutable)
3607 struct ib_port_attr attr;
3610 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE |
3611 RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
3613 err = ib_query_port(ibdev, port_num, &attr);
3617 immutable->pkey_tbl_len = attr.pkey_tbl_len;
3618 immutable->gid_tbl_len = attr.gid_tbl_len;
3619 immutable->max_mad_size = IB_MGMT_MAD_SIZE;