1 /* QLogic qedr NIC Driver
2 * Copyright (c) 2015-2016 QLogic Corporation
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and /or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 #include <linux/dma-mapping.h>
33 #include <linux/crc32.h>
37 #include <linux/iommu.h>
39 #include <rdma/ib_verbs.h>
40 #include <rdma/ib_user_verbs.h>
41 #include <rdma/iw_cm.h>
42 #include <rdma/ib_umem.h>
43 #include <rdma/ib_addr.h>
44 #include <rdma/ib_cache.h>
46 #include <linux/qed/common_hsi.h>
47 #include "qedr_hsi_rdma.h"
48 #include <linux/qed/qed_if.h>
51 #include <rdma/qedr-abi.h>
54 #define DB_ADDR_SHIFT(addr) ((addr) << DB_PWM_ADDR_OFFSET_SHIFT)
56 int qedr_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey)
58 if (index > QEDR_ROCE_PKEY_TABLE_LEN)
61 *pkey = QEDR_ROCE_PKEY_DEFAULT;
65 int qedr_query_gid(struct ib_device *ibdev, u8 port, int index,
68 struct qedr_dev *dev = get_qedr_dev(ibdev);
71 if (!rdma_cap_roce_gid_table(ibdev, port))
74 rc = ib_get_cached_gid(ibdev, port, index, sgid, NULL);
76 memcpy(sgid, &zgid, sizeof(*sgid));
80 DP_DEBUG(dev, QEDR_MSG_INIT, "query gid: index=%d %llx:%llx\n", index,
81 sgid->global.interface_id, sgid->global.subnet_prefix);
86 int qedr_add_gid(struct ib_device *device, u8 port_num,
87 unsigned int index, const union ib_gid *gid,
88 const struct ib_gid_attr *attr, void **context)
90 if (!rdma_cap_roce_gid_table(device, port_num))
93 if (port_num > QEDR_MAX_PORT)
102 int qedr_del_gid(struct ib_device *device, u8 port_num,
103 unsigned int index, void **context)
105 if (!rdma_cap_roce_gid_table(device, port_num))
108 if (port_num > QEDR_MAX_PORT)
117 int qedr_query_device(struct ib_device *ibdev,
118 struct ib_device_attr *attr, struct ib_udata *udata)
120 struct qedr_dev *dev = get_qedr_dev(ibdev);
121 struct qedr_device_attr *qattr = &dev->attr;
123 if (!dev->rdma_ctx) {
125 "qedr_query_device called with invalid params rdma_ctx=%p\n",
130 memset(attr, 0, sizeof(*attr));
132 attr->fw_ver = qattr->fw_ver;
133 attr->sys_image_guid = qattr->sys_image_guid;
134 attr->max_mr_size = qattr->max_mr_size;
135 attr->page_size_cap = qattr->page_size_caps;
136 attr->vendor_id = qattr->vendor_id;
137 attr->vendor_part_id = qattr->vendor_part_id;
138 attr->hw_ver = qattr->hw_ver;
139 attr->max_qp = qattr->max_qp;
140 attr->max_qp_wr = max_t(u32, qattr->max_sqe, qattr->max_rqe);
141 attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD |
142 IB_DEVICE_RC_RNR_NAK_GEN |
143 IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_MGT_EXTENSIONS;
145 attr->max_sge = qattr->max_sge;
146 attr->max_sge_rd = qattr->max_sge;
147 attr->max_cq = qattr->max_cq;
148 attr->max_cqe = qattr->max_cqe;
149 attr->max_mr = qattr->max_mr;
150 attr->max_mw = qattr->max_mw;
151 attr->max_pd = qattr->max_pd;
152 attr->atomic_cap = dev->atomic_cap;
153 attr->max_fmr = qattr->max_fmr;
154 attr->max_map_per_fmr = 16;
155 attr->max_qp_init_rd_atom =
156 1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1);
157 attr->max_qp_rd_atom =
158 min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1),
159 attr->max_qp_init_rd_atom);
161 attr->max_srq = qattr->max_srq;
162 attr->max_srq_sge = qattr->max_srq_sge;
163 attr->max_srq_wr = qattr->max_srq_wr;
165 attr->local_ca_ack_delay = qattr->dev_ack_delay;
166 attr->max_fast_reg_page_list_len = qattr->max_mr / 8;
167 attr->max_pkeys = QEDR_ROCE_PKEY_MAX;
168 attr->max_ah = qattr->max_ah;
173 #define QEDR_SPEED_SDR (1)
174 #define QEDR_SPEED_DDR (2)
175 #define QEDR_SPEED_QDR (4)
176 #define QEDR_SPEED_FDR10 (8)
177 #define QEDR_SPEED_FDR (16)
178 #define QEDR_SPEED_EDR (32)
180 static inline void get_link_speed_and_width(int speed, u8 *ib_speed,
185 *ib_speed = QEDR_SPEED_SDR;
186 *ib_width = IB_WIDTH_1X;
189 *ib_speed = QEDR_SPEED_QDR;
190 *ib_width = IB_WIDTH_1X;
194 *ib_speed = QEDR_SPEED_DDR;
195 *ib_width = IB_WIDTH_4X;
199 *ib_speed = QEDR_SPEED_EDR;
200 *ib_width = IB_WIDTH_1X;
204 *ib_speed = QEDR_SPEED_QDR;
205 *ib_width = IB_WIDTH_4X;
209 *ib_speed = QEDR_SPEED_QDR;
210 *ib_width = IB_WIDTH_4X;
214 *ib_speed = QEDR_SPEED_EDR;
215 *ib_width = IB_WIDTH_4X;
220 *ib_speed = QEDR_SPEED_SDR;
221 *ib_width = IB_WIDTH_1X;
225 int qedr_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *attr)
227 struct qedr_dev *dev;
228 struct qed_rdma_port *rdma_port;
230 dev = get_qedr_dev(ibdev);
232 DP_ERR(dev, "invalid_port=0x%x\n", port);
236 if (!dev->rdma_ctx) {
237 DP_ERR(dev, "rdma_ctx is NULL\n");
241 rdma_port = dev->ops->rdma_query_port(dev->rdma_ctx);
243 /* *attr being zeroed by the caller, avoid zeroing it here */
244 if (rdma_port->port_state == QED_RDMA_PORT_UP) {
245 attr->state = IB_PORT_ACTIVE;
246 attr->phys_state = 5;
248 attr->state = IB_PORT_DOWN;
249 attr->phys_state = 3;
251 attr->max_mtu = IB_MTU_4096;
252 attr->active_mtu = iboe_get_mtu(dev->ndev->mtu);
257 attr->port_cap_flags = IB_PORT_IP_BASED_GIDS;
258 attr->gid_tbl_len = QEDR_MAX_SGID;
259 attr->pkey_tbl_len = QEDR_ROCE_PKEY_TABLE_LEN;
260 attr->bad_pkey_cntr = rdma_port->pkey_bad_counter;
261 attr->qkey_viol_cntr = 0;
262 get_link_speed_and_width(rdma_port->link_speed,
263 &attr->active_speed, &attr->active_width);
264 attr->max_msg_sz = rdma_port->max_msg_size;
265 attr->max_vl_num = 4;
270 int qedr_modify_port(struct ib_device *ibdev, u8 port, int mask,
271 struct ib_port_modify *props)
273 struct qedr_dev *dev;
275 dev = get_qedr_dev(ibdev);
277 DP_ERR(dev, "invalid_port=0x%x\n", port);
284 static int qedr_add_mmap(struct qedr_ucontext *uctx, u64 phy_addr,
289 mm = kzalloc(sizeof(*mm), GFP_KERNEL);
293 mm->key.phy_addr = phy_addr;
294 /* This function might be called with a length which is not a multiple
295 * of PAGE_SIZE, while the mapping is PAGE_SIZE grained and the kernel
296 * forces this granularity by increasing the requested size if needed.
297 * When qedr_mmap is called, it will search the list with the updated
298 * length as a key. To prevent search failures, the length is rounded up
299 * in advance to PAGE_SIZE.
301 mm->key.len = roundup(len, PAGE_SIZE);
302 INIT_LIST_HEAD(&mm->entry);
304 mutex_lock(&uctx->mm_list_lock);
305 list_add(&mm->entry, &uctx->mm_head);
306 mutex_unlock(&uctx->mm_list_lock);
308 DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
309 "added (addr=0x%llx,len=0x%lx) for ctx=%p\n",
310 (unsigned long long)mm->key.phy_addr,
311 (unsigned long)mm->key.len, uctx);
316 static bool qedr_search_mmap(struct qedr_ucontext *uctx, u64 phy_addr,
322 mutex_lock(&uctx->mm_list_lock);
323 list_for_each_entry(mm, &uctx->mm_head, entry) {
324 if (len != mm->key.len || phy_addr != mm->key.phy_addr)
330 mutex_unlock(&uctx->mm_list_lock);
331 DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
332 "searched for (addr=0x%llx,len=0x%lx) for ctx=%p, result=%d\n",
333 mm->key.phy_addr, mm->key.len, uctx, found);
338 struct ib_ucontext *qedr_alloc_ucontext(struct ib_device *ibdev,
339 struct ib_udata *udata)
342 struct qedr_ucontext *ctx;
343 struct qedr_alloc_ucontext_resp uresp;
344 struct qedr_dev *dev = get_qedr_dev(ibdev);
345 struct qed_rdma_add_user_out_params oparams;
348 return ERR_PTR(-EFAULT);
350 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
352 return ERR_PTR(-ENOMEM);
354 rc = dev->ops->rdma_add_user(dev->rdma_ctx, &oparams);
357 "failed to allocate a DPI for a new RoCE application, rc=%d. To overcome this consider to increase the number of DPIs, increase the doorbell BAR size or just close unnecessary RoCE applications. In order to increase the number of DPIs consult the qedr readme\n",
362 ctx->dpi = oparams.dpi;
363 ctx->dpi_addr = oparams.dpi_addr;
364 ctx->dpi_phys_addr = oparams.dpi_phys_addr;
365 ctx->dpi_size = oparams.dpi_size;
366 INIT_LIST_HEAD(&ctx->mm_head);
367 mutex_init(&ctx->mm_list_lock);
369 memset(&uresp, 0, sizeof(uresp));
371 uresp.db_pa = ctx->dpi_phys_addr;
372 uresp.db_size = ctx->dpi_size;
373 uresp.max_send_wr = dev->attr.max_sqe;
374 uresp.max_recv_wr = dev->attr.max_rqe;
375 uresp.max_srq_wr = dev->attr.max_srq_wr;
376 uresp.sges_per_send_wr = QEDR_MAX_SQE_ELEMENTS_PER_SQE;
377 uresp.sges_per_recv_wr = QEDR_MAX_RQE_ELEMENTS_PER_RQE;
378 uresp.sges_per_srq_wr = dev->attr.max_srq_sge;
379 uresp.max_cqes = QEDR_MAX_CQES;
381 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
387 rc = qedr_add_mmap(ctx, ctx->dpi_phys_addr, ctx->dpi_size);
391 DP_DEBUG(dev, QEDR_MSG_INIT, "Allocating user context %p\n",
393 return &ctx->ibucontext;
400 int qedr_dealloc_ucontext(struct ib_ucontext *ibctx)
402 struct qedr_ucontext *uctx = get_qedr_ucontext(ibctx);
403 struct qedr_mm *mm, *tmp;
406 DP_DEBUG(uctx->dev, QEDR_MSG_INIT, "Deallocating user context %p\n",
408 uctx->dev->ops->rdma_remove_user(uctx->dev->rdma_ctx, uctx->dpi);
410 list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) {
411 DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
412 "deleted (addr=0x%llx,len=0x%lx) for ctx=%p\n",
413 mm->key.phy_addr, mm->key.len, uctx);
414 list_del(&mm->entry);
422 int qedr_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
424 struct qedr_ucontext *ucontext = get_qedr_ucontext(context);
425 struct qedr_dev *dev = get_qedr_dev(context->device);
426 unsigned long vm_page = vma->vm_pgoff << PAGE_SHIFT;
427 u64 unmapped_db = dev->db_phys_addr;
428 unsigned long len = (vma->vm_end - vma->vm_start);
432 DP_DEBUG(dev, QEDR_MSG_INIT,
433 "qedr_mmap called vm_page=0x%lx vm_pgoff=0x%lx unmapped_db=0x%llx db_size=%x, len=%lx\n",
434 vm_page, vma->vm_pgoff, unmapped_db, dev->db_size, len);
435 if (vma->vm_start & (PAGE_SIZE - 1)) {
436 DP_ERR(dev, "Vma_start not page aligned = %ld\n",
441 found = qedr_search_mmap(ucontext, vm_page, len);
443 DP_ERR(dev, "Vma_pgoff not found in mapped array = %ld\n",
448 DP_DEBUG(dev, QEDR_MSG_INIT, "Mapping doorbell bar\n");
450 if ((vm_page >= unmapped_db) && (vm_page <= (unmapped_db +
452 DP_DEBUG(dev, QEDR_MSG_INIT, "Mapping doorbell bar\n");
453 if (vma->vm_flags & VM_READ) {
454 DP_ERR(dev, "Trying to map doorbell bar for read\n");
458 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
460 rc = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
461 PAGE_SIZE, vma->vm_page_prot);
463 DP_DEBUG(dev, QEDR_MSG_INIT, "Mapping chains\n");
464 rc = remap_pfn_range(vma, vma->vm_start,
465 vma->vm_pgoff, len, vma->vm_page_prot);
467 DP_DEBUG(dev, QEDR_MSG_INIT, "qedr_mmap return code: %d\n", rc);
471 struct ib_pd *qedr_alloc_pd(struct ib_device *ibdev,
472 struct ib_ucontext *context, struct ib_udata *udata)
474 struct qedr_dev *dev = get_qedr_dev(ibdev);
479 DP_DEBUG(dev, QEDR_MSG_INIT, "Function called from: %s\n",
480 (udata && context) ? "User Lib" : "Kernel");
482 if (!dev->rdma_ctx) {
483 DP_ERR(dev, "invlaid RDMA context\n");
484 return ERR_PTR(-EINVAL);
487 pd = kzalloc(sizeof(*pd), GFP_KERNEL);
489 return ERR_PTR(-ENOMEM);
491 rc = dev->ops->rdma_alloc_pd(dev->rdma_ctx, &pd_id);
497 if (udata && context) {
498 struct qedr_alloc_pd_uresp uresp;
502 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
504 DP_ERR(dev, "copy error pd_id=0x%x.\n", pd_id);
505 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd_id);
509 pd->uctx = get_qedr_ucontext(context);
520 int qedr_dealloc_pd(struct ib_pd *ibpd)
522 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
523 struct qedr_pd *pd = get_qedr_pd(ibpd);
526 pr_err("Invalid PD received in dealloc_pd\n");
530 DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id);
531 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id);
538 static void qedr_free_pbl(struct qedr_dev *dev,
539 struct qedr_pbl_info *pbl_info, struct qedr_pbl *pbl)
541 struct pci_dev *pdev = dev->pdev;
544 for (i = 0; i < pbl_info->num_pbls; i++) {
547 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
548 pbl[i].va, pbl[i].pa);
554 #define MIN_FW_PBL_PAGE_SIZE (4 * 1024)
555 #define MAX_FW_PBL_PAGE_SIZE (64 * 1024)
557 #define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64))
558 #define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE)
559 #define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE * MAX_PBES_ON_PAGE)
561 static struct qedr_pbl *qedr_alloc_pbl_tbl(struct qedr_dev *dev,
562 struct qedr_pbl_info *pbl_info,
565 struct pci_dev *pdev = dev->pdev;
566 struct qedr_pbl *pbl_table;
567 dma_addr_t *pbl_main_tbl;
572 pbl_table = kcalloc(pbl_info->num_pbls, sizeof(*pbl_table), flags);
574 return ERR_PTR(-ENOMEM);
576 for (i = 0; i < pbl_info->num_pbls; i++) {
577 va = dma_alloc_coherent(&pdev->dev, pbl_info->pbl_size,
582 memset(va, 0, pbl_info->pbl_size);
583 pbl_table[i].va = va;
584 pbl_table[i].pa = pa;
587 /* Two-Layer PBLs, if we have more than one pbl we need to initialize
588 * the first one with physical pointers to all of the rest
590 pbl_main_tbl = (dma_addr_t *)pbl_table[0].va;
591 for (i = 0; i < pbl_info->num_pbls - 1; i++)
592 pbl_main_tbl[i] = pbl_table[i + 1].pa;
597 for (i--; i >= 0; i--)
598 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
599 pbl_table[i].va, pbl_table[i].pa);
601 qedr_free_pbl(dev, pbl_info, pbl_table);
603 return ERR_PTR(-ENOMEM);
606 static int qedr_prepare_pbl_tbl(struct qedr_dev *dev,
607 struct qedr_pbl_info *pbl_info,
608 u32 num_pbes, int two_layer_capable)
614 if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) {
615 if (num_pbes > MAX_PBES_TWO_LAYER) {
616 DP_ERR(dev, "prepare pbl table: too many pages %d\n",
621 /* calculate required pbl page size */
622 pbl_size = MIN_FW_PBL_PAGE_SIZE;
623 pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) *
624 NUM_PBES_ON_PAGE(pbl_size);
626 while (pbl_capacity < num_pbes) {
628 pbl_capacity = pbl_size / sizeof(u64);
629 pbl_capacity = pbl_capacity * pbl_capacity;
632 num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size));
633 num_pbls++; /* One for the layer0 ( points to the pbls) */
634 pbl_info->two_layered = true;
636 /* One layered PBL */
638 pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE,
639 roundup_pow_of_two((num_pbes * sizeof(u64))));
640 pbl_info->two_layered = false;
643 pbl_info->num_pbls = num_pbls;
644 pbl_info->pbl_size = pbl_size;
645 pbl_info->num_pbes = num_pbes;
647 DP_DEBUG(dev, QEDR_MSG_MR,
648 "prepare pbl table: num_pbes=%d, num_pbls=%d, pbl_size=%d\n",
649 pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size);
654 static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem,
655 struct qedr_pbl *pbl,
656 struct qedr_pbl_info *pbl_info, u32 pg_shift)
658 int shift, pg_cnt, pages, pbe_cnt, total_num_pbes = 0;
659 u32 fw_pg_cnt, fw_pg_per_umem_pg;
660 struct qedr_pbl *pbl_tbl;
661 struct scatterlist *sg;
666 if (!pbl_info->num_pbes)
669 /* If we have a two layered pbl, the first pbl points to the rest
670 * of the pbls and the first entry lays on the second pbl in the table
672 if (pbl_info->two_layered)
677 pbe = (struct regpair *)pbl_tbl->va;
679 DP_ERR(dev, "cannot populate PBL due to a NULL PBE\n");
685 shift = umem->page_shift;
687 fw_pg_per_umem_pg = BIT(umem->page_shift - pg_shift);
689 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
690 pages = sg_dma_len(sg) >> shift;
691 pg_addr = sg_dma_address(sg);
692 for (pg_cnt = 0; pg_cnt < pages; pg_cnt++) {
693 for (fw_pg_cnt = 0; fw_pg_cnt < fw_pg_per_umem_pg;) {
694 pbe->lo = cpu_to_le32(pg_addr);
695 pbe->hi = cpu_to_le32(upper_32_bits(pg_addr));
697 pg_addr += BIT(pg_shift);
702 if (total_num_pbes == pbl_info->num_pbes)
705 /* If the given pbl is full storing the pbes,
709 (pbl_info->pbl_size / sizeof(u64))) {
711 pbe = (struct regpair *)pbl_tbl->va;
721 static int qedr_copy_cq_uresp(struct qedr_dev *dev,
722 struct qedr_cq *cq, struct ib_udata *udata)
724 struct qedr_create_cq_uresp uresp;
727 memset(&uresp, 0, sizeof(uresp));
729 uresp.db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
730 uresp.icid = cq->icid;
732 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
734 DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid);
739 static void consume_cqe(struct qedr_cq *cq)
741 if (cq->latest_cqe == cq->toggle_cqe)
742 cq->pbl_toggle ^= RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
744 cq->latest_cqe = qed_chain_consume(&cq->pbl);
747 static inline int qedr_align_cq_entries(int entries)
749 u64 size, aligned_size;
751 /* We allocate an extra entry that we don't report to the FW. */
752 size = (entries + 1) * QEDR_CQE_SIZE;
753 aligned_size = ALIGN(size, PAGE_SIZE);
755 return aligned_size / QEDR_CQE_SIZE;
758 static inline int qedr_init_user_queue(struct ib_ucontext *ib_ctx,
759 struct qedr_dev *dev,
760 struct qedr_userq *q,
761 u64 buf_addr, size_t buf_len,
762 int access, int dmasync)
767 q->buf_addr = buf_addr;
768 q->buf_len = buf_len;
769 q->umem = ib_umem_get(ib_ctx, q->buf_addr, q->buf_len, access, dmasync);
770 if (IS_ERR(q->umem)) {
771 DP_ERR(dev, "create user queue: failed ib_umem_get, got %ld\n",
773 return PTR_ERR(q->umem);
776 fw_pages = ib_umem_page_count(q->umem) <<
777 (q->umem->page_shift - FW_PAGE_SHIFT);
779 rc = qedr_prepare_pbl_tbl(dev, &q->pbl_info, fw_pages, 0);
783 q->pbl_tbl = qedr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL);
784 if (IS_ERR(q->pbl_tbl)) {
785 rc = PTR_ERR(q->pbl_tbl);
789 qedr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info,
795 ib_umem_release(q->umem);
800 static inline void qedr_init_cq_params(struct qedr_cq *cq,
801 struct qedr_ucontext *ctx,
802 struct qedr_dev *dev, int vector,
803 int chain_entries, int page_cnt,
805 struct qed_rdma_create_cq_in_params
808 memset(params, 0, sizeof(*params));
809 params->cq_handle_hi = upper_32_bits((uintptr_t)cq);
810 params->cq_handle_lo = lower_32_bits((uintptr_t)cq);
811 params->cnq_id = vector;
812 params->cq_size = chain_entries - 1;
813 params->dpi = (ctx) ? ctx->dpi : dev->dpi;
814 params->pbl_num_pages = page_cnt;
815 params->pbl_ptr = pbl_ptr;
816 params->pbl_two_level = 0;
819 static void doorbell_cq(struct qedr_cq *cq, u32 cons, u8 flags)
821 /* Flush data before signalling doorbell */
823 cq->db.data.agg_flags = flags;
824 cq->db.data.value = cpu_to_le32(cons);
825 writeq(cq->db.raw, cq->db_addr);
827 /* Make sure write would stick */
831 int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
833 struct qedr_cq *cq = get_qedr_cq(ibcq);
834 unsigned long sflags;
835 struct qedr_dev *dev;
837 dev = get_qedr_dev(ibcq->device);
841 "warning: arm was invoked after destroy for cq %p (icid=%d)\n",
847 if (cq->cq_type == QEDR_CQ_TYPE_GSI)
850 spin_lock_irqsave(&cq->cq_lock, sflags);
854 if (flags & IB_CQ_SOLICITED)
855 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD;
857 if (flags & IB_CQ_NEXT_COMP)
858 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD;
860 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
862 spin_unlock_irqrestore(&cq->cq_lock, sflags);
867 struct ib_cq *qedr_create_cq(struct ib_device *ibdev,
868 const struct ib_cq_init_attr *attr,
869 struct ib_ucontext *ib_ctx, struct ib_udata *udata)
871 struct qedr_ucontext *ctx = get_qedr_ucontext(ib_ctx);
872 struct qed_rdma_destroy_cq_out_params destroy_oparams;
873 struct qed_rdma_destroy_cq_in_params destroy_iparams;
874 struct qedr_dev *dev = get_qedr_dev(ibdev);
875 struct qed_rdma_create_cq_in_params params;
876 struct qedr_create_cq_ureq ureq;
877 int vector = attr->comp_vector;
878 int entries = attr->cqe;
886 DP_DEBUG(dev, QEDR_MSG_INIT,
887 "create_cq: called from %s. entries=%d, vector=%d\n",
888 udata ? "User Lib" : "Kernel", entries, vector);
890 if (entries > QEDR_MAX_CQES) {
892 "create cq: the number of entries %d is too high. Must be equal or below %d.\n",
893 entries, QEDR_MAX_CQES);
894 return ERR_PTR(-EINVAL);
897 chain_entries = qedr_align_cq_entries(entries);
898 chain_entries = min_t(int, chain_entries, QEDR_MAX_CQES);
900 cq = kzalloc(sizeof(*cq), GFP_KERNEL);
902 return ERR_PTR(-ENOMEM);
905 memset(&ureq, 0, sizeof(ureq));
906 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) {
908 "create cq: problem copying data from user space\n");
914 "create cq: cannot create a cq with 0 entries\n");
918 cq->cq_type = QEDR_CQ_TYPE_USER;
920 rc = qedr_init_user_queue(ib_ctx, dev, &cq->q, ureq.addr,
921 ureq.len, IB_ACCESS_LOCAL_WRITE, 1);
925 pbl_ptr = cq->q.pbl_tbl->pa;
926 page_cnt = cq->q.pbl_info.num_pbes;
928 cq->ibcq.cqe = chain_entries;
930 cq->cq_type = QEDR_CQ_TYPE_KERNEL;
932 rc = dev->ops->common->chain_alloc(dev->cdev,
933 QED_CHAIN_USE_TO_CONSUME,
935 QED_CHAIN_CNT_TYPE_U32,
937 sizeof(union rdma_cqe),
942 page_cnt = qed_chain_get_page_cnt(&cq->pbl);
943 pbl_ptr = qed_chain_get_pbl_phys(&cq->pbl);
944 cq->ibcq.cqe = cq->pbl.capacity;
947 qedr_init_cq_params(cq, ctx, dev, vector, chain_entries, page_cnt,
950 rc = dev->ops->rdma_create_cq(dev->rdma_ctx, ¶ms, &icid);
955 cq->sig = QEDR_CQ_MAGIC_NUMBER;
956 spin_lock_init(&cq->cq_lock);
959 rc = qedr_copy_cq_uresp(dev, cq, udata);
963 /* Generate doorbell address. */
964 cq->db_addr = dev->db_addr +
965 DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
966 cq->db.data.icid = cq->icid;
967 cq->db.data.params = DB_AGG_CMD_SET <<
968 RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT;
970 /* point to the very last element, passing it we will toggle */
971 cq->toggle_cqe = qed_chain_get_last_elem(&cq->pbl);
972 cq->pbl_toggle = RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
973 cq->latest_cqe = NULL;
975 cq->cq_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
978 DP_DEBUG(dev, QEDR_MSG_CQ,
979 "create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n",
980 cq->icid, cq, params.cq_size);
985 destroy_iparams.icid = cq->icid;
986 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams,
990 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
992 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
995 ib_umem_release(cq->q.umem);
998 return ERR_PTR(-EINVAL);
1001 int qedr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata)
1003 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
1004 struct qedr_cq *cq = get_qedr_cq(ibcq);
1006 DP_ERR(dev, "cq %p RESIZE NOT SUPPORTED\n", cq);
1011 #define QEDR_DESTROY_CQ_MAX_ITERATIONS (10)
1012 #define QEDR_DESTROY_CQ_ITER_DURATION (10)
1014 int qedr_destroy_cq(struct ib_cq *ibcq)
1016 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
1017 struct qed_rdma_destroy_cq_out_params oparams;
1018 struct qed_rdma_destroy_cq_in_params iparams;
1019 struct qedr_cq *cq = get_qedr_cq(ibcq);
1023 DP_DEBUG(dev, QEDR_MSG_CQ, "destroy cq %p (icid=%d)\n", cq, cq->icid);
1027 /* GSIs CQs are handled by driver, so they don't exist in the FW */
1028 if (cq->cq_type == QEDR_CQ_TYPE_GSI)
1031 iparams.icid = cq->icid;
1032 rc = dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams);
1036 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
1038 if (ibcq->uobject && ibcq->uobject->context) {
1039 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
1040 ib_umem_release(cq->q.umem);
1043 /* We don't want the IRQ handler to handle a non-existing CQ so we
1044 * wait until all CNQ interrupts, if any, are received. This will always
1045 * happen and will always happen very fast. If not, then a serious error
1046 * has occured. That is why we can use a long delay.
1047 * We spin for a short time so we don’t lose time on context switching
1048 * in case all the completions are handled in that span. Otherwise
1049 * we sleep for a while and check again. Since the CNQ may be
1050 * associated with (only) the current CPU we use msleep to allow the
1051 * current CPU to be freed.
1052 * The CNQ notification is increased in qedr_irq_handler().
1054 iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1055 while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1056 udelay(QEDR_DESTROY_CQ_ITER_DURATION);
1060 iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1061 while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1062 msleep(QEDR_DESTROY_CQ_ITER_DURATION);
1066 if (oparams.num_cq_notif != cq->cnq_notif)
1069 /* Note that we don't need to have explicit code to wait for the
1070 * completion of the event handler because it is invoked from the EQ.
1071 * Since the destroy CQ ramrod has also been received on the EQ we can
1072 * be certain that there's no event handler in process.
1083 "CQ %p (icid=%d) not freed, expecting %d ints but got %d ints\n",
1084 cq, cq->icid, oparams.num_cq_notif, cq->cnq_notif);
1089 static inline int get_gid_info_from_table(struct ib_qp *ibqp,
1090 struct ib_qp_attr *attr,
1092 struct qed_rdma_modify_qp_in_params
1095 enum rdma_network_type nw_type;
1096 struct ib_gid_attr gid_attr;
1097 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
1103 rc = ib_get_cached_gid(ibqp->device,
1104 rdma_ah_get_port_num(&attr->ah_attr),
1105 grh->sgid_index, &gid, &gid_attr);
1109 if (!memcmp(&gid, &zgid, sizeof(gid)))
1112 if (gid_attr.ndev) {
1113 qp_params->vlan_id = rdma_vlan_dev_vlan_id(gid_attr.ndev);
1115 dev_put(gid_attr.ndev);
1116 nw_type = ib_gid_to_network_type(gid_attr.gid_type, &gid);
1118 case RDMA_NETWORK_IPV6:
1119 memcpy(&qp_params->sgid.bytes[0], &gid.raw[0],
1120 sizeof(qp_params->sgid));
1121 memcpy(&qp_params->dgid.bytes[0],
1123 sizeof(qp_params->dgid));
1124 qp_params->roce_mode = ROCE_V2_IPV6;
1125 SET_FIELD(qp_params->modify_flags,
1126 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1128 case RDMA_NETWORK_IB:
1129 memcpy(&qp_params->sgid.bytes[0], &gid.raw[0],
1130 sizeof(qp_params->sgid));
1131 memcpy(&qp_params->dgid.bytes[0],
1133 sizeof(qp_params->dgid));
1134 qp_params->roce_mode = ROCE_V1;
1136 case RDMA_NETWORK_IPV4:
1137 memset(&qp_params->sgid, 0, sizeof(qp_params->sgid));
1138 memset(&qp_params->dgid, 0, sizeof(qp_params->dgid));
1139 ipv4_addr = qedr_get_ipv4_from_gid(gid.raw);
1140 qp_params->sgid.ipv4_addr = ipv4_addr;
1142 qedr_get_ipv4_from_gid(grh->dgid.raw);
1143 qp_params->dgid.ipv4_addr = ipv4_addr;
1144 SET_FIELD(qp_params->modify_flags,
1145 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1146 qp_params->roce_mode = ROCE_V2_IPV4;
1151 for (i = 0; i < 4; i++) {
1152 qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]);
1153 qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]);
1156 if (qp_params->vlan_id >= VLAN_CFI_MASK)
1157 qp_params->vlan_id = 0;
1162 static int qedr_check_qp_attrs(struct ib_pd *ibpd, struct qedr_dev *dev,
1163 struct ib_qp_init_attr *attrs)
1165 struct qedr_device_attr *qattr = &dev->attr;
1167 /* QP0... attrs->qp_type == IB_QPT_GSI */
1168 if (attrs->qp_type != IB_QPT_RC && attrs->qp_type != IB_QPT_GSI) {
1169 DP_DEBUG(dev, QEDR_MSG_QP,
1170 "create qp: unsupported qp type=0x%x requested\n",
1175 if (attrs->cap.max_send_wr > qattr->max_sqe) {
1177 "create qp: cannot create a SQ with %d elements (max_send_wr=0x%x)\n",
1178 attrs->cap.max_send_wr, qattr->max_sqe);
1182 if (attrs->cap.max_inline_data > qattr->max_inline) {
1184 "create qp: unsupported inline data size=0x%x requested (max_inline=0x%x)\n",
1185 attrs->cap.max_inline_data, qattr->max_inline);
1189 if (attrs->cap.max_send_sge > qattr->max_sge) {
1191 "create qp: unsupported send_sge=0x%x requested (max_send_sge=0x%x)\n",
1192 attrs->cap.max_send_sge, qattr->max_sge);
1196 if (attrs->cap.max_recv_sge > qattr->max_sge) {
1198 "create qp: unsupported recv_sge=0x%x requested (max_recv_sge=0x%x)\n",
1199 attrs->cap.max_recv_sge, qattr->max_sge);
1203 /* Unprivileged user space cannot create special QP */
1204 if (ibpd->uobject && attrs->qp_type == IB_QPT_GSI) {
1206 "create qp: userspace can't create special QPs of type=0x%x\n",
1214 static void qedr_copy_rq_uresp(struct qedr_create_qp_uresp *uresp,
1217 uresp->rq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1218 uresp->rq_icid = qp->icid;
1221 static void qedr_copy_sq_uresp(struct qedr_create_qp_uresp *uresp,
1224 uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1225 uresp->sq_icid = qp->icid + 1;
1228 static int qedr_copy_qp_uresp(struct qedr_dev *dev,
1229 struct qedr_qp *qp, struct ib_udata *udata)
1231 struct qedr_create_qp_uresp uresp;
1234 memset(&uresp, 0, sizeof(uresp));
1235 qedr_copy_sq_uresp(&uresp, qp);
1236 qedr_copy_rq_uresp(&uresp, qp);
1238 uresp.atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE;
1239 uresp.qp_id = qp->qp_id;
1241 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1244 "create qp: failed a copy to user space with qp icid=0x%x.\n",
1250 static void qedr_set_common_qp_params(struct qedr_dev *dev,
1253 struct ib_qp_init_attr *attrs)
1255 spin_lock_init(&qp->q_lock);
1257 qp->qp_type = attrs->qp_type;
1258 qp->max_inline_data = attrs->cap.max_inline_data;
1259 qp->sq.max_sges = attrs->cap.max_send_sge;
1260 qp->state = QED_ROCE_QP_STATE_RESET;
1261 qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false;
1262 qp->sq_cq = get_qedr_cq(attrs->send_cq);
1263 qp->rq_cq = get_qedr_cq(attrs->recv_cq);
1265 qp->rq.max_sges = attrs->cap.max_recv_sge;
1267 DP_DEBUG(dev, QEDR_MSG_QP,
1268 "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n",
1269 qp->rq.max_sges, qp->rq_cq->icid);
1270 DP_DEBUG(dev, QEDR_MSG_QP,
1271 "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d, state = %d, signaled = %d, use_srq=%d\n",
1272 pd->pd_id, qp->qp_type, qp->max_inline_data,
1273 qp->state, qp->signaled, (attrs->srq) ? 1 : 0);
1274 DP_DEBUG(dev, QEDR_MSG_QP,
1275 "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n",
1276 qp->sq.max_sges, qp->sq_cq->icid);
1279 static void qedr_set_roce_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
1281 qp->sq.db = dev->db_addr +
1282 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1283 qp->sq.db_data.data.icid = qp->icid + 1;
1284 qp->rq.db = dev->db_addr +
1285 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1286 qp->rq.db_data.data.icid = qp->icid;
1290 qedr_init_common_qp_in_params(struct qedr_dev *dev,
1293 struct ib_qp_init_attr *attrs,
1294 bool fmr_and_reserved_lkey,
1295 struct qed_rdma_create_qp_in_params *params)
1297 /* QP handle to be written in an async event */
1298 params->qp_handle_async_lo = lower_32_bits((uintptr_t) qp);
1299 params->qp_handle_async_hi = upper_32_bits((uintptr_t) qp);
1301 params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR);
1302 params->fmr_and_reserved_lkey = fmr_and_reserved_lkey;
1303 params->pd = pd->pd_id;
1304 params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi;
1305 params->sq_cq_id = get_qedr_cq(attrs->send_cq)->icid;
1306 params->stats_queue = 0;
1307 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1309 params->use_srq = false;
1312 static inline void qedr_qp_user_print(struct qedr_dev *dev, struct qedr_qp *qp)
1314 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: successfully created user QP. "
1323 qp->usq.buf_len, qp->urq.buf_addr, qp->urq.buf_len);
1326 static void qedr_cleanup_user(struct qedr_dev *dev, struct qedr_qp *qp)
1329 ib_umem_release(qp->usq.umem);
1330 qp->usq.umem = NULL;
1333 ib_umem_release(qp->urq.umem);
1334 qp->urq.umem = NULL;
1337 static int qedr_create_user_qp(struct qedr_dev *dev,
1340 struct ib_udata *udata,
1341 struct ib_qp_init_attr *attrs)
1343 struct qed_rdma_create_qp_in_params in_params;
1344 struct qed_rdma_create_qp_out_params out_params;
1345 struct qedr_pd *pd = get_qedr_pd(ibpd);
1346 struct ib_ucontext *ib_ctx = NULL;
1347 struct qedr_ucontext *ctx = NULL;
1348 struct qedr_create_qp_ureq ureq;
1351 ib_ctx = ibpd->uobject->context;
1352 ctx = get_qedr_ucontext(ib_ctx);
1354 memset(&ureq, 0, sizeof(ureq));
1355 rc = ib_copy_from_udata(&ureq, udata, sizeof(ureq));
1357 DP_ERR(dev, "Problem copying data from user space\n");
1361 /* SQ - read access only (0), dma sync not required (0) */
1362 rc = qedr_init_user_queue(ib_ctx, dev, &qp->usq, ureq.sq_addr,
1367 /* RQ - read access only (0), dma sync not required (0) */
1368 rc = qedr_init_user_queue(ib_ctx, dev, &qp->urq, ureq.rq_addr,
1374 memset(&in_params, 0, sizeof(in_params));
1375 qedr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params);
1376 in_params.qp_handle_lo = ureq.qp_handle_lo;
1377 in_params.qp_handle_hi = ureq.qp_handle_hi;
1378 in_params.sq_num_pages = qp->usq.pbl_info.num_pbes;
1379 in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa;
1380 in_params.rq_num_pages = qp->urq.pbl_info.num_pbes;
1381 in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa;
1383 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1384 &in_params, &out_params);
1391 qp->qp_id = out_params.qp_id;
1392 qp->icid = out_params.icid;
1394 rc = qedr_copy_qp_uresp(dev, qp, udata);
1398 qedr_qp_user_print(dev, qp);
1402 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
1404 DP_ERR(dev, "create qp: fatal fault. rc=%d", rc);
1407 qedr_cleanup_user(dev, qp);
1412 qedr_roce_create_kernel_qp(struct qedr_dev *dev,
1414 struct qed_rdma_create_qp_in_params *in_params,
1415 u32 n_sq_elems, u32 n_rq_elems)
1417 struct qed_rdma_create_qp_out_params out_params;
1420 rc = dev->ops->common->chain_alloc(dev->cdev,
1421 QED_CHAIN_USE_TO_PRODUCE,
1423 QED_CHAIN_CNT_TYPE_U32,
1425 QEDR_SQE_ELEMENT_SIZE,
1431 in_params->sq_num_pages = qed_chain_get_page_cnt(&qp->sq.pbl);
1432 in_params->sq_pbl_ptr = qed_chain_get_pbl_phys(&qp->sq.pbl);
1434 rc = dev->ops->common->chain_alloc(dev->cdev,
1435 QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1437 QED_CHAIN_CNT_TYPE_U32,
1439 QEDR_RQE_ELEMENT_SIZE,
1444 in_params->rq_num_pages = qed_chain_get_page_cnt(&qp->rq.pbl);
1445 in_params->rq_pbl_ptr = qed_chain_get_pbl_phys(&qp->rq.pbl);
1447 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1448 in_params, &out_params);
1453 qp->qp_id = out_params.qp_id;
1454 qp->icid = out_params.icid;
1456 qedr_set_roce_db_info(dev, qp);
1461 static void qedr_cleanup_kernel(struct qedr_dev *dev, struct qedr_qp *qp)
1463 dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl);
1464 kfree(qp->wqe_wr_id);
1466 dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl);
1467 kfree(qp->rqe_wr_id);
1470 static int qedr_create_kernel_qp(struct qedr_dev *dev,
1473 struct ib_qp_init_attr *attrs)
1475 struct qed_rdma_create_qp_in_params in_params;
1476 struct qedr_pd *pd = get_qedr_pd(ibpd);
1482 memset(&in_params, 0, sizeof(in_params));
1484 /* A single work request may take up to QEDR_MAX_SQ_WQE_SIZE elements in
1485 * the ring. The ring should allow at least a single WR, even if the
1486 * user requested none, due to allocation issues.
1487 * We should add an extra WR since the prod and cons indices of
1488 * wqe_wr_id are managed in such a way that the WQ is considered full
1489 * when (prod+1)%max_wr==cons. We currently don't do that because we
1490 * double the number of entries due an iSER issue that pushes far more
1491 * WRs than indicated. If we decline its ib_post_send() then we get
1492 * error prints in the dmesg we'd like to avoid.
1494 qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier,
1497 qp->wqe_wr_id = kzalloc(qp->sq.max_wr * sizeof(*qp->wqe_wr_id),
1499 if (!qp->wqe_wr_id) {
1500 DP_ERR(dev, "create qp: failed SQ shadow memory allocation\n");
1504 /* QP handle to be written in CQE */
1505 in_params.qp_handle_lo = lower_32_bits((uintptr_t) qp);
1506 in_params.qp_handle_hi = upper_32_bits((uintptr_t) qp);
1508 /* A single work request may take up to QEDR_MAX_RQ_WQE_SIZE elements in
1509 * the ring. There ring should allow at least a single WR, even if the
1510 * user requested none, due to allocation issues.
1512 qp->rq.max_wr = (u16) max_t(u32, attrs->cap.max_recv_wr, 1);
1514 /* Allocate driver internal RQ array */
1515 qp->rqe_wr_id = kzalloc(qp->rq.max_wr * sizeof(*qp->rqe_wr_id),
1517 if (!qp->rqe_wr_id) {
1519 "create qp: failed RQ shadow memory allocation\n");
1520 kfree(qp->wqe_wr_id);
1524 qedr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params);
1526 n_sq_entries = attrs->cap.max_send_wr;
1527 n_sq_entries = min_t(u32, n_sq_entries, dev->attr.max_sqe);
1528 n_sq_entries = max_t(u32, n_sq_entries, 1);
1529 n_sq_elems = n_sq_entries * QEDR_MAX_SQE_ELEMENTS_PER_SQE;
1531 n_rq_elems = qp->rq.max_wr * QEDR_MAX_RQE_ELEMENTS_PER_RQE;
1533 rc = qedr_roce_create_kernel_qp(dev, qp, &in_params,
1534 n_sq_elems, n_rq_elems);
1536 qedr_cleanup_kernel(dev, qp);
1541 struct ib_qp *qedr_create_qp(struct ib_pd *ibpd,
1542 struct ib_qp_init_attr *attrs,
1543 struct ib_udata *udata)
1545 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
1546 struct qedr_pd *pd = get_qedr_pd(ibpd);
1551 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: called from %s, pd=%p\n",
1552 udata ? "user library" : "kernel", pd);
1554 rc = qedr_check_qp_attrs(ibpd, dev, attrs);
1559 return ERR_PTR(-EINVAL);
1561 DP_DEBUG(dev, QEDR_MSG_QP,
1562 "create qp: called from %s, event_handler=%p, eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n",
1563 udata ? "user library" : "kernel", attrs->event_handler, pd,
1564 get_qedr_cq(attrs->send_cq),
1565 get_qedr_cq(attrs->send_cq)->icid,
1566 get_qedr_cq(attrs->recv_cq),
1567 get_qedr_cq(attrs->recv_cq)->icid);
1569 qp = kzalloc(sizeof(*qp), GFP_KERNEL);
1571 DP_ERR(dev, "create qp: failed allocating memory\n");
1572 return ERR_PTR(-ENOMEM);
1575 qedr_set_common_qp_params(dev, qp, pd, attrs);
1577 if (attrs->qp_type == IB_QPT_GSI) {
1578 ibqp = qedr_create_gsi_qp(dev, attrs, qp);
1585 rc = qedr_create_user_qp(dev, qp, ibpd, udata, attrs);
1587 rc = qedr_create_kernel_qp(dev, qp, ibpd, attrs);
1592 qp->ibqp.qp_num = qp->qp_id;
1599 return ERR_PTR(-EFAULT);
1602 static enum ib_qp_state qedr_get_ibqp_state(enum qed_roce_qp_state qp_state)
1605 case QED_ROCE_QP_STATE_RESET:
1606 return IB_QPS_RESET;
1607 case QED_ROCE_QP_STATE_INIT:
1609 case QED_ROCE_QP_STATE_RTR:
1611 case QED_ROCE_QP_STATE_RTS:
1613 case QED_ROCE_QP_STATE_SQD:
1615 case QED_ROCE_QP_STATE_ERR:
1617 case QED_ROCE_QP_STATE_SQE:
1623 static enum qed_roce_qp_state qedr_get_state_from_ibqp(
1624 enum ib_qp_state qp_state)
1628 return QED_ROCE_QP_STATE_RESET;
1630 return QED_ROCE_QP_STATE_INIT;
1632 return QED_ROCE_QP_STATE_RTR;
1634 return QED_ROCE_QP_STATE_RTS;
1636 return QED_ROCE_QP_STATE_SQD;
1638 return QED_ROCE_QP_STATE_ERR;
1640 return QED_ROCE_QP_STATE_ERR;
1644 static void qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info *qph)
1646 qed_chain_reset(&qph->pbl);
1650 qph->db_data.data.value = cpu_to_le16(0);
1653 static int qedr_update_qp_state(struct qedr_dev *dev,
1655 enum qed_roce_qp_state new_state)
1659 if (new_state == qp->state)
1662 switch (qp->state) {
1663 case QED_ROCE_QP_STATE_RESET:
1664 switch (new_state) {
1665 case QED_ROCE_QP_STATE_INIT:
1666 qp->prev_wqe_size = 0;
1667 qedr_reset_qp_hwq_info(&qp->sq);
1668 qedr_reset_qp_hwq_info(&qp->rq);
1675 case QED_ROCE_QP_STATE_INIT:
1676 switch (new_state) {
1677 case QED_ROCE_QP_STATE_RTR:
1678 /* Update doorbell (in case post_recv was
1679 * done before move to RTR)
1682 writel(qp->rq.db_data.raw, qp->rq.db);
1683 /* Make sure write takes effect */
1686 case QED_ROCE_QP_STATE_ERR:
1689 /* Invalid state change. */
1694 case QED_ROCE_QP_STATE_RTR:
1696 switch (new_state) {
1697 case QED_ROCE_QP_STATE_RTS:
1699 case QED_ROCE_QP_STATE_ERR:
1702 /* Invalid state change. */
1707 case QED_ROCE_QP_STATE_RTS:
1709 switch (new_state) {
1710 case QED_ROCE_QP_STATE_SQD:
1712 case QED_ROCE_QP_STATE_ERR:
1715 /* Invalid state change. */
1720 case QED_ROCE_QP_STATE_SQD:
1722 switch (new_state) {
1723 case QED_ROCE_QP_STATE_RTS:
1724 case QED_ROCE_QP_STATE_ERR:
1727 /* Invalid state change. */
1732 case QED_ROCE_QP_STATE_ERR:
1734 switch (new_state) {
1735 case QED_ROCE_QP_STATE_RESET:
1736 if ((qp->rq.prod != qp->rq.cons) ||
1737 (qp->sq.prod != qp->sq.cons)) {
1739 "Error->Reset with rq/sq not empty rq.prod=%x rq.cons=%x sq.prod=%x sq.cons=%x\n",
1740 qp->rq.prod, qp->rq.cons, qp->sq.prod,
1758 int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
1759 int attr_mask, struct ib_udata *udata)
1761 struct qedr_qp *qp = get_qedr_qp(ibqp);
1762 struct qed_rdma_modify_qp_in_params qp_params = { 0 };
1763 struct qedr_dev *dev = get_qedr_dev(&qp->dev->ibdev);
1764 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
1765 enum ib_qp_state old_qp_state, new_qp_state;
1768 DP_DEBUG(dev, QEDR_MSG_QP,
1769 "modify qp: qp %p attr_mask=0x%x, state=%d", qp, attr_mask,
1772 old_qp_state = qedr_get_ibqp_state(qp->state);
1773 if (attr_mask & IB_QP_STATE)
1774 new_qp_state = attr->qp_state;
1776 new_qp_state = old_qp_state;
1778 if (!ib_modify_qp_is_ok
1779 (old_qp_state, new_qp_state, ibqp->qp_type, attr_mask,
1780 IB_LINK_LAYER_ETHERNET)) {
1782 "modify qp: invalid attribute mask=0x%x specified for\n"
1783 "qpn=0x%x of type=0x%x old_qp_state=0x%x, new_qp_state=0x%x\n",
1784 attr_mask, qp->qp_id, ibqp->qp_type, old_qp_state,
1790 /* Translate the masks... */
1791 if (attr_mask & IB_QP_STATE) {
1792 SET_FIELD(qp_params.modify_flags,
1793 QED_RDMA_MODIFY_QP_VALID_NEW_STATE, 1);
1794 qp_params.new_state = qedr_get_state_from_ibqp(attr->qp_state);
1797 if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY)
1798 qp_params.sqd_async = true;
1800 if (attr_mask & IB_QP_PKEY_INDEX) {
1801 SET_FIELD(qp_params.modify_flags,
1802 QED_ROCE_MODIFY_QP_VALID_PKEY, 1);
1803 if (attr->pkey_index >= QEDR_ROCE_PKEY_TABLE_LEN) {
1808 qp_params.pkey = QEDR_ROCE_PKEY_DEFAULT;
1811 if (attr_mask & IB_QP_QKEY)
1812 qp->qkey = attr->qkey;
1814 if (attr_mask & IB_QP_ACCESS_FLAGS) {
1815 SET_FIELD(qp_params.modify_flags,
1816 QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1);
1817 qp_params.incoming_rdma_read_en = attr->qp_access_flags &
1818 IB_ACCESS_REMOTE_READ;
1819 qp_params.incoming_rdma_write_en = attr->qp_access_flags &
1820 IB_ACCESS_REMOTE_WRITE;
1821 qp_params.incoming_atomic_en = attr->qp_access_flags &
1822 IB_ACCESS_REMOTE_ATOMIC;
1825 if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) {
1826 if (attr_mask & IB_QP_PATH_MTU) {
1827 if (attr->path_mtu < IB_MTU_256 ||
1828 attr->path_mtu > IB_MTU_4096) {
1829 pr_err("error: Only MTU sizes of 256, 512, 1024, 2048 and 4096 are supported by RoCE\n");
1833 qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu),
1834 ib_mtu_enum_to_int(iboe_get_mtu
1840 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
1841 pr_err("Fixing zeroed MTU to qp->mtu = %d\n", qp->mtu);
1844 SET_FIELD(qp_params.modify_flags,
1845 QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, 1);
1847 qp_params.traffic_class_tos = grh->traffic_class;
1848 qp_params.flow_label = grh->flow_label;
1849 qp_params.hop_limit_ttl = grh->hop_limit;
1851 qp->sgid_idx = grh->sgid_index;
1853 rc = get_gid_info_from_table(ibqp, attr, attr_mask, &qp_params);
1856 "modify qp: problems with GID index %d (rc=%d)\n",
1857 grh->sgid_index, rc);
1861 rc = qedr_get_dmac(dev, &attr->ah_attr,
1862 qp_params.remote_mac_addr);
1866 qp_params.use_local_mac = true;
1867 ether_addr_copy(qp_params.local_mac_addr, dev->ndev->dev_addr);
1869 DP_DEBUG(dev, QEDR_MSG_QP, "dgid=%x:%x:%x:%x\n",
1870 qp_params.dgid.dwords[0], qp_params.dgid.dwords[1],
1871 qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]);
1872 DP_DEBUG(dev, QEDR_MSG_QP, "sgid=%x:%x:%x:%x\n",
1873 qp_params.sgid.dwords[0], qp_params.sgid.dwords[1],
1874 qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]);
1875 DP_DEBUG(dev, QEDR_MSG_QP, "remote_mac=[%pM]\n",
1876 qp_params.remote_mac_addr);
1878 qp_params.mtu = qp->mtu;
1879 qp_params.lb_indication = false;
1882 if (!qp_params.mtu) {
1883 /* Stay with current MTU */
1885 qp_params.mtu = qp->mtu;
1888 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
1891 if (attr_mask & IB_QP_TIMEOUT) {
1892 SET_FIELD(qp_params.modify_flags,
1893 QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1);
1895 qp_params.ack_timeout = attr->timeout;
1896 if (attr->timeout) {
1899 temp = 4096 * (1UL << attr->timeout) / 1000 / 1000;
1900 /* FW requires [msec] */
1901 qp_params.ack_timeout = temp;
1904 qp_params.ack_timeout = 0;
1907 if (attr_mask & IB_QP_RETRY_CNT) {
1908 SET_FIELD(qp_params.modify_flags,
1909 QED_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1);
1910 qp_params.retry_cnt = attr->retry_cnt;
1913 if (attr_mask & IB_QP_RNR_RETRY) {
1914 SET_FIELD(qp_params.modify_flags,
1915 QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, 1);
1916 qp_params.rnr_retry_cnt = attr->rnr_retry;
1919 if (attr_mask & IB_QP_RQ_PSN) {
1920 SET_FIELD(qp_params.modify_flags,
1921 QED_ROCE_MODIFY_QP_VALID_RQ_PSN, 1);
1922 qp_params.rq_psn = attr->rq_psn;
1923 qp->rq_psn = attr->rq_psn;
1926 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
1927 if (attr->max_rd_atomic > dev->attr.max_qp_req_rd_atomic_resc) {
1930 "unsupported max_rd_atomic=%d, supported=%d\n",
1931 attr->max_rd_atomic,
1932 dev->attr.max_qp_req_rd_atomic_resc);
1936 SET_FIELD(qp_params.modify_flags,
1937 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, 1);
1938 qp_params.max_rd_atomic_req = attr->max_rd_atomic;
1941 if (attr_mask & IB_QP_MIN_RNR_TIMER) {
1942 SET_FIELD(qp_params.modify_flags,
1943 QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, 1);
1944 qp_params.min_rnr_nak_timer = attr->min_rnr_timer;
1947 if (attr_mask & IB_QP_SQ_PSN) {
1948 SET_FIELD(qp_params.modify_flags,
1949 QED_ROCE_MODIFY_QP_VALID_SQ_PSN, 1);
1950 qp_params.sq_psn = attr->sq_psn;
1951 qp->sq_psn = attr->sq_psn;
1954 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
1955 if (attr->max_dest_rd_atomic >
1956 dev->attr.max_qp_resp_rd_atomic_resc) {
1958 "unsupported max_dest_rd_atomic=%d, supported=%d\n",
1959 attr->max_dest_rd_atomic,
1960 dev->attr.max_qp_resp_rd_atomic_resc);
1966 SET_FIELD(qp_params.modify_flags,
1967 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, 1);
1968 qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic;
1971 if (attr_mask & IB_QP_DEST_QPN) {
1972 SET_FIELD(qp_params.modify_flags,
1973 QED_ROCE_MODIFY_QP_VALID_DEST_QP, 1);
1975 qp_params.dest_qp = attr->dest_qp_num;
1976 qp->dest_qp_num = attr->dest_qp_num;
1979 if (qp->qp_type != IB_QPT_GSI)
1980 rc = dev->ops->rdma_modify_qp(dev->rdma_ctx,
1981 qp->qed_qp, &qp_params);
1983 if (attr_mask & IB_QP_STATE) {
1984 if ((qp->qp_type != IB_QPT_GSI) && (!udata))
1985 rc = qedr_update_qp_state(dev, qp, qp_params.new_state);
1986 qp->state = qp_params.new_state;
1993 static int qedr_to_ib_qp_acc_flags(struct qed_rdma_query_qp_out_params *params)
1995 int ib_qp_acc_flags = 0;
1997 if (params->incoming_rdma_write_en)
1998 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE;
1999 if (params->incoming_rdma_read_en)
2000 ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ;
2001 if (params->incoming_atomic_en)
2002 ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC;
2003 ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE;
2004 return ib_qp_acc_flags;
2007 int qedr_query_qp(struct ib_qp *ibqp,
2008 struct ib_qp_attr *qp_attr,
2009 int attr_mask, struct ib_qp_init_attr *qp_init_attr)
2011 struct qed_rdma_query_qp_out_params params;
2012 struct qedr_qp *qp = get_qedr_qp(ibqp);
2013 struct qedr_dev *dev = qp->dev;
2016 memset(¶ms, 0, sizeof(params));
2018 rc = dev->ops->rdma_query_qp(dev->rdma_ctx, qp->qed_qp, ¶ms);
2022 memset(qp_attr, 0, sizeof(*qp_attr));
2023 memset(qp_init_attr, 0, sizeof(*qp_init_attr));
2025 qp_attr->qp_state = qedr_get_ibqp_state(params.state);
2026 qp_attr->cur_qp_state = qedr_get_ibqp_state(params.state);
2027 qp_attr->path_mtu = ib_mtu_int_to_enum(params.mtu);
2028 qp_attr->path_mig_state = IB_MIG_MIGRATED;
2029 qp_attr->rq_psn = params.rq_psn;
2030 qp_attr->sq_psn = params.sq_psn;
2031 qp_attr->dest_qp_num = params.dest_qp;
2033 qp_attr->qp_access_flags = qedr_to_ib_qp_acc_flags(¶ms);
2035 qp_attr->cap.max_send_wr = qp->sq.max_wr;
2036 qp_attr->cap.max_recv_wr = qp->rq.max_wr;
2037 qp_attr->cap.max_send_sge = qp->sq.max_sges;
2038 qp_attr->cap.max_recv_sge = qp->rq.max_sges;
2039 qp_attr->cap.max_inline_data = ROCE_REQ_MAX_INLINE_DATA_SIZE;
2040 qp_init_attr->cap = qp_attr->cap;
2042 qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
2043 rdma_ah_set_grh(&qp_attr->ah_attr, NULL,
2044 params.flow_label, qp->sgid_idx,
2045 params.hop_limit_ttl, params.traffic_class_tos);
2046 rdma_ah_set_dgid_raw(&qp_attr->ah_attr, ¶ms.dgid.bytes[0]);
2047 rdma_ah_set_port_num(&qp_attr->ah_attr, 1);
2048 rdma_ah_set_sl(&qp_attr->ah_attr, 0);
2049 qp_attr->timeout = params.timeout;
2050 qp_attr->rnr_retry = params.rnr_retry;
2051 qp_attr->retry_cnt = params.retry_cnt;
2052 qp_attr->min_rnr_timer = params.min_rnr_nak_timer;
2053 qp_attr->pkey_index = params.pkey_index;
2054 qp_attr->port_num = 1;
2055 rdma_ah_set_path_bits(&qp_attr->ah_attr, 0);
2056 rdma_ah_set_static_rate(&qp_attr->ah_attr, 0);
2057 qp_attr->alt_pkey_index = 0;
2058 qp_attr->alt_port_num = 0;
2059 qp_attr->alt_timeout = 0;
2060 memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr));
2062 qp_attr->sq_draining = (params.state == QED_ROCE_QP_STATE_SQD) ? 1 : 0;
2063 qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic;
2064 qp_attr->max_rd_atomic = params.max_rd_atomic;
2065 qp_attr->en_sqd_async_notify = (params.sqd_async) ? 1 : 0;
2067 DP_DEBUG(dev, QEDR_MSG_QP, "QEDR_QUERY_QP: max_inline_data=%d\n",
2068 qp_attr->cap.max_inline_data);
2074 int qedr_free_qp_resources(struct qedr_dev *dev, struct qedr_qp *qp)
2078 if (qp->qp_type != IB_QPT_GSI) {
2079 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2084 if (qp->ibqp.uobject && qp->ibqp.uobject->context)
2085 qedr_cleanup_user(dev, qp);
2087 qedr_cleanup_kernel(dev, qp);
2092 int qedr_destroy_qp(struct ib_qp *ibqp)
2094 struct qedr_qp *qp = get_qedr_qp(ibqp);
2095 struct qedr_dev *dev = qp->dev;
2096 struct ib_qp_attr attr;
2100 DP_DEBUG(dev, QEDR_MSG_QP, "destroy qp: destroying %p, qp type=%d\n",
2103 if ((qp->state != QED_ROCE_QP_STATE_RESET) &&
2104 (qp->state != QED_ROCE_QP_STATE_ERR) &&
2105 (qp->state != QED_ROCE_QP_STATE_INIT)) {
2107 attr.qp_state = IB_QPS_ERR;
2108 attr_mask |= IB_QP_STATE;
2110 /* Change the QP state to ERROR */
2111 qedr_modify_qp(ibqp, &attr, attr_mask, NULL);
2114 if (qp->qp_type == IB_QPT_GSI)
2115 qedr_destroy_gsi_qp(dev);
2117 qedr_free_qp_resources(dev, qp);
2124 struct ib_ah *qedr_create_ah(struct ib_pd *ibpd, struct rdma_ah_attr *attr,
2125 struct ib_udata *udata)
2129 ah = kzalloc(sizeof(*ah), GFP_ATOMIC);
2131 return ERR_PTR(-ENOMEM);
2138 int qedr_destroy_ah(struct ib_ah *ibah)
2140 struct qedr_ah *ah = get_qedr_ah(ibah);
2146 static void free_mr_info(struct qedr_dev *dev, struct mr_info *info)
2148 struct qedr_pbl *pbl, *tmp;
2150 if (info->pbl_table)
2151 list_add_tail(&info->pbl_table->list_entry,
2152 &info->free_pbl_list);
2154 if (!list_empty(&info->inuse_pbl_list))
2155 list_splice(&info->inuse_pbl_list, &info->free_pbl_list);
2157 list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) {
2158 list_del(&pbl->list_entry);
2159 qedr_free_pbl(dev, &info->pbl_info, pbl);
2163 static int init_mr_info(struct qedr_dev *dev, struct mr_info *info,
2164 size_t page_list_len, bool two_layered)
2166 struct qedr_pbl *tmp;
2169 INIT_LIST_HEAD(&info->free_pbl_list);
2170 INIT_LIST_HEAD(&info->inuse_pbl_list);
2172 rc = qedr_prepare_pbl_tbl(dev, &info->pbl_info,
2173 page_list_len, two_layered);
2177 info->pbl_table = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2178 if (IS_ERR(info->pbl_table)) {
2179 rc = PTR_ERR(info->pbl_table);
2183 DP_DEBUG(dev, QEDR_MSG_MR, "pbl_table_pa = %pa\n",
2184 &info->pbl_table->pa);
2186 /* in usual case we use 2 PBLs, so we add one to free
2187 * list and allocating another one
2189 tmp = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2191 DP_DEBUG(dev, QEDR_MSG_MR, "Extra PBL is not allocated\n");
2195 list_add_tail(&tmp->list_entry, &info->free_pbl_list);
2197 DP_DEBUG(dev, QEDR_MSG_MR, "extra pbl_table_pa = %pa\n", &tmp->pa);
2201 free_mr_info(dev, info);
2206 struct ib_mr *qedr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
2207 u64 usr_addr, int acc, struct ib_udata *udata)
2209 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2214 pd = get_qedr_pd(ibpd);
2215 DP_DEBUG(dev, QEDR_MSG_MR,
2216 "qedr_register user mr pd = %d start = %lld, len = %lld, usr_addr = %lld, acc = %d\n",
2217 pd->pd_id, start, len, usr_addr, acc);
2219 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE))
2220 return ERR_PTR(-EINVAL);
2222 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2226 mr->type = QEDR_MR_USER;
2228 mr->umem = ib_umem_get(ibpd->uobject->context, start, len, acc, 0);
2229 if (IS_ERR(mr->umem)) {
2234 rc = init_mr_info(dev, &mr->info, ib_umem_page_count(mr->umem), 1);
2238 qedr_populate_pbls(dev, mr->umem, mr->info.pbl_table,
2239 &mr->info.pbl_info, mr->umem->page_shift);
2241 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2243 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2247 /* Index only, 18 bit long, lkey = itid << 8 | key */
2248 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
2250 mr->hw_mr.pd = pd->pd_id;
2251 mr->hw_mr.local_read = 1;
2252 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
2253 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
2254 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
2255 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
2256 mr->hw_mr.mw_bind = false;
2257 mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa;
2258 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
2259 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
2260 mr->hw_mr.page_size_log = mr->umem->page_shift;
2261 mr->hw_mr.fbo = ib_umem_offset(mr->umem);
2262 mr->hw_mr.length = len;
2263 mr->hw_mr.vaddr = usr_addr;
2264 mr->hw_mr.zbva = false;
2265 mr->hw_mr.phy_mr = false;
2266 mr->hw_mr.dma_mr = false;
2268 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2270 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2274 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2275 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
2276 mr->hw_mr.remote_atomic)
2277 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2279 DP_DEBUG(dev, QEDR_MSG_MR, "register user mr lkey: %x\n",
2284 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2286 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
2292 int qedr_dereg_mr(struct ib_mr *ib_mr)
2294 struct qedr_mr *mr = get_qedr_mr(ib_mr);
2295 struct qedr_dev *dev = get_qedr_dev(ib_mr->device);
2298 rc = dev->ops->rdma_deregister_tid(dev->rdma_ctx, mr->hw_mr.itid);
2302 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2304 if ((mr->type != QEDR_MR_DMA) && (mr->type != QEDR_MR_FRMR))
2305 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
2307 /* it could be user registered memory. */
2309 ib_umem_release(mr->umem);
2316 static struct qedr_mr *__qedr_alloc_mr(struct ib_pd *ibpd,
2317 int max_page_list_len)
2319 struct qedr_pd *pd = get_qedr_pd(ibpd);
2320 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2324 DP_DEBUG(dev, QEDR_MSG_MR,
2325 "qedr_alloc_frmr pd = %d max_page_list_len= %d\n", pd->pd_id,
2328 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2333 mr->type = QEDR_MR_FRMR;
2335 rc = init_mr_info(dev, &mr->info, max_page_list_len, 1);
2339 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2341 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2345 /* Index only, 18 bit long, lkey = itid << 8 | key */
2346 mr->hw_mr.tid_type = QED_RDMA_TID_FMR;
2348 mr->hw_mr.pd = pd->pd_id;
2349 mr->hw_mr.local_read = 1;
2350 mr->hw_mr.local_write = 0;
2351 mr->hw_mr.remote_read = 0;
2352 mr->hw_mr.remote_write = 0;
2353 mr->hw_mr.remote_atomic = 0;
2354 mr->hw_mr.mw_bind = false;
2355 mr->hw_mr.pbl_ptr = 0;
2356 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
2357 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
2359 mr->hw_mr.length = 0;
2360 mr->hw_mr.vaddr = 0;
2361 mr->hw_mr.zbva = false;
2362 mr->hw_mr.phy_mr = true;
2363 mr->hw_mr.dma_mr = false;
2365 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2367 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2371 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2372 mr->ibmr.rkey = mr->ibmr.lkey;
2374 DP_DEBUG(dev, QEDR_MSG_MR, "alloc frmr: %x\n", mr->ibmr.lkey);
2378 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2384 struct ib_mr *qedr_alloc_mr(struct ib_pd *ibpd,
2385 enum ib_mr_type mr_type, u32 max_num_sg)
2387 struct qedr_dev *dev;
2390 if (mr_type != IB_MR_TYPE_MEM_REG)
2391 return ERR_PTR(-EINVAL);
2393 mr = __qedr_alloc_mr(ibpd, max_num_sg);
2396 return ERR_PTR(-EINVAL);
2403 static int qedr_set_page(struct ib_mr *ibmr, u64 addr)
2405 struct qedr_mr *mr = get_qedr_mr(ibmr);
2406 struct qedr_pbl *pbl_table;
2407 struct regpair *pbe;
2410 if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) {
2411 DP_ERR(mr->dev, "qedr_set_page failes when %d\n", mr->npages);
2415 DP_DEBUG(mr->dev, QEDR_MSG_MR, "qedr_set_page pages[%d] = 0x%llx\n",
2418 pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64);
2419 pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page);
2420 pbe = (struct regpair *)pbl_table->va;
2421 pbe += mr->npages % pbes_in_page;
2422 pbe->lo = cpu_to_le32((u32)addr);
2423 pbe->hi = cpu_to_le32((u32)upper_32_bits(addr));
2430 static void handle_completed_mrs(struct qedr_dev *dev, struct mr_info *info)
2432 int work = info->completed - info->completed_handled - 1;
2434 DP_DEBUG(dev, QEDR_MSG_MR, "Special FMR work = %d\n", work);
2435 while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) {
2436 struct qedr_pbl *pbl;
2438 /* Free all the page list that are possible to be freed
2439 * (all the ones that were invalidated), under the assumption
2440 * that if an FMR was completed successfully that means that
2441 * if there was an invalidate operation before it also ended
2443 pbl = list_first_entry(&info->inuse_pbl_list,
2444 struct qedr_pbl, list_entry);
2445 list_move_tail(&pbl->list_entry, &info->free_pbl_list);
2446 info->completed_handled++;
2450 int qedr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
2451 int sg_nents, unsigned int *sg_offset)
2453 struct qedr_mr *mr = get_qedr_mr(ibmr);
2457 handle_completed_mrs(mr->dev, &mr->info);
2458 return ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qedr_set_page);
2461 struct ib_mr *qedr_get_dma_mr(struct ib_pd *ibpd, int acc)
2463 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2464 struct qedr_pd *pd = get_qedr_pd(ibpd);
2468 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2470 return ERR_PTR(-ENOMEM);
2472 mr->type = QEDR_MR_DMA;
2474 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2476 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2480 /* index only, 18 bit long, lkey = itid << 8 | key */
2481 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
2482 mr->hw_mr.pd = pd->pd_id;
2483 mr->hw_mr.local_read = 1;
2484 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
2485 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
2486 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
2487 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
2488 mr->hw_mr.dma_mr = true;
2490 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2492 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2496 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2497 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
2498 mr->hw_mr.remote_atomic)
2499 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2501 DP_DEBUG(dev, QEDR_MSG_MR, "get dma mr: lkey = %x\n", mr->ibmr.lkey);
2505 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2511 static inline int qedr_wq_is_full(struct qedr_qp_hwq_info *wq)
2513 return (((wq->prod + 1) % wq->max_wr) == wq->cons);
2516 static int sge_data_len(struct ib_sge *sg_list, int num_sge)
2520 for (i = 0; i < num_sge; i++)
2521 len += sg_list[i].length;
2526 static void swap_wqe_data64(u64 *p)
2530 for (i = 0; i < QEDR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++)
2531 *p = cpu_to_be64(cpu_to_le64(*p));
2534 static u32 qedr_prepare_sq_inline_data(struct qedr_dev *dev,
2535 struct qedr_qp *qp, u8 *wqe_size,
2536 struct ib_send_wr *wr,
2537 struct ib_send_wr **bad_wr, u8 *bits,
2540 u32 data_size = sge_data_len(wr->sg_list, wr->num_sge);
2541 char *seg_prt, *wqe;
2544 if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) {
2545 DP_ERR(dev, "Too much inline data in WR: %d\n", data_size);
2559 /* Copy data inline */
2560 for (i = 0; i < wr->num_sge; i++) {
2561 u32 len = wr->sg_list[i].length;
2562 void *src = (void *)(uintptr_t)wr->sg_list[i].addr;
2567 /* New segment required */
2569 wqe = (char *)qed_chain_produce(&qp->sq.pbl);
2571 seg_siz = sizeof(struct rdma_sq_common_wqe);
2575 /* Calculate currently allowed length */
2576 cur = min_t(u32, len, seg_siz);
2577 memcpy(seg_prt, src, cur);
2579 /* Update segment variables */
2583 /* Update sge variables */
2587 /* Swap fully-completed segments */
2589 swap_wqe_data64((u64 *)wqe);
2593 /* swap last not completed segment */
2595 swap_wqe_data64((u64 *)wqe);
2600 #define RQ_SGE_SET(sge, vaddr, vlength, vflags) \
2602 DMA_REGPAIR_LE(sge->addr, vaddr); \
2603 (sge)->length = cpu_to_le32(vlength); \
2604 (sge)->flags = cpu_to_le32(vflags); \
2607 #define SRQ_HDR_SET(hdr, vwr_id, num_sge) \
2609 DMA_REGPAIR_LE(hdr->wr_id, vwr_id); \
2610 (hdr)->num_sges = num_sge; \
2613 #define SRQ_SGE_SET(sge, vaddr, vlength, vlkey) \
2615 DMA_REGPAIR_LE(sge->addr, vaddr); \
2616 (sge)->length = cpu_to_le32(vlength); \
2617 (sge)->l_key = cpu_to_le32(vlkey); \
2620 static u32 qedr_prepare_sq_sges(struct qedr_qp *qp, u8 *wqe_size,
2621 struct ib_send_wr *wr)
2626 for (i = 0; i < wr->num_sge; i++) {
2627 struct rdma_sq_sge *sge = qed_chain_produce(&qp->sq.pbl);
2629 DMA_REGPAIR_LE(sge->addr, wr->sg_list[i].addr);
2630 sge->l_key = cpu_to_le32(wr->sg_list[i].lkey);
2631 sge->length = cpu_to_le32(wr->sg_list[i].length);
2632 data_size += wr->sg_list[i].length;
2636 *wqe_size += wr->num_sge;
2641 static u32 qedr_prepare_sq_rdma_data(struct qedr_dev *dev,
2643 struct rdma_sq_rdma_wqe_1st *rwqe,
2644 struct rdma_sq_rdma_wqe_2nd *rwqe2,
2645 struct ib_send_wr *wr,
2646 struct ib_send_wr **bad_wr)
2648 rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey);
2649 DMA_REGPAIR_LE(rwqe2->remote_va, rdma_wr(wr)->remote_addr);
2651 if (wr->send_flags & IB_SEND_INLINE &&
2652 (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM ||
2653 wr->opcode == IB_WR_RDMA_WRITE)) {
2656 SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1);
2657 return qedr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, wr,
2658 bad_wr, &rwqe->flags, flags);
2661 return qedr_prepare_sq_sges(qp, &rwqe->wqe_size, wr);
2664 static u32 qedr_prepare_sq_send_data(struct qedr_dev *dev,
2666 struct rdma_sq_send_wqe_1st *swqe,
2667 struct rdma_sq_send_wqe_2st *swqe2,
2668 struct ib_send_wr *wr,
2669 struct ib_send_wr **bad_wr)
2671 memset(swqe2, 0, sizeof(*swqe2));
2672 if (wr->send_flags & IB_SEND_INLINE) {
2675 SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1);
2676 return qedr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, wr,
2677 bad_wr, &swqe->flags, flags);
2680 return qedr_prepare_sq_sges(qp, &swqe->wqe_size, wr);
2683 static int qedr_prepare_reg(struct qedr_qp *qp,
2684 struct rdma_sq_fmr_wqe_1st *fwqe1,
2685 struct ib_reg_wr *wr)
2687 struct qedr_mr *mr = get_qedr_mr(wr->mr);
2688 struct rdma_sq_fmr_wqe_2nd *fwqe2;
2690 fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)qed_chain_produce(&qp->sq.pbl);
2691 fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova);
2692 fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova);
2693 fwqe1->l_key = wr->key;
2695 fwqe2->access_ctrl = 0;
2697 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ,
2698 !!(wr->access & IB_ACCESS_REMOTE_READ));
2699 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE,
2700 !!(wr->access & IB_ACCESS_REMOTE_WRITE));
2701 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC,
2702 !!(wr->access & IB_ACCESS_REMOTE_ATOMIC));
2703 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1);
2704 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE,
2705 !!(wr->access & IB_ACCESS_LOCAL_WRITE));
2706 fwqe2->fmr_ctrl = 0;
2708 SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG,
2709 ilog2(mr->ibmr.page_size) - 12);
2711 fwqe2->length_hi = 0;
2712 fwqe2->length_lo = mr->ibmr.length;
2713 fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa);
2714 fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa);
2716 qp->wqe_wr_id[qp->sq.prod].mr = mr;
2721 static enum ib_wc_opcode qedr_ib_to_wc_opcode(enum ib_wr_opcode opcode)
2724 case IB_WR_RDMA_WRITE:
2725 case IB_WR_RDMA_WRITE_WITH_IMM:
2726 return IB_WC_RDMA_WRITE;
2727 case IB_WR_SEND_WITH_IMM:
2729 case IB_WR_SEND_WITH_INV:
2731 case IB_WR_RDMA_READ:
2732 return IB_WC_RDMA_READ;
2733 case IB_WR_ATOMIC_CMP_AND_SWP:
2734 return IB_WC_COMP_SWAP;
2735 case IB_WR_ATOMIC_FETCH_AND_ADD:
2736 return IB_WC_FETCH_ADD;
2738 return IB_WC_REG_MR;
2739 case IB_WR_LOCAL_INV:
2740 return IB_WC_LOCAL_INV;
2746 static inline bool qedr_can_post_send(struct qedr_qp *qp, struct ib_send_wr *wr)
2748 int wq_is_full, err_wr, pbl_is_full;
2749 struct qedr_dev *dev = qp->dev;
2751 /* prevent SQ overflow and/or processing of a bad WR */
2752 err_wr = wr->num_sge > qp->sq.max_sges;
2753 wq_is_full = qedr_wq_is_full(&qp->sq);
2754 pbl_is_full = qed_chain_get_elem_left_u32(&qp->sq.pbl) <
2755 QEDR_MAX_SQE_ELEMENTS_PER_SQE;
2756 if (wq_is_full || err_wr || pbl_is_full) {
2757 if (wq_is_full && !(qp->err_bitmap & QEDR_QP_ERR_SQ_FULL)) {
2759 "error: WQ is full. Post send on QP %p failed (this error appears only once)\n",
2761 qp->err_bitmap |= QEDR_QP_ERR_SQ_FULL;
2764 if (err_wr && !(qp->err_bitmap & QEDR_QP_ERR_BAD_SR)) {
2766 "error: WR is bad. Post send on QP %p failed (this error appears only once)\n",
2768 qp->err_bitmap |= QEDR_QP_ERR_BAD_SR;
2772 !(qp->err_bitmap & QEDR_QP_ERR_SQ_PBL_FULL)) {
2774 "error: WQ PBL is full. Post send on QP %p failed (this error appears only once)\n",
2776 qp->err_bitmap |= QEDR_QP_ERR_SQ_PBL_FULL;
2783 static int __qedr_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
2784 struct ib_send_wr **bad_wr)
2786 struct qedr_dev *dev = get_qedr_dev(ibqp->device);
2787 struct qedr_qp *qp = get_qedr_qp(ibqp);
2788 struct rdma_sq_atomic_wqe_1st *awqe1;
2789 struct rdma_sq_atomic_wqe_2nd *awqe2;
2790 struct rdma_sq_atomic_wqe_3rd *awqe3;
2791 struct rdma_sq_send_wqe_2st *swqe2;
2792 struct rdma_sq_local_inv_wqe *iwqe;
2793 struct rdma_sq_rdma_wqe_2nd *rwqe2;
2794 struct rdma_sq_send_wqe_1st *swqe;
2795 struct rdma_sq_rdma_wqe_1st *rwqe;
2796 struct rdma_sq_fmr_wqe_1st *fwqe1;
2797 struct rdma_sq_common_wqe *wqe;
2802 if (!qedr_can_post_send(qp, wr)) {
2807 wqe = qed_chain_produce(&qp->sq.pbl);
2808 qp->wqe_wr_id[qp->sq.prod].signaled =
2809 !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled;
2812 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG,
2813 !!(wr->send_flags & IB_SEND_SOLICITED));
2814 comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || qp->signaled;
2815 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp);
2816 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG,
2817 !!(wr->send_flags & IB_SEND_FENCE));
2818 wqe->prev_wqe_size = qp->prev_wqe_size;
2820 qp->wqe_wr_id[qp->sq.prod].opcode = qedr_ib_to_wc_opcode(wr->opcode);
2822 switch (wr->opcode) {
2823 case IB_WR_SEND_WITH_IMM:
2824 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM;
2825 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
2827 swqe2 = qed_chain_produce(&qp->sq.pbl);
2829 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.imm_data);
2830 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
2832 swqe->length = cpu_to_le32(length);
2833 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
2834 qp->prev_wqe_size = swqe->wqe_size;
2835 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
2838 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND;
2839 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
2842 swqe2 = qed_chain_produce(&qp->sq.pbl);
2843 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
2845 swqe->length = cpu_to_le32(length);
2846 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
2847 qp->prev_wqe_size = swqe->wqe_size;
2848 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
2850 case IB_WR_SEND_WITH_INV:
2851 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE;
2852 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
2853 swqe2 = qed_chain_produce(&qp->sq.pbl);
2855 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.invalidate_rkey);
2856 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
2858 swqe->length = cpu_to_le32(length);
2859 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
2860 qp->prev_wqe_size = swqe->wqe_size;
2861 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
2864 case IB_WR_RDMA_WRITE_WITH_IMM:
2865 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM;
2866 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
2869 rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data));
2870 rwqe2 = qed_chain_produce(&qp->sq.pbl);
2871 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
2873 rwqe->length = cpu_to_le32(length);
2874 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
2875 qp->prev_wqe_size = rwqe->wqe_size;
2876 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
2878 case IB_WR_RDMA_WRITE:
2879 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR;
2880 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
2883 rwqe2 = qed_chain_produce(&qp->sq.pbl);
2884 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
2886 rwqe->length = cpu_to_le32(length);
2887 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
2888 qp->prev_wqe_size = rwqe->wqe_size;
2889 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
2891 case IB_WR_RDMA_READ_WITH_INV:
2893 "RDMA READ WITH INVALIDATE not supported\n");
2898 case IB_WR_RDMA_READ:
2899 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD;
2900 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
2903 rwqe2 = qed_chain_produce(&qp->sq.pbl);
2904 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
2906 rwqe->length = cpu_to_le32(length);
2907 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
2908 qp->prev_wqe_size = rwqe->wqe_size;
2909 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
2912 case IB_WR_ATOMIC_CMP_AND_SWP:
2913 case IB_WR_ATOMIC_FETCH_AND_ADD:
2914 awqe1 = (struct rdma_sq_atomic_wqe_1st *)wqe;
2915 awqe1->wqe_size = 4;
2917 awqe2 = qed_chain_produce(&qp->sq.pbl);
2918 DMA_REGPAIR_LE(awqe2->remote_va, atomic_wr(wr)->remote_addr);
2919 awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey);
2921 awqe3 = qed_chain_produce(&qp->sq.pbl);
2923 if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
2924 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD;
2925 DMA_REGPAIR_LE(awqe3->swap_data,
2926 atomic_wr(wr)->compare_add);
2928 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP;
2929 DMA_REGPAIR_LE(awqe3->swap_data,
2930 atomic_wr(wr)->swap);
2931 DMA_REGPAIR_LE(awqe3->cmp_data,
2932 atomic_wr(wr)->compare_add);
2935 qedr_prepare_sq_sges(qp, NULL, wr);
2937 qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->wqe_size;
2938 qp->prev_wqe_size = awqe1->wqe_size;
2941 case IB_WR_LOCAL_INV:
2942 iwqe = (struct rdma_sq_local_inv_wqe *)wqe;
2945 iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE;
2946 iwqe->inv_l_key = wr->ex.invalidate_rkey;
2947 qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->wqe_size;
2948 qp->prev_wqe_size = iwqe->wqe_size;
2951 DP_DEBUG(dev, QEDR_MSG_CQ, "REG_MR\n");
2952 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR;
2953 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe;
2954 fwqe1->wqe_size = 2;
2956 rc = qedr_prepare_reg(qp, fwqe1, reg_wr(wr));
2958 DP_ERR(dev, "IB_REG_MR failed rc=%d\n", rc);
2963 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size;
2964 qp->prev_wqe_size = fwqe1->wqe_size;
2967 DP_ERR(dev, "invalid opcode 0x%x!\n", wr->opcode);
2976 /* Restore prod to its position before
2977 * this WR was processed
2979 value = le16_to_cpu(qp->sq.db_data.data.value);
2980 qed_chain_set_prod(&qp->sq.pbl, value, wqe);
2982 /* Restore prev_wqe_size */
2983 qp->prev_wqe_size = wqe->prev_wqe_size;
2985 DP_ERR(dev, "POST SEND FAILED\n");
2991 int qedr_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
2992 struct ib_send_wr **bad_wr)
2994 struct qedr_dev *dev = get_qedr_dev(ibqp->device);
2995 struct qedr_qp *qp = get_qedr_qp(ibqp);
2996 unsigned long flags;
3001 if (qp->qp_type == IB_QPT_GSI)
3002 return qedr_gsi_post_send(ibqp, wr, bad_wr);
3004 spin_lock_irqsave(&qp->q_lock, flags);
3006 if ((qp->state != QED_ROCE_QP_STATE_RTS) &&
3007 (qp->state != QED_ROCE_QP_STATE_ERR) &&
3008 (qp->state != QED_ROCE_QP_STATE_SQD)) {
3009 spin_unlock_irqrestore(&qp->q_lock, flags);
3011 DP_DEBUG(dev, QEDR_MSG_CQ,
3012 "QP in wrong state! QP icid=0x%x state %d\n",
3013 qp->icid, qp->state);
3018 rc = __qedr_post_send(ibqp, wr, bad_wr);
3022 qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id;
3024 qedr_inc_sw_prod(&qp->sq);
3026 qp->sq.db_data.data.value++;
3032 * If there was a failure in the first WR then it will be triggered in
3033 * vane. However this is not harmful (as long as the producer value is
3034 * unchanged). For performance reasons we avoid checking for this
3035 * redundant doorbell.
3038 writel(qp->sq.db_data.raw, qp->sq.db);
3040 /* Make sure write sticks */
3043 spin_unlock_irqrestore(&qp->q_lock, flags);
3048 int qedr_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr,
3049 struct ib_recv_wr **bad_wr)
3051 struct qedr_qp *qp = get_qedr_qp(ibqp);
3052 struct qedr_dev *dev = qp->dev;
3053 unsigned long flags;
3056 if (qp->qp_type == IB_QPT_GSI)
3057 return qedr_gsi_post_recv(ibqp, wr, bad_wr);
3059 spin_lock_irqsave(&qp->q_lock, flags);
3061 if (qp->state == QED_ROCE_QP_STATE_RESET) {
3062 spin_unlock_irqrestore(&qp->q_lock, flags);
3070 if (qed_chain_get_elem_left_u32(&qp->rq.pbl) <
3071 QEDR_MAX_RQE_ELEMENTS_PER_RQE ||
3072 wr->num_sge > qp->rq.max_sges) {
3073 DP_ERR(dev, "Can't post WR (%d < %d) || (%d > %d)\n",
3074 qed_chain_get_elem_left_u32(&qp->rq.pbl),
3075 QEDR_MAX_RQE_ELEMENTS_PER_RQE, wr->num_sge,
3081 for (i = 0; i < wr->num_sge; i++) {
3083 struct rdma_rq_sge *rqe =
3084 qed_chain_produce(&qp->rq.pbl);
3086 /* First one must include the number
3087 * of SGE in the list
3090 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES,
3093 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY,
3094 wr->sg_list[i].lkey);
3096 RQ_SGE_SET(rqe, wr->sg_list[i].addr,
3097 wr->sg_list[i].length, flags);
3100 /* Special case of no sges. FW requires between 1-4 sges...
3101 * in this case we need to post 1 sge with length zero. this is
3102 * because rdma write with immediate consumes an RQ.
3106 struct rdma_rq_sge *rqe =
3107 qed_chain_produce(&qp->rq.pbl);
3109 /* First one must include the number
3110 * of SGE in the list
3112 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY, 0);
3113 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1);
3115 RQ_SGE_SET(rqe, 0, 0, flags);
3119 qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id;
3120 qp->rqe_wr_id[qp->rq.prod].wqe_size = i;
3122 qedr_inc_sw_prod(&qp->rq);
3124 /* Flush all the writes before signalling doorbell */
3127 qp->rq.db_data.data.value++;
3129 writel(qp->rq.db_data.raw, qp->rq.db);
3131 /* Make sure write sticks */
3137 spin_unlock_irqrestore(&qp->q_lock, flags);
3142 static int is_valid_cqe(struct qedr_cq *cq, union rdma_cqe *cqe)
3144 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3146 return (resp_cqe->flags & RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK) ==
3150 static struct qedr_qp *cqe_get_qp(union rdma_cqe *cqe)
3152 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3155 qp = (struct qedr_qp *)(uintptr_t)HILO_GEN(resp_cqe->qp_handle.hi,
3156 resp_cqe->qp_handle.lo,
3161 static enum rdma_cqe_type cqe_get_type(union rdma_cqe *cqe)
3163 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3165 return GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE);
3168 /* Return latest CQE (needs processing) */
3169 static union rdma_cqe *get_cqe(struct qedr_cq *cq)
3171 return cq->latest_cqe;
3174 /* In fmr we need to increase the number of fmr completed counter for the fmr
3175 * algorithm determining whether we can free a pbl or not.
3176 * we need to perform this whether the work request was signaled or not. for
3177 * this purpose we call this function from the condition that checks if a wr
3178 * should be skipped, to make sure we don't miss it ( possibly this fmr
3179 * operation was not signalted)
3181 static inline void qedr_chk_if_fmr(struct qedr_qp *qp)
3183 if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR)
3184 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
3187 static int process_req(struct qedr_dev *dev, struct qedr_qp *qp,
3188 struct qedr_cq *cq, int num_entries,
3189 struct ib_wc *wc, u16 hw_cons, enum ib_wc_status status,
3194 while (num_entries && qp->sq.wqe_cons != hw_cons) {
3195 if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) {
3196 qedr_chk_if_fmr(qp);
3202 wc->status = status;
3205 wc->src_qp = qp->id;
3208 wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id;
3209 wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode;
3211 switch (wc->opcode) {
3212 case IB_WC_RDMA_WRITE:
3213 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
3215 case IB_WC_COMP_SWAP:
3216 case IB_WC_FETCH_ADD:
3220 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
3222 case IB_WC_RDMA_READ:
3224 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
3234 while (qp->wqe_wr_id[qp->sq.cons].wqe_size--)
3235 qed_chain_consume(&qp->sq.pbl);
3236 qedr_inc_sw_cons(&qp->sq);
3242 static int qedr_poll_cq_req(struct qedr_dev *dev,
3243 struct qedr_qp *qp, struct qedr_cq *cq,
3244 int num_entries, struct ib_wc *wc,
3245 struct rdma_cqe_requester *req)
3249 switch (req->status) {
3250 case RDMA_CQE_REQ_STS_OK:
3251 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
3254 case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR:
3255 if (qp->state != QED_ROCE_QP_STATE_ERR)
3257 "Error: POLL CQ with RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3258 cq->icid, qp->icid);
3259 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
3260 IB_WC_WR_FLUSH_ERR, 1);
3263 /* process all WQE before the cosumer */
3264 qp->state = QED_ROCE_QP_STATE_ERR;
3265 cnt = process_req(dev, qp, cq, num_entries, wc,
3266 req->sq_cons - 1, IB_WC_SUCCESS, 0);
3268 /* if we have extra WC fill it with actual error info */
3269 if (cnt < num_entries) {
3270 enum ib_wc_status wc_status;
3272 switch (req->status) {
3273 case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR:
3275 "Error: POLL CQ with RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3276 cq->icid, qp->icid);
3277 wc_status = IB_WC_BAD_RESP_ERR;
3279 case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR:
3281 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3282 cq->icid, qp->icid);
3283 wc_status = IB_WC_LOC_LEN_ERR;
3285 case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR:
3287 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3288 cq->icid, qp->icid);
3289 wc_status = IB_WC_LOC_QP_OP_ERR;
3291 case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR:
3293 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3294 cq->icid, qp->icid);
3295 wc_status = IB_WC_LOC_PROT_ERR;
3297 case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR:
3299 "Error: POLL CQ with RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3300 cq->icid, qp->icid);
3301 wc_status = IB_WC_MW_BIND_ERR;
3303 case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR:
3305 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3306 cq->icid, qp->icid);
3307 wc_status = IB_WC_REM_INV_REQ_ERR;
3309 case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR:
3311 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3312 cq->icid, qp->icid);
3313 wc_status = IB_WC_REM_ACCESS_ERR;
3315 case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR:
3317 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3318 cq->icid, qp->icid);
3319 wc_status = IB_WC_REM_OP_ERR;
3321 case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR:
3323 "Error: POLL CQ with RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3324 cq->icid, qp->icid);
3325 wc_status = IB_WC_RNR_RETRY_EXC_ERR;
3327 case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR:
3329 "Error: POLL CQ with ROCE_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3330 cq->icid, qp->icid);
3331 wc_status = IB_WC_RETRY_EXC_ERR;
3335 "Error: POLL CQ with IB_WC_GENERAL_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3336 cq->icid, qp->icid);
3337 wc_status = IB_WC_GENERAL_ERR;
3339 cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons,
3347 static inline int qedr_cqe_resp_status_to_ib(u8 status)
3350 case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR:
3351 return IB_WC_LOC_ACCESS_ERR;
3352 case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR:
3353 return IB_WC_LOC_LEN_ERR;
3354 case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR:
3355 return IB_WC_LOC_QP_OP_ERR;
3356 case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR:
3357 return IB_WC_LOC_PROT_ERR;
3358 case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR:
3359 return IB_WC_MW_BIND_ERR;
3360 case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR:
3361 return IB_WC_REM_INV_RD_REQ_ERR;
3362 case RDMA_CQE_RESP_STS_OK:
3363 return IB_WC_SUCCESS;
3365 return IB_WC_GENERAL_ERR;
3369 static inline int qedr_set_ok_cqe_resp_wc(struct rdma_cqe_responder *resp,
3372 wc->status = IB_WC_SUCCESS;
3373 wc->byte_len = le32_to_cpu(resp->length);
3375 if (resp->flags & QEDR_RESP_IMM) {
3376 wc->ex.imm_data = le32_to_cpu(resp->imm_data_or_inv_r_Key);
3377 wc->wc_flags |= IB_WC_WITH_IMM;
3379 if (resp->flags & QEDR_RESP_RDMA)
3380 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
3382 if (resp->flags & QEDR_RESP_INV)
3385 } else if (resp->flags & QEDR_RESP_INV) {
3386 wc->ex.imm_data = le32_to_cpu(resp->imm_data_or_inv_r_Key);
3387 wc->wc_flags |= IB_WC_WITH_INVALIDATE;
3389 if (resp->flags & QEDR_RESP_RDMA)
3392 } else if (resp->flags & QEDR_RESP_RDMA) {
3399 static void __process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
3400 struct qedr_cq *cq, struct ib_wc *wc,
3401 struct rdma_cqe_responder *resp, u64 wr_id)
3403 /* Must fill fields before qedr_set_ok_cqe_resp_wc() */
3404 wc->opcode = IB_WC_RECV;
3407 if (likely(resp->status == RDMA_CQE_RESP_STS_OK)) {
3408 if (qedr_set_ok_cqe_resp_wc(resp, wc))
3410 "CQ %p (icid=%d) has invalid CQE responder flags=0x%x\n",
3411 cq, cq->icid, resp->flags);
3414 wc->status = qedr_cqe_resp_status_to_ib(resp->status);
3415 if (wc->status == IB_WC_GENERAL_ERR)
3417 "CQ %p (icid=%d) contains an invalid CQE status %d\n",
3418 cq, cq->icid, resp->status);
3421 /* Fill the rest of the WC */
3423 wc->src_qp = qp->id;
3428 static int process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
3429 struct qedr_cq *cq, struct ib_wc *wc,
3430 struct rdma_cqe_responder *resp)
3432 u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
3434 __process_resp_one(dev, qp, cq, wc, resp, wr_id);
3436 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
3437 qed_chain_consume(&qp->rq.pbl);
3438 qedr_inc_sw_cons(&qp->rq);
3443 static int process_resp_flush(struct qedr_qp *qp, struct qedr_cq *cq,
3444 int num_entries, struct ib_wc *wc, u16 hw_cons)
3448 while (num_entries && qp->rq.wqe_cons != hw_cons) {
3450 wc->status = IB_WC_WR_FLUSH_ERR;
3453 wc->src_qp = qp->id;
3455 wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
3460 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
3461 qed_chain_consume(&qp->rq.pbl);
3462 qedr_inc_sw_cons(&qp->rq);
3468 static void try_consume_resp_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
3469 struct rdma_cqe_responder *resp, int *update)
3471 if (le16_to_cpu(resp->rq_cons) == qp->rq.wqe_cons) {
3477 static int qedr_poll_cq_resp(struct qedr_dev *dev, struct qedr_qp *qp,
3478 struct qedr_cq *cq, int num_entries,
3479 struct ib_wc *wc, struct rdma_cqe_responder *resp,
3484 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
3485 cnt = process_resp_flush(qp, cq, num_entries, wc,
3487 try_consume_resp_cqe(cq, qp, resp, update);
3489 cnt = process_resp_one(dev, qp, cq, wc, resp);
3497 static void try_consume_req_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
3498 struct rdma_cqe_requester *req, int *update)
3500 if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) {
3506 int qedr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
3508 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
3509 struct qedr_cq *cq = get_qedr_cq(ibcq);
3510 union rdma_cqe *cqe = cq->latest_cqe;
3511 u32 old_cons, new_cons;
3512 unsigned long flags;
3516 if (cq->destroyed) {
3518 "warning: poll was invoked after destroy for cq %p (icid=%d)\n",
3523 if (cq->cq_type == QEDR_CQ_TYPE_GSI)
3524 return qedr_gsi_poll_cq(ibcq, num_entries, wc);
3526 spin_lock_irqsave(&cq->cq_lock, flags);
3527 old_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
3528 while (num_entries && is_valid_cqe(cq, cqe)) {
3532 /* prevent speculative reads of any field of CQE */
3535 qp = cqe_get_qp(cqe);
3537 WARN(1, "Error: CQE QP pointer is NULL. CQE=%p\n", cqe);
3543 switch (cqe_get_type(cqe)) {
3544 case RDMA_CQE_TYPE_REQUESTER:
3545 cnt = qedr_poll_cq_req(dev, qp, cq, num_entries, wc,
3547 try_consume_req_cqe(cq, qp, &cqe->req, &update);
3549 case RDMA_CQE_TYPE_RESPONDER_RQ:
3550 cnt = qedr_poll_cq_resp(dev, qp, cq, num_entries, wc,
3551 &cqe->resp, &update);
3553 case RDMA_CQE_TYPE_INVALID:
3555 DP_ERR(dev, "Error: invalid CQE type = %d\n",
3564 new_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
3566 cq->cq_cons += new_cons - old_cons;
3569 /* doorbell notifies abount latest VALID entry,
3570 * but chain already point to the next INVALID one
3572 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
3574 spin_unlock_irqrestore(&cq->cq_lock, flags);
3578 int qedr_process_mad(struct ib_device *ibdev, int process_mad_flags,
3580 const struct ib_wc *in_wc,
3581 const struct ib_grh *in_grh,
3582 const struct ib_mad_hdr *mad_hdr,
3583 size_t in_mad_size, struct ib_mad_hdr *out_mad,
3584 size_t *out_mad_size, u16 *out_mad_pkey_index)
3586 struct qedr_dev *dev = get_qedr_dev(ibdev);
3588 DP_DEBUG(dev, QEDR_MSG_GSI,
3589 "QEDR_PROCESS_MAD in_mad %x %x %x %x %x %x %x %x\n",
3590 mad_hdr->attr_id, mad_hdr->base_version, mad_hdr->attr_mod,
3591 mad_hdr->class_specific, mad_hdr->class_version,
3592 mad_hdr->method, mad_hdr->mgmt_class, mad_hdr->status);
3593 return IB_MAD_RESULT_SUCCESS;
3596 int qedr_port_immutable(struct ib_device *ibdev, u8 port_num,
3597 struct ib_port_immutable *immutable)
3599 struct ib_port_attr attr;
3602 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE |
3603 RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
3605 err = ib_query_port(ibdev, port_num, &attr);
3609 immutable->pkey_tbl_len = attr.pkey_tbl_len;
3610 immutable->gid_tbl_len = attr.gid_tbl_len;
3611 immutable->max_mad_size = IB_MGMT_MAD_SIZE;