drivers/infiniband/hw/qedr/verbs.c

   1 /* QLogic qedr NIC Driver
   2  * Copyright (c) 2015-2016  QLogic Corporation
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and /or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32 #include <linux/dma-mapping.h>
  33 #include <linux/crc32.h>
  34 #include <net/ip.h>
  35 #include <net/ipv6.h>
  36 #include <net/udp.h>
  37 #include <linux/iommu.h>
  38
  39 #include <rdma/ib_verbs.h>
  40 #include <rdma/ib_user_verbs.h>
  41 #include <rdma/iw_cm.h>
  42 #include <rdma/ib_umem.h>
  43 #include <rdma/ib_addr.h>
  44 #include <rdma/ib_cache.h>
  45 #include <rdma/uverbs_ioctl.h>
  46
  47 #include <linux/qed/common_hsi.h>
  48 #include "qedr_hsi_rdma.h"
  49 #include <linux/qed/qed_if.h>
  50 #include "qedr.h"
  51 #include "verbs.h"
  52 #include <rdma/qedr-abi.h>
  53 #include "qedr_roce_cm.h"
  54 #include "qedr_iw_cm.h"
  55
  56 #define QEDR_SRQ_WQE_ELEM_SIZE  sizeof(union rdma_srq_elm)
  57 #define RDMA_MAX_SGE_PER_SRQ    (4)
  58 #define RDMA_MAX_SRQ_WQE_SIZE   (RDMA_MAX_SGE_PER_SRQ + 1)
  59
  60 #define DB_ADDR_SHIFT(addr)             ((addr) << DB_PWM_ADDR_OFFSET_SHIFT)
  61
  62 enum {
  63         QEDR_USER_MMAP_IO_WC = 0,
  64         QEDR_USER_MMAP_PHYS_PAGE,
  65 };
  66
  67 static inline int qedr_ib_copy_to_udata(struct ib_udata *udata, void *src,
  68                                         size_t len)
  69 {
  70         size_t min_len = min_t(size_t, len, udata->outlen);
  71
  72         return ib_copy_to_udata(udata, src, min_len);
  73 }
  74
  75 int qedr_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey)
  76 {
  77         if (index >= QEDR_ROCE_PKEY_TABLE_LEN)
  78                 return -EINVAL;
  79
  80         *pkey = QEDR_ROCE_PKEY_DEFAULT;
  81         return 0;
  82 }
  83
  84 int qedr_iw_query_gid(struct ib_device *ibdev, u8 port,
  85                       int index, union ib_gid *sgid)
  86 {
  87         struct qedr_dev *dev = get_qedr_dev(ibdev);
  88
  89         memset(sgid->raw, 0, sizeof(sgid->raw));
  90         ether_addr_copy(sgid->raw, dev->ndev->dev_addr);
  91
  92         DP_DEBUG(dev, QEDR_MSG_INIT, "QUERY sgid[%d]=%llx:%llx\n", index,
  93                  sgid->global.interface_id, sgid->global.subnet_prefix);
  94
  95         return 0;
  96 }
  97
  98 int qedr_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr)
  99 {
 100         struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
 101         struct qedr_device_attr *qattr = &dev->attr;
 102         struct qedr_srq *srq = get_qedr_srq(ibsrq);
 103
 104         srq_attr->srq_limit = srq->srq_limit;
 105         srq_attr->max_wr = qattr->max_srq_wr;
 106         srq_attr->max_sge = qattr->max_sge;
 107
 108         return 0;
 109 }
 110
 111 int qedr_query_device(struct ib_device *ibdev,
 112                       struct ib_device_attr *attr, struct ib_udata *udata)
 113 {
 114         struct qedr_dev *dev = get_qedr_dev(ibdev);
 115         struct qedr_device_attr *qattr = &dev->attr;
 116
 117         if (!dev->rdma_ctx) {
 118                 DP_ERR(dev,
 119                        "qedr_query_device called with invalid params rdma_ctx=%p\n",
 120                        dev->rdma_ctx);
 121                 return -EINVAL;
 122         }
 123
 124         memset(attr, 0, sizeof(*attr));
 125
 126         attr->fw_ver = qattr->fw_ver;
 127         attr->sys_image_guid = qattr->sys_image_guid;
 128         attr->max_mr_size = qattr->max_mr_size;
 129         attr->page_size_cap = qattr->page_size_caps;
 130         attr->vendor_id = qattr->vendor_id;
 131         attr->vendor_part_id = qattr->vendor_part_id;
 132         attr->hw_ver = qattr->hw_ver;
 133         attr->max_qp = qattr->max_qp;
 134         attr->max_qp_wr = max_t(u32, qattr->max_sqe, qattr->max_rqe);
 135         attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD |
 136             IB_DEVICE_RC_RNR_NAK_GEN |
 137             IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_MGT_EXTENSIONS;
 138
 139         attr->max_send_sge = qattr->max_sge;
 140         attr->max_recv_sge = qattr->max_sge;
 141         attr->max_sge_rd = qattr->max_sge;
 142         attr->max_cq = qattr->max_cq;
 143         attr->max_cqe = qattr->max_cqe;
 144         attr->max_mr = qattr->max_mr;
 145         attr->max_mw = qattr->max_mw;
 146         attr->max_pd = qattr->max_pd;
 147         attr->atomic_cap = dev->atomic_cap;
 148         attr->max_fmr = qattr->max_fmr;
 149         attr->max_map_per_fmr = 16;
 150         attr->max_qp_init_rd_atom =
 151             1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1);
 152         attr->max_qp_rd_atom =
 153             min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1),
 154                 attr->max_qp_init_rd_atom);
 155
 156         attr->max_srq = qattr->max_srq;
 157         attr->max_srq_sge = qattr->max_srq_sge;
 158         attr->max_srq_wr = qattr->max_srq_wr;
 159
 160         attr->local_ca_ack_delay = qattr->dev_ack_delay;
 161         attr->max_fast_reg_page_list_len = qattr->max_mr / 8;
 162         attr->max_pkeys = QEDR_ROCE_PKEY_MAX;
 163         attr->max_ah = qattr->max_ah;
 164
 165         return 0;
 166 }
 167
 168 static inline void get_link_speed_and_width(int speed, u8 *ib_speed,
 169                                             u8 *ib_width)
 170 {
 171         switch (speed) {
 172         case 1000:
 173                 *ib_speed = IB_SPEED_SDR;
 174                 *ib_width = IB_WIDTH_1X;
 175                 break;
 176         case 10000:
 177                 *ib_speed = IB_SPEED_QDR;
 178                 *ib_width = IB_WIDTH_1X;
 179                 break;
 180
 181         case 20000:
 182                 *ib_speed = IB_SPEED_DDR;
 183                 *ib_width = IB_WIDTH_4X;
 184                 break;
 185
 186         case 25000:
 187                 *ib_speed = IB_SPEED_EDR;
 188                 *ib_width = IB_WIDTH_1X;
 189                 break;
 190
 191         case 40000:
 192                 *ib_speed = IB_SPEED_QDR;
 193                 *ib_width = IB_WIDTH_4X;
 194                 break;
 195
 196         case 50000:
 197                 *ib_speed = IB_SPEED_HDR;
 198                 *ib_width = IB_WIDTH_1X;
 199                 break;
 200
 201         case 100000:
 202                 *ib_speed = IB_SPEED_EDR;
 203                 *ib_width = IB_WIDTH_4X;
 204                 break;
 205
 206         default:
 207                 /* Unsupported */
 208                 *ib_speed = IB_SPEED_SDR;
 209                 *ib_width = IB_WIDTH_1X;
 210         }
 211 }
 212
 213 int qedr_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *attr)
 214 {
 215         struct qedr_dev *dev;
 216         struct qed_rdma_port *rdma_port;
 217
 218         dev = get_qedr_dev(ibdev);
 219
 220         if (!dev->rdma_ctx) {
 221                 DP_ERR(dev, "rdma_ctx is NULL\n");
 222                 return -EINVAL;
 223         }
 224
 225         rdma_port = dev->ops->rdma_query_port(dev->rdma_ctx);
 226
 227         /* *attr being zeroed by the caller, avoid zeroing it here */
 228         if (rdma_port->port_state == QED_RDMA_PORT_UP) {
 229                 attr->state = IB_PORT_ACTIVE;
 230                 attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
 231         } else {
 232                 attr->state = IB_PORT_DOWN;
 233                 attr->phys_state = IB_PORT_PHYS_STATE_DISABLED;
 234         }
 235         attr->max_mtu = IB_MTU_4096;
 236         attr->active_mtu = iboe_get_mtu(dev->ndev->mtu);
 237         attr->lid = 0;
 238         attr->lmc = 0;
 239         attr->sm_lid = 0;
 240         attr->sm_sl = 0;
 241         attr->ip_gids = true;
 242         if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
 243                 attr->gid_tbl_len = 1;
 244                 attr->pkey_tbl_len = 1;
 245         } else {
 246                 attr->gid_tbl_len = QEDR_MAX_SGID;
 247                 attr->pkey_tbl_len = QEDR_ROCE_PKEY_TABLE_LEN;
 248         }
 249         attr->bad_pkey_cntr = rdma_port->pkey_bad_counter;
 250         attr->qkey_viol_cntr = 0;
 251         get_link_speed_and_width(rdma_port->link_speed,
 252                                  &attr->active_speed, &attr->active_width);
 253         attr->max_msg_sz = rdma_port->max_msg_size;
 254         attr->max_vl_num = 4;
 255
 256         return 0;
 257 }
 258
 259 int qedr_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata)
 260 {
 261         struct ib_device *ibdev = uctx->device;
 262         int rc;
 263         struct qedr_ucontext *ctx = get_qedr_ucontext(uctx);
 264         struct qedr_alloc_ucontext_resp uresp = {};
 265         struct qedr_alloc_ucontext_req ureq = {};
 266         struct qedr_dev *dev = get_qedr_dev(ibdev);
 267         struct qed_rdma_add_user_out_params oparams;
 268         struct qedr_user_mmap_entry *entry;
 269
 270         if (!udata)
 271                 return -EFAULT;
 272
 273         if (udata->inlen) {
 274                 rc = ib_copy_from_udata(&ureq, udata,
 275                                         min(sizeof(ureq), udata->inlen));
 276                 if (rc) {
 277                         DP_ERR(dev, "Problem copying data from user space\n");
 278                         return -EFAULT;
 279                 }
 280
 281                 ctx->db_rec = !!(ureq.context_flags & QEDR_ALLOC_UCTX_DB_REC);
 282         }
 283
 284         rc = dev->ops->rdma_add_user(dev->rdma_ctx, &oparams);
 285         if (rc) {
 286                 DP_ERR(dev,
 287                        "failed to allocate a DPI for a new RoCE application, rc=%d. To overcome this consider to increase the number of DPIs, increase the doorbell BAR size or just close unnecessary RoCE applications. In order to increase the number of DPIs consult the qedr readme\n",
 288                        rc);
 289                 return rc;
 290         }
 291
 292         ctx->dpi = oparams.dpi;
 293         ctx->dpi_addr = oparams.dpi_addr;
 294         ctx->dpi_phys_addr = oparams.dpi_phys_addr;
 295         ctx->dpi_size = oparams.dpi_size;
 296         entry = kzalloc(sizeof(*entry), GFP_KERNEL);
 297         if (!entry) {
 298                 rc = -ENOMEM;
 299                 goto err;
 300         }
 301
 302         entry->io_address = ctx->dpi_phys_addr;
 303         entry->length = ctx->dpi_size;
 304         entry->mmap_flag = QEDR_USER_MMAP_IO_WC;
 305         entry->dpi = ctx->dpi;
 306         entry->dev = dev;
 307         rc = rdma_user_mmap_entry_insert(uctx, &entry->rdma_entry,
 308                                          ctx->dpi_size);
 309         if (rc) {
 310                 kfree(entry);
 311                 goto err;
 312         }
 313         ctx->db_mmap_entry = &entry->rdma_entry;
 314
 315         uresp.dpm_enabled = dev->user_dpm_enabled;
 316         uresp.wids_enabled = 1;
 317         uresp.wid_count = oparams.wid_count;
 318         uresp.db_pa = rdma_user_mmap_get_offset(ctx->db_mmap_entry);
 319         uresp.db_size = ctx->dpi_size;
 320         uresp.max_send_wr = dev->attr.max_sqe;
 321         uresp.max_recv_wr = dev->attr.max_rqe;
 322         uresp.max_srq_wr = dev->attr.max_srq_wr;
 323         uresp.sges_per_send_wr = QEDR_MAX_SQE_ELEMENTS_PER_SQE;
 324         uresp.sges_per_recv_wr = QEDR_MAX_RQE_ELEMENTS_PER_RQE;
 325         uresp.sges_per_srq_wr = dev->attr.max_srq_sge;
 326         uresp.max_cqes = QEDR_MAX_CQES;
 327
 328         rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
 329         if (rc)
 330                 goto err;
 331
 332         ctx->dev = dev;
 333
 334         DP_DEBUG(dev, QEDR_MSG_INIT, "Allocating user context %p\n",
 335                  &ctx->ibucontext);
 336         return 0;
 337
 338 err:
 339         if (!ctx->db_mmap_entry)
 340                 dev->ops->rdma_remove_user(dev->rdma_ctx, ctx->dpi);
 341         else
 342                 rdma_user_mmap_entry_remove(ctx->db_mmap_entry);
 343
 344         return rc;
 345 }
 346
 347 void qedr_dealloc_ucontext(struct ib_ucontext *ibctx)
 348 {
 349         struct qedr_ucontext *uctx = get_qedr_ucontext(ibctx);
 350
 351         DP_DEBUG(uctx->dev, QEDR_MSG_INIT, "Deallocating user context %p\n",
 352                  uctx);
 353
 354         rdma_user_mmap_entry_remove(uctx->db_mmap_entry);
 355 }
 356
 357 void qedr_mmap_free(struct rdma_user_mmap_entry *rdma_entry)
 358 {
 359         struct qedr_user_mmap_entry *entry = get_qedr_mmap_entry(rdma_entry);
 360         struct qedr_dev *dev = entry->dev;
 361
 362         if (entry->mmap_flag == QEDR_USER_MMAP_PHYS_PAGE)
 363                 free_page((unsigned long)entry->address);
 364         else if (entry->mmap_flag == QEDR_USER_MMAP_IO_WC)
 365                 dev->ops->rdma_remove_user(dev->rdma_ctx, entry->dpi);
 366
 367         kfree(entry);
 368 }
 369
 370 int qedr_mmap(struct ib_ucontext *ucontext, struct vm_area_struct *vma)
 371 {
 372         struct ib_device *dev = ucontext->device;
 373         size_t length = vma->vm_end - vma->vm_start;
 374         struct rdma_user_mmap_entry *rdma_entry;
 375         struct qedr_user_mmap_entry *entry;
 376         int rc = 0;
 377         u64 pfn;
 378
 379         ibdev_dbg(dev,
 380                   "start %#lx, end %#lx, length = %#zx, pgoff = %#lx\n",
 381                   vma->vm_start, vma->vm_end, length, vma->vm_pgoff);
 382
 383         rdma_entry = rdma_user_mmap_entry_get(ucontext, vma);
 384         if (!rdma_entry) {
 385                 ibdev_dbg(dev, "pgoff[%#lx] does not have valid entry\n",
 386                           vma->vm_pgoff);
 387                 return -EINVAL;
 388         }
 389         entry = get_qedr_mmap_entry(rdma_entry);
 390         ibdev_dbg(dev,
 391                   "Mapping address[%#llx], length[%#zx], mmap_flag[%d]\n",
 392                   entry->io_address, length, entry->mmap_flag);
 393
 394         switch (entry->mmap_flag) {
 395         case QEDR_USER_MMAP_IO_WC:
 396                 pfn = entry->io_address >> PAGE_SHIFT;
 397                 rc = rdma_user_mmap_io(ucontext, vma, pfn, length,
 398                                        pgprot_writecombine(vma->vm_page_prot),
 399                                        rdma_entry);
 400                 break;
 401         case QEDR_USER_MMAP_PHYS_PAGE:
 402                 rc = vm_insert_page(vma, vma->vm_start,
 403                                     virt_to_page(entry->address));
 404                 break;
 405         default:
 406                 rc = -EINVAL;
 407         }
 408
 409         if (rc)
 410                 ibdev_dbg(dev,
 411                           "Couldn't mmap address[%#llx] length[%#zx] mmap_flag[%d] err[%d]\n",
 412                           entry->io_address, length, entry->mmap_flag, rc);
 413
 414         rdma_user_mmap_entry_put(rdma_entry);
 415         return rc;
 416 }
 417
 418 int qedr_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 419 {
 420         struct ib_device *ibdev = ibpd->device;
 421         struct qedr_dev *dev = get_qedr_dev(ibdev);
 422         struct qedr_pd *pd = get_qedr_pd(ibpd);
 423         u16 pd_id;
 424         int rc;
 425
 426         DP_DEBUG(dev, QEDR_MSG_INIT, "Function called from: %s\n",
 427                  udata ? "User Lib" : "Kernel");
 428
 429         if (!dev->rdma_ctx) {
 430                 DP_ERR(dev, "invalid RDMA context\n");
 431                 return -EINVAL;
 432         }
 433
 434         rc = dev->ops->rdma_alloc_pd(dev->rdma_ctx, &pd_id);
 435         if (rc)
 436                 return rc;
 437
 438         pd->pd_id = pd_id;
 439
 440         if (udata) {
 441                 struct qedr_alloc_pd_uresp uresp = {
 442                         .pd_id = pd_id,
 443                 };
 444                 struct qedr_ucontext *context = rdma_udata_to_drv_context(
 445                         udata, struct qedr_ucontext, ibucontext);
 446
 447                 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
 448                 if (rc) {
 449                         DP_ERR(dev, "copy error pd_id=0x%x.\n", pd_id);
 450                         dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd_id);
 451                         return rc;
 452                 }
 453
 454                 pd->uctx = context;
 455                 pd->uctx->pd = pd;
 456         }
 457
 458         return 0;
 459 }
 460
 461 void qedr_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 462 {
 463         struct qedr_dev *dev = get_qedr_dev(ibpd->device);
 464         struct qedr_pd *pd = get_qedr_pd(ibpd);
 465
 466         DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id);
 467         dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id);
 468 }
 469
 470 static void qedr_free_pbl(struct qedr_dev *dev,
 471                           struct qedr_pbl_info *pbl_info, struct qedr_pbl *pbl)
 472 {
 473         struct pci_dev *pdev = dev->pdev;
 474         int i;
 475
 476         for (i = 0; i < pbl_info->num_pbls; i++) {
 477                 if (!pbl[i].va)
 478                         continue;
 479                 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
 480                                   pbl[i].va, pbl[i].pa);
 481         }
 482
 483         kfree(pbl);
 484 }
 485
 486 #define MIN_FW_PBL_PAGE_SIZE (4 * 1024)
 487 #define MAX_FW_PBL_PAGE_SIZE (64 * 1024)
 488
 489 #define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64))
 490 #define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE)
 491 #define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE * MAX_PBES_ON_PAGE)
 492
 493 static struct qedr_pbl *qedr_alloc_pbl_tbl(struct qedr_dev *dev,
 494                                            struct qedr_pbl_info *pbl_info,
 495                                            gfp_t flags)
 496 {
 497         struct pci_dev *pdev = dev->pdev;
 498         struct qedr_pbl *pbl_table;
 499         dma_addr_t *pbl_main_tbl;
 500         dma_addr_t pa;
 501         void *va;
 502         int i;
 503
 504         pbl_table = kcalloc(pbl_info->num_pbls, sizeof(*pbl_table), flags);
 505         if (!pbl_table)
 506                 return ERR_PTR(-ENOMEM);
 507
 508         for (i = 0; i < pbl_info->num_pbls; i++) {
 509                 va = dma_alloc_coherent(&pdev->dev, pbl_info->pbl_size, &pa,
 510                                         flags);
 511                 if (!va)
 512                         goto err;
 513
 514                 pbl_table[i].va = va;
 515                 pbl_table[i].pa = pa;
 516         }
 517
 518         /* Two-Layer PBLs, if we have more than one pbl we need to initialize
 519          * the first one with physical pointers to all of the rest
 520          */
 521         pbl_main_tbl = (dma_addr_t *)pbl_table[0].va;
 522         for (i = 0; i < pbl_info->num_pbls - 1; i++)
 523                 pbl_main_tbl[i] = pbl_table[i + 1].pa;
 524
 525         return pbl_table;
 526
 527 err:
 528         for (i--; i >= 0; i--)
 529                 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
 530                                   pbl_table[i].va, pbl_table[i].pa);
 531
 532         qedr_free_pbl(dev, pbl_info, pbl_table);
 533
 534         return ERR_PTR(-ENOMEM);
 535 }
 536
 537 static int qedr_prepare_pbl_tbl(struct qedr_dev *dev,
 538                                 struct qedr_pbl_info *pbl_info,
 539                                 u32 num_pbes, int two_layer_capable)
 540 {
 541         u32 pbl_capacity;
 542         u32 pbl_size;
 543         u32 num_pbls;
 544
 545         if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) {
 546                 if (num_pbes > MAX_PBES_TWO_LAYER) {
 547                         DP_ERR(dev, "prepare pbl table: too many pages %d\n",
 548                                num_pbes);
 549                         return -EINVAL;
 550                 }
 551
 552                 /* calculate required pbl page size */
 553                 pbl_size = MIN_FW_PBL_PAGE_SIZE;
 554                 pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) *
 555                                NUM_PBES_ON_PAGE(pbl_size);
 556
 557                 while (pbl_capacity < num_pbes) {
 558                         pbl_size *= 2;
 559                         pbl_capacity = pbl_size / sizeof(u64);
 560                         pbl_capacity = pbl_capacity * pbl_capacity;
 561                 }
 562
 563                 num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size));
 564                 num_pbls++;     /* One for the layer0 ( points to the pbls) */
 565                 pbl_info->two_layered = true;
 566         } else {
 567                 /* One layered PBL */
 568                 num_pbls = 1;
 569                 pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE,
 570                                  roundup_pow_of_two((num_pbes * sizeof(u64))));
 571                 pbl_info->two_layered = false;
 572         }
 573
 574         pbl_info->num_pbls = num_pbls;
 575         pbl_info->pbl_size = pbl_size;
 576         pbl_info->num_pbes = num_pbes;
 577
 578         DP_DEBUG(dev, QEDR_MSG_MR,
 579                  "prepare pbl table: num_pbes=%d, num_pbls=%d, pbl_size=%d\n",
 580                  pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size);
 581
 582         return 0;
 583 }
 584
 585 static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem,
 586                                struct qedr_pbl *pbl,
 587                                struct qedr_pbl_info *pbl_info, u32 pg_shift)
 588 {
 589         int pbe_cnt, total_num_pbes = 0;
 590         u32 fw_pg_cnt, fw_pg_per_umem_pg;
 591         struct qedr_pbl *pbl_tbl;
 592         struct sg_dma_page_iter sg_iter;
 593         struct regpair *pbe;
 594         u64 pg_addr;
 595
 596         if (!pbl_info->num_pbes)
 597                 return;
 598
 599         /* If we have a two layered pbl, the first pbl points to the rest
 600          * of the pbls and the first entry lays on the second pbl in the table
 601          */
 602         if (pbl_info->two_layered)
 603                 pbl_tbl = &pbl[1];
 604         else
 605                 pbl_tbl = pbl;
 606
 607         pbe = (struct regpair *)pbl_tbl->va;
 608         if (!pbe) {
 609                 DP_ERR(dev, "cannot populate PBL due to a NULL PBE\n");
 610                 return;
 611         }
 612
 613         pbe_cnt = 0;
 614
 615         fw_pg_per_umem_pg = BIT(PAGE_SHIFT - pg_shift);
 616
 617         for_each_sg_dma_page (umem->sg_head.sgl, &sg_iter, umem->nmap, 0) {
 618                 pg_addr = sg_page_iter_dma_address(&sg_iter);
 619                 for (fw_pg_cnt = 0; fw_pg_cnt < fw_pg_per_umem_pg;) {
 620                         pbe->lo = cpu_to_le32(pg_addr);
 621                         pbe->hi = cpu_to_le32(upper_32_bits(pg_addr));
 622
 623                         pg_addr += BIT(pg_shift);
 624                         pbe_cnt++;
 625                         total_num_pbes++;
 626                         pbe++;
 627
 628                         if (total_num_pbes == pbl_info->num_pbes)
 629                                 return;
 630
 631                         /* If the given pbl is full storing the pbes,
 632                          * move to next pbl.
 633                          */
 634                         if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) {
 635                                 pbl_tbl++;
 636                                 pbe = (struct regpair *)pbl_tbl->va;
 637                                 pbe_cnt = 0;
 638                         }
 639
 640                         fw_pg_cnt++;
 641                 }
 642         }
 643 }
 644
 645 static int qedr_db_recovery_add(struct qedr_dev *dev,
 646                                 void __iomem *db_addr,
 647                                 void *db_data,
 648                                 enum qed_db_rec_width db_width,
 649                                 enum qed_db_rec_space db_space)
 650 {
 651         if (!db_data) {
 652                 DP_DEBUG(dev, QEDR_MSG_INIT, "avoiding db rec since old lib\n");
 653                 return 0;
 654         }
 655
 656         return dev->ops->common->db_recovery_add(dev->cdev, db_addr, db_data,
 657                                                  db_width, db_space);
 658 }
 659
 660 static void qedr_db_recovery_del(struct qedr_dev *dev,
 661                                  void __iomem *db_addr,
 662                                  void *db_data)
 663 {
 664         if (!db_data) {
 665                 DP_DEBUG(dev, QEDR_MSG_INIT, "avoiding db rec since old lib\n");
 666                 return;
 667         }
 668
 669         /* Ignore return code as there is not much we can do about it. Error
 670          * log will be printed inside.
 671          */
 672         dev->ops->common->db_recovery_del(dev->cdev, db_addr, db_data);
 673 }
 674
 675 static int qedr_copy_cq_uresp(struct qedr_dev *dev,
 676                               struct qedr_cq *cq, struct ib_udata *udata,
 677                               u32 db_offset)
 678 {
 679         struct qedr_create_cq_uresp uresp;
 680         int rc;
 681
 682         memset(&uresp, 0, sizeof(uresp));
 683
 684         uresp.db_offset = db_offset;
 685         uresp.icid = cq->icid;
 686         if (cq->q.db_mmap_entry)
 687                 uresp.db_rec_addr =
 688                         rdma_user_mmap_get_offset(cq->q.db_mmap_entry);
 689
 690         rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
 691         if (rc)
 692                 DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid);
 693
 694         return rc;
 695 }
 696
 697 static void consume_cqe(struct qedr_cq *cq)
 698 {
 699         if (cq->latest_cqe == cq->toggle_cqe)
 700                 cq->pbl_toggle ^= RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
 701
 702         cq->latest_cqe = qed_chain_consume(&cq->pbl);
 703 }
 704
 705 static inline int qedr_align_cq_entries(int entries)
 706 {
 707         u64 size, aligned_size;
 708
 709         /* We allocate an extra entry that we don't report to the FW. */
 710         size = (entries + 1) * QEDR_CQE_SIZE;
 711         aligned_size = ALIGN(size, PAGE_SIZE);
 712
 713         return aligned_size / QEDR_CQE_SIZE;
 714 }
 715
 716 static int qedr_init_user_db_rec(struct ib_udata *udata,
 717                                  struct qedr_dev *dev, struct qedr_userq *q,
 718                                  bool requires_db_rec)
 719 {
 720         struct qedr_ucontext *uctx =
 721                 rdma_udata_to_drv_context(udata, struct qedr_ucontext,
 722                                           ibucontext);
 723         struct qedr_user_mmap_entry *entry;
 724         int rc;
 725
 726         /* Aborting for non doorbell userqueue (SRQ) or non-supporting lib */
 727         if (requires_db_rec == 0 || !uctx->db_rec)
 728                 return 0;
 729
 730         /* Allocate a page for doorbell recovery, add to mmap */
 731         q->db_rec_data = (void *)get_zeroed_page(GFP_USER);
 732         if (!q->db_rec_data) {
 733                 DP_ERR(dev, "get_zeroed_page failed\n");
 734                 return -ENOMEM;
 735         }
 736
 737         entry = kzalloc(sizeof(*entry), GFP_KERNEL);
 738         if (!entry)
 739                 goto err_free_db_data;
 740
 741         entry->address = q->db_rec_data;
 742         entry->length = PAGE_SIZE;
 743         entry->mmap_flag = QEDR_USER_MMAP_PHYS_PAGE;
 744         rc = rdma_user_mmap_entry_insert(&uctx->ibucontext,
 745                                          &entry->rdma_entry,
 746                                          PAGE_SIZE);
 747         if (rc)
 748                 goto err_free_entry;
 749
 750         q->db_mmap_entry = &entry->rdma_entry;
 751
 752         return 0;
 753
 754 err_free_entry:
 755         kfree(entry);
 756
 757 err_free_db_data:
 758         free_page((unsigned long)q->db_rec_data);
 759         q->db_rec_data = NULL;
 760         return -ENOMEM;
 761 }
 762
 763 static inline int qedr_init_user_queue(struct ib_udata *udata,
 764                                        struct qedr_dev *dev,
 765                                        struct qedr_userq *q, u64 buf_addr,
 766                                        size_t buf_len, bool requires_db_rec,
 767                                        int access,
 768                                        int alloc_and_init)
 769 {
 770         u32 fw_pages;
 771         int rc;
 772
 773         q->buf_addr = buf_addr;
 774         q->buf_len = buf_len;
 775         q->umem = ib_umem_get(&dev->ibdev, q->buf_addr, q->buf_len, access);
 776         if (IS_ERR(q->umem)) {
 777                 DP_ERR(dev, "create user queue: failed ib_umem_get, got %ld\n",
 778                        PTR_ERR(q->umem));
 779                 return PTR_ERR(q->umem);
 780         }
 781
 782         fw_pages = ib_umem_page_count(q->umem) <<
 783             (PAGE_SHIFT - FW_PAGE_SHIFT);
 784
 785         rc = qedr_prepare_pbl_tbl(dev, &q->pbl_info, fw_pages, 0);
 786         if (rc)
 787                 goto err0;
 788
 789         if (alloc_and_init) {
 790                 q->pbl_tbl = qedr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL);
 791                 if (IS_ERR(q->pbl_tbl)) {
 792                         rc = PTR_ERR(q->pbl_tbl);
 793                         goto err0;
 794                 }
 795                 qedr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info,
 796                                    FW_PAGE_SHIFT);
 797         } else {
 798                 q->pbl_tbl = kzalloc(sizeof(*q->pbl_tbl), GFP_KERNEL);
 799                 if (!q->pbl_tbl) {
 800                         rc = -ENOMEM;
 801                         goto err0;
 802                 }
 803         }
 804
 805         /* mmap the user address used to store doorbell data for recovery */
 806         return qedr_init_user_db_rec(udata, dev, q, requires_db_rec);
 807
 808 err0:
 809         ib_umem_release(q->umem);
 810         q->umem = NULL;
 811
 812         return rc;
 813 }
 814
 815 static inline void qedr_init_cq_params(struct qedr_cq *cq,
 816                                        struct qedr_ucontext *ctx,
 817                                        struct qedr_dev *dev, int vector,
 818                                        int chain_entries, int page_cnt,
 819                                        u64 pbl_ptr,
 820                                        struct qed_rdma_create_cq_in_params
 821                                        *params)
 822 {
 823         memset(params, 0, sizeof(*params));
 824         params->cq_handle_hi = upper_32_bits((uintptr_t)cq);
 825         params->cq_handle_lo = lower_32_bits((uintptr_t)cq);
 826         params->cnq_id = vector;
 827         params->cq_size = chain_entries - 1;
 828         params->dpi = (ctx) ? ctx->dpi : dev->dpi;
 829         params->pbl_num_pages = page_cnt;
 830         params->pbl_ptr = pbl_ptr;
 831         params->pbl_two_level = 0;
 832 }
 833
 834 static void doorbell_cq(struct qedr_cq *cq, u32 cons, u8 flags)
 835 {
 836         cq->db.data.agg_flags = flags;
 837         cq->db.data.value = cpu_to_le32(cons);
 838         writeq(cq->db.raw, cq->db_addr);
 839 }
 840
 841 int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
 842 {
 843         struct qedr_cq *cq = get_qedr_cq(ibcq);
 844         unsigned long sflags;
 845         struct qedr_dev *dev;
 846
 847         dev = get_qedr_dev(ibcq->device);
 848
 849         if (cq->destroyed) {
 850                 DP_ERR(dev,
 851                        "warning: arm was invoked after destroy for cq %p (icid=%d)\n",
 852                        cq, cq->icid);
 853                 return -EINVAL;
 854         }
 855
 856
 857         if (cq->cq_type == QEDR_CQ_TYPE_GSI)
 858                 return 0;
 859
 860         spin_lock_irqsave(&cq->cq_lock, sflags);
 861
 862         cq->arm_flags = 0;
 863
 864         if (flags & IB_CQ_SOLICITED)
 865                 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD;
 866
 867         if (flags & IB_CQ_NEXT_COMP)
 868                 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD;
 869
 870         doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
 871
 872         spin_unlock_irqrestore(&cq->cq_lock, sflags);
 873
 874         return 0;
 875 }
 876
 877 int qedr_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 878                    struct ib_udata *udata)
 879 {
 880         struct ib_device *ibdev = ibcq->device;
 881         struct qedr_ucontext *ctx = rdma_udata_to_drv_context(
 882                 udata, struct qedr_ucontext, ibucontext);
 883         struct qed_rdma_destroy_cq_out_params destroy_oparams;
 884         struct qed_rdma_destroy_cq_in_params destroy_iparams;
 885         struct qedr_dev *dev = get_qedr_dev(ibdev);
 886         struct qed_rdma_create_cq_in_params params;
 887         struct qedr_create_cq_ureq ureq = {};
 888         int vector = attr->comp_vector;
 889         int entries = attr->cqe;
 890         struct qedr_cq *cq = get_qedr_cq(ibcq);
 891         int chain_entries;
 892         u32 db_offset;
 893         int page_cnt;
 894         u64 pbl_ptr;
 895         u16 icid;
 896         int rc;
 897
 898         DP_DEBUG(dev, QEDR_MSG_INIT,
 899                  "create_cq: called from %s. entries=%d, vector=%d\n",
 900                  udata ? "User Lib" : "Kernel", entries, vector);
 901
 902         if (entries > QEDR_MAX_CQES) {
 903                 DP_ERR(dev,
 904                        "create cq: the number of entries %d is too high. Must be equal or below %d.\n",
 905                        entries, QEDR_MAX_CQES);
 906                 return -EINVAL;
 907         }
 908
 909         chain_entries = qedr_align_cq_entries(entries);
 910         chain_entries = min_t(int, chain_entries, QEDR_MAX_CQES);
 911
 912         /* calc db offset. user will add DPI base, kernel will add db addr */
 913         db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
 914
 915         if (udata) {
 916                 if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq),
 917                                                          udata->inlen))) {
 918                         DP_ERR(dev,
 919                                "create cq: problem copying data from user space\n");
 920                         goto err0;
 921                 }
 922
 923                 if (!ureq.len) {
 924                         DP_ERR(dev,
 925                                "create cq: cannot create a cq with 0 entries\n");
 926                         goto err0;
 927                 }
 928
 929                 cq->cq_type = QEDR_CQ_TYPE_USER;
 930
 931                 rc = qedr_init_user_queue(udata, dev, &cq->q, ureq.addr,
 932                                           ureq.len, true, IB_ACCESS_LOCAL_WRITE,
 933                                           1);
 934                 if (rc)
 935                         goto err0;
 936
 937                 pbl_ptr = cq->q.pbl_tbl->pa;
 938                 page_cnt = cq->q.pbl_info.num_pbes;
 939
 940                 cq->ibcq.cqe = chain_entries;
 941                 cq->q.db_addr = ctx->dpi_addr + db_offset;
 942         } else {
 943                 cq->cq_type = QEDR_CQ_TYPE_KERNEL;
 944
 945                 rc = dev->ops->common->chain_alloc(dev->cdev,
 946                                                    QED_CHAIN_USE_TO_CONSUME,
 947                                                    QED_CHAIN_MODE_PBL,
 948                                                    QED_CHAIN_CNT_TYPE_U32,
 949                                                    chain_entries,
 950                                                    sizeof(union rdma_cqe),
 951                                                    &cq->pbl, NULL);
 952                 if (rc)
 953                         goto err0;
 954
 955                 page_cnt = qed_chain_get_page_cnt(&cq->pbl);
 956                 pbl_ptr = qed_chain_get_pbl_phys(&cq->pbl);
 957                 cq->ibcq.cqe = cq->pbl.capacity;
 958         }
 959
 960         qedr_init_cq_params(cq, ctx, dev, vector, chain_entries, page_cnt,
 961                             pbl_ptr, &params);
 962
 963         rc = dev->ops->rdma_create_cq(dev->rdma_ctx, &params, &icid);
 964         if (rc)
 965                 goto err1;
 966
 967         cq->icid = icid;
 968         cq->sig = QEDR_CQ_MAGIC_NUMBER;
 969         spin_lock_init(&cq->cq_lock);
 970
 971         if (udata) {
 972                 rc = qedr_copy_cq_uresp(dev, cq, udata, db_offset);
 973                 if (rc)
 974                         goto err2;
 975
 976                 rc = qedr_db_recovery_add(dev, cq->q.db_addr,
 977                                           &cq->q.db_rec_data->db_data,
 978                                           DB_REC_WIDTH_64B,
 979                                           DB_REC_USER);
 980                 if (rc)
 981                         goto err2;
 982
 983         } else {
 984                 /* Generate doorbell address. */
 985                 cq->db.data.icid = cq->icid;
 986                 cq->db_addr = dev->db_addr + db_offset;
 987                 cq->db.data.params = DB_AGG_CMD_SET <<
 988                     RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT;
 989
 990                 /* point to the very last element, passing it we will toggle */
 991                 cq->toggle_cqe = qed_chain_get_last_elem(&cq->pbl);
 992                 cq->pbl_toggle = RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
 993                 cq->latest_cqe = NULL;
 994                 consume_cqe(cq);
 995                 cq->cq_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
 996
 997                 rc = qedr_db_recovery_add(dev, cq->db_addr, &cq->db.data,
 998                                           DB_REC_WIDTH_64B, DB_REC_KERNEL);
 999                 if (rc)
1000                         goto err2;
1001         }
1002
1003         DP_DEBUG(dev, QEDR_MSG_CQ,
1004                  "create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n",
1005                  cq->icid, cq, params.cq_size);
1006
1007         return 0;
1008
1009 err2:
1010         destroy_iparams.icid = cq->icid;
1011         dev->ops->rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams,
1012                                   &destroy_oparams);
1013 err1:
1014         if (udata) {
1015                 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
1016                 ib_umem_release(cq->q.umem);
1017                 if (cq->q.db_mmap_entry)
1018                         rdma_user_mmap_entry_remove(cq->q.db_mmap_entry);
1019         } else {
1020                 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
1021         }
1022 err0:
1023         return -EINVAL;
1024 }
1025
1026 int qedr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata)
1027 {
1028         struct qedr_dev *dev = get_qedr_dev(ibcq->device);
1029         struct qedr_cq *cq = get_qedr_cq(ibcq);
1030
1031         DP_ERR(dev, "cq %p RESIZE NOT SUPPORTED\n", cq);
1032
1033         return 0;
1034 }
1035
1036 #define QEDR_DESTROY_CQ_MAX_ITERATIONS          (10)
1037 #define QEDR_DESTROY_CQ_ITER_DURATION           (10)
1038
1039 void qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
1040 {
1041         struct qedr_dev *dev = get_qedr_dev(ibcq->device);
1042         struct qed_rdma_destroy_cq_out_params oparams;
1043         struct qed_rdma_destroy_cq_in_params iparams;
1044         struct qedr_cq *cq = get_qedr_cq(ibcq);
1045         int iter;
1046
1047         DP_DEBUG(dev, QEDR_MSG_CQ, "destroy cq %p (icid=%d)\n", cq, cq->icid);
1048
1049         cq->destroyed = 1;
1050
1051         /* GSIs CQs are handled by driver, so they don't exist in the FW */
1052         if (cq->cq_type == QEDR_CQ_TYPE_GSI) {
1053                 qedr_db_recovery_del(dev, cq->db_addr, &cq->db.data);
1054                 return;
1055         }
1056
1057         iparams.icid = cq->icid;
1058         dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams);
1059         dev->ops->common->chain_free(dev->cdev, &cq->pbl);
1060
1061         if (udata) {
1062                 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
1063                 ib_umem_release(cq->q.umem);
1064
1065                 if (cq->q.db_rec_data) {
1066                         qedr_db_recovery_del(dev, cq->q.db_addr,
1067                                              &cq->q.db_rec_data->db_data);
1068                         rdma_user_mmap_entry_remove(cq->q.db_mmap_entry);
1069                 }
1070         } else {
1071                 qedr_db_recovery_del(dev, cq->db_addr, &cq->db.data);
1072         }
1073
1074         /* We don't want the IRQ handler to handle a non-existing CQ so we
1075          * wait until all CNQ interrupts, if any, are received. This will always
1076          * happen and will always happen very fast. If not, then a serious error
1077          * has occured. That is why we can use a long delay.
1078          * We spin for a short time so we don’t lose time on context switching
1079          * in case all the completions are handled in that span. Otherwise
1080          * we sleep for a while and check again. Since the CNQ may be
1081          * associated with (only) the current CPU we use msleep to allow the
1082          * current CPU to be freed.
1083          * The CNQ notification is increased in qedr_irq_handler().
1084          */
1085         iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1086         while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1087                 udelay(QEDR_DESTROY_CQ_ITER_DURATION);
1088                 iter--;
1089         }
1090
1091         iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1092         while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1093                 msleep(QEDR_DESTROY_CQ_ITER_DURATION);
1094                 iter--;
1095         }
1096
1097         /* Note that we don't need to have explicit code to wait for the
1098          * completion of the event handler because it is invoked from the EQ.
1099          * Since the destroy CQ ramrod has also been received on the EQ we can
1100          * be certain that there's no event handler in process.
1101          */
1102 }
1103
1104 static inline int get_gid_info_from_table(struct ib_qp *ibqp,
1105                                           struct ib_qp_attr *attr,
1106                                           int attr_mask,
1107                                           struct qed_rdma_modify_qp_in_params
1108                                           *qp_params)
1109 {
1110         const struct ib_gid_attr *gid_attr;
1111         enum rdma_network_type nw_type;
1112         const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
1113         u32 ipv4_addr;
1114         int ret;
1115         int i;
1116
1117         gid_attr = grh->sgid_attr;
1118         ret = rdma_read_gid_l2_fields(gid_attr, &qp_params->vlan_id, NULL);
1119         if (ret)
1120                 return ret;
1121
1122         nw_type = rdma_gid_attr_network_type(gid_attr);
1123         switch (nw_type) {
1124         case RDMA_NETWORK_IPV6:
1125                 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0],
1126                        sizeof(qp_params->sgid));
1127                 memcpy(&qp_params->dgid.bytes[0],
1128                        &grh->dgid,
1129                        sizeof(qp_params->dgid));
1130                 qp_params->roce_mode = ROCE_V2_IPV6;
1131                 SET_FIELD(qp_params->modify_flags,
1132                           QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1133                 break;
1134         case RDMA_NETWORK_IB:
1135                 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0],
1136                        sizeof(qp_params->sgid));
1137                 memcpy(&qp_params->dgid.bytes[0],
1138                        &grh->dgid,
1139                        sizeof(qp_params->dgid));
1140                 qp_params->roce_mode = ROCE_V1;
1141                 break;
1142         case RDMA_NETWORK_IPV4:
1143                 memset(&qp_params->sgid, 0, sizeof(qp_params->sgid));
1144                 memset(&qp_params->dgid, 0, sizeof(qp_params->dgid));
1145                 ipv4_addr = qedr_get_ipv4_from_gid(gid_attr->gid.raw);
1146                 qp_params->sgid.ipv4_addr = ipv4_addr;
1147                 ipv4_addr =
1148                     qedr_get_ipv4_from_gid(grh->dgid.raw);
1149                 qp_params->dgid.ipv4_addr = ipv4_addr;
1150                 SET_FIELD(qp_params->modify_flags,
1151                           QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1152                 qp_params->roce_mode = ROCE_V2_IPV4;
1153                 break;
1154         }
1155
1156         for (i = 0; i < 4; i++) {
1157                 qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]);
1158                 qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]);
1159         }
1160
1161         if (qp_params->vlan_id >= VLAN_CFI_MASK)
1162                 qp_params->vlan_id = 0;
1163
1164         return 0;
1165 }
1166
1167 static int qedr_check_qp_attrs(struct ib_pd *ibpd, struct qedr_dev *dev,
1168                                struct ib_qp_init_attr *attrs,
1169                                struct ib_udata *udata)
1170 {
1171         struct qedr_device_attr *qattr = &dev->attr;
1172
1173         /* QP0... attrs->qp_type == IB_QPT_GSI */
1174         if (attrs->qp_type != IB_QPT_RC && attrs->qp_type != IB_QPT_GSI) {
1175                 DP_DEBUG(dev, QEDR_MSG_QP,
1176                          "create qp: unsupported qp type=0x%x requested\n",
1177                          attrs->qp_type);
1178                 return -EINVAL;
1179         }
1180
1181         if (attrs->cap.max_send_wr > qattr->max_sqe) {
1182                 DP_ERR(dev,
1183                        "create qp: cannot create a SQ with %d elements (max_send_wr=0x%x)\n",
1184                        attrs->cap.max_send_wr, qattr->max_sqe);
1185                 return -EINVAL;
1186         }
1187
1188         if (attrs->cap.max_inline_data > qattr->max_inline) {
1189                 DP_ERR(dev,
1190                        "create qp: unsupported inline data size=0x%x requested (max_inline=0x%x)\n",
1191                        attrs->cap.max_inline_data, qattr->max_inline);
1192                 return -EINVAL;
1193         }
1194
1195         if (attrs->cap.max_send_sge > qattr->max_sge) {
1196                 DP_ERR(dev,
1197                        "create qp: unsupported send_sge=0x%x requested (max_send_sge=0x%x)\n",
1198                        attrs->cap.max_send_sge, qattr->max_sge);
1199                 return -EINVAL;
1200         }
1201
1202         if (attrs->cap.max_recv_sge > qattr->max_sge) {
1203                 DP_ERR(dev,
1204                        "create qp: unsupported recv_sge=0x%x requested (max_recv_sge=0x%x)\n",
1205                        attrs->cap.max_recv_sge, qattr->max_sge);
1206                 return -EINVAL;
1207         }
1208
1209         /* Unprivileged user space cannot create special QP */
1210         if (udata && attrs->qp_type == IB_QPT_GSI) {
1211                 DP_ERR(dev,
1212                        "create qp: userspace can't create special QPs of type=0x%x\n",
1213                        attrs->qp_type);
1214                 return -EINVAL;
1215         }
1216
1217         return 0;
1218 }
1219
1220 static int qedr_copy_srq_uresp(struct qedr_dev *dev,
1221                                struct qedr_srq *srq, struct ib_udata *udata)
1222 {
1223         struct qedr_create_srq_uresp uresp = {};
1224         int rc;
1225
1226         uresp.srq_id = srq->srq_id;
1227
1228         rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1229         if (rc)
1230                 DP_ERR(dev, "create srq: problem copying data to user space\n");
1231
1232         return rc;
1233 }
1234
1235 static void qedr_copy_rq_uresp(struct qedr_dev *dev,
1236                               struct qedr_create_qp_uresp *uresp,
1237                               struct qedr_qp *qp)
1238 {
1239         /* iWARP requires two doorbells per RQ. */
1240         if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1241                 uresp->rq_db_offset =
1242                     DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
1243                 uresp->rq_db2_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
1244         } else {
1245                 uresp->rq_db_offset =
1246                     DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1247         }
1248
1249         uresp->rq_icid = qp->icid;
1250         if (qp->urq.db_mmap_entry)
1251                 uresp->rq_db_rec_addr =
1252                         rdma_user_mmap_get_offset(qp->urq.db_mmap_entry);
1253 }
1254
1255 static void qedr_copy_sq_uresp(struct qedr_dev *dev,
1256                                struct qedr_create_qp_uresp *uresp,
1257                                struct qedr_qp *qp)
1258 {
1259         uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1260
1261         /* iWARP uses the same cid for rq and sq */
1262         if (rdma_protocol_iwarp(&dev->ibdev, 1))
1263                 uresp->sq_icid = qp->icid;
1264         else
1265                 uresp->sq_icid = qp->icid + 1;
1266
1267         if (qp->usq.db_mmap_entry)
1268                 uresp->sq_db_rec_addr =
1269                         rdma_user_mmap_get_offset(qp->usq.db_mmap_entry);
1270 }
1271
1272 static int qedr_copy_qp_uresp(struct qedr_dev *dev,
1273                               struct qedr_qp *qp, struct ib_udata *udata,
1274                               struct qedr_create_qp_uresp *uresp)
1275 {
1276         int rc;
1277
1278         memset(uresp, 0, sizeof(*uresp));
1279         qedr_copy_sq_uresp(dev, uresp, qp);
1280         qedr_copy_rq_uresp(dev, uresp, qp);
1281
1282         uresp->atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE;
1283         uresp->qp_id = qp->qp_id;
1284
1285         rc = qedr_ib_copy_to_udata(udata, uresp, sizeof(*uresp));
1286         if (rc)
1287                 DP_ERR(dev,
1288                        "create qp: failed a copy to user space with qp icid=0x%x.\n",
1289                        qp->icid);
1290
1291         return rc;
1292 }
1293
1294 static void qedr_set_common_qp_params(struct qedr_dev *dev,
1295                                       struct qedr_qp *qp,
1296                                       struct qedr_pd *pd,
1297                                       struct ib_qp_init_attr *attrs)
1298 {
1299         spin_lock_init(&qp->q_lock);
1300         if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1301                 kref_init(&qp->refcnt);
1302                 init_completion(&qp->iwarp_cm_comp);
1303         }
1304         qp->pd = pd;
1305         qp->qp_type = attrs->qp_type;
1306         qp->max_inline_data = attrs->cap.max_inline_data;
1307         qp->sq.max_sges = attrs->cap.max_send_sge;
1308         qp->state = QED_ROCE_QP_STATE_RESET;
1309         qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false;
1310         qp->sq_cq = get_qedr_cq(attrs->send_cq);
1311         qp->dev = dev;
1312
1313         if (attrs->srq) {
1314                 qp->srq = get_qedr_srq(attrs->srq);
1315         } else {
1316                 qp->rq_cq = get_qedr_cq(attrs->recv_cq);
1317                 qp->rq.max_sges = attrs->cap.max_recv_sge;
1318                 DP_DEBUG(dev, QEDR_MSG_QP,
1319                          "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n",
1320                          qp->rq.max_sges, qp->rq_cq->icid);
1321         }
1322
1323         DP_DEBUG(dev, QEDR_MSG_QP,
1324                  "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d, state = %d, signaled = %d, use_srq=%d\n",
1325                  pd->pd_id, qp->qp_type, qp->max_inline_data,
1326                  qp->state, qp->signaled, (attrs->srq) ? 1 : 0);
1327         DP_DEBUG(dev, QEDR_MSG_QP,
1328                  "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n",
1329                  qp->sq.max_sges, qp->sq_cq->icid);
1330 }
1331
1332 static int qedr_set_roce_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
1333 {
1334         int rc;
1335
1336         qp->sq.db = dev->db_addr +
1337                     DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1338         qp->sq.db_data.data.icid = qp->icid + 1;
1339         rc = qedr_db_recovery_add(dev, qp->sq.db,
1340                                   &qp->sq.db_data,
1341                                   DB_REC_WIDTH_32B,
1342                                   DB_REC_KERNEL);
1343         if (rc)
1344                 return rc;
1345
1346         if (!qp->srq) {
1347                 qp->rq.db = dev->db_addr +
1348                             DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1349                 qp->rq.db_data.data.icid = qp->icid;
1350
1351                 rc = qedr_db_recovery_add(dev, qp->rq.db,
1352                                           &qp->rq.db_data,
1353                                           DB_REC_WIDTH_32B,
1354                                           DB_REC_KERNEL);
1355                 if (rc)
1356                         qedr_db_recovery_del(dev, qp->sq.db,
1357                                              &qp->sq.db_data);
1358         }
1359
1360         return rc;
1361 }
1362
1363 static int qedr_check_srq_params(struct qedr_dev *dev,
1364                                  struct ib_srq_init_attr *attrs,
1365                                  struct ib_udata *udata)
1366 {
1367         struct qedr_device_attr *qattr = &dev->attr;
1368
1369         if (attrs->attr.max_wr > qattr->max_srq_wr) {
1370                 DP_ERR(dev,
1371                        "create srq: unsupported srq_wr=0x%x requested (max_srq_wr=0x%x)\n",
1372                        attrs->attr.max_wr, qattr->max_srq_wr);
1373                 return -EINVAL;
1374         }
1375
1376         if (attrs->attr.max_sge > qattr->max_sge) {
1377                 DP_ERR(dev,
1378                        "create srq: unsupported sge=0x%x requested (max_srq_sge=0x%x)\n",
1379                        attrs->attr.max_sge, qattr->max_sge);
1380                 return -EINVAL;
1381         }
1382
1383         return 0;
1384 }
1385
1386 static void qedr_free_srq_user_params(struct qedr_srq *srq)
1387 {
1388         qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
1389         ib_umem_release(srq->usrq.umem);
1390         ib_umem_release(srq->prod_umem);
1391 }
1392
1393 static void qedr_free_srq_kernel_params(struct qedr_srq *srq)
1394 {
1395         struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq;
1396         struct qedr_dev *dev = srq->dev;
1397
1398         dev->ops->common->chain_free(dev->cdev, &hw_srq->pbl);
1399
1400         dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers),
1401                           hw_srq->virt_prod_pair_addr,
1402                           hw_srq->phy_prod_pair_addr);
1403 }
1404
1405 static int qedr_init_srq_user_params(struct ib_udata *udata,
1406                                      struct qedr_srq *srq,
1407                                      struct qedr_create_srq_ureq *ureq,
1408                                      int access)
1409 {
1410         struct scatterlist *sg;
1411         int rc;
1412
1413         rc = qedr_init_user_queue(udata, srq->dev, &srq->usrq, ureq->srq_addr,
1414                                   ureq->srq_len, false, access, 1);
1415         if (rc)
1416                 return rc;
1417
1418         srq->prod_umem = ib_umem_get(srq->ibsrq.device, ureq->prod_pair_addr,
1419                                      sizeof(struct rdma_srq_producers), access);
1420         if (IS_ERR(srq->prod_umem)) {
1421                 qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
1422                 ib_umem_release(srq->usrq.umem);
1423                 DP_ERR(srq->dev,
1424                        "create srq: failed ib_umem_get for producer, got %ld\n",
1425                        PTR_ERR(srq->prod_umem));
1426                 return PTR_ERR(srq->prod_umem);
1427         }
1428
1429         sg = srq->prod_umem->sg_head.sgl;
1430         srq->hw_srq.phy_prod_pair_addr = sg_dma_address(sg);
1431
1432         return 0;
1433 }
1434
1435 static int qedr_alloc_srq_kernel_params(struct qedr_srq *srq,
1436                                         struct qedr_dev *dev,
1437                                         struct ib_srq_init_attr *init_attr)
1438 {
1439         struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq;
1440         dma_addr_t phy_prod_pair_addr;
1441         u32 num_elems;
1442         void *va;
1443         int rc;
1444
1445         va = dma_alloc_coherent(&dev->pdev->dev,
1446                                 sizeof(struct rdma_srq_producers),
1447                                 &phy_prod_pair_addr, GFP_KERNEL);
1448         if (!va) {
1449                 DP_ERR(dev,
1450                        "create srq: failed to allocate dma memory for producer\n");
1451                 return -ENOMEM;
1452         }
1453
1454         hw_srq->phy_prod_pair_addr = phy_prod_pair_addr;
1455         hw_srq->virt_prod_pair_addr = va;
1456
1457         num_elems = init_attr->attr.max_wr * RDMA_MAX_SRQ_WQE_SIZE;
1458         rc = dev->ops->common->chain_alloc(dev->cdev,
1459                                            QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1460                                            QED_CHAIN_MODE_PBL,
1461                                            QED_CHAIN_CNT_TYPE_U32,
1462                                            num_elems,
1463                                            QEDR_SRQ_WQE_ELEM_SIZE,
1464                                            &hw_srq->pbl, NULL);
1465         if (rc)
1466                 goto err0;
1467
1468         hw_srq->num_elems = num_elems;
1469
1470         return 0;
1471
1472 err0:
1473         dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers),
1474                           va, phy_prod_pair_addr);
1475         return rc;
1476 }
1477
1478 int qedr_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr,
1479                     struct ib_udata *udata)
1480 {
1481         struct qed_rdma_destroy_srq_in_params destroy_in_params;
1482         struct qed_rdma_create_srq_in_params in_params = {};
1483         struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1484         struct qed_rdma_create_srq_out_params out_params;
1485         struct qedr_pd *pd = get_qedr_pd(ibsrq->pd);
1486         struct qedr_create_srq_ureq ureq = {};
1487         u64 pbl_base_addr, phy_prod_pair_addr;
1488         struct qedr_srq_hwq_info *hw_srq;
1489         u32 page_cnt, page_size;
1490         struct qedr_srq *srq = get_qedr_srq(ibsrq);
1491         int rc = 0;
1492
1493         DP_DEBUG(dev, QEDR_MSG_QP,
1494                  "create SRQ called from %s (pd %p)\n",
1495                  (udata) ? "User lib" : "kernel", pd);
1496
1497         rc = qedr_check_srq_params(dev, init_attr, udata);
1498         if (rc)
1499                 return -EINVAL;
1500
1501         srq->dev = dev;
1502         hw_srq = &srq->hw_srq;
1503         spin_lock_init(&srq->lock);
1504
1505         hw_srq->max_wr = init_attr->attr.max_wr;
1506         hw_srq->max_sges = init_attr->attr.max_sge;
1507
1508         if (udata) {
1509                 if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq),
1510                                                          udata->inlen))) {
1511                         DP_ERR(dev,
1512                                "create srq: problem copying data from user space\n");
1513                         goto err0;
1514                 }
1515
1516                 rc = qedr_init_srq_user_params(udata, srq, &ureq, 0);
1517                 if (rc)
1518                         goto err0;
1519
1520                 page_cnt = srq->usrq.pbl_info.num_pbes;
1521                 pbl_base_addr = srq->usrq.pbl_tbl->pa;
1522                 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
1523                 page_size = PAGE_SIZE;
1524         } else {
1525                 struct qed_chain *pbl;
1526
1527                 rc = qedr_alloc_srq_kernel_params(srq, dev, init_attr);
1528                 if (rc)
1529                         goto err0;
1530
1531                 pbl = &hw_srq->pbl;
1532                 page_cnt = qed_chain_get_page_cnt(pbl);
1533                 pbl_base_addr = qed_chain_get_pbl_phys(pbl);
1534                 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
1535                 page_size = QED_CHAIN_PAGE_SIZE;
1536         }
1537
1538         in_params.pd_id = pd->pd_id;
1539         in_params.pbl_base_addr = pbl_base_addr;
1540         in_params.prod_pair_addr = phy_prod_pair_addr;
1541         in_params.num_pages = page_cnt;
1542         in_params.page_size = page_size;
1543
1544         rc = dev->ops->rdma_create_srq(dev->rdma_ctx, &in_params, &out_params);
1545         if (rc)
1546                 goto err1;
1547
1548         srq->srq_id = out_params.srq_id;
1549
1550         if (udata) {
1551                 rc = qedr_copy_srq_uresp(dev, srq, udata);
1552                 if (rc)
1553                         goto err2;
1554         }
1555
1556         rc = xa_insert_irq(&dev->srqs, srq->srq_id, srq, GFP_KERNEL);
1557         if (rc)
1558                 goto err2;
1559
1560         DP_DEBUG(dev, QEDR_MSG_SRQ,
1561                  "create srq: created srq with srq_id=0x%0x\n", srq->srq_id);
1562         return 0;
1563
1564 err2:
1565         destroy_in_params.srq_id = srq->srq_id;
1566
1567         dev->ops->rdma_destroy_srq(dev->rdma_ctx, &destroy_in_params);
1568 err1:
1569         if (udata)
1570                 qedr_free_srq_user_params(srq);
1571         else
1572                 qedr_free_srq_kernel_params(srq);
1573 err0:
1574         return -EFAULT;
1575 }
1576
1577 void qedr_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata)
1578 {
1579         struct qed_rdma_destroy_srq_in_params in_params = {};
1580         struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1581         struct qedr_srq *srq = get_qedr_srq(ibsrq);
1582
1583         xa_erase_irq(&dev->srqs, srq->srq_id);
1584         in_params.srq_id = srq->srq_id;
1585         dev->ops->rdma_destroy_srq(dev->rdma_ctx, &in_params);
1586
1587         if (ibsrq->uobject)
1588                 qedr_free_srq_user_params(srq);
1589         else
1590                 qedr_free_srq_kernel_params(srq);
1591
1592         DP_DEBUG(dev, QEDR_MSG_SRQ,
1593                  "destroy srq: destroyed srq with srq_id=0x%0x\n",
1594                  srq->srq_id);
1595 }
1596
1597 int qedr_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
1598                     enum ib_srq_attr_mask attr_mask, struct ib_udata *udata)
1599 {
1600         struct qed_rdma_modify_srq_in_params in_params = {};
1601         struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1602         struct qedr_srq *srq = get_qedr_srq(ibsrq);
1603         int rc;
1604
1605         if (attr_mask & IB_SRQ_MAX_WR) {
1606                 DP_ERR(dev,
1607                        "modify srq: invalid attribute mask=0x%x specified for %p\n",
1608                        attr_mask, srq);
1609                 return -EINVAL;
1610         }
1611
1612         if (attr_mask & IB_SRQ_LIMIT) {
1613                 if (attr->srq_limit >= srq->hw_srq.max_wr) {
1614                         DP_ERR(dev,
1615                                "modify srq: invalid srq_limit=0x%x (max_srq_limit=0x%x)\n",
1616                                attr->srq_limit, srq->hw_srq.max_wr);
1617                         return -EINVAL;
1618                 }
1619
1620                 in_params.srq_id = srq->srq_id;
1621                 in_params.wqe_limit = attr->srq_limit;
1622                 rc = dev->ops->rdma_modify_srq(dev->rdma_ctx, &in_params);
1623                 if (rc)
1624                         return rc;
1625         }
1626
1627         srq->srq_limit = attr->srq_limit;
1628
1629         DP_DEBUG(dev, QEDR_MSG_SRQ,
1630                  "modify srq: modified srq with srq_id=0x%0x\n", srq->srq_id);
1631
1632         return 0;
1633 }
1634
1635 static inline void
1636 qedr_init_common_qp_in_params(struct qedr_dev *dev,
1637                               struct qedr_pd *pd,
1638                               struct qedr_qp *qp,
1639                               struct ib_qp_init_attr *attrs,
1640                               bool fmr_and_reserved_lkey,
1641                               struct qed_rdma_create_qp_in_params *params)
1642 {
1643         /* QP handle to be written in an async event */
1644         params->qp_handle_async_lo = lower_32_bits((uintptr_t) qp);
1645         params->qp_handle_async_hi = upper_32_bits((uintptr_t) qp);
1646
1647         params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR);
1648         params->fmr_and_reserved_lkey = fmr_and_reserved_lkey;
1649         params->pd = pd->pd_id;
1650         params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi;
1651         params->sq_cq_id = get_qedr_cq(attrs->send_cq)->icid;
1652         params->stats_queue = 0;
1653         params->srq_id = 0;
1654         params->use_srq = false;
1655
1656         if (!qp->srq) {
1657                 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1658
1659         } else {
1660                 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1661                 params->srq_id = qp->srq->srq_id;
1662                 params->use_srq = true;
1663         }
1664 }
1665
1666 static inline void qedr_qp_user_print(struct qedr_dev *dev, struct qedr_qp *qp)
1667 {
1668         DP_DEBUG(dev, QEDR_MSG_QP, "create qp: successfully created user QP. "
1669                  "qp=%p. "
1670                  "sq_addr=0x%llx, "
1671                  "sq_len=%zd, "
1672                  "rq_addr=0x%llx, "
1673                  "rq_len=%zd"
1674                  "\n",
1675                  qp,
1676                  qp->usq.buf_addr,
1677                  qp->usq.buf_len, qp->urq.buf_addr, qp->urq.buf_len);
1678 }
1679
1680 static inline void
1681 qedr_iwarp_populate_user_qp(struct qedr_dev *dev,
1682                             struct qedr_qp *qp,
1683                             struct qed_rdma_create_qp_out_params *out_params)
1684 {
1685         qp->usq.pbl_tbl->va = out_params->sq_pbl_virt;
1686         qp->usq.pbl_tbl->pa = out_params->sq_pbl_phys;
1687
1688         qedr_populate_pbls(dev, qp->usq.umem, qp->usq.pbl_tbl,
1689                            &qp->usq.pbl_info, FW_PAGE_SHIFT);
1690         if (!qp->srq) {
1691                 qp->urq.pbl_tbl->va = out_params->rq_pbl_virt;
1692                 qp->urq.pbl_tbl->pa = out_params->rq_pbl_phys;
1693         }
1694
1695         qedr_populate_pbls(dev, qp->urq.umem, qp->urq.pbl_tbl,
1696                            &qp->urq.pbl_info, FW_PAGE_SHIFT);
1697 }
1698
1699 static void qedr_cleanup_user(struct qedr_dev *dev,
1700                               struct qedr_ucontext *ctx,
1701                               struct qedr_qp *qp)
1702 {
1703         ib_umem_release(qp->usq.umem);
1704         qp->usq.umem = NULL;
1705
1706         ib_umem_release(qp->urq.umem);
1707         qp->urq.umem = NULL;
1708
1709         if (rdma_protocol_roce(&dev->ibdev, 1)) {
1710                 qedr_free_pbl(dev, &qp->usq.pbl_info, qp->usq.pbl_tbl);
1711                 qedr_free_pbl(dev, &qp->urq.pbl_info, qp->urq.pbl_tbl);
1712         } else {
1713                 kfree(qp->usq.pbl_tbl);
1714                 kfree(qp->urq.pbl_tbl);
1715         }
1716
1717         if (qp->usq.db_rec_data) {
1718                 qedr_db_recovery_del(dev, qp->usq.db_addr,
1719                                      &qp->usq.db_rec_data->db_data);
1720                 rdma_user_mmap_entry_remove(qp->usq.db_mmap_entry);
1721         }
1722
1723         if (qp->urq.db_rec_data) {
1724                 qedr_db_recovery_del(dev, qp->urq.db_addr,
1725                                      &qp->urq.db_rec_data->db_data);
1726                 rdma_user_mmap_entry_remove(qp->urq.db_mmap_entry);
1727         }
1728
1729         if (rdma_protocol_iwarp(&dev->ibdev, 1))
1730                 qedr_db_recovery_del(dev, qp->urq.db_rec_db2_addr,
1731                                      &qp->urq.db_rec_db2_data);
1732 }
1733
1734 static int qedr_create_user_qp(struct qedr_dev *dev,
1735                                struct qedr_qp *qp,
1736                                struct ib_pd *ibpd,
1737                                struct ib_udata *udata,
1738                                struct ib_qp_init_attr *attrs)
1739 {
1740         struct qed_rdma_create_qp_in_params in_params;
1741         struct qed_rdma_create_qp_out_params out_params;
1742         struct qedr_pd *pd = get_qedr_pd(ibpd);
1743         struct qedr_create_qp_uresp uresp;
1744         struct qedr_ucontext *ctx = NULL;
1745         struct qedr_create_qp_ureq ureq;
1746         int alloc_and_init = rdma_protocol_roce(&dev->ibdev, 1);
1747         int rc = -EINVAL;
1748
1749         qp->create_type = QEDR_QP_CREATE_USER;
1750         memset(&ureq, 0, sizeof(ureq));
1751         rc = ib_copy_from_udata(&ureq, udata, min(sizeof(ureq), udata->inlen));
1752         if (rc) {
1753                 DP_ERR(dev, "Problem copying data from user space\n");
1754                 return rc;
1755         }
1756
1757         /* SQ - read access only (0) */
1758         rc = qedr_init_user_queue(udata, dev, &qp->usq, ureq.sq_addr,
1759                                   ureq.sq_len, true, 0, alloc_and_init);
1760         if (rc)
1761                 return rc;
1762
1763         if (!qp->srq) {
1764                 /* RQ - read access only (0) */
1765                 rc = qedr_init_user_queue(udata, dev, &qp->urq, ureq.rq_addr,
1766                                           ureq.rq_len, true, 0, alloc_and_init);
1767                 if (rc)
1768                         return rc;
1769         }
1770
1771         memset(&in_params, 0, sizeof(in_params));
1772         qedr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params);
1773         in_params.qp_handle_lo = ureq.qp_handle_lo;
1774         in_params.qp_handle_hi = ureq.qp_handle_hi;
1775         in_params.sq_num_pages = qp->usq.pbl_info.num_pbes;
1776         in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa;
1777         if (!qp->srq) {
1778                 in_params.rq_num_pages = qp->urq.pbl_info.num_pbes;
1779                 in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa;
1780         }
1781
1782         qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1783                                               &in_params, &out_params);
1784
1785         if (!qp->qed_qp) {
1786                 rc = -ENOMEM;
1787                 goto err1;
1788         }
1789
1790         if (rdma_protocol_iwarp(&dev->ibdev, 1))
1791                 qedr_iwarp_populate_user_qp(dev, qp, &out_params);
1792
1793         qp->qp_id = out_params.qp_id;
1794         qp->icid = out_params.icid;
1795
1796         rc = qedr_copy_qp_uresp(dev, qp, udata, &uresp);
1797         if (rc)
1798                 goto err;
1799
1800         /* db offset was calculated in copy_qp_uresp, now set in the user q */
1801         ctx = pd->uctx;
1802         qp->usq.db_addr = ctx->dpi_addr + uresp.sq_db_offset;
1803         qp->urq.db_addr = ctx->dpi_addr + uresp.rq_db_offset;
1804
1805         if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1806                 qp->urq.db_rec_db2_addr = ctx->dpi_addr + uresp.rq_db2_offset;
1807
1808                 /* calculate the db_rec_db2 data since it is constant so no
1809                  *  need to reflect from user
1810                  */
1811                 qp->urq.db_rec_db2_data.data.icid = cpu_to_le16(qp->icid);
1812                 qp->urq.db_rec_db2_data.data.value =
1813                         cpu_to_le16(DQ_TCM_IWARP_POST_RQ_CF_CMD);
1814         }
1815
1816         rc = qedr_db_recovery_add(dev, qp->usq.db_addr,
1817                                   &qp->usq.db_rec_data->db_data,
1818                                   DB_REC_WIDTH_32B,
1819                                   DB_REC_USER);
1820         if (rc)
1821                 goto err;
1822
1823         rc = qedr_db_recovery_add(dev, qp->urq.db_addr,
1824                                   &qp->urq.db_rec_data->db_data,
1825                                   DB_REC_WIDTH_32B,
1826                                   DB_REC_USER);
1827         if (rc)
1828                 goto err;
1829
1830         if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1831                 rc = qedr_db_recovery_add(dev, qp->urq.db_rec_db2_addr,
1832                                           &qp->urq.db_rec_db2_data,
1833                                           DB_REC_WIDTH_32B,
1834                                           DB_REC_USER);
1835                 if (rc)
1836                         goto err;
1837         }
1838         qedr_qp_user_print(dev, qp);
1839
1840         return rc;
1841 err:
1842         rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
1843         if (rc)
1844                 DP_ERR(dev, "create qp: fatal fault. rc=%d", rc);
1845
1846 err1:
1847         qedr_cleanup_user(dev, ctx, qp);
1848         return rc;
1849 }
1850
1851 static int qedr_set_iwarp_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
1852 {
1853         int rc;
1854
1855         qp->sq.db = dev->db_addr +
1856             DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1857         qp->sq.db_data.data.icid = qp->icid;
1858
1859         rc = qedr_db_recovery_add(dev, qp->sq.db,
1860                                   &qp->sq.db_data,
1861                                   DB_REC_WIDTH_32B,
1862                                   DB_REC_KERNEL);
1863         if (rc)
1864                 return rc;
1865
1866         qp->rq.db = dev->db_addr +
1867                     DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
1868         qp->rq.db_data.data.icid = qp->icid;
1869         qp->rq.iwarp_db2 = dev->db_addr +
1870                            DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
1871         qp->rq.iwarp_db2_data.data.icid = qp->icid;
1872         qp->rq.iwarp_db2_data.data.value = DQ_TCM_IWARP_POST_RQ_CF_CMD;
1873
1874         rc = qedr_db_recovery_add(dev, qp->rq.db,
1875                                   &qp->rq.db_data,
1876                                   DB_REC_WIDTH_32B,
1877                                   DB_REC_KERNEL);
1878         if (rc)
1879                 return rc;
1880
1881         rc = qedr_db_recovery_add(dev, qp->rq.iwarp_db2,
1882                                   &qp->rq.iwarp_db2_data,
1883                                   DB_REC_WIDTH_32B,
1884                                   DB_REC_KERNEL);
1885         return rc;
1886 }
1887
1888 static int
1889 qedr_roce_create_kernel_qp(struct qedr_dev *dev,
1890                            struct qedr_qp *qp,
1891                            struct qed_rdma_create_qp_in_params *in_params,
1892                            u32 n_sq_elems, u32 n_rq_elems)
1893 {
1894         struct qed_rdma_create_qp_out_params out_params;
1895         int rc;
1896
1897         rc = dev->ops->common->chain_alloc(dev->cdev,
1898                                            QED_CHAIN_USE_TO_PRODUCE,
1899                                            QED_CHAIN_MODE_PBL,
1900                                            QED_CHAIN_CNT_TYPE_U32,
1901                                            n_sq_elems,
1902                                            QEDR_SQE_ELEMENT_SIZE,
1903                                            &qp->sq.pbl, NULL);
1904
1905         if (rc)
1906                 return rc;
1907
1908         in_params->sq_num_pages = qed_chain_get_page_cnt(&qp->sq.pbl);
1909         in_params->sq_pbl_ptr = qed_chain_get_pbl_phys(&qp->sq.pbl);
1910
1911         rc = dev->ops->common->chain_alloc(dev->cdev,
1912                                            QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1913                                            QED_CHAIN_MODE_PBL,
1914                                            QED_CHAIN_CNT_TYPE_U32,
1915                                            n_rq_elems,
1916                                            QEDR_RQE_ELEMENT_SIZE,
1917                                            &qp->rq.pbl, NULL);
1918         if (rc)
1919                 return rc;
1920
1921         in_params->rq_num_pages = qed_chain_get_page_cnt(&qp->rq.pbl);
1922         in_params->rq_pbl_ptr = qed_chain_get_pbl_phys(&qp->rq.pbl);
1923
1924         qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1925                                               in_params, &out_params);
1926
1927         if (!qp->qed_qp)
1928                 return -EINVAL;
1929
1930         qp->qp_id = out_params.qp_id;
1931         qp->icid = out_params.icid;
1932
1933         return qedr_set_roce_db_info(dev, qp);
1934 }
1935
1936 static int
1937 qedr_iwarp_create_kernel_qp(struct qedr_dev *dev,
1938                             struct qedr_qp *qp,
1939                             struct qed_rdma_create_qp_in_params *in_params,
1940                             u32 n_sq_elems, u32 n_rq_elems)
1941 {
1942         struct qed_rdma_create_qp_out_params out_params;
1943         struct qed_chain_ext_pbl ext_pbl;
1944         int rc;
1945
1946         in_params->sq_num_pages = QED_CHAIN_PAGE_CNT(n_sq_elems,
1947                                                      QEDR_SQE_ELEMENT_SIZE,
1948                                                      QED_CHAIN_MODE_PBL);
1949         in_params->rq_num_pages = QED_CHAIN_PAGE_CNT(n_rq_elems,
1950                                                      QEDR_RQE_ELEMENT_SIZE,
1951                                                      QED_CHAIN_MODE_PBL);
1952
1953         qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1954                                               in_params, &out_params);
1955
1956         if (!qp->qed_qp)
1957                 return -EINVAL;
1958
1959         /* Now we allocate the chain */
1960         ext_pbl.p_pbl_virt = out_params.sq_pbl_virt;
1961         ext_pbl.p_pbl_phys = out_params.sq_pbl_phys;
1962
1963         rc = dev->ops->common->chain_alloc(dev->cdev,
1964                                            QED_CHAIN_USE_TO_PRODUCE,
1965                                            QED_CHAIN_MODE_PBL,
1966                                            QED_CHAIN_CNT_TYPE_U32,
1967                                            n_sq_elems,
1968                                            QEDR_SQE_ELEMENT_SIZE,
1969                                            &qp->sq.pbl, &ext_pbl);
1970
1971         if (rc)
1972                 goto err;
1973
1974         ext_pbl.p_pbl_virt = out_params.rq_pbl_virt;
1975         ext_pbl.p_pbl_phys = out_params.rq_pbl_phys;
1976
1977         rc = dev->ops->common->chain_alloc(dev->cdev,
1978                                            QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1979                                            QED_CHAIN_MODE_PBL,
1980                                            QED_CHAIN_CNT_TYPE_U32,
1981                                            n_rq_elems,
1982                                            QEDR_RQE_ELEMENT_SIZE,
1983                                            &qp->rq.pbl, &ext_pbl);
1984
1985         if (rc)
1986                 goto err;
1987
1988         qp->qp_id = out_params.qp_id;
1989         qp->icid = out_params.icid;
1990
1991         return qedr_set_iwarp_db_info(dev, qp);
1992
1993 err:
1994         dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
1995
1996         return rc;
1997 }
1998
1999 static void qedr_cleanup_kernel(struct qedr_dev *dev, struct qedr_qp *qp)
2000 {
2001         dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl);
2002         kfree(qp->wqe_wr_id);
2003
2004         dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl);
2005         kfree(qp->rqe_wr_id);
2006
2007         /* GSI qp is not registered to db mechanism so no need to delete */
2008         if (qp->qp_type == IB_QPT_GSI)
2009                 return;
2010
2011         qedr_db_recovery_del(dev, qp->sq.db, &qp->sq.db_data);
2012
2013         if (!qp->srq) {
2014                 qedr_db_recovery_del(dev, qp->rq.db, &qp->rq.db_data);
2015
2016                 if (rdma_protocol_iwarp(&dev->ibdev, 1))
2017                         qedr_db_recovery_del(dev, qp->rq.iwarp_db2,
2018                                              &qp->rq.iwarp_db2_data);
2019         }
2020 }
2021
2022 static int qedr_create_kernel_qp(struct qedr_dev *dev,
2023                                  struct qedr_qp *qp,
2024                                  struct ib_pd *ibpd,
2025                                  struct ib_qp_init_attr *attrs)
2026 {
2027         struct qed_rdma_create_qp_in_params in_params;
2028         struct qedr_pd *pd = get_qedr_pd(ibpd);
2029         int rc = -EINVAL;
2030         u32 n_rq_elems;
2031         u32 n_sq_elems;
2032         u32 n_sq_entries;
2033
2034         memset(&in_params, 0, sizeof(in_params));
2035         qp->create_type = QEDR_QP_CREATE_KERNEL;
2036
2037         /* A single work request may take up to QEDR_MAX_SQ_WQE_SIZE elements in
2038          * the ring. The ring should allow at least a single WR, even if the
2039          * user requested none, due to allocation issues.
2040          * We should add an extra WR since the prod and cons indices of
2041          * wqe_wr_id are managed in such a way that the WQ is considered full
2042          * when (prod+1)%max_wr==cons. We currently don't do that because we
2043          * double the number of entries due an iSER issue that pushes far more
2044          * WRs than indicated. If we decline its ib_post_send() then we get
2045          * error prints in the dmesg we'd like to avoid.
2046          */
2047         qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier,
2048                               dev->attr.max_sqe);
2049
2050         qp->wqe_wr_id = kcalloc(qp->sq.max_wr, sizeof(*qp->wqe_wr_id),
2051                                 GFP_KERNEL);
2052         if (!qp->wqe_wr_id) {
2053                 DP_ERR(dev, "create qp: failed SQ shadow memory allocation\n");
2054                 return -ENOMEM;
2055         }
2056
2057         /* QP handle to be written in CQE */
2058         in_params.qp_handle_lo = lower_32_bits((uintptr_t) qp);
2059         in_params.qp_handle_hi = upper_32_bits((uintptr_t) qp);
2060
2061         /* A single work request may take up to QEDR_MAX_RQ_WQE_SIZE elements in
2062          * the ring. There ring should allow at least a single WR, even if the
2063          * user requested none, due to allocation issues.
2064          */
2065         qp->rq.max_wr = (u16) max_t(u32, attrs->cap.max_recv_wr, 1);
2066
2067         /* Allocate driver internal RQ array */
2068         qp->rqe_wr_id = kcalloc(qp->rq.max_wr, sizeof(*qp->rqe_wr_id),
2069                                 GFP_KERNEL);
2070         if (!qp->rqe_wr_id) {
2071                 DP_ERR(dev,
2072                        "create qp: failed RQ shadow memory allocation\n");
2073                 kfree(qp->wqe_wr_id);
2074                 return -ENOMEM;
2075         }
2076
2077         qedr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params);
2078
2079         n_sq_entries = attrs->cap.max_send_wr;
2080         n_sq_entries = min_t(u32, n_sq_entries, dev->attr.max_sqe);
2081         n_sq_entries = max_t(u32, n_sq_entries, 1);
2082         n_sq_elems = n_sq_entries * QEDR_MAX_SQE_ELEMENTS_PER_SQE;
2083
2084         n_rq_elems = qp->rq.max_wr * QEDR_MAX_RQE_ELEMENTS_PER_RQE;
2085
2086         if (rdma_protocol_iwarp(&dev->ibdev, 1))
2087                 rc = qedr_iwarp_create_kernel_qp(dev, qp, &in_params,
2088                                                  n_sq_elems, n_rq_elems);
2089         else
2090                 rc = qedr_roce_create_kernel_qp(dev, qp, &in_params,
2091                                                 n_sq_elems, n_rq_elems);
2092         if (rc)
2093                 qedr_cleanup_kernel(dev, qp);
2094
2095         return rc;
2096 }
2097
2098 struct ib_qp *qedr_create_qp(struct ib_pd *ibpd,
2099                              struct ib_qp_init_attr *attrs,
2100                              struct ib_udata *udata)
2101 {
2102         struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2103         struct qedr_pd *pd = get_qedr_pd(ibpd);
2104         struct qedr_qp *qp;
2105         struct ib_qp *ibqp;
2106         int rc = 0;
2107
2108         DP_DEBUG(dev, QEDR_MSG_QP, "create qp: called from %s, pd=%p\n",
2109                  udata ? "user library" : "kernel", pd);
2110
2111         rc = qedr_check_qp_attrs(ibpd, dev, attrs, udata);
2112         if (rc)
2113                 return ERR_PTR(rc);
2114
2115         DP_DEBUG(dev, QEDR_MSG_QP,
2116                  "create qp: called from %s, event_handler=%p, eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n",
2117                  udata ? "user library" : "kernel", attrs->event_handler, pd,
2118                  get_qedr_cq(attrs->send_cq),
2119                  get_qedr_cq(attrs->send_cq)->icid,
2120                  get_qedr_cq(attrs->recv_cq),
2121                  attrs->recv_cq ? get_qedr_cq(attrs->recv_cq)->icid : 0);
2122
2123         qp = kzalloc(sizeof(*qp), GFP_KERNEL);
2124         if (!qp) {
2125                 DP_ERR(dev, "create qp: failed allocating memory\n");
2126                 return ERR_PTR(-ENOMEM);
2127         }
2128
2129         qedr_set_common_qp_params(dev, qp, pd, attrs);
2130
2131         if (attrs->qp_type == IB_QPT_GSI) {
2132                 ibqp = qedr_create_gsi_qp(dev, attrs, qp);
2133                 if (IS_ERR(ibqp))
2134                         kfree(qp);
2135                 return ibqp;
2136         }
2137
2138         if (udata)
2139                 rc = qedr_create_user_qp(dev, qp, ibpd, udata, attrs);
2140         else
2141                 rc = qedr_create_kernel_qp(dev, qp, ibpd, attrs);
2142
2143         if (rc)
2144                 goto err;
2145
2146         qp->ibqp.qp_num = qp->qp_id;
2147
2148         if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
2149                 rc = xa_insert(&dev->qps, qp->qp_id, qp, GFP_KERNEL);
2150                 if (rc)
2151                         goto err;
2152         }
2153
2154         return &qp->ibqp;
2155
2156 err:
2157         kfree(qp);
2158
2159         return ERR_PTR(-EFAULT);
2160 }
2161
2162 static enum ib_qp_state qedr_get_ibqp_state(enum qed_roce_qp_state qp_state)
2163 {
2164         switch (qp_state) {
2165         case QED_ROCE_QP_STATE_RESET:
2166                 return IB_QPS_RESET;
2167         case QED_ROCE_QP_STATE_INIT:
2168                 return IB_QPS_INIT;
2169         case QED_ROCE_QP_STATE_RTR:
2170                 return IB_QPS_RTR;
2171         case QED_ROCE_QP_STATE_RTS:
2172                 return IB_QPS_RTS;
2173         case QED_ROCE_QP_STATE_SQD:
2174                 return IB_QPS_SQD;
2175         case QED_ROCE_QP_STATE_ERR:
2176                 return IB_QPS_ERR;
2177         case QED_ROCE_QP_STATE_SQE:
2178                 return IB_QPS_SQE;
2179         }
2180         return IB_QPS_ERR;
2181 }
2182
2183 static enum qed_roce_qp_state qedr_get_state_from_ibqp(
2184                                         enum ib_qp_state qp_state)
2185 {
2186         switch (qp_state) {
2187         case IB_QPS_RESET:
2188                 return QED_ROCE_QP_STATE_RESET;
2189         case IB_QPS_INIT:
2190                 return QED_ROCE_QP_STATE_INIT;
2191         case IB_QPS_RTR:
2192                 return QED_ROCE_QP_STATE_RTR;
2193         case IB_QPS_RTS:
2194                 return QED_ROCE_QP_STATE_RTS;
2195         case IB_QPS_SQD:
2196                 return QED_ROCE_QP_STATE_SQD;
2197         case IB_QPS_ERR:
2198                 return QED_ROCE_QP_STATE_ERR;
2199         default:
2200                 return QED_ROCE_QP_STATE_ERR;
2201         }
2202 }
2203
2204 static void qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info *qph)
2205 {
2206         qed_chain_reset(&qph->pbl);
2207         qph->prod = 0;
2208         qph->cons = 0;
2209         qph->wqe_cons = 0;
2210         qph->db_data.data.value = cpu_to_le16(0);
2211 }
2212
2213 static int qedr_update_qp_state(struct qedr_dev *dev,
2214                                 struct qedr_qp *qp,
2215                                 enum qed_roce_qp_state cur_state,
2216                                 enum qed_roce_qp_state new_state)
2217 {
2218         int status = 0;
2219
2220         if (new_state == cur_state)
2221                 return 0;
2222
2223         switch (cur_state) {
2224         case QED_ROCE_QP_STATE_RESET:
2225                 switch (new_state) {
2226                 case QED_ROCE_QP_STATE_INIT:
2227                         qp->prev_wqe_size = 0;
2228                         qedr_reset_qp_hwq_info(&qp->sq);
2229                         qedr_reset_qp_hwq_info(&qp->rq);
2230                         break;
2231                 default:
2232                         status = -EINVAL;
2233                         break;
2234                 }
2235                 break;
2236         case QED_ROCE_QP_STATE_INIT:
2237                 switch (new_state) {
2238                 case QED_ROCE_QP_STATE_RTR:
2239                         /* Update doorbell (in case post_recv was
2240                          * done before move to RTR)
2241                          */
2242
2243                         if (rdma_protocol_roce(&dev->ibdev, 1)) {
2244                                 writel(qp->rq.db_data.raw, qp->rq.db);
2245                         }
2246                         break;
2247                 case QED_ROCE_QP_STATE_ERR:
2248                         break;
2249                 default:
2250                         /* Invalid state change. */
2251                         status = -EINVAL;
2252                         break;
2253                 }
2254                 break;
2255         case QED_ROCE_QP_STATE_RTR:
2256                 /* RTR->XXX */
2257                 switch (new_state) {
2258                 case QED_ROCE_QP_STATE_RTS:
2259                         break;
2260                 case QED_ROCE_QP_STATE_ERR:
2261                         break;
2262                 default:
2263                         /* Invalid state change. */
2264                         status = -EINVAL;
2265                         break;
2266                 }
2267                 break;
2268         case QED_ROCE_QP_STATE_RTS:
2269                 /* RTS->XXX */
2270                 switch (new_state) {
2271                 case QED_ROCE_QP_STATE_SQD:
2272                         break;
2273                 case QED_ROCE_QP_STATE_ERR:
2274                         break;
2275                 default:
2276                         /* Invalid state change. */
2277                         status = -EINVAL;
2278                         break;
2279                 }
2280                 break;
2281         case QED_ROCE_QP_STATE_SQD:
2282                 /* SQD->XXX */
2283                 switch (new_state) {
2284                 case QED_ROCE_QP_STATE_RTS:
2285                 case QED_ROCE_QP_STATE_ERR:
2286                         break;
2287                 default:
2288                         /* Invalid state change. */
2289                         status = -EINVAL;
2290                         break;
2291                 }
2292                 break;
2293         case QED_ROCE_QP_STATE_ERR:
2294                 /* ERR->XXX */
2295                 switch (new_state) {
2296                 case QED_ROCE_QP_STATE_RESET:
2297                         if ((qp->rq.prod != qp->rq.cons) ||
2298                             (qp->sq.prod != qp->sq.cons)) {
2299                                 DP_NOTICE(dev,
2300                                           "Error->Reset with rq/sq not empty rq.prod=%x rq.cons=%x sq.prod=%x sq.cons=%x\n",
2301                                           qp->rq.prod, qp->rq.cons, qp->sq.prod,
2302                                           qp->sq.cons);
2303                                 status = -EINVAL;
2304                         }
2305                         break;
2306                 default:
2307                         status = -EINVAL;
2308                         break;
2309                 }
2310                 break;
2311         default:
2312                 status = -EINVAL;
2313                 break;
2314         }
2315
2316         return status;
2317 }
2318
2319 int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
2320                    int attr_mask, struct ib_udata *udata)
2321 {
2322         struct qedr_qp *qp = get_qedr_qp(ibqp);
2323         struct qed_rdma_modify_qp_in_params qp_params = { 0 };
2324         struct qedr_dev *dev = get_qedr_dev(&qp->dev->ibdev);
2325         const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
2326         enum ib_qp_state old_qp_state, new_qp_state;
2327         enum qed_roce_qp_state cur_state;
2328         int rc = 0;
2329
2330         DP_DEBUG(dev, QEDR_MSG_QP,
2331                  "modify qp: qp %p attr_mask=0x%x, state=%d", qp, attr_mask,
2332                  attr->qp_state);
2333
2334         old_qp_state = qedr_get_ibqp_state(qp->state);
2335         if (attr_mask & IB_QP_STATE)
2336                 new_qp_state = attr->qp_state;
2337         else
2338                 new_qp_state = old_qp_state;
2339
2340         if (rdma_protocol_roce(&dev->ibdev, 1)) {
2341                 if (!ib_modify_qp_is_ok(old_qp_state, new_qp_state,
2342                                         ibqp->qp_type, attr_mask)) {
2343                         DP_ERR(dev,
2344                                "modify qp: invalid attribute mask=0x%x specified for\n"
2345                                "qpn=0x%x of type=0x%x old_qp_state=0x%x, new_qp_state=0x%x\n",
2346                                attr_mask, qp->qp_id, ibqp->qp_type,
2347                                old_qp_state, new_qp_state);
2348                         rc = -EINVAL;
2349                         goto err;
2350                 }
2351         }
2352
2353         /* Translate the masks... */
2354         if (attr_mask & IB_QP_STATE) {
2355                 SET_FIELD(qp_params.modify_flags,
2356                           QED_RDMA_MODIFY_QP_VALID_NEW_STATE, 1);
2357                 qp_params.new_state = qedr_get_state_from_ibqp(attr->qp_state);
2358         }
2359
2360         if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY)
2361                 qp_params.sqd_async = true;
2362
2363         if (attr_mask & IB_QP_PKEY_INDEX) {
2364                 SET_FIELD(qp_params.modify_flags,
2365                           QED_ROCE_MODIFY_QP_VALID_PKEY, 1);
2366                 if (attr->pkey_index >= QEDR_ROCE_PKEY_TABLE_LEN) {
2367                         rc = -EINVAL;
2368                         goto err;
2369                 }
2370
2371                 qp_params.pkey = QEDR_ROCE_PKEY_DEFAULT;
2372         }
2373
2374         if (attr_mask & IB_QP_QKEY)
2375                 qp->qkey = attr->qkey;
2376
2377         if (attr_mask & IB_QP_ACCESS_FLAGS) {
2378                 SET_FIELD(qp_params.modify_flags,
2379                           QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1);
2380                 qp_params.incoming_rdma_read_en = attr->qp_access_flags &
2381                                                   IB_ACCESS_REMOTE_READ;
2382                 qp_params.incoming_rdma_write_en = attr->qp_access_flags &
2383                                                    IB_ACCESS_REMOTE_WRITE;
2384                 qp_params.incoming_atomic_en = attr->qp_access_flags &
2385                                                IB_ACCESS_REMOTE_ATOMIC;
2386         }
2387
2388         if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) {
2389                 if (rdma_protocol_iwarp(&dev->ibdev, 1))
2390                         return -EINVAL;
2391
2392                 if (attr_mask & IB_QP_PATH_MTU) {
2393                         if (attr->path_mtu < IB_MTU_256 ||
2394                             attr->path_mtu > IB_MTU_4096) {
2395                                 pr_err("error: Only MTU sizes of 256, 512, 1024, 2048 and 4096 are supported by RoCE\n");
2396                                 rc = -EINVAL;
2397                                 goto err;
2398                         }
2399                         qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu),
2400                                       ib_mtu_enum_to_int(iboe_get_mtu
2401                                                          (dev->ndev->mtu)));
2402                 }
2403
2404                 if (!qp->mtu) {
2405                         qp->mtu =
2406                         ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
2407                         pr_err("Fixing zeroed MTU to qp->mtu = %d\n", qp->mtu);
2408                 }
2409
2410                 SET_FIELD(qp_params.modify_flags,
2411                           QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, 1);
2412
2413                 qp_params.traffic_class_tos = grh->traffic_class;
2414                 qp_params.flow_label = grh->flow_label;
2415                 qp_params.hop_limit_ttl = grh->hop_limit;
2416
2417                 qp->sgid_idx = grh->sgid_index;
2418
2419                 rc = get_gid_info_from_table(ibqp, attr, attr_mask, &qp_params);
2420                 if (rc) {
2421                         DP_ERR(dev,
2422                                "modify qp: problems with GID index %d (rc=%d)\n",
2423                                grh->sgid_index, rc);
2424                         return rc;
2425                 }
2426
2427                 rc = qedr_get_dmac(dev, &attr->ah_attr,
2428                                    qp_params.remote_mac_addr);
2429                 if (rc)
2430                         return rc;
2431
2432                 qp_params.use_local_mac = true;
2433                 ether_addr_copy(qp_params.local_mac_addr, dev->ndev->dev_addr);
2434
2435                 DP_DEBUG(dev, QEDR_MSG_QP, "dgid=%x:%x:%x:%x\n",
2436                          qp_params.dgid.dwords[0], qp_params.dgid.dwords[1],
2437                          qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]);
2438                 DP_DEBUG(dev, QEDR_MSG_QP, "sgid=%x:%x:%x:%x\n",
2439                          qp_params.sgid.dwords[0], qp_params.sgid.dwords[1],
2440                          qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]);
2441                 DP_DEBUG(dev, QEDR_MSG_QP, "remote_mac=[%pM]\n",
2442                          qp_params.remote_mac_addr);
2443
2444                 qp_params.mtu = qp->mtu;
2445                 qp_params.lb_indication = false;
2446         }
2447
2448         if (!qp_params.mtu) {
2449                 /* Stay with current MTU */
2450                 if (qp->mtu)
2451                         qp_params.mtu = qp->mtu;
2452                 else
2453                         qp_params.mtu =
2454                             ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
2455         }
2456
2457         if (attr_mask & IB_QP_TIMEOUT) {
2458                 SET_FIELD(qp_params.modify_flags,
2459                           QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1);
2460
2461                 /* The received timeout value is an exponent used like this:
2462                  *    "12.7.34 LOCAL ACK TIMEOUT
2463                  *    Value representing the transport (ACK) timeout for use by
2464                  *    the remote, expressed as: 4.096 * 2^timeout [usec]"
2465                  * The FW expects timeout in msec so we need to divide the usec
2466                  * result by 1000. We'll approximate 1000~2^10, and 4.096 ~ 2^2,
2467                  * so we get: 2^2 * 2^timeout / 2^10 = 2^(timeout - 8).
2468                  * The value of zero means infinite so we use a 'max_t' to make
2469                  * sure that sub 1 msec values will be configured as 1 msec.
2470                  */
2471                 if (attr->timeout)
2472                         qp_params.ack_timeout =
2473                                         1 << max_t(int, attr->timeout - 8, 0);
2474                 else
2475                         qp_params.ack_timeout = 0;
2476         }
2477
2478         if (attr_mask & IB_QP_RETRY_CNT) {
2479                 SET_FIELD(qp_params.modify_flags,
2480                           QED_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1);
2481                 qp_params.retry_cnt = attr->retry_cnt;
2482         }
2483
2484         if (attr_mask & IB_QP_RNR_RETRY) {
2485                 SET_FIELD(qp_params.modify_flags,
2486                           QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, 1);
2487                 qp_params.rnr_retry_cnt = attr->rnr_retry;
2488         }
2489
2490         if (attr_mask & IB_QP_RQ_PSN) {
2491                 SET_FIELD(qp_params.modify_flags,
2492                           QED_ROCE_MODIFY_QP_VALID_RQ_PSN, 1);
2493                 qp_params.rq_psn = attr->rq_psn;
2494                 qp->rq_psn = attr->rq_psn;
2495         }
2496
2497         if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
2498                 if (attr->max_rd_atomic > dev->attr.max_qp_req_rd_atomic_resc) {
2499                         rc = -EINVAL;
2500                         DP_ERR(dev,
2501                                "unsupported max_rd_atomic=%d, supported=%d\n",
2502                                attr->max_rd_atomic,
2503                                dev->attr.max_qp_req_rd_atomic_resc);
2504                         goto err;
2505                 }
2506
2507                 SET_FIELD(qp_params.modify_flags,
2508                           QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, 1);
2509                 qp_params.max_rd_atomic_req = attr->max_rd_atomic;
2510         }
2511
2512         if (attr_mask & IB_QP_MIN_RNR_TIMER) {
2513                 SET_FIELD(qp_params.modify_flags,
2514                           QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, 1);
2515                 qp_params.min_rnr_nak_timer = attr->min_rnr_timer;
2516         }
2517
2518         if (attr_mask & IB_QP_SQ_PSN) {
2519                 SET_FIELD(qp_params.modify_flags,
2520                           QED_ROCE_MODIFY_QP_VALID_SQ_PSN, 1);
2521                 qp_params.sq_psn = attr->sq_psn;
2522                 qp->sq_psn = attr->sq_psn;
2523         }
2524
2525         if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
2526                 if (attr->max_dest_rd_atomic >
2527                     dev->attr.max_qp_resp_rd_atomic_resc) {
2528                         DP_ERR(dev,
2529                                "unsupported max_dest_rd_atomic=%d, supported=%d\n",
2530                                attr->max_dest_rd_atomic,
2531                                dev->attr.max_qp_resp_rd_atomic_resc);
2532
2533                         rc = -EINVAL;
2534                         goto err;
2535                 }
2536
2537                 SET_FIELD(qp_params.modify_flags,
2538                           QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, 1);
2539                 qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic;
2540         }
2541
2542         if (attr_mask & IB_QP_DEST_QPN) {
2543                 SET_FIELD(qp_params.modify_flags,
2544                           QED_ROCE_MODIFY_QP_VALID_DEST_QP, 1);
2545
2546                 qp_params.dest_qp = attr->dest_qp_num;
2547                 qp->dest_qp_num = attr->dest_qp_num;
2548         }
2549
2550         cur_state = qp->state;
2551
2552         /* Update the QP state before the actual ramrod to prevent a race with
2553          * fast path. Modifying the QP state to error will cause the device to
2554          * flush the CQEs and while polling the flushed CQEs will considered as
2555          * a potential issue if the QP isn't in error state.
2556          */
2557         if ((attr_mask & IB_QP_STATE) && qp->qp_type != IB_QPT_GSI &&
2558             !udata && qp_params.new_state == QED_ROCE_QP_STATE_ERR)
2559                 qp->state = QED_ROCE_QP_STATE_ERR;
2560
2561         if (qp->qp_type != IB_QPT_GSI)
2562                 rc = dev->ops->rdma_modify_qp(dev->rdma_ctx,
2563                                               qp->qed_qp, &qp_params);
2564
2565         if (attr_mask & IB_QP_STATE) {
2566                 if ((qp->qp_type != IB_QPT_GSI) && (!udata))
2567                         rc = qedr_update_qp_state(dev, qp, cur_state,
2568                                                   qp_params.new_state);
2569                 qp->state = qp_params.new_state;
2570         }
2571
2572 err:
2573         return rc;
2574 }
2575
2576 static int qedr_to_ib_qp_acc_flags(struct qed_rdma_query_qp_out_params *params)
2577 {
2578         int ib_qp_acc_flags = 0;
2579
2580         if (params->incoming_rdma_write_en)
2581                 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE;
2582         if (params->incoming_rdma_read_en)
2583                 ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ;
2584         if (params->incoming_atomic_en)
2585                 ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC;
2586         ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE;
2587         return ib_qp_acc_flags;
2588 }
2589
2590 int qedr_query_qp(struct ib_qp *ibqp,
2591                   struct ib_qp_attr *qp_attr,
2592                   int attr_mask, struct ib_qp_init_attr *qp_init_attr)
2593 {
2594         struct qed_rdma_query_qp_out_params params;
2595         struct qedr_qp *qp = get_qedr_qp(ibqp);
2596         struct qedr_dev *dev = qp->dev;
2597         int rc = 0;
2598
2599         memset(&params, 0, sizeof(params));
2600
2601         rc = dev->ops->rdma_query_qp(dev->rdma_ctx, qp->qed_qp, &params);
2602         if (rc)
2603                 goto err;
2604
2605         memset(qp_attr, 0, sizeof(*qp_attr));
2606         memset(qp_init_attr, 0, sizeof(*qp_init_attr));
2607
2608         qp_attr->qp_state = qedr_get_ibqp_state(params.state);
2609         qp_attr->cur_qp_state = qedr_get_ibqp_state(params.state);
2610         qp_attr->path_mtu = ib_mtu_int_to_enum(params.mtu);
2611         qp_attr->path_mig_state = IB_MIG_MIGRATED;
2612         qp_attr->rq_psn = params.rq_psn;
2613         qp_attr->sq_psn = params.sq_psn;
2614         qp_attr->dest_qp_num = params.dest_qp;
2615
2616         qp_attr->qp_access_flags = qedr_to_ib_qp_acc_flags(&params);
2617
2618         qp_attr->cap.max_send_wr = qp->sq.max_wr;
2619         qp_attr->cap.max_recv_wr = qp->rq.max_wr;
2620         qp_attr->cap.max_send_sge = qp->sq.max_sges;
2621         qp_attr->cap.max_recv_sge = qp->rq.max_sges;
2622         qp_attr->cap.max_inline_data = ROCE_REQ_MAX_INLINE_DATA_SIZE;
2623         qp_init_attr->cap = qp_attr->cap;
2624
2625         qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
2626         rdma_ah_set_grh(&qp_attr->ah_attr, NULL,
2627                         params.flow_label, qp->sgid_idx,
2628                         params.hop_limit_ttl, params.traffic_class_tos);
2629         rdma_ah_set_dgid_raw(&qp_attr->ah_attr, &params.dgid.bytes[0]);
2630         rdma_ah_set_port_num(&qp_attr->ah_attr, 1);
2631         rdma_ah_set_sl(&qp_attr->ah_attr, 0);
2632         qp_attr->timeout = params.timeout;
2633         qp_attr->rnr_retry = params.rnr_retry;
2634         qp_attr->retry_cnt = params.retry_cnt;
2635         qp_attr->min_rnr_timer = params.min_rnr_nak_timer;
2636         qp_attr->pkey_index = params.pkey_index;
2637         qp_attr->port_num = 1;
2638         rdma_ah_set_path_bits(&qp_attr->ah_attr, 0);
2639         rdma_ah_set_static_rate(&qp_attr->ah_attr, 0);
2640         qp_attr->alt_pkey_index = 0;
2641         qp_attr->alt_port_num = 0;
2642         qp_attr->alt_timeout = 0;
2643         memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr));
2644
2645         qp_attr->sq_draining = (params.state == QED_ROCE_QP_STATE_SQD) ? 1 : 0;
2646         qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic;
2647         qp_attr->max_rd_atomic = params.max_rd_atomic;
2648         qp_attr->en_sqd_async_notify = (params.sqd_async) ? 1 : 0;
2649
2650         DP_DEBUG(dev, QEDR_MSG_QP, "QEDR_QUERY_QP: max_inline_data=%d\n",
2651                  qp_attr->cap.max_inline_data);
2652
2653 err:
2654         return rc;
2655 }
2656
2657 static int qedr_free_qp_resources(struct qedr_dev *dev, struct qedr_qp *qp,
2658                                   struct ib_udata *udata)
2659 {
2660         struct qedr_ucontext *ctx =
2661                 rdma_udata_to_drv_context(udata, struct qedr_ucontext,
2662                                           ibucontext);
2663         int rc;
2664
2665         if (qp->qp_type != IB_QPT_GSI) {
2666                 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2667                 if (rc)
2668                         return rc;
2669         }
2670
2671         if (qp->create_type == QEDR_QP_CREATE_USER)
2672                 qedr_cleanup_user(dev, ctx, qp);
2673         else
2674                 qedr_cleanup_kernel(dev, qp);
2675
2676         return 0;
2677 }
2678
2679 int qedr_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
2680 {
2681         struct qedr_qp *qp = get_qedr_qp(ibqp);
2682         struct qedr_dev *dev = qp->dev;
2683         struct ib_qp_attr attr;
2684         int attr_mask = 0;
2685
2686         DP_DEBUG(dev, QEDR_MSG_QP, "destroy qp: destroying %p, qp type=%d\n",
2687                  qp, qp->qp_type);
2688
2689         if (rdma_protocol_roce(&dev->ibdev, 1)) {
2690                 if ((qp->state != QED_ROCE_QP_STATE_RESET) &&
2691                     (qp->state != QED_ROCE_QP_STATE_ERR) &&
2692                     (qp->state != QED_ROCE_QP_STATE_INIT)) {
2693
2694                         attr.qp_state = IB_QPS_ERR;
2695                         attr_mask |= IB_QP_STATE;
2696
2697                         /* Change the QP state to ERROR */
2698                         qedr_modify_qp(ibqp, &attr, attr_mask, NULL);
2699                 }
2700         } else {
2701                 /* If connection establishment started the WAIT_FOR_CONNECT
2702                  * bit will be on and we need to Wait for the establishment
2703                  * to complete before destroying the qp.
2704                  */
2705                 if (test_and_set_bit(QEDR_IWARP_CM_WAIT_FOR_CONNECT,
2706                                      &qp->iwarp_cm_flags))
2707                         wait_for_completion(&qp->iwarp_cm_comp);
2708
2709                 /* If graceful disconnect started, the WAIT_FOR_DISCONNECT
2710                  * bit will be on, and we need to wait for the disconnect to
2711                  * complete before continuing. We can use the same completion,
2712                  * iwarp_cm_comp, since this is the only place that waits for
2713                  * this completion and it is sequential. In addition,
2714                  * disconnect can't occur before the connection is fully
2715                  * established, therefore if WAIT_FOR_DISCONNECT is on it
2716                  * means WAIT_FOR_CONNECT is also on and the completion for
2717                  * CONNECT already occurred.
2718                  */
2719                 if (test_and_set_bit(QEDR_IWARP_CM_WAIT_FOR_DISCONNECT,
2720                                      &qp->iwarp_cm_flags))
2721                         wait_for_completion(&qp->iwarp_cm_comp);
2722         }
2723
2724         if (qp->qp_type == IB_QPT_GSI)
2725                 qedr_destroy_gsi_qp(dev);
2726
2727         /* We need to remove the entry from the xarray before we release the
2728          * qp_id to avoid a race of the qp_id being reallocated and failing
2729          * on xa_insert
2730          */
2731         if (rdma_protocol_iwarp(&dev->ibdev, 1))
2732                 xa_erase(&dev->qps, qp->qp_id);
2733
2734         qedr_free_qp_resources(dev, qp, udata);
2735
2736         if (rdma_protocol_iwarp(&dev->ibdev, 1))
2737                 qedr_iw_qp_rem_ref(&qp->ibqp);
2738
2739         return 0;
2740 }
2741
2742 int qedr_create_ah(struct ib_ah *ibah, struct rdma_ah_attr *attr, u32 flags,
2743                    struct ib_udata *udata)
2744 {
2745         struct qedr_ah *ah = get_qedr_ah(ibah);
2746
2747         rdma_copy_ah_attr(&ah->attr, attr);
2748
2749         return 0;
2750 }
2751
2752 void qedr_destroy_ah(struct ib_ah *ibah, u32 flags)
2753 {
2754         struct qedr_ah *ah = get_qedr_ah(ibah);
2755
2756         rdma_destroy_ah_attr(&ah->attr);
2757 }
2758
2759 static void free_mr_info(struct qedr_dev *dev, struct mr_info *info)
2760 {
2761         struct qedr_pbl *pbl, *tmp;
2762
2763         if (info->pbl_table)
2764                 list_add_tail(&info->pbl_table->list_entry,
2765                               &info->free_pbl_list);
2766
2767         if (!list_empty(&info->inuse_pbl_list))
2768                 list_splice(&info->inuse_pbl_list, &info->free_pbl_list);
2769
2770         list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) {
2771                 list_del(&pbl->list_entry);
2772                 qedr_free_pbl(dev, &info->pbl_info, pbl);
2773         }
2774 }
2775
2776 static int init_mr_info(struct qedr_dev *dev, struct mr_info *info,
2777                         size_t page_list_len, bool two_layered)
2778 {
2779         struct qedr_pbl *tmp;
2780         int rc;
2781
2782         INIT_LIST_HEAD(&info->free_pbl_list);
2783         INIT_LIST_HEAD(&info->inuse_pbl_list);
2784
2785         rc = qedr_prepare_pbl_tbl(dev, &info->pbl_info,
2786                                   page_list_len, two_layered);
2787         if (rc)
2788                 goto done;
2789
2790         info->pbl_table = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2791         if (IS_ERR(info->pbl_table)) {
2792                 rc = PTR_ERR(info->pbl_table);
2793                 goto done;
2794         }
2795
2796         DP_DEBUG(dev, QEDR_MSG_MR, "pbl_table_pa = %pa\n",
2797                  &info->pbl_table->pa);
2798
2799         /* in usual case we use 2 PBLs, so we add one to free
2800          * list and allocating another one
2801          */
2802         tmp = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2803         if (IS_ERR(tmp)) {
2804                 DP_DEBUG(dev, QEDR_MSG_MR, "Extra PBL is not allocated\n");
2805                 goto done;
2806         }
2807
2808         list_add_tail(&tmp->list_entry, &info->free_pbl_list);
2809
2810         DP_DEBUG(dev, QEDR_MSG_MR, "extra pbl_table_pa = %pa\n", &tmp->pa);
2811
2812 done:
2813         if (rc)
2814                 free_mr_info(dev, info);
2815
2816         return rc;
2817 }
2818
2819 struct ib_mr *qedr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
2820                                u64 usr_addr, int acc, struct ib_udata *udata)
2821 {
2822         struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2823         struct qedr_mr *mr;
2824         struct qedr_pd *pd;
2825         int rc = -ENOMEM;
2826
2827         pd = get_qedr_pd(ibpd);
2828         DP_DEBUG(dev, QEDR_MSG_MR,
2829                  "qedr_register user mr pd = %d start = %lld, len = %lld, usr_addr = %lld, acc = %d\n",
2830                  pd->pd_id, start, len, usr_addr, acc);
2831
2832         if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE))
2833                 return ERR_PTR(-EINVAL);
2834
2835         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2836         if (!mr)
2837                 return ERR_PTR(rc);
2838
2839         mr->type = QEDR_MR_USER;
2840
2841         mr->umem = ib_umem_get(ibpd->device, start, len, acc);
2842         if (IS_ERR(mr->umem)) {
2843                 rc = -EFAULT;
2844                 goto err0;
2845         }
2846
2847         rc = init_mr_info(dev, &mr->info, ib_umem_page_count(mr->umem), 1);
2848         if (rc)
2849                 goto err1;
2850
2851         qedr_populate_pbls(dev, mr->umem, mr->info.pbl_table,
2852                            &mr->info.pbl_info, PAGE_SHIFT);
2853
2854         rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2855         if (rc) {
2856                 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2857                 goto err1;
2858         }
2859
2860         /* Index only, 18 bit long, lkey = itid << 8 | key */
2861         mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
2862         mr->hw_mr.key = 0;
2863         mr->hw_mr.pd = pd->pd_id;
2864         mr->hw_mr.local_read = 1;
2865         mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
2866         mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
2867         mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
2868         mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
2869         mr->hw_mr.mw_bind = false;
2870         mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa;
2871         mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
2872         mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
2873         mr->hw_mr.page_size_log = PAGE_SHIFT;
2874         mr->hw_mr.fbo = ib_umem_offset(mr->umem);
2875         mr->hw_mr.length = len;
2876         mr->hw_mr.vaddr = usr_addr;
2877         mr->hw_mr.zbva = false;
2878         mr->hw_mr.phy_mr = false;
2879         mr->hw_mr.dma_mr = false;
2880
2881         rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2882         if (rc) {
2883                 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2884                 goto err2;
2885         }
2886
2887         mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2888         if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
2889             mr->hw_mr.remote_atomic)
2890                 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2891
2892         DP_DEBUG(dev, QEDR_MSG_MR, "register user mr lkey: %x\n",
2893                  mr->ibmr.lkey);
2894         return &mr->ibmr;
2895
2896 err2:
2897         dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2898 err1:
2899         qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
2900 err0:
2901         kfree(mr);
2902         return ERR_PTR(rc);
2903 }
2904
2905 int qedr_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
2906 {
2907         struct qedr_mr *mr = get_qedr_mr(ib_mr);
2908         struct qedr_dev *dev = get_qedr_dev(ib_mr->device);
2909         int rc = 0;
2910
2911         rc = dev->ops->rdma_deregister_tid(dev->rdma_ctx, mr->hw_mr.itid);
2912         if (rc)
2913                 return rc;
2914
2915         dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2916
2917         if (mr->type != QEDR_MR_DMA)
2918                 free_mr_info(dev, &mr->info);
2919
2920         /* it could be user registered memory. */
2921         ib_umem_release(mr->umem);
2922
2923         kfree(mr);
2924
2925         return rc;
2926 }
2927
2928 static struct qedr_mr *__qedr_alloc_mr(struct ib_pd *ibpd,
2929                                        int max_page_list_len)
2930 {
2931         struct qedr_pd *pd = get_qedr_pd(ibpd);
2932         struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2933         struct qedr_mr *mr;
2934         int rc = -ENOMEM;
2935
2936         DP_DEBUG(dev, QEDR_MSG_MR,
2937                  "qedr_alloc_frmr pd = %d max_page_list_len= %d\n", pd->pd_id,
2938                  max_page_list_len);
2939
2940         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2941         if (!mr)
2942                 return ERR_PTR(rc);
2943
2944         mr->dev = dev;
2945         mr->type = QEDR_MR_FRMR;
2946
2947         rc = init_mr_info(dev, &mr->info, max_page_list_len, 1);
2948         if (rc)
2949                 goto err0;
2950
2951         rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2952         if (rc) {
2953                 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2954                 goto err0;
2955         }
2956
2957         /* Index only, 18 bit long, lkey = itid << 8 | key */
2958         mr->hw_mr.tid_type = QED_RDMA_TID_FMR;
2959         mr->hw_mr.key = 0;
2960         mr->hw_mr.pd = pd->pd_id;
2961         mr->hw_mr.local_read = 1;
2962         mr->hw_mr.local_write = 0;
2963         mr->hw_mr.remote_read = 0;
2964         mr->hw_mr.remote_write = 0;
2965         mr->hw_mr.remote_atomic = 0;
2966         mr->hw_mr.mw_bind = false;
2967         mr->hw_mr.pbl_ptr = 0;
2968         mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
2969         mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
2970         mr->hw_mr.fbo = 0;
2971         mr->hw_mr.length = 0;
2972         mr->hw_mr.vaddr = 0;
2973         mr->hw_mr.zbva = false;
2974         mr->hw_mr.phy_mr = true;
2975         mr->hw_mr.dma_mr = false;
2976
2977         rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2978         if (rc) {
2979                 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2980                 goto err1;
2981         }
2982
2983         mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2984         mr->ibmr.rkey = mr->ibmr.lkey;
2985
2986         DP_DEBUG(dev, QEDR_MSG_MR, "alloc frmr: %x\n", mr->ibmr.lkey);
2987         return mr;
2988
2989 err1:
2990         dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2991 err0:
2992         kfree(mr);
2993         return ERR_PTR(rc);
2994 }
2995
2996 struct ib_mr *qedr_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
2997                             u32 max_num_sg, struct ib_udata *udata)
2998 {
2999         struct qedr_mr *mr;
3000
3001         if (mr_type != IB_MR_TYPE_MEM_REG)
3002                 return ERR_PTR(-EINVAL);
3003
3004         mr = __qedr_alloc_mr(ibpd, max_num_sg);
3005
3006         if (IS_ERR(mr))
3007                 return ERR_PTR(-EINVAL);
3008
3009         return &mr->ibmr;
3010 }
3011
3012 static int qedr_set_page(struct ib_mr *ibmr, u64 addr)
3013 {
3014         struct qedr_mr *mr = get_qedr_mr(ibmr);
3015         struct qedr_pbl *pbl_table;
3016         struct regpair *pbe;
3017         u32 pbes_in_page;
3018
3019         if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) {
3020                 DP_ERR(mr->dev, "qedr_set_page fails when %d\n", mr->npages);
3021                 return -ENOMEM;
3022         }
3023
3024         DP_DEBUG(mr->dev, QEDR_MSG_MR, "qedr_set_page pages[%d] = 0x%llx\n",
3025                  mr->npages, addr);
3026
3027         pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64);
3028         pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page);
3029         pbe = (struct regpair *)pbl_table->va;
3030         pbe +=  mr->npages % pbes_in_page;
3031         pbe->lo = cpu_to_le32((u32)addr);
3032         pbe->hi = cpu_to_le32((u32)upper_32_bits(addr));
3033
3034         mr->npages++;
3035
3036         return 0;
3037 }
3038
3039 static void handle_completed_mrs(struct qedr_dev *dev, struct mr_info *info)
3040 {
3041         int work = info->completed - info->completed_handled - 1;
3042
3043         DP_DEBUG(dev, QEDR_MSG_MR, "Special FMR work = %d\n", work);
3044         while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) {
3045                 struct qedr_pbl *pbl;
3046
3047                 /* Free all the page list that are possible to be freed
3048                  * (all the ones that were invalidated), under the assumption
3049                  * that if an FMR was completed successfully that means that
3050                  * if there was an invalidate operation before it also ended
3051                  */
3052                 pbl = list_first_entry(&info->inuse_pbl_list,
3053                                        struct qedr_pbl, list_entry);
3054                 list_move_tail(&pbl->list_entry, &info->free_pbl_list);
3055                 info->completed_handled++;
3056         }
3057 }
3058
3059 int qedr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
3060                    int sg_nents, unsigned int *sg_offset)
3061 {
3062         struct qedr_mr *mr = get_qedr_mr(ibmr);
3063
3064         mr->npages = 0;
3065
3066         handle_completed_mrs(mr->dev, &mr->info);
3067         return ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qedr_set_page);
3068 }
3069
3070 struct ib_mr *qedr_get_dma_mr(struct ib_pd *ibpd, int acc)
3071 {
3072         struct qedr_dev *dev = get_qedr_dev(ibpd->device);
3073         struct qedr_pd *pd = get_qedr_pd(ibpd);
3074         struct qedr_mr *mr;
3075         int rc;
3076
3077         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
3078         if (!mr)
3079                 return ERR_PTR(-ENOMEM);
3080
3081         mr->type = QEDR_MR_DMA;
3082
3083         rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
3084         if (rc) {
3085                 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
3086                 goto err1;
3087         }
3088
3089         /* index only, 18 bit long, lkey = itid << 8 | key */
3090         mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
3091         mr->hw_mr.pd = pd->pd_id;
3092         mr->hw_mr.local_read = 1;
3093         mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
3094         mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
3095         mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
3096         mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
3097         mr->hw_mr.dma_mr = true;
3098
3099         rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
3100         if (rc) {
3101                 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
3102                 goto err2;
3103         }
3104
3105         mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3106         if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
3107             mr->hw_mr.remote_atomic)
3108                 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3109
3110         DP_DEBUG(dev, QEDR_MSG_MR, "get dma mr: lkey = %x\n", mr->ibmr.lkey);
3111         return &mr->ibmr;
3112
3113 err2:
3114         dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3115 err1:
3116         kfree(mr);
3117         return ERR_PTR(rc);
3118 }
3119
3120 static inline int qedr_wq_is_full(struct qedr_qp_hwq_info *wq)
3121 {
3122         return (((wq->prod + 1) % wq->max_wr) == wq->cons);
3123 }
3124
3125 static int sge_data_len(struct ib_sge *sg_list, int num_sge)
3126 {
3127         int i, len = 0;
3128
3129         for (i = 0; i < num_sge; i++)
3130                 len += sg_list[i].length;
3131
3132         return len;
3133 }
3134
3135 static void swap_wqe_data64(u64 *p)
3136 {
3137         int i;
3138
3139         for (i = 0; i < QEDR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++)
3140                 *p = cpu_to_be64(cpu_to_le64(*p));
3141 }
3142
3143 static u32 qedr_prepare_sq_inline_data(struct qedr_dev *dev,
3144                                        struct qedr_qp *qp, u8 *wqe_size,
3145                                        const struct ib_send_wr *wr,
3146                                        const struct ib_send_wr **bad_wr,
3147                                        u8 *bits, u8 bit)
3148 {
3149         u32 data_size = sge_data_len(wr->sg_list, wr->num_sge);
3150         char *seg_prt, *wqe;
3151         int i, seg_siz;
3152
3153         if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) {
3154                 DP_ERR(dev, "Too much inline data in WR: %d\n", data_size);
3155                 *bad_wr = wr;
3156                 return 0;
3157         }
3158
3159         if (!data_size)
3160                 return data_size;
3161
3162         *bits |= bit;
3163
3164         seg_prt = NULL;
3165         wqe = NULL;
3166         seg_siz = 0;
3167
3168         /* Copy data inline */
3169         for (i = 0; i < wr->num_sge; i++) {
3170                 u32 len = wr->sg_list[i].length;
3171                 void *src = (void *)(uintptr_t)wr->sg_list[i].addr;
3172
3173                 while (len > 0) {
3174                         u32 cur;
3175
3176                         /* New segment required */
3177                         if (!seg_siz) {
3178                                 wqe = (char *)qed_chain_produce(&qp->sq.pbl);
3179                                 seg_prt = wqe;
3180                                 seg_siz = sizeof(struct rdma_sq_common_wqe);
3181                                 (*wqe_size)++;
3182                         }
3183
3184                         /* Calculate currently allowed length */
3185                         cur = min_t(u32, len, seg_siz);
3186                         memcpy(seg_prt, src, cur);
3187
3188                         /* Update segment variables */
3189                         seg_prt += cur;
3190                         seg_siz -= cur;
3191
3192                         /* Update sge variables */
3193                         src += cur;
3194                         len -= cur;
3195
3196                         /* Swap fully-completed segments */
3197                         if (!seg_siz)
3198                                 swap_wqe_data64((u64 *)wqe);
3199                 }
3200         }
3201
3202         /* swap last not completed segment */
3203         if (seg_siz)
3204                 swap_wqe_data64((u64 *)wqe);
3205
3206         return data_size;
3207 }
3208
3209 #define RQ_SGE_SET(sge, vaddr, vlength, vflags)                 \
3210         do {                                                    \
3211                 DMA_REGPAIR_LE(sge->addr, vaddr);               \
3212                 (sge)->length = cpu_to_le32(vlength);           \
3213                 (sge)->flags = cpu_to_le32(vflags);             \
3214         } while (0)
3215
3216 #define SRQ_HDR_SET(hdr, vwr_id, num_sge)                       \
3217         do {                                                    \
3218                 DMA_REGPAIR_LE(hdr->wr_id, vwr_id);             \
3219                 (hdr)->num_sges = num_sge;                      \
3220         } while (0)
3221
3222 #define SRQ_SGE_SET(sge, vaddr, vlength, vlkey)                 \
3223         do {                                                    \
3224                 DMA_REGPAIR_LE(sge->addr, vaddr);               \
3225                 (sge)->length = cpu_to_le32(vlength);           \
3226                 (sge)->l_key = cpu_to_le32(vlkey);              \
3227         } while (0)
3228
3229 static u32 qedr_prepare_sq_sges(struct qedr_qp *qp, u8 *wqe_size,
3230                                 const struct ib_send_wr *wr)
3231 {
3232         u32 data_size = 0;
3233         int i;
3234
3235         for (i = 0; i < wr->num_sge; i++) {
3236                 struct rdma_sq_sge *sge = qed_chain_produce(&qp->sq.pbl);
3237
3238                 DMA_REGPAIR_LE(sge->addr, wr->sg_list[i].addr);
3239                 sge->l_key = cpu_to_le32(wr->sg_list[i].lkey);
3240                 sge->length = cpu_to_le32(wr->sg_list[i].length);
3241                 data_size += wr->sg_list[i].length;
3242         }
3243
3244         if (wqe_size)
3245                 *wqe_size += wr->num_sge;
3246
3247         return data_size;
3248 }
3249
3250 static u32 qedr_prepare_sq_rdma_data(struct qedr_dev *dev,
3251                                      struct qedr_qp *qp,
3252                                      struct rdma_sq_rdma_wqe_1st *rwqe,
3253                                      struct rdma_sq_rdma_wqe_2nd *rwqe2,
3254                                      const struct ib_send_wr *wr,
3255                                      const struct ib_send_wr **bad_wr)
3256 {
3257         rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey);
3258         DMA_REGPAIR_LE(rwqe2->remote_va, rdma_wr(wr)->remote_addr);
3259
3260         if (wr->send_flags & IB_SEND_INLINE &&
3261             (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM ||
3262              wr->opcode == IB_WR_RDMA_WRITE)) {
3263                 u8 flags = 0;
3264
3265                 SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1);
3266                 return qedr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, wr,
3267                                                    bad_wr, &rwqe->flags, flags);
3268         }
3269
3270         return qedr_prepare_sq_sges(qp, &rwqe->wqe_size, wr);
3271 }
3272
3273 static u32 qedr_prepare_sq_send_data(struct qedr_dev *dev,
3274                                      struct qedr_qp *qp,
3275                                      struct rdma_sq_send_wqe_1st *swqe,
3276                                      struct rdma_sq_send_wqe_2st *swqe2,
3277                                      const struct ib_send_wr *wr,
3278                                      const struct ib_send_wr **bad_wr)
3279 {
3280         memset(swqe2, 0, sizeof(*swqe2));
3281         if (wr->send_flags & IB_SEND_INLINE) {
3282                 u8 flags = 0;
3283
3284                 SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1);
3285                 return qedr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, wr,
3286                                                    bad_wr, &swqe->flags, flags);
3287         }
3288
3289         return qedr_prepare_sq_sges(qp, &swqe->wqe_size, wr);
3290 }
3291
3292 static int qedr_prepare_reg(struct qedr_qp *qp,
3293                             struct rdma_sq_fmr_wqe_1st *fwqe1,
3294                             const struct ib_reg_wr *wr)
3295 {
3296         struct qedr_mr *mr = get_qedr_mr(wr->mr);
3297         struct rdma_sq_fmr_wqe_2nd *fwqe2;
3298
3299         fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)qed_chain_produce(&qp->sq.pbl);
3300         fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova);
3301         fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova);
3302         fwqe1->l_key = wr->key;
3303
3304         fwqe2->access_ctrl = 0;
3305
3306         SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ,
3307                    !!(wr->access & IB_ACCESS_REMOTE_READ));
3308         SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE,
3309                    !!(wr->access & IB_ACCESS_REMOTE_WRITE));
3310         SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC,
3311                    !!(wr->access & IB_ACCESS_REMOTE_ATOMIC));
3312         SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1);
3313         SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE,
3314                    !!(wr->access & IB_ACCESS_LOCAL_WRITE));
3315         fwqe2->fmr_ctrl = 0;
3316
3317         SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG,
3318                    ilog2(mr->ibmr.page_size) - 12);
3319
3320         fwqe2->length_hi = 0;
3321         fwqe2->length_lo = mr->ibmr.length;
3322         fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa);
3323         fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa);
3324
3325         qp->wqe_wr_id[qp->sq.prod].mr = mr;
3326
3327         return 0;
3328 }
3329
3330 static enum ib_wc_opcode qedr_ib_to_wc_opcode(enum ib_wr_opcode opcode)
3331 {
3332         switch (opcode) {
3333         case IB_WR_RDMA_WRITE:
3334         case IB_WR_RDMA_WRITE_WITH_IMM:
3335                 return IB_WC_RDMA_WRITE;
3336         case IB_WR_SEND_WITH_IMM:
3337         case IB_WR_SEND:
3338         case IB_WR_SEND_WITH_INV:
3339                 return IB_WC_SEND;
3340         case IB_WR_RDMA_READ:
3341         case IB_WR_RDMA_READ_WITH_INV:
3342                 return IB_WC_RDMA_READ;
3343         case IB_WR_ATOMIC_CMP_AND_SWP:
3344                 return IB_WC_COMP_SWAP;
3345         case IB_WR_ATOMIC_FETCH_AND_ADD:
3346                 return IB_WC_FETCH_ADD;
3347         case IB_WR_REG_MR:
3348                 return IB_WC_REG_MR;
3349         case IB_WR_LOCAL_INV:
3350                 return IB_WC_LOCAL_INV;
3351         default:
3352                 return IB_WC_SEND;
3353         }
3354 }
3355
3356 static inline bool qedr_can_post_send(struct qedr_qp *qp,
3357                                       const struct ib_send_wr *wr)
3358 {
3359         int wq_is_full, err_wr, pbl_is_full;
3360         struct qedr_dev *dev = qp->dev;
3361
3362         /* prevent SQ overflow and/or processing of a bad WR */
3363         err_wr = wr->num_sge > qp->sq.max_sges;
3364         wq_is_full = qedr_wq_is_full(&qp->sq);
3365         pbl_is_full = qed_chain_get_elem_left_u32(&qp->sq.pbl) <
3366                       QEDR_MAX_SQE_ELEMENTS_PER_SQE;
3367         if (wq_is_full || err_wr || pbl_is_full) {
3368                 if (wq_is_full && !(qp->err_bitmap & QEDR_QP_ERR_SQ_FULL)) {
3369                         DP_ERR(dev,
3370                                "error: WQ is full. Post send on QP %p failed (this error appears only once)\n",
3371                                qp);
3372                         qp->err_bitmap |= QEDR_QP_ERR_SQ_FULL;
3373                 }
3374
3375                 if (err_wr && !(qp->err_bitmap & QEDR_QP_ERR_BAD_SR)) {
3376                         DP_ERR(dev,
3377                                "error: WR is bad. Post send on QP %p failed (this error appears only once)\n",
3378                                qp);
3379                         qp->err_bitmap |= QEDR_QP_ERR_BAD_SR;
3380                 }
3381
3382                 if (pbl_is_full &&
3383                     !(qp->err_bitmap & QEDR_QP_ERR_SQ_PBL_FULL)) {
3384                         DP_ERR(dev,
3385                                "error: WQ PBL is full. Post send on QP %p failed (this error appears only once)\n",
3386                                qp);
3387                         qp->err_bitmap |= QEDR_QP_ERR_SQ_PBL_FULL;
3388                 }
3389                 return false;
3390         }
3391         return true;
3392 }
3393
3394 static int __qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
3395                             const struct ib_send_wr **bad_wr)
3396 {
3397         struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3398         struct qedr_qp *qp = get_qedr_qp(ibqp);
3399         struct rdma_sq_atomic_wqe_1st *awqe1;
3400         struct rdma_sq_atomic_wqe_2nd *awqe2;
3401         struct rdma_sq_atomic_wqe_3rd *awqe3;
3402         struct rdma_sq_send_wqe_2st *swqe2;
3403         struct rdma_sq_local_inv_wqe *iwqe;
3404         struct rdma_sq_rdma_wqe_2nd *rwqe2;
3405         struct rdma_sq_send_wqe_1st *swqe;
3406         struct rdma_sq_rdma_wqe_1st *rwqe;
3407         struct rdma_sq_fmr_wqe_1st *fwqe1;
3408         struct rdma_sq_common_wqe *wqe;
3409         u32 length;
3410         int rc = 0;
3411         bool comp;
3412
3413         if (!qedr_can_post_send(qp, wr)) {
3414                 *bad_wr = wr;
3415                 return -ENOMEM;
3416         }
3417
3418         wqe = qed_chain_produce(&qp->sq.pbl);
3419         qp->wqe_wr_id[qp->sq.prod].signaled =
3420                 !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled;
3421
3422         wqe->flags = 0;
3423         SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG,
3424                    !!(wr->send_flags & IB_SEND_SOLICITED));
3425         comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || qp->signaled;
3426         SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp);
3427         SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG,
3428                    !!(wr->send_flags & IB_SEND_FENCE));
3429         wqe->prev_wqe_size = qp->prev_wqe_size;
3430
3431         qp->wqe_wr_id[qp->sq.prod].opcode = qedr_ib_to_wc_opcode(wr->opcode);
3432
3433         switch (wr->opcode) {
3434         case IB_WR_SEND_WITH_IMM:
3435                 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
3436                         rc = -EINVAL;
3437                         *bad_wr = wr;
3438                         break;
3439                 }
3440                 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM;
3441                 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3442                 swqe->wqe_size = 2;
3443                 swqe2 = qed_chain_produce(&qp->sq.pbl);
3444
3445                 swqe->inv_key_or_imm_data = cpu_to_le32(be32_to_cpu(wr->ex.imm_data));
3446                 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3447                                                    wr, bad_wr);
3448                 swqe->length = cpu_to_le32(length);
3449                 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3450                 qp->prev_wqe_size = swqe->wqe_size;
3451                 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3452                 break;
3453         case IB_WR_SEND:
3454                 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND;
3455                 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3456
3457                 swqe->wqe_size = 2;
3458                 swqe2 = qed_chain_produce(&qp->sq.pbl);
3459                 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3460                                                    wr, bad_wr);
3461                 swqe->length = cpu_to_le32(length);
3462                 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3463                 qp->prev_wqe_size = swqe->wqe_size;
3464                 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3465                 break;
3466         case IB_WR_SEND_WITH_INV:
3467                 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE;
3468                 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3469                 swqe2 = qed_chain_produce(&qp->sq.pbl);
3470                 swqe->wqe_size = 2;
3471                 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.invalidate_rkey);
3472                 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3473                                                    wr, bad_wr);
3474                 swqe->length = cpu_to_le32(length);
3475                 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3476                 qp->prev_wqe_size = swqe->wqe_size;
3477                 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3478                 break;
3479
3480         case IB_WR_RDMA_WRITE_WITH_IMM:
3481                 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
3482                         rc = -EINVAL;
3483                         *bad_wr = wr;
3484                         break;
3485                 }
3486                 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM;
3487                 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3488
3489                 rwqe->wqe_size = 2;
3490                 rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data));
3491                 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3492                 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3493                                                    wr, bad_wr);
3494                 rwqe->length = cpu_to_le32(length);
3495                 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3496                 qp->prev_wqe_size = rwqe->wqe_size;
3497                 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3498                 break;
3499         case IB_WR_RDMA_WRITE:
3500                 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR;
3501                 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3502
3503                 rwqe->wqe_size = 2;
3504                 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3505                 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3506                                                    wr, bad_wr);
3507                 rwqe->length = cpu_to_le32(length);
3508                 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3509                 qp->prev_wqe_size = rwqe->wqe_size;
3510                 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3511                 break;
3512         case IB_WR_RDMA_READ_WITH_INV:
3513                 SET_FIELD2(wqe->flags, RDMA_SQ_RDMA_WQE_1ST_READ_INV_FLG, 1);
3514                 /* fallthrough -- same is identical to RDMA READ */
3515
3516         case IB_WR_RDMA_READ:
3517                 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD;
3518                 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3519
3520                 rwqe->wqe_size = 2;
3521                 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3522                 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3523                                                    wr, bad_wr);
3524                 rwqe->length = cpu_to_le32(length);
3525                 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3526                 qp->prev_wqe_size = rwqe->wqe_size;
3527                 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3528                 break;
3529
3530         case IB_WR_ATOMIC_CMP_AND_SWP:
3531         case IB_WR_ATOMIC_FETCH_AND_ADD:
3532                 awqe1 = (struct rdma_sq_atomic_wqe_1st *)wqe;
3533                 awqe1->wqe_size = 4;
3534
3535                 awqe2 = qed_chain_produce(&qp->sq.pbl);
3536                 DMA_REGPAIR_LE(awqe2->remote_va, atomic_wr(wr)->remote_addr);
3537                 awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey);
3538
3539                 awqe3 = qed_chain_produce(&qp->sq.pbl);
3540
3541                 if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
3542                         wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD;
3543                         DMA_REGPAIR_LE(awqe3->swap_data,
3544                                        atomic_wr(wr)->compare_add);
3545                 } else {
3546                         wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP;
3547                         DMA_REGPAIR_LE(awqe3->swap_data,
3548                                        atomic_wr(wr)->swap);
3549                         DMA_REGPAIR_LE(awqe3->cmp_data,
3550                                        atomic_wr(wr)->compare_add);
3551                 }
3552
3553                 qedr_prepare_sq_sges(qp, NULL, wr);
3554
3555                 qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->wqe_size;
3556                 qp->prev_wqe_size = awqe1->wqe_size;
3557                 break;
3558
3559         case IB_WR_LOCAL_INV:
3560                 iwqe = (struct rdma_sq_local_inv_wqe *)wqe;
3561                 iwqe->wqe_size = 1;
3562
3563                 iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE;
3564                 iwqe->inv_l_key = wr->ex.invalidate_rkey;
3565                 qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->wqe_size;
3566                 qp->prev_wqe_size = iwqe->wqe_size;
3567                 break;
3568         case IB_WR_REG_MR:
3569                 DP_DEBUG(dev, QEDR_MSG_CQ, "REG_MR\n");
3570                 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR;
3571                 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe;
3572                 fwqe1->wqe_size = 2;
3573
3574                 rc = qedr_prepare_reg(qp, fwqe1, reg_wr(wr));
3575                 if (rc) {
3576                         DP_ERR(dev, "IB_REG_MR failed rc=%d\n", rc);
3577                         *bad_wr = wr;
3578                         break;
3579                 }
3580
3581                 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size;
3582                 qp->prev_wqe_size = fwqe1->wqe_size;
3583                 break;
3584         default:
3585                 DP_ERR(dev, "invalid opcode 0x%x!\n", wr->opcode);
3586                 rc = -EINVAL;
3587                 *bad_wr = wr;
3588                 break;
3589         }
3590
3591         if (*bad_wr) {
3592                 u16 value;
3593
3594                 /* Restore prod to its position before
3595                  * this WR was processed
3596                  */
3597                 value = le16_to_cpu(qp->sq.db_data.data.value);
3598                 qed_chain_set_prod(&qp->sq.pbl, value, wqe);
3599
3600                 /* Restore prev_wqe_size */
3601                 qp->prev_wqe_size = wqe->prev_wqe_size;
3602                 rc = -EINVAL;
3603                 DP_ERR(dev, "POST SEND FAILED\n");
3604         }
3605
3606         return rc;
3607 }
3608
3609 int qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
3610                    const struct ib_send_wr **bad_wr)
3611 {
3612         struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3613         struct qedr_qp *qp = get_qedr_qp(ibqp);
3614         unsigned long flags;
3615         int rc = 0;
3616
3617         *bad_wr = NULL;
3618
3619         if (qp->qp_type == IB_QPT_GSI)
3620                 return qedr_gsi_post_send(ibqp, wr, bad_wr);
3621
3622         spin_lock_irqsave(&qp->q_lock, flags);
3623
3624         if (rdma_protocol_roce(&dev->ibdev, 1)) {
3625                 if ((qp->state != QED_ROCE_QP_STATE_RTS) &&
3626                     (qp->state != QED_ROCE_QP_STATE_ERR) &&
3627                     (qp->state != QED_ROCE_QP_STATE_SQD)) {
3628                         spin_unlock_irqrestore(&qp->q_lock, flags);
3629                         *bad_wr = wr;
3630                         DP_DEBUG(dev, QEDR_MSG_CQ,
3631                                  "QP in wrong state! QP icid=0x%x state %d\n",
3632                                  qp->icid, qp->state);
3633                         return -EINVAL;
3634                 }
3635         }
3636
3637         while (wr) {
3638                 rc = __qedr_post_send(ibqp, wr, bad_wr);
3639                 if (rc)
3640                         break;
3641
3642                 qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id;
3643
3644                 qedr_inc_sw_prod(&qp->sq);
3645
3646                 qp->sq.db_data.data.value++;
3647
3648                 wr = wr->next;
3649         }
3650
3651         /* Trigger doorbell
3652          * If there was a failure in the first WR then it will be triggered in
3653          * vane. However this is not harmful (as long as the producer value is
3654          * unchanged). For performance reasons we avoid checking for this
3655          * redundant doorbell.
3656          *
3657          * qp->wqe_wr_id is accessed during qedr_poll_cq, as
3658          * soon as we give the doorbell, we could get a completion
3659          * for this wr, therefore we need to make sure that the
3660          * memory is updated before giving the doorbell.
3661          * During qedr_poll_cq, rmb is called before accessing the
3662          * cqe. This covers for the smp_rmb as well.
3663          */
3664         smp_wmb();
3665         writel(qp->sq.db_data.raw, qp->sq.db);
3666
3667         spin_unlock_irqrestore(&qp->q_lock, flags);
3668
3669         return rc;
3670 }
3671
3672 static u32 qedr_srq_elem_left(struct qedr_srq_hwq_info *hw_srq)
3673 {
3674         u32 used;
3675
3676         /* Calculate number of elements used based on producer
3677          * count and consumer count and subtract it from max
3678          * work request supported so that we get elements left.
3679          */
3680         used = hw_srq->wr_prod_cnt - hw_srq->wr_cons_cnt;
3681
3682         return hw_srq->max_wr - used;
3683 }
3684
3685 int qedr_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,
3686                        const struct ib_recv_wr **bad_wr)
3687 {
3688         struct qedr_srq *srq = get_qedr_srq(ibsrq);
3689         struct qedr_srq_hwq_info *hw_srq;
3690         struct qedr_dev *dev = srq->dev;
3691         struct qed_chain *pbl;
3692         unsigned long flags;
3693         int status = 0;
3694         u32 num_sge;
3695         u32 offset;
3696
3697         spin_lock_irqsave(&srq->lock, flags);
3698
3699         hw_srq = &srq->hw_srq;
3700         pbl = &srq->hw_srq.pbl;
3701         while (wr) {
3702                 struct rdma_srq_wqe_header *hdr;
3703                 int i;
3704
3705                 if (!qedr_srq_elem_left(hw_srq) ||
3706                     wr->num_sge > srq->hw_srq.max_sges) {
3707                         DP_ERR(dev, "Can't post WR  (%d,%d) || (%d > %d)\n",
3708                                hw_srq->wr_prod_cnt, hw_srq->wr_cons_cnt,
3709                                wr->num_sge, srq->hw_srq.max_sges);
3710                         status = -ENOMEM;
3711                         *bad_wr = wr;
3712                         break;
3713                 }
3714
3715                 hdr = qed_chain_produce(pbl);
3716                 num_sge = wr->num_sge;
3717                 /* Set number of sge and work request id in header */
3718                 SRQ_HDR_SET(hdr, wr->wr_id, num_sge);
3719
3720                 srq->hw_srq.wr_prod_cnt++;
3721                 hw_srq->wqe_prod++;
3722                 hw_srq->sge_prod++;
3723
3724                 DP_DEBUG(dev, QEDR_MSG_SRQ,
3725                          "SRQ WR: SGEs: %d with wr_id[%d] = %llx\n",
3726                          wr->num_sge, hw_srq->wqe_prod, wr->wr_id);
3727
3728                 for (i = 0; i < wr->num_sge; i++) {
3729                         struct rdma_srq_sge *srq_sge = qed_chain_produce(pbl);
3730
3731                         /* Set SGE length, lkey and address */
3732                         SRQ_SGE_SET(srq_sge, wr->sg_list[i].addr,
3733                                     wr->sg_list[i].length, wr->sg_list[i].lkey);
3734
3735                         DP_DEBUG(dev, QEDR_MSG_SRQ,
3736                                  "[%d]: len %d key %x addr %x:%x\n",
3737                                  i, srq_sge->length, srq_sge->l_key,
3738                                  srq_sge->addr.hi, srq_sge->addr.lo);
3739                         hw_srq->sge_prod++;
3740                 }
3741
3742                 /* Flush WQE and SGE information before
3743                  * updating producer.
3744                  */
3745                 wmb();
3746
3747                 /* SRQ producer is 8 bytes. Need to update SGE producer index
3748                  * in first 4 bytes and need to update WQE producer in
3749                  * next 4 bytes.
3750                  */
3751                 *srq->hw_srq.virt_prod_pair_addr = hw_srq->sge_prod;
3752                 offset = offsetof(struct rdma_srq_producers, wqe_prod);
3753                 *((u8 *)srq->hw_srq.virt_prod_pair_addr + offset) =
3754                         hw_srq->wqe_prod;
3755
3756                 /* Flush producer after updating it. */
3757                 wmb();
3758                 wr = wr->next;
3759         }
3760
3761         DP_DEBUG(dev, QEDR_MSG_SRQ, "POST: Elements in S-RQ: %d\n",
3762                  qed_chain_get_elem_left(pbl));
3763         spin_unlock_irqrestore(&srq->lock, flags);
3764
3765         return status;
3766 }
3767
3768 int qedr_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
3769                    const struct ib_recv_wr **bad_wr)
3770 {
3771         struct qedr_qp *qp = get_qedr_qp(ibqp);
3772         struct qedr_dev *dev = qp->dev;
3773         unsigned long flags;
3774         int status = 0;
3775
3776         if (qp->qp_type == IB_QPT_GSI)
3777                 return qedr_gsi_post_recv(ibqp, wr, bad_wr);
3778
3779         spin_lock_irqsave(&qp->q_lock, flags);
3780
3781         if (qp->state == QED_ROCE_QP_STATE_RESET) {
3782                 spin_unlock_irqrestore(&qp->q_lock, flags);
3783                 *bad_wr = wr;
3784                 return -EINVAL;
3785         }
3786
3787         while (wr) {
3788                 int i;
3789
3790                 if (qed_chain_get_elem_left_u32(&qp->rq.pbl) <
3791                     QEDR_MAX_RQE_ELEMENTS_PER_RQE ||
3792                     wr->num_sge > qp->rq.max_sges) {
3793                         DP_ERR(dev, "Can't post WR  (%d < %d) || (%d > %d)\n",
3794                                qed_chain_get_elem_left_u32(&qp->rq.pbl),
3795                                QEDR_MAX_RQE_ELEMENTS_PER_RQE, wr->num_sge,
3796                                qp->rq.max_sges);
3797                         status = -ENOMEM;
3798                         *bad_wr = wr;
3799                         break;
3800                 }
3801                 for (i = 0; i < wr->num_sge; i++) {
3802                         u32 flags = 0;
3803                         struct rdma_rq_sge *rqe =
3804                             qed_chain_produce(&qp->rq.pbl);
3805
3806                         /* First one must include the number
3807                          * of SGE in the list
3808                          */
3809                         if (!i)
3810                                 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES,
3811                                           wr->num_sge);
3812
3813                         SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO,
3814                                   wr->sg_list[i].lkey);
3815
3816                         RQ_SGE_SET(rqe, wr->sg_list[i].addr,
3817                                    wr->sg_list[i].length, flags);
3818                 }
3819
3820                 /* Special case of no sges. FW requires between 1-4 sges...
3821                  * in this case we need to post 1 sge with length zero. this is
3822                  * because rdma write with immediate consumes an RQ.
3823                  */
3824                 if (!wr->num_sge) {
3825                         u32 flags = 0;
3826                         struct rdma_rq_sge *rqe =
3827                             qed_chain_produce(&qp->rq.pbl);
3828
3829                         /* First one must include the number
3830                          * of SGE in the list
3831                          */
3832                         SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO, 0);
3833                         SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1);
3834
3835                         RQ_SGE_SET(rqe, 0, 0, flags);
3836                         i = 1;
3837                 }
3838
3839                 qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id;
3840                 qp->rqe_wr_id[qp->rq.prod].wqe_size = i;
3841
3842                 qedr_inc_sw_prod(&qp->rq);
3843
3844                 /* qp->rqe_wr_id is accessed during qedr_poll_cq, as
3845                  * soon as we give the doorbell, we could get a completion
3846                  * for this wr, therefore we need to make sure that the
3847                  * memory is update before giving the doorbell.
3848                  * During qedr_poll_cq, rmb is called before accessing the
3849                  * cqe. This covers for the smp_rmb as well.
3850                  */
3851                 smp_wmb();
3852
3853                 qp->rq.db_data.data.value++;
3854
3855                 writel(qp->rq.db_data.raw, qp->rq.db);
3856
3857                 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
3858                         writel(qp->rq.iwarp_db2_data.raw, qp->rq.iwarp_db2);
3859                 }
3860
3861                 wr = wr->next;
3862         }
3863
3864         spin_unlock_irqrestore(&qp->q_lock, flags);
3865
3866         return status;
3867 }
3868
3869 static int is_valid_cqe(struct qedr_cq *cq, union rdma_cqe *cqe)
3870 {
3871         struct rdma_cqe_requester *resp_cqe = &cqe->req;
3872
3873         return (resp_cqe->flags & RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK) ==
3874                 cq->pbl_toggle;
3875 }
3876
3877 static struct qedr_qp *cqe_get_qp(union rdma_cqe *cqe)
3878 {
3879         struct rdma_cqe_requester *resp_cqe = &cqe->req;
3880         struct qedr_qp *qp;
3881
3882         qp = (struct qedr_qp *)(uintptr_t)HILO_GEN(resp_cqe->qp_handle.hi,
3883                                                    resp_cqe->qp_handle.lo,
3884                                                    u64);
3885         return qp;
3886 }
3887
3888 static enum rdma_cqe_type cqe_get_type(union rdma_cqe *cqe)
3889 {
3890         struct rdma_cqe_requester *resp_cqe = &cqe->req;
3891
3892         return GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE);
3893 }
3894
3895 /* Return latest CQE (needs processing) */
3896 static union rdma_cqe *get_cqe(struct qedr_cq *cq)
3897 {
3898         return cq->latest_cqe;
3899 }
3900
3901 /* In fmr we need to increase the number of fmr completed counter for the fmr
3902  * algorithm determining whether we can free a pbl or not.
3903  * we need to perform this whether the work request was signaled or not. for
3904  * this purpose we call this function from the condition that checks if a wr
3905  * should be skipped, to make sure we don't miss it ( possibly this fmr
3906  * operation was not signalted)
3907  */
3908 static inline void qedr_chk_if_fmr(struct qedr_qp *qp)
3909 {
3910         if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR)
3911                 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
3912 }
3913
3914 static int process_req(struct qedr_dev *dev, struct qedr_qp *qp,
3915                        struct qedr_cq *cq, int num_entries,
3916                        struct ib_wc *wc, u16 hw_cons, enum ib_wc_status status,
3917                        int force)
3918 {
3919         u16 cnt = 0;
3920
3921         while (num_entries && qp->sq.wqe_cons != hw_cons) {
3922                 if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) {
3923                         qedr_chk_if_fmr(qp);
3924                         /* skip WC */
3925                         goto next_cqe;
3926                 }
3927
3928                 /* fill WC */
3929                 wc->status = status;
3930                 wc->vendor_err = 0;
3931                 wc->wc_flags = 0;
3932                 wc->src_qp = qp->id;
3933                 wc->qp = &qp->ibqp;
3934
3935                 wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id;
3936                 wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode;
3937
3938                 switch (wc->opcode) {
3939                 case IB_WC_RDMA_WRITE:
3940                         wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
3941                         break;
3942                 case IB_WC_COMP_SWAP:
3943                 case IB_WC_FETCH_ADD:
3944                         wc->byte_len = 8;
3945                         break;
3946                 case IB_WC_REG_MR:
3947                         qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
3948                         break;
3949                 case IB_WC_RDMA_READ:
3950                 case IB_WC_SEND:
3951                         wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
3952                         break;
3953                 default:
3954                         break;
3955                 }
3956
3957                 num_entries--;
3958                 wc++;
3959                 cnt++;
3960 next_cqe:
3961                 while (qp->wqe_wr_id[qp->sq.cons].wqe_size--)
3962                         qed_chain_consume(&qp->sq.pbl);
3963                 qedr_inc_sw_cons(&qp->sq);
3964         }
3965
3966         return cnt;
3967 }
3968
3969 static int qedr_poll_cq_req(struct qedr_dev *dev,
3970                             struct qedr_qp *qp, struct qedr_cq *cq,
3971                             int num_entries, struct ib_wc *wc,
3972                             struct rdma_cqe_requester *req)
3973 {
3974         int cnt = 0;
3975
3976         switch (req->status) {
3977         case RDMA_CQE_REQ_STS_OK:
3978                 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
3979                                   IB_WC_SUCCESS, 0);
3980                 break;
3981         case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR:
3982                 if (qp->state != QED_ROCE_QP_STATE_ERR)
3983                         DP_DEBUG(dev, QEDR_MSG_CQ,
3984                                  "Error: POLL CQ with RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3985                                  cq->icid, qp->icid);
3986                 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
3987                                   IB_WC_WR_FLUSH_ERR, 1);
3988                 break;
3989         default:
3990                 /* process all WQE before the cosumer */
3991                 qp->state = QED_ROCE_QP_STATE_ERR;
3992                 cnt = process_req(dev, qp, cq, num_entries, wc,
3993                                   req->sq_cons - 1, IB_WC_SUCCESS, 0);
3994                 wc += cnt;
3995                 /* if we have extra WC fill it with actual error info */
3996                 if (cnt < num_entries) {
3997                         enum ib_wc_status wc_status;
3998
3999                         switch (req->status) {
4000                         case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR:
4001                                 DP_ERR(dev,
4002                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4003                                        cq->icid, qp->icid);
4004                                 wc_status = IB_WC_BAD_RESP_ERR;
4005                                 break;
4006                         case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR:
4007                                 DP_ERR(dev,
4008                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4009                                        cq->icid, qp->icid);
4010                                 wc_status = IB_WC_LOC_LEN_ERR;
4011                                 break;
4012                         case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR:
4013                                 DP_ERR(dev,
4014                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4015                                        cq->icid, qp->icid);
4016                                 wc_status = IB_WC_LOC_QP_OP_ERR;
4017                                 break;
4018                         case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR:
4019                                 DP_ERR(dev,
4020                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4021                                        cq->icid, qp->icid);
4022                                 wc_status = IB_WC_LOC_PROT_ERR;
4023                                 break;
4024                         case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR:
4025                                 DP_ERR(dev,
4026                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4027                                        cq->icid, qp->icid);
4028                                 wc_status = IB_WC_MW_BIND_ERR;
4029                                 break;
4030                         case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR:
4031                                 DP_ERR(dev,
4032                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4033                                        cq->icid, qp->icid);
4034                                 wc_status = IB_WC_REM_INV_REQ_ERR;
4035                                 break;
4036                         case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR:
4037                                 DP_ERR(dev,
4038                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4039                                        cq->icid, qp->icid);
4040                                 wc_status = IB_WC_REM_ACCESS_ERR;
4041                                 break;
4042                         case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR:
4043                                 DP_ERR(dev,
4044                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4045                                        cq->icid, qp->icid);
4046                                 wc_status = IB_WC_REM_OP_ERR;
4047                                 break;
4048                         case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR:
4049                                 DP_ERR(dev,
4050                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4051                                        cq->icid, qp->icid);
4052                                 wc_status = IB_WC_RNR_RETRY_EXC_ERR;
4053                                 break;
4054                         case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR:
4055                                 DP_ERR(dev,
4056                                        "Error: POLL CQ with ROCE_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4057                                        cq->icid, qp->icid);
4058                                 wc_status = IB_WC_RETRY_EXC_ERR;
4059                                 break;
4060                         default:
4061                                 DP_ERR(dev,
4062                                        "Error: POLL CQ with IB_WC_GENERAL_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4063                                        cq->icid, qp->icid);
4064                                 wc_status = IB_WC_GENERAL_ERR;
4065                         }
4066                         cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons,
4067                                            wc_status, 1);
4068                 }
4069         }
4070
4071         return cnt;
4072 }
4073
4074 static inline int qedr_cqe_resp_status_to_ib(u8 status)
4075 {
4076         switch (status) {
4077         case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR:
4078                 return IB_WC_LOC_ACCESS_ERR;
4079         case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR:
4080                 return IB_WC_LOC_LEN_ERR;
4081         case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR:
4082                 return IB_WC_LOC_QP_OP_ERR;
4083         case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR:
4084                 return IB_WC_LOC_PROT_ERR;
4085         case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR:
4086                 return IB_WC_MW_BIND_ERR;
4087         case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR:
4088                 return IB_WC_REM_INV_RD_REQ_ERR;
4089         case RDMA_CQE_RESP_STS_OK:
4090                 return IB_WC_SUCCESS;
4091         default:
4092                 return IB_WC_GENERAL_ERR;
4093         }
4094 }
4095
4096 static inline int qedr_set_ok_cqe_resp_wc(struct rdma_cqe_responder *resp,
4097                                           struct ib_wc *wc)
4098 {
4099         wc->status = IB_WC_SUCCESS;
4100         wc->byte_len = le32_to_cpu(resp->length);
4101
4102         if (resp->flags & QEDR_RESP_IMM) {
4103                 wc->ex.imm_data = cpu_to_be32(le32_to_cpu(resp->imm_data_or_inv_r_Key));
4104                 wc->wc_flags |= IB_WC_WITH_IMM;
4105
4106                 if (resp->flags & QEDR_RESP_RDMA)
4107                         wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
4108
4109                 if (resp->flags & QEDR_RESP_INV)
4110                         return -EINVAL;
4111
4112         } else if (resp->flags & QEDR_RESP_INV) {
4113                 wc->ex.imm_data = le32_to_cpu(resp->imm_data_or_inv_r_Key);
4114                 wc->wc_flags |= IB_WC_WITH_INVALIDATE;
4115
4116                 if (resp->flags & QEDR_RESP_RDMA)
4117                         return -EINVAL;
4118
4119         } else if (resp->flags & QEDR_RESP_RDMA) {
4120                 return -EINVAL;
4121         }
4122
4123         return 0;
4124 }
4125
4126 static void __process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
4127                                struct qedr_cq *cq, struct ib_wc *wc,
4128                                struct rdma_cqe_responder *resp, u64 wr_id)
4129 {
4130         /* Must fill fields before qedr_set_ok_cqe_resp_wc() */
4131         wc->opcode = IB_WC_RECV;
4132         wc->wc_flags = 0;
4133
4134         if (likely(resp->status == RDMA_CQE_RESP_STS_OK)) {
4135                 if (qedr_set_ok_cqe_resp_wc(resp, wc))
4136                         DP_ERR(dev,
4137                                "CQ %p (icid=%d) has invalid CQE responder flags=0x%x\n",
4138                                cq, cq->icid, resp->flags);
4139
4140         } else {
4141                 wc->status = qedr_cqe_resp_status_to_ib(resp->status);
4142                 if (wc->status == IB_WC_GENERAL_ERR)
4143                         DP_ERR(dev,
4144                                "CQ %p (icid=%d) contains an invalid CQE status %d\n",
4145                                cq, cq->icid, resp->status);
4146         }
4147
4148         /* Fill the rest of the WC */
4149         wc->vendor_err = 0;
4150         wc->src_qp = qp->id;
4151         wc->qp = &qp->ibqp;
4152         wc->wr_id = wr_id;
4153 }
4154
4155 static int process_resp_one_srq(struct qedr_dev *dev, struct qedr_qp *qp,
4156                                 struct qedr_cq *cq, struct ib_wc *wc,
4157                                 struct rdma_cqe_responder *resp)
4158 {
4159         struct qedr_srq *srq = qp->srq;
4160         u64 wr_id;
4161
4162         wr_id = HILO_GEN(le32_to_cpu(resp->srq_wr_id.hi),
4163                          le32_to_cpu(resp->srq_wr_id.lo), u64);
4164
4165         if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
4166                 wc->status = IB_WC_WR_FLUSH_ERR;
4167                 wc->vendor_err = 0;
4168                 wc->wr_id = wr_id;
4169                 wc->byte_len = 0;
4170                 wc->src_qp = qp->id;
4171                 wc->qp = &qp->ibqp;
4172                 wc->wr_id = wr_id;
4173         } else {
4174                 __process_resp_one(dev, qp, cq, wc, resp, wr_id);
4175         }
4176         srq->hw_srq.wr_cons_cnt++;
4177
4178         return 1;
4179 }
4180 static int process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
4181                             struct qedr_cq *cq, struct ib_wc *wc,
4182                             struct rdma_cqe_responder *resp)
4183 {
4184         u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
4185
4186         __process_resp_one(dev, qp, cq, wc, resp, wr_id);
4187
4188         while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
4189                 qed_chain_consume(&qp->rq.pbl);
4190         qedr_inc_sw_cons(&qp->rq);
4191
4192         return 1;
4193 }
4194
4195 static int process_resp_flush(struct qedr_qp *qp, struct qedr_cq *cq,
4196                               int num_entries, struct ib_wc *wc, u16 hw_cons)
4197 {
4198         u16 cnt = 0;
4199
4200         while (num_entries && qp->rq.wqe_cons != hw_cons) {
4201                 /* fill WC */
4202                 wc->status = IB_WC_WR_FLUSH_ERR;
4203                 wc->vendor_err = 0;
4204                 wc->wc_flags = 0;
4205                 wc->src_qp = qp->id;
4206                 wc->byte_len = 0;
4207                 wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
4208                 wc->qp = &qp->ibqp;
4209                 num_entries--;
4210                 wc++;
4211                 cnt++;
4212                 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
4213                         qed_chain_consume(&qp->rq.pbl);
4214                 qedr_inc_sw_cons(&qp->rq);
4215         }
4216
4217         return cnt;
4218 }
4219
4220 static void try_consume_resp_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
4221                                  struct rdma_cqe_responder *resp, int *update)
4222 {
4223         if (le16_to_cpu(resp->rq_cons_or_srq_id) == qp->rq.wqe_cons) {
4224                 consume_cqe(cq);
4225                 *update |= 1;
4226         }
4227 }
4228
4229 static int qedr_poll_cq_resp_srq(struct qedr_dev *dev, struct qedr_qp *qp,
4230                                  struct qedr_cq *cq, int num_entries,
4231                                  struct ib_wc *wc,
4232                                  struct rdma_cqe_responder *resp)
4233 {
4234         int cnt;
4235
4236         cnt = process_resp_one_srq(dev, qp, cq, wc, resp);
4237         consume_cqe(cq);
4238
4239         return cnt;
4240 }
4241
4242 static int qedr_poll_cq_resp(struct qedr_dev *dev, struct qedr_qp *qp,
4243                              struct qedr_cq *cq, int num_entries,
4244                              struct ib_wc *wc, struct rdma_cqe_responder *resp,
4245                              int *update)
4246 {
4247         int cnt;
4248
4249         if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
4250                 cnt = process_resp_flush(qp, cq, num_entries, wc,
4251                                          resp->rq_cons_or_srq_id);
4252                 try_consume_resp_cqe(cq, qp, resp, update);
4253         } else {
4254                 cnt = process_resp_one(dev, qp, cq, wc, resp);
4255                 consume_cqe(cq);
4256                 *update |= 1;
4257         }
4258
4259         return cnt;
4260 }
4261
4262 static void try_consume_req_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
4263                                 struct rdma_cqe_requester *req, int *update)
4264 {
4265         if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) {
4266                 consume_cqe(cq);
4267                 *update |= 1;
4268         }
4269 }
4270
4271 int qedr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
4272 {
4273         struct qedr_dev *dev = get_qedr_dev(ibcq->device);
4274         struct qedr_cq *cq = get_qedr_cq(ibcq);
4275         union rdma_cqe *cqe;
4276         u32 old_cons, new_cons;
4277         unsigned long flags;
4278         int update = 0;
4279         int done = 0;
4280
4281         if (cq->destroyed) {
4282                 DP_ERR(dev,
4283                        "warning: poll was invoked after destroy for cq %p (icid=%d)\n",
4284                        cq, cq->icid);
4285                 return 0;
4286         }
4287
4288         if (cq->cq_type == QEDR_CQ_TYPE_GSI)
4289                 return qedr_gsi_poll_cq(ibcq, num_entries, wc);
4290
4291         spin_lock_irqsave(&cq->cq_lock, flags);
4292         cqe = cq->latest_cqe;
4293         old_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
4294         while (num_entries && is_valid_cqe(cq, cqe)) {
4295                 struct qedr_qp *qp;
4296                 int cnt = 0;
4297
4298                 /* prevent speculative reads of any field of CQE */
4299                 rmb();
4300
4301                 qp = cqe_get_qp(cqe);
4302                 if (!qp) {
4303                         WARN(1, "Error: CQE QP pointer is NULL. CQE=%p\n", cqe);
4304                         break;
4305                 }
4306
4307                 wc->qp = &qp->ibqp;
4308
4309                 switch (cqe_get_type(cqe)) {
4310                 case RDMA_CQE_TYPE_REQUESTER:
4311                         cnt = qedr_poll_cq_req(dev, qp, cq, num_entries, wc,
4312                                                &cqe->req);
4313                         try_consume_req_cqe(cq, qp, &cqe->req, &update);
4314                         break;
4315                 case RDMA_CQE_TYPE_RESPONDER_RQ:
4316                         cnt = qedr_poll_cq_resp(dev, qp, cq, num_entries, wc,
4317                                                 &cqe->resp, &update);
4318                         break;
4319                 case RDMA_CQE_TYPE_RESPONDER_SRQ:
4320                         cnt = qedr_poll_cq_resp_srq(dev, qp, cq, num_entries,
4321                                                     wc, &cqe->resp);
4322                         update = 1;
4323                         break;
4324                 case RDMA_CQE_TYPE_INVALID:
4325                 default:
4326                         DP_ERR(dev, "Error: invalid CQE type = %d\n",
4327                                cqe_get_type(cqe));
4328                 }
4329                 num_entries -= cnt;
4330                 wc += cnt;
4331                 done += cnt;
4332
4333                 cqe = get_cqe(cq);
4334         }
4335         new_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
4336
4337         cq->cq_cons += new_cons - old_cons;
4338
4339         if (update)
4340                 /* doorbell notifies abount latest VALID entry,
4341                  * but chain already point to the next INVALID one
4342                  */
4343                 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
4344
4345         spin_unlock_irqrestore(&cq->cq_lock, flags);
4346         return done;
4347 }
4348
4349 int qedr_process_mad(struct ib_device *ibdev, int process_mad_flags,
4350                      u8 port_num, const struct ib_wc *in_wc,
4351                      const struct ib_grh *in_grh, const struct ib_mad *in,
4352                      struct ib_mad *out_mad, size_t *out_mad_size,
4353                      u16 *out_mad_pkey_index)
4354 {
4355         return IB_MAD_RESULT_SUCCESS;
4356 }