drivers/infiniband/hw/ocrdma/ocrdma_verbs.c

   1 /* This file is part of the Emulex RoCE Device Driver for
   2  * RoCE (RDMA over Converged Ethernet) adapters.
   3  * Copyright (C) 2012-2015 Emulex. All rights reserved.
   4  * EMULEX and SLI are trademarks of Emulex.
   5  * www.emulex.com
   6  *
   7  * This software is available to you under a choice of one of two licenses.
   8  * You may choose to be licensed under the terms of the GNU General Public
   9  * License (GPL) Version 2, available from the file COPYING in the main
  10  * directory of this source tree, or the BSD license below:
  11  *
  12  * Redistribution and use in source and binary forms, with or without
  13  * modification, are permitted provided that the following conditions
  14  * are met:
  15  *
  16  * - Redistributions of source code must retain the above copyright notice,
  17  *   this list of conditions and the following disclaimer.
  18  *
  19  * - Redistributions in binary form must reproduce the above copyright
  20  *   notice, this list of conditions and the following disclaimer in
  21  *   the documentation and/or other materials provided with the distribution.
  22  *
  23  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  24  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,THE
  25  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  26  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
  27  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  28  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  29  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
  30  * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  31  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  32  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
  33  * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  34  *
  35  * Contact Information:
  36  * linux-drivers@emulex.com
  37  *
  38  * Emulex
  39  * 3333 Susan Street
  40  * Costa Mesa, CA 92626
  41  */
  42
  43 #include <linux/dma-mapping.h>
  44 #include <rdma/ib_verbs.h>
  45 #include <rdma/ib_user_verbs.h>
  46 #include <rdma/iw_cm.h>
  47 #include <rdma/ib_umem.h>
  48 #include <rdma/ib_addr.h>
  49 #include <rdma/ib_cache.h>
  50 #include <rdma/uverbs_ioctl.h>
  51
  52 #include "ocrdma.h"
  53 #include "ocrdma_hw.h"
  54 #include "ocrdma_verbs.h"
  55 #include <rdma/ocrdma-abi.h>
  56
  57 int ocrdma_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey)
  58 {
  59         if (index > 0)
  60                 return -EINVAL;
  61
  62         *pkey = 0xffff;
  63         return 0;
  64 }
  65
  66 int ocrdma_query_device(struct ib_device *ibdev, struct ib_device_attr *attr,
  67                         struct ib_udata *uhw)
  68 {
  69         struct ocrdma_dev *dev = get_ocrdma_dev(ibdev);
  70
  71         if (uhw->inlen || uhw->outlen)
  72                 return -EINVAL;
  73
  74         memset(attr, 0, sizeof *attr);
  75         memcpy(&attr->fw_ver, &dev->attr.fw_ver[0],
  76                min(sizeof(dev->attr.fw_ver), sizeof(attr->fw_ver)));
  77         ocrdma_get_guid(dev, (u8 *)&attr->sys_image_guid);
  78         attr->max_mr_size = dev->attr.max_mr_size;
  79         attr->page_size_cap = 0xffff000;
  80         attr->vendor_id = dev->nic_info.pdev->vendor;
  81         attr->vendor_part_id = dev->nic_info.pdev->device;
  82         attr->hw_ver = dev->asic_id;
  83         attr->max_qp = dev->attr.max_qp;
  84         attr->max_ah = OCRDMA_MAX_AH;
  85         attr->max_qp_wr = dev->attr.max_wqe;
  86
  87         attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD |
  88                                         IB_DEVICE_RC_RNR_NAK_GEN |
  89                                         IB_DEVICE_SHUTDOWN_PORT |
  90                                         IB_DEVICE_SYS_IMAGE_GUID |
  91                                         IB_DEVICE_LOCAL_DMA_LKEY |
  92                                         IB_DEVICE_MEM_MGT_EXTENSIONS;
  93         attr->max_send_sge = dev->attr.max_send_sge;
  94         attr->max_recv_sge = dev->attr.max_recv_sge;
  95         attr->max_sge_rd = dev->attr.max_rdma_sge;
  96         attr->max_cq = dev->attr.max_cq;
  97         attr->max_cqe = dev->attr.max_cqe;
  98         attr->max_mr = dev->attr.max_mr;
  99         attr->max_mw = dev->attr.max_mw;
 100         attr->max_pd = dev->attr.max_pd;
 101         attr->atomic_cap = 0;
 102         attr->max_fmr = 0;
 103         attr->max_map_per_fmr = 0;
 104         attr->max_qp_rd_atom =
 105             min(dev->attr.max_ord_per_qp, dev->attr.max_ird_per_qp);
 106         attr->max_qp_init_rd_atom = dev->attr.max_ord_per_qp;
 107         attr->max_srq = dev->attr.max_srq;
 108         attr->max_srq_sge = dev->attr.max_srq_sge;
 109         attr->max_srq_wr = dev->attr.max_rqe;
 110         attr->local_ca_ack_delay = dev->attr.local_ca_ack_delay;
 111         attr->max_fast_reg_page_list_len = dev->attr.max_pages_per_frmr;
 112         attr->max_pkeys = 1;
 113         return 0;
 114 }
 115
 116 static inline void get_link_speed_and_width(struct ocrdma_dev *dev,
 117                                             u8 *ib_speed, u8 *ib_width)
 118 {
 119         int status;
 120         u8 speed;
 121
 122         status = ocrdma_mbx_get_link_speed(dev, &speed, NULL);
 123         if (status)
 124                 speed = OCRDMA_PHYS_LINK_SPEED_ZERO;
 125
 126         switch (speed) {
 127         case OCRDMA_PHYS_LINK_SPEED_1GBPS:
 128                 *ib_speed = IB_SPEED_SDR;
 129                 *ib_width = IB_WIDTH_1X;
 130                 break;
 131
 132         case OCRDMA_PHYS_LINK_SPEED_10GBPS:
 133                 *ib_speed = IB_SPEED_QDR;
 134                 *ib_width = IB_WIDTH_1X;
 135                 break;
 136
 137         case OCRDMA_PHYS_LINK_SPEED_20GBPS:
 138                 *ib_speed = IB_SPEED_DDR;
 139                 *ib_width = IB_WIDTH_4X;
 140                 break;
 141
 142         case OCRDMA_PHYS_LINK_SPEED_40GBPS:
 143                 *ib_speed = IB_SPEED_QDR;
 144                 *ib_width = IB_WIDTH_4X;
 145                 break;
 146
 147         default:
 148                 /* Unsupported */
 149                 *ib_speed = IB_SPEED_SDR;
 150                 *ib_width = IB_WIDTH_1X;
 151         }
 152 }
 153
 154 int ocrdma_query_port(struct ib_device *ibdev,
 155                       u8 port, struct ib_port_attr *props)
 156 {
 157         enum ib_port_state port_state;
 158         struct ocrdma_dev *dev;
 159         struct net_device *netdev;
 160
 161         /* props being zeroed by the caller, avoid zeroing it here */
 162         dev = get_ocrdma_dev(ibdev);
 163         netdev = dev->nic_info.netdev;
 164         if (netif_running(netdev) && netif_oper_up(netdev)) {
 165                 port_state = IB_PORT_ACTIVE;
 166                 props->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
 167         } else {
 168                 port_state = IB_PORT_DOWN;
 169                 props->phys_state = IB_PORT_PHYS_STATE_DISABLED;
 170         }
 171         props->max_mtu = IB_MTU_4096;
 172         props->active_mtu = iboe_get_mtu(netdev->mtu);
 173         props->lid = 0;
 174         props->lmc = 0;
 175         props->sm_lid = 0;
 176         props->sm_sl = 0;
 177         props->state = port_state;
 178         props->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP |
 179                                 IB_PORT_DEVICE_MGMT_SUP |
 180                                 IB_PORT_VENDOR_CLASS_SUP;
 181         props->ip_gids = true;
 182         props->gid_tbl_len = OCRDMA_MAX_SGID;
 183         props->pkey_tbl_len = 1;
 184         props->bad_pkey_cntr = 0;
 185         props->qkey_viol_cntr = 0;
 186         get_link_speed_and_width(dev, &props->active_speed,
 187                                  &props->active_width);
 188         props->max_msg_sz = 0x80000000;
 189         props->max_vl_num = 4;
 190         return 0;
 191 }
 192
 193 static int ocrdma_add_mmap(struct ocrdma_ucontext *uctx, u64 phy_addr,
 194                            unsigned long len)
 195 {
 196         struct ocrdma_mm *mm;
 197
 198         mm = kzalloc(sizeof(*mm), GFP_KERNEL);
 199         if (mm == NULL)
 200                 return -ENOMEM;
 201         mm->key.phy_addr = phy_addr;
 202         mm->key.len = len;
 203         INIT_LIST_HEAD(&mm->entry);
 204
 205         mutex_lock(&uctx->mm_list_lock);
 206         list_add_tail(&mm->entry, &uctx->mm_head);
 207         mutex_unlock(&uctx->mm_list_lock);
 208         return 0;
 209 }
 210
 211 static void ocrdma_del_mmap(struct ocrdma_ucontext *uctx, u64 phy_addr,
 212                             unsigned long len)
 213 {
 214         struct ocrdma_mm *mm, *tmp;
 215
 216         mutex_lock(&uctx->mm_list_lock);
 217         list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) {
 218                 if (len != mm->key.len && phy_addr != mm->key.phy_addr)
 219                         continue;
 220
 221                 list_del(&mm->entry);
 222                 kfree(mm);
 223                 break;
 224         }
 225         mutex_unlock(&uctx->mm_list_lock);
 226 }
 227
 228 static bool ocrdma_search_mmap(struct ocrdma_ucontext *uctx, u64 phy_addr,
 229                               unsigned long len)
 230 {
 231         bool found = false;
 232         struct ocrdma_mm *mm;
 233
 234         mutex_lock(&uctx->mm_list_lock);
 235         list_for_each_entry(mm, &uctx->mm_head, entry) {
 236                 if (len != mm->key.len && phy_addr != mm->key.phy_addr)
 237                         continue;
 238
 239                 found = true;
 240                 break;
 241         }
 242         mutex_unlock(&uctx->mm_list_lock);
 243         return found;
 244 }
 245
 246
 247 static u16 _ocrdma_pd_mgr_get_bitmap(struct ocrdma_dev *dev, bool dpp_pool)
 248 {
 249         u16 pd_bitmap_idx = 0;
 250         const unsigned long *pd_bitmap;
 251
 252         if (dpp_pool) {
 253                 pd_bitmap = dev->pd_mgr->pd_dpp_bitmap;
 254                 pd_bitmap_idx = find_first_zero_bit(pd_bitmap,
 255                                                     dev->pd_mgr->max_dpp_pd);
 256                 __set_bit(pd_bitmap_idx, dev->pd_mgr->pd_dpp_bitmap);
 257                 dev->pd_mgr->pd_dpp_count++;
 258                 if (dev->pd_mgr->pd_dpp_count > dev->pd_mgr->pd_dpp_thrsh)
 259                         dev->pd_mgr->pd_dpp_thrsh = dev->pd_mgr->pd_dpp_count;
 260         } else {
 261                 pd_bitmap = dev->pd_mgr->pd_norm_bitmap;
 262                 pd_bitmap_idx = find_first_zero_bit(pd_bitmap,
 263                                                     dev->pd_mgr->max_normal_pd);
 264                 __set_bit(pd_bitmap_idx, dev->pd_mgr->pd_norm_bitmap);
 265                 dev->pd_mgr->pd_norm_count++;
 266                 if (dev->pd_mgr->pd_norm_count > dev->pd_mgr->pd_norm_thrsh)
 267                         dev->pd_mgr->pd_norm_thrsh = dev->pd_mgr->pd_norm_count;
 268         }
 269         return pd_bitmap_idx;
 270 }
 271
 272 static int _ocrdma_pd_mgr_put_bitmap(struct ocrdma_dev *dev, u16 pd_id,
 273                                         bool dpp_pool)
 274 {
 275         u16 pd_count;
 276         u16 pd_bit_index;
 277
 278         pd_count = dpp_pool ? dev->pd_mgr->pd_dpp_count :
 279                               dev->pd_mgr->pd_norm_count;
 280         if (pd_count == 0)
 281                 return -EINVAL;
 282
 283         if (dpp_pool) {
 284                 pd_bit_index = pd_id - dev->pd_mgr->pd_dpp_start;
 285                 if (pd_bit_index >= dev->pd_mgr->max_dpp_pd) {
 286                         return -EINVAL;
 287                 } else {
 288                         __clear_bit(pd_bit_index, dev->pd_mgr->pd_dpp_bitmap);
 289                         dev->pd_mgr->pd_dpp_count--;
 290                 }
 291         } else {
 292                 pd_bit_index = pd_id - dev->pd_mgr->pd_norm_start;
 293                 if (pd_bit_index >= dev->pd_mgr->max_normal_pd) {
 294                         return -EINVAL;
 295                 } else {
 296                         __clear_bit(pd_bit_index, dev->pd_mgr->pd_norm_bitmap);
 297                         dev->pd_mgr->pd_norm_count--;
 298                 }
 299         }
 300
 301         return 0;
 302 }
 303
 304 static int ocrdma_put_pd_num(struct ocrdma_dev *dev, u16 pd_id,
 305                                    bool dpp_pool)
 306 {
 307         int status;
 308
 309         mutex_lock(&dev->dev_lock);
 310         status = _ocrdma_pd_mgr_put_bitmap(dev, pd_id, dpp_pool);
 311         mutex_unlock(&dev->dev_lock);
 312         return status;
 313 }
 314
 315 static int ocrdma_get_pd_num(struct ocrdma_dev *dev, struct ocrdma_pd *pd)
 316 {
 317         u16 pd_idx = 0;
 318         int status = 0;
 319
 320         mutex_lock(&dev->dev_lock);
 321         if (pd->dpp_enabled) {
 322                 /* try allocating DPP PD, if not available then normal PD */
 323                 if (dev->pd_mgr->pd_dpp_count < dev->pd_mgr->max_dpp_pd) {
 324                         pd_idx = _ocrdma_pd_mgr_get_bitmap(dev, true);
 325                         pd->id = dev->pd_mgr->pd_dpp_start + pd_idx;
 326                         pd->dpp_page = dev->pd_mgr->dpp_page_index + pd_idx;
 327                 } else if (dev->pd_mgr->pd_norm_count <
 328                            dev->pd_mgr->max_normal_pd) {
 329                         pd_idx = _ocrdma_pd_mgr_get_bitmap(dev, false);
 330                         pd->id = dev->pd_mgr->pd_norm_start + pd_idx;
 331                         pd->dpp_enabled = false;
 332                 } else {
 333                         status = -EINVAL;
 334                 }
 335         } else {
 336                 if (dev->pd_mgr->pd_norm_count < dev->pd_mgr->max_normal_pd) {
 337                         pd_idx = _ocrdma_pd_mgr_get_bitmap(dev, false);
 338                         pd->id = dev->pd_mgr->pd_norm_start + pd_idx;
 339                 } else {
 340                         status = -EINVAL;
 341                 }
 342         }
 343         mutex_unlock(&dev->dev_lock);
 344         return status;
 345 }
 346
 347 /*
 348  * NOTE:
 349  *
 350  * ocrdma_ucontext must be used here because this function is also
 351  * called from ocrdma_alloc_ucontext where ib_udata does not have
 352  * valid ib_ucontext pointer. ib_uverbs_get_context does not call
 353  * uobj_{alloc|get_xxx} helpers which are used to store the
 354  * ib_ucontext in uverbs_attr_bundle wrapping the ib_udata. so
 355  * ib_udata does NOT imply valid ib_ucontext here!
 356  */
 357 static int _ocrdma_alloc_pd(struct ocrdma_dev *dev, struct ocrdma_pd *pd,
 358                             struct ocrdma_ucontext *uctx,
 359                             struct ib_udata *udata)
 360 {
 361         int status;
 362
 363         if (udata && uctx && dev->attr.max_dpp_pds) {
 364                 pd->dpp_enabled =
 365                         ocrdma_get_asic_type(dev) == OCRDMA_ASIC_GEN_SKH_R;
 366                 pd->num_dpp_qp =
 367                         pd->dpp_enabled ? (dev->nic_info.db_page_size /
 368                                            dev->attr.wqe_size) : 0;
 369         }
 370
 371         if (dev->pd_mgr->pd_prealloc_valid)
 372                 return ocrdma_get_pd_num(dev, pd);
 373
 374 retry:
 375         status = ocrdma_mbx_alloc_pd(dev, pd);
 376         if (status) {
 377                 if (pd->dpp_enabled) {
 378                         pd->dpp_enabled = false;
 379                         pd->num_dpp_qp = 0;
 380                         goto retry;
 381                 }
 382                 return status;
 383         }
 384
 385         return 0;
 386 }
 387
 388 static inline int is_ucontext_pd(struct ocrdma_ucontext *uctx,
 389                                  struct ocrdma_pd *pd)
 390 {
 391         return (uctx->cntxt_pd == pd);
 392 }
 393
 394 static void _ocrdma_dealloc_pd(struct ocrdma_dev *dev,
 395                               struct ocrdma_pd *pd)
 396 {
 397         if (dev->pd_mgr->pd_prealloc_valid)
 398                 ocrdma_put_pd_num(dev, pd->id, pd->dpp_enabled);
 399         else
 400                 ocrdma_mbx_dealloc_pd(dev, pd);
 401 }
 402
 403 static int ocrdma_alloc_ucontext_pd(struct ocrdma_dev *dev,
 404                                     struct ocrdma_ucontext *uctx,
 405                                     struct ib_udata *udata)
 406 {
 407         struct ib_device *ibdev = &dev->ibdev;
 408         struct ib_pd *pd;
 409         int status;
 410
 411         pd = rdma_zalloc_drv_obj(ibdev, ib_pd);
 412         if (!pd)
 413                 return -ENOMEM;
 414
 415         pd->device  = ibdev;
 416         uctx->cntxt_pd = get_ocrdma_pd(pd);
 417
 418         status = _ocrdma_alloc_pd(dev, uctx->cntxt_pd, uctx, udata);
 419         if (status) {
 420                 kfree(uctx->cntxt_pd);
 421                 goto err;
 422         }
 423
 424         uctx->cntxt_pd->uctx = uctx;
 425         uctx->cntxt_pd->ibpd.device = &dev->ibdev;
 426 err:
 427         return status;
 428 }
 429
 430 static void ocrdma_dealloc_ucontext_pd(struct ocrdma_ucontext *uctx)
 431 {
 432         struct ocrdma_pd *pd = uctx->cntxt_pd;
 433         struct ocrdma_dev *dev = get_ocrdma_dev(pd->ibpd.device);
 434
 435         if (uctx->pd_in_use) {
 436                 pr_err("%s(%d) Freeing in use pdid=0x%x.\n",
 437                        __func__, dev->id, pd->id);
 438         }
 439         kfree(uctx->cntxt_pd);
 440         uctx->cntxt_pd = NULL;
 441         _ocrdma_dealloc_pd(dev, pd);
 442 }
 443
 444 static struct ocrdma_pd *ocrdma_get_ucontext_pd(struct ocrdma_ucontext *uctx)
 445 {
 446         struct ocrdma_pd *pd = NULL;
 447
 448         mutex_lock(&uctx->mm_list_lock);
 449         if (!uctx->pd_in_use) {
 450                 uctx->pd_in_use = true;
 451                 pd = uctx->cntxt_pd;
 452         }
 453         mutex_unlock(&uctx->mm_list_lock);
 454
 455         return pd;
 456 }
 457
 458 static void ocrdma_release_ucontext_pd(struct ocrdma_ucontext *uctx)
 459 {
 460         mutex_lock(&uctx->mm_list_lock);
 461         uctx->pd_in_use = false;
 462         mutex_unlock(&uctx->mm_list_lock);
 463 }
 464
 465 int ocrdma_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata)
 466 {
 467         struct ib_device *ibdev = uctx->device;
 468         int status;
 469         struct ocrdma_ucontext *ctx = get_ocrdma_ucontext(uctx);
 470         struct ocrdma_alloc_ucontext_resp resp = {};
 471         struct ocrdma_dev *dev = get_ocrdma_dev(ibdev);
 472         struct pci_dev *pdev = dev->nic_info.pdev;
 473         u32 map_len = roundup(sizeof(u32) * 2048, PAGE_SIZE);
 474
 475         if (!udata)
 476                 return -EFAULT;
 477         INIT_LIST_HEAD(&ctx->mm_head);
 478         mutex_init(&ctx->mm_list_lock);
 479
 480         ctx->ah_tbl.va = dma_alloc_coherent(&pdev->dev, map_len,
 481                                             &ctx->ah_tbl.pa, GFP_KERNEL);
 482         if (!ctx->ah_tbl.va)
 483                 return -ENOMEM;
 484
 485         ctx->ah_tbl.len = map_len;
 486
 487         resp.ah_tbl_len = ctx->ah_tbl.len;
 488         resp.ah_tbl_page = virt_to_phys(ctx->ah_tbl.va);
 489
 490         status = ocrdma_add_mmap(ctx, resp.ah_tbl_page, resp.ah_tbl_len);
 491         if (status)
 492                 goto map_err;
 493
 494         status = ocrdma_alloc_ucontext_pd(dev, ctx, udata);
 495         if (status)
 496                 goto pd_err;
 497
 498         resp.dev_id = dev->id;
 499         resp.max_inline_data = dev->attr.max_inline_data;
 500         resp.wqe_size = dev->attr.wqe_size;
 501         resp.rqe_size = dev->attr.rqe_size;
 502         resp.dpp_wqe_size = dev->attr.wqe_size;
 503
 504         memcpy(resp.fw_ver, dev->attr.fw_ver, sizeof(resp.fw_ver));
 505         status = ib_copy_to_udata(udata, &resp, sizeof(resp));
 506         if (status)
 507                 goto cpy_err;
 508         return 0;
 509
 510 cpy_err:
 511         ocrdma_dealloc_ucontext_pd(ctx);
 512 pd_err:
 513         ocrdma_del_mmap(ctx, ctx->ah_tbl.pa, ctx->ah_tbl.len);
 514 map_err:
 515         dma_free_coherent(&pdev->dev, ctx->ah_tbl.len, ctx->ah_tbl.va,
 516                           ctx->ah_tbl.pa);
 517         return status;
 518 }
 519
 520 void ocrdma_dealloc_ucontext(struct ib_ucontext *ibctx)
 521 {
 522         struct ocrdma_mm *mm, *tmp;
 523         struct ocrdma_ucontext *uctx = get_ocrdma_ucontext(ibctx);
 524         struct ocrdma_dev *dev = get_ocrdma_dev(ibctx->device);
 525         struct pci_dev *pdev = dev->nic_info.pdev;
 526
 527         ocrdma_dealloc_ucontext_pd(uctx);
 528
 529         ocrdma_del_mmap(uctx, uctx->ah_tbl.pa, uctx->ah_tbl.len);
 530         dma_free_coherent(&pdev->dev, uctx->ah_tbl.len, uctx->ah_tbl.va,
 531                           uctx->ah_tbl.pa);
 532
 533         list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) {
 534                 list_del(&mm->entry);
 535                 kfree(mm);
 536         }
 537 }
 538
 539 int ocrdma_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
 540 {
 541         struct ocrdma_ucontext *ucontext = get_ocrdma_ucontext(context);
 542         struct ocrdma_dev *dev = get_ocrdma_dev(context->device);
 543         unsigned long vm_page = vma->vm_pgoff << PAGE_SHIFT;
 544         u64 unmapped_db = (u64) dev->nic_info.unmapped_db;
 545         unsigned long len = (vma->vm_end - vma->vm_start);
 546         int status;
 547         bool found;
 548
 549         if (vma->vm_start & (PAGE_SIZE - 1))
 550                 return -EINVAL;
 551         found = ocrdma_search_mmap(ucontext, vma->vm_pgoff << PAGE_SHIFT, len);
 552         if (!found)
 553                 return -EINVAL;
 554
 555         if ((vm_page >= unmapped_db) && (vm_page <= (unmapped_db +
 556                 dev->nic_info.db_total_size)) &&
 557                 (len <= dev->nic_info.db_page_size)) {
 558                 if (vma->vm_flags & VM_READ)
 559                         return -EPERM;
 560
 561                 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
 562                 status = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
 563                                             len, vma->vm_page_prot);
 564         } else if (dev->nic_info.dpp_unmapped_len &&
 565                 (vm_page >= (u64) dev->nic_info.dpp_unmapped_addr) &&
 566                 (vm_page <= (u64) (dev->nic_info.dpp_unmapped_addr +
 567                         dev->nic_info.dpp_unmapped_len)) &&
 568                 (len <= dev->nic_info.dpp_unmapped_len)) {
 569                 if (vma->vm_flags & VM_READ)
 570                         return -EPERM;
 571
 572                 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
 573                 status = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
 574                                             len, vma->vm_page_prot);
 575         } else {
 576                 status = remap_pfn_range(vma, vma->vm_start,
 577                                          vma->vm_pgoff, len, vma->vm_page_prot);
 578         }
 579         return status;
 580 }
 581
 582 static int ocrdma_copy_pd_uresp(struct ocrdma_dev *dev, struct ocrdma_pd *pd,
 583                                 struct ib_udata *udata)
 584 {
 585         int status;
 586         u64 db_page_addr;
 587         u64 dpp_page_addr = 0;
 588         u32 db_page_size;
 589         struct ocrdma_alloc_pd_uresp rsp;
 590         struct ocrdma_ucontext *uctx = rdma_udata_to_drv_context(
 591                 udata, struct ocrdma_ucontext, ibucontext);
 592
 593         memset(&rsp, 0, sizeof(rsp));
 594         rsp.id = pd->id;
 595         rsp.dpp_enabled = pd->dpp_enabled;
 596         db_page_addr = ocrdma_get_db_addr(dev, pd->id);
 597         db_page_size = dev->nic_info.db_page_size;
 598
 599         status = ocrdma_add_mmap(uctx, db_page_addr, db_page_size);
 600         if (status)
 601                 return status;
 602
 603         if (pd->dpp_enabled) {
 604                 dpp_page_addr = dev->nic_info.dpp_unmapped_addr +
 605                                 (pd->id * PAGE_SIZE);
 606                 status = ocrdma_add_mmap(uctx, dpp_page_addr,
 607                                  PAGE_SIZE);
 608                 if (status)
 609                         goto dpp_map_err;
 610                 rsp.dpp_page_addr_hi = upper_32_bits(dpp_page_addr);
 611                 rsp.dpp_page_addr_lo = dpp_page_addr;
 612         }
 613
 614         status = ib_copy_to_udata(udata, &rsp, sizeof(rsp));
 615         if (status)
 616                 goto ucopy_err;
 617
 618         pd->uctx = uctx;
 619         return 0;
 620
 621 ucopy_err:
 622         if (pd->dpp_enabled)
 623                 ocrdma_del_mmap(pd->uctx, dpp_page_addr, PAGE_SIZE);
 624 dpp_map_err:
 625         ocrdma_del_mmap(pd->uctx, db_page_addr, db_page_size);
 626         return status;
 627 }
 628
 629 int ocrdma_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 630 {
 631         struct ib_device *ibdev = ibpd->device;
 632         struct ocrdma_dev *dev = get_ocrdma_dev(ibdev);
 633         struct ocrdma_pd *pd;
 634         int status;
 635         u8 is_uctx_pd = false;
 636         struct ocrdma_ucontext *uctx = rdma_udata_to_drv_context(
 637                 udata, struct ocrdma_ucontext, ibucontext);
 638
 639         if (udata) {
 640                 pd = ocrdma_get_ucontext_pd(uctx);
 641                 if (pd) {
 642                         is_uctx_pd = true;
 643                         goto pd_mapping;
 644                 }
 645         }
 646
 647         pd = get_ocrdma_pd(ibpd);
 648         status = _ocrdma_alloc_pd(dev, pd, uctx, udata);
 649         if (status)
 650                 goto exit;
 651
 652 pd_mapping:
 653         if (udata) {
 654                 status = ocrdma_copy_pd_uresp(dev, pd, udata);
 655                 if (status)
 656                         goto err;
 657         }
 658         return 0;
 659
 660 err:
 661         if (is_uctx_pd)
 662                 ocrdma_release_ucontext_pd(uctx);
 663         else
 664                 _ocrdma_dealloc_pd(dev, pd);
 665 exit:
 666         return status;
 667 }
 668
 669 void ocrdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 670 {
 671         struct ocrdma_pd *pd = get_ocrdma_pd(ibpd);
 672         struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device);
 673         struct ocrdma_ucontext *uctx = NULL;
 674         u64 usr_db;
 675
 676         uctx = pd->uctx;
 677         if (uctx) {
 678                 u64 dpp_db = dev->nic_info.dpp_unmapped_addr +
 679                         (pd->id * PAGE_SIZE);
 680                 if (pd->dpp_enabled)
 681                         ocrdma_del_mmap(pd->uctx, dpp_db, PAGE_SIZE);
 682                 usr_db = ocrdma_get_db_addr(dev, pd->id);
 683                 ocrdma_del_mmap(pd->uctx, usr_db, dev->nic_info.db_page_size);
 684
 685                 if (is_ucontext_pd(uctx, pd)) {
 686                         ocrdma_release_ucontext_pd(uctx);
 687                         return;
 688                 }
 689         }
 690         _ocrdma_dealloc_pd(dev, pd);
 691 }
 692
 693 static int ocrdma_alloc_lkey(struct ocrdma_dev *dev, struct ocrdma_mr *mr,
 694                             u32 pdid, int acc, u32 num_pbls, u32 addr_check)
 695 {
 696         int status;
 697
 698         mr->hwmr.fr_mr = 0;
 699         mr->hwmr.local_rd = 1;
 700         mr->hwmr.remote_rd = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
 701         mr->hwmr.remote_wr = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
 702         mr->hwmr.local_wr = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
 703         mr->hwmr.mw_bind = (acc & IB_ACCESS_MW_BIND) ? 1 : 0;
 704         mr->hwmr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
 705         mr->hwmr.num_pbls = num_pbls;
 706
 707         status = ocrdma_mbx_alloc_lkey(dev, &mr->hwmr, pdid, addr_check);
 708         if (status)
 709                 return status;
 710
 711         mr->ibmr.lkey = mr->hwmr.lkey;
 712         if (mr->hwmr.remote_wr || mr->hwmr.remote_rd)
 713                 mr->ibmr.rkey = mr->hwmr.lkey;
 714         return 0;
 715 }
 716
 717 struct ib_mr *ocrdma_get_dma_mr(struct ib_pd *ibpd, int acc)
 718 {
 719         int status;
 720         struct ocrdma_mr *mr;
 721         struct ocrdma_pd *pd = get_ocrdma_pd(ibpd);
 722         struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device);
 723
 724         if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE)) {
 725                 pr_err("%s err, invalid access rights\n", __func__);
 726                 return ERR_PTR(-EINVAL);
 727         }
 728
 729         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
 730         if (!mr)
 731                 return ERR_PTR(-ENOMEM);
 732
 733         status = ocrdma_alloc_lkey(dev, mr, pd->id, acc, 0,
 734                                    OCRDMA_ADDR_CHECK_DISABLE);
 735         if (status) {
 736                 kfree(mr);
 737                 return ERR_PTR(status);
 738         }
 739
 740         return &mr->ibmr;
 741 }
 742
 743 static void ocrdma_free_mr_pbl_tbl(struct ocrdma_dev *dev,
 744                                    struct ocrdma_hw_mr *mr)
 745 {
 746         struct pci_dev *pdev = dev->nic_info.pdev;
 747         int i = 0;
 748
 749         if (mr->pbl_table) {
 750                 for (i = 0; i < mr->num_pbls; i++) {
 751                         if (!mr->pbl_table[i].va)
 752                                 continue;
 753                         dma_free_coherent(&pdev->dev, mr->pbl_size,
 754                                           mr->pbl_table[i].va,
 755                                           mr->pbl_table[i].pa);
 756                 }
 757                 kfree(mr->pbl_table);
 758                 mr->pbl_table = NULL;
 759         }
 760 }
 761
 762 static int ocrdma_get_pbl_info(struct ocrdma_dev *dev, struct ocrdma_mr *mr,
 763                               u32 num_pbes)
 764 {
 765         u32 num_pbls = 0;
 766         u32 idx = 0;
 767         int status = 0;
 768         u32 pbl_size;
 769
 770         do {
 771                 pbl_size = OCRDMA_MIN_HPAGE_SIZE * (1 << idx);
 772                 if (pbl_size > MAX_OCRDMA_PBL_SIZE) {
 773                         status = -EFAULT;
 774                         break;
 775                 }
 776                 num_pbls = roundup(num_pbes, (pbl_size / sizeof(u64)));
 777                 num_pbls = num_pbls / (pbl_size / sizeof(u64));
 778                 idx++;
 779         } while (num_pbls >= dev->attr.max_num_mr_pbl);
 780
 781         mr->hwmr.num_pbes = num_pbes;
 782         mr->hwmr.num_pbls = num_pbls;
 783         mr->hwmr.pbl_size = pbl_size;
 784         return status;
 785 }
 786
 787 static int ocrdma_build_pbl_tbl(struct ocrdma_dev *dev, struct ocrdma_hw_mr *mr)
 788 {
 789         int status = 0;
 790         int i;
 791         u32 dma_len = mr->pbl_size;
 792         struct pci_dev *pdev = dev->nic_info.pdev;
 793         void *va;
 794         dma_addr_t pa;
 795
 796         mr->pbl_table = kcalloc(mr->num_pbls, sizeof(struct ocrdma_pbl),
 797                                 GFP_KERNEL);
 798
 799         if (!mr->pbl_table)
 800                 return -ENOMEM;
 801
 802         for (i = 0; i < mr->num_pbls; i++) {
 803                 va = dma_alloc_coherent(&pdev->dev, dma_len, &pa, GFP_KERNEL);
 804                 if (!va) {
 805                         ocrdma_free_mr_pbl_tbl(dev, mr);
 806                         status = -ENOMEM;
 807                         break;
 808                 }
 809                 mr->pbl_table[i].va = va;
 810                 mr->pbl_table[i].pa = pa;
 811         }
 812         return status;
 813 }
 814
 815 static void build_user_pbes(struct ocrdma_dev *dev, struct ocrdma_mr *mr,
 816                             u32 num_pbes)
 817 {
 818         struct ocrdma_pbe *pbe;
 819         struct sg_dma_page_iter sg_iter;
 820         struct ocrdma_pbl *pbl_tbl = mr->hwmr.pbl_table;
 821         struct ib_umem *umem = mr->umem;
 822         int pbe_cnt, total_num_pbes = 0;
 823         u64 pg_addr;
 824
 825         if (!mr->hwmr.num_pbes)
 826                 return;
 827
 828         pbe = (struct ocrdma_pbe *)pbl_tbl->va;
 829         pbe_cnt = 0;
 830
 831         for_each_sg_dma_page (umem->sg_head.sgl, &sg_iter, umem->nmap, 0) {
 832                 /* store the page address in pbe */
 833                 pg_addr = sg_page_iter_dma_address(&sg_iter);
 834                 pbe->pa_lo = cpu_to_le32(pg_addr);
 835                 pbe->pa_hi = cpu_to_le32(upper_32_bits(pg_addr));
 836                 pbe_cnt += 1;
 837                 total_num_pbes += 1;
 838                 pbe++;
 839
 840                 /* if done building pbes, issue the mbx cmd. */
 841                 if (total_num_pbes == num_pbes)
 842                         return;
 843
 844                 /* if the given pbl is full storing the pbes,
 845                  * move to next pbl.
 846                  */
 847                 if (pbe_cnt == (mr->hwmr.pbl_size / sizeof(u64))) {
 848                         pbl_tbl++;
 849                         pbe = (struct ocrdma_pbe *)pbl_tbl->va;
 850                         pbe_cnt = 0;
 851                 }
 852         }
 853 }
 854
 855 struct ib_mr *ocrdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
 856                                  u64 usr_addr, int acc, struct ib_udata *udata)
 857 {
 858         int status = -ENOMEM;
 859         struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device);
 860         struct ocrdma_mr *mr;
 861         struct ocrdma_pd *pd;
 862         u32 num_pbes;
 863
 864         pd = get_ocrdma_pd(ibpd);
 865
 866         if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE))
 867                 return ERR_PTR(-EINVAL);
 868
 869         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
 870         if (!mr)
 871                 return ERR_PTR(status);
 872         mr->umem = ib_umem_get(ibpd->device, start, len, acc);
 873         if (IS_ERR(mr->umem)) {
 874                 status = -EFAULT;
 875                 goto umem_err;
 876         }
 877         num_pbes = ib_umem_page_count(mr->umem);
 878         status = ocrdma_get_pbl_info(dev, mr, num_pbes);
 879         if (status)
 880                 goto umem_err;
 881
 882         mr->hwmr.pbe_size = PAGE_SIZE;
 883         mr->hwmr.fbo = ib_umem_offset(mr->umem);
 884         mr->hwmr.va = usr_addr;
 885         mr->hwmr.len = len;
 886         mr->hwmr.remote_wr = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
 887         mr->hwmr.remote_rd = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
 888         mr->hwmr.local_wr = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
 889         mr->hwmr.local_rd = 1;
 890         mr->hwmr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
 891         status = ocrdma_build_pbl_tbl(dev, &mr->hwmr);
 892         if (status)
 893                 goto umem_err;
 894         build_user_pbes(dev, mr, num_pbes);
 895         status = ocrdma_reg_mr(dev, &mr->hwmr, pd->id, acc);
 896         if (status)
 897                 goto mbx_err;
 898         mr->ibmr.lkey = mr->hwmr.lkey;
 899         if (mr->hwmr.remote_wr || mr->hwmr.remote_rd)
 900                 mr->ibmr.rkey = mr->hwmr.lkey;
 901
 902         return &mr->ibmr;
 903
 904 mbx_err:
 905         ocrdma_free_mr_pbl_tbl(dev, &mr->hwmr);
 906 umem_err:
 907         kfree(mr);
 908         return ERR_PTR(status);
 909 }
 910
 911 int ocrdma_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
 912 {
 913         struct ocrdma_mr *mr = get_ocrdma_mr(ib_mr);
 914         struct ocrdma_dev *dev = get_ocrdma_dev(ib_mr->device);
 915
 916         (void) ocrdma_mbx_dealloc_lkey(dev, mr->hwmr.fr_mr, mr->hwmr.lkey);
 917
 918         kfree(mr->pages);
 919         ocrdma_free_mr_pbl_tbl(dev, &mr->hwmr);
 920
 921         /* it could be user registered memory. */
 922         ib_umem_release(mr->umem);
 923         kfree(mr);
 924
 925         /* Don't stop cleanup, in case FW is unresponsive */
 926         if (dev->mqe_ctx.fw_error_state) {
 927                 pr_err("%s(%d) fw not responding.\n",
 928                        __func__, dev->id);
 929         }
 930         return 0;
 931 }
 932
 933 static int ocrdma_copy_cq_uresp(struct ocrdma_dev *dev, struct ocrdma_cq *cq,
 934                                 struct ib_udata *udata)
 935 {
 936         int status;
 937         struct ocrdma_ucontext *uctx = rdma_udata_to_drv_context(
 938                 udata, struct ocrdma_ucontext, ibucontext);
 939         struct ocrdma_create_cq_uresp uresp;
 940
 941         /* this must be user flow! */
 942         if (!udata)
 943                 return -EINVAL;
 944
 945         memset(&uresp, 0, sizeof(uresp));
 946         uresp.cq_id = cq->id;
 947         uresp.page_size = PAGE_ALIGN(cq->len);
 948         uresp.num_pages = 1;
 949         uresp.max_hw_cqe = cq->max_hw_cqe;
 950         uresp.page_addr[0] = virt_to_phys(cq->va);
 951         uresp.db_page_addr =  ocrdma_get_db_addr(dev, uctx->cntxt_pd->id);
 952         uresp.db_page_size = dev->nic_info.db_page_size;
 953         uresp.phase_change = cq->phase_change ? 1 : 0;
 954         status = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
 955         if (status) {
 956                 pr_err("%s(%d) copy error cqid=0x%x.\n",
 957                        __func__, dev->id, cq->id);
 958                 goto err;
 959         }
 960         status = ocrdma_add_mmap(uctx, uresp.db_page_addr, uresp.db_page_size);
 961         if (status)
 962                 goto err;
 963         status = ocrdma_add_mmap(uctx, uresp.page_addr[0], uresp.page_size);
 964         if (status) {
 965                 ocrdma_del_mmap(uctx, uresp.db_page_addr, uresp.db_page_size);
 966                 goto err;
 967         }
 968         cq->ucontext = uctx;
 969 err:
 970         return status;
 971 }
 972
 973 int ocrdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 974                      struct ib_udata *udata)
 975 {
 976         struct ib_device *ibdev = ibcq->device;
 977         int entries = attr->cqe;
 978         struct ocrdma_cq *cq = get_ocrdma_cq(ibcq);
 979         struct ocrdma_dev *dev = get_ocrdma_dev(ibdev);
 980         struct ocrdma_ucontext *uctx = rdma_udata_to_drv_context(
 981                 udata, struct ocrdma_ucontext, ibucontext);
 982         u16 pd_id = 0;
 983         int status;
 984         struct ocrdma_create_cq_ureq ureq;
 985
 986         if (attr->flags)
 987                 return -EINVAL;
 988
 989         if (udata) {
 990                 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq)))
 991                         return -EFAULT;
 992         } else
 993                 ureq.dpp_cq = 0;
 994
 995         spin_lock_init(&cq->cq_lock);
 996         spin_lock_init(&cq->comp_handler_lock);
 997         INIT_LIST_HEAD(&cq->sq_head);
 998         INIT_LIST_HEAD(&cq->rq_head);
 999
1000         if (udata)
1001                 pd_id = uctx->cntxt_pd->id;
1002
1003         status = ocrdma_mbx_create_cq(dev, cq, entries, ureq.dpp_cq, pd_id);
1004         if (status)
1005                 return status;
1006
1007         if (udata) {
1008                 status = ocrdma_copy_cq_uresp(dev, cq, udata);
1009                 if (status)
1010                         goto ctx_err;
1011         }
1012         cq->phase = OCRDMA_CQE_VALID;
1013         dev->cq_tbl[cq->id] = cq;
1014         return 0;
1015
1016 ctx_err:
1017         ocrdma_mbx_destroy_cq(dev, cq);
1018         return status;
1019 }
1020
1021 int ocrdma_resize_cq(struct ib_cq *ibcq, int new_cnt,
1022                      struct ib_udata *udata)
1023 {
1024         int status = 0;
1025         struct ocrdma_cq *cq = get_ocrdma_cq(ibcq);
1026
1027         if (new_cnt < 1 || new_cnt > cq->max_hw_cqe) {
1028                 status = -EINVAL;
1029                 return status;
1030         }
1031         ibcq->cqe = new_cnt;
1032         return status;
1033 }
1034
1035 static void ocrdma_flush_cq(struct ocrdma_cq *cq)
1036 {
1037         int cqe_cnt;
1038         int valid_count = 0;
1039         unsigned long flags;
1040
1041         struct ocrdma_dev *dev = get_ocrdma_dev(cq->ibcq.device);
1042         struct ocrdma_cqe *cqe = NULL;
1043
1044         cqe = cq->va;
1045         cqe_cnt = cq->cqe_cnt;
1046
1047         /* Last irq might have scheduled a polling thread
1048          * sync-up with it before hard flushing.
1049          */
1050         spin_lock_irqsave(&cq->cq_lock, flags);
1051         while (cqe_cnt) {
1052                 if (is_cqe_valid(cq, cqe))
1053                         valid_count++;
1054                 cqe++;
1055                 cqe_cnt--;
1056         }
1057         ocrdma_ring_cq_db(dev, cq->id, false, false, valid_count);
1058         spin_unlock_irqrestore(&cq->cq_lock, flags);
1059 }
1060
1061 void ocrdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
1062 {
1063         struct ocrdma_cq *cq = get_ocrdma_cq(ibcq);
1064         struct ocrdma_eq *eq = NULL;
1065         struct ocrdma_dev *dev = get_ocrdma_dev(ibcq->device);
1066         int pdid = 0;
1067         u32 irq, indx;
1068
1069         dev->cq_tbl[cq->id] = NULL;
1070         indx = ocrdma_get_eq_table_index(dev, cq->eqn);
1071
1072         eq = &dev->eq_tbl[indx];
1073         irq = ocrdma_get_irq(dev, eq);
1074         synchronize_irq(irq);
1075         ocrdma_flush_cq(cq);
1076
1077         ocrdma_mbx_destroy_cq(dev, cq);
1078         if (cq->ucontext) {
1079                 pdid = cq->ucontext->cntxt_pd->id;
1080                 ocrdma_del_mmap(cq->ucontext, (u64) cq->pa,
1081                                 PAGE_ALIGN(cq->len));
1082                 ocrdma_del_mmap(cq->ucontext,
1083                                 ocrdma_get_db_addr(dev, pdid),
1084                                 dev->nic_info.db_page_size);
1085         }
1086 }
1087
1088 static int ocrdma_add_qpn_map(struct ocrdma_dev *dev, struct ocrdma_qp *qp)
1089 {
1090         int status = -EINVAL;
1091
1092         if (qp->id < OCRDMA_MAX_QP && dev->qp_tbl[qp->id] == NULL) {
1093                 dev->qp_tbl[qp->id] = qp;
1094                 status = 0;
1095         }
1096         return status;
1097 }
1098
1099 static void ocrdma_del_qpn_map(struct ocrdma_dev *dev, struct ocrdma_qp *qp)
1100 {
1101         dev->qp_tbl[qp->id] = NULL;
1102 }
1103
1104 static int ocrdma_check_qp_params(struct ib_pd *ibpd, struct ocrdma_dev *dev,
1105                                   struct ib_qp_init_attr *attrs,
1106                                   struct ib_udata *udata)
1107 {
1108         if ((attrs->qp_type != IB_QPT_GSI) &&
1109             (attrs->qp_type != IB_QPT_RC) &&
1110             (attrs->qp_type != IB_QPT_UC) &&
1111             (attrs->qp_type != IB_QPT_UD)) {
1112                 pr_err("%s(%d) unsupported qp type=0x%x requested\n",
1113                        __func__, dev->id, attrs->qp_type);
1114                 return -EINVAL;
1115         }
1116         /* Skip the check for QP1 to support CM size of 128 */
1117         if ((attrs->qp_type != IB_QPT_GSI) &&
1118             (attrs->cap.max_send_wr > dev->attr.max_wqe)) {
1119                 pr_err("%s(%d) unsupported send_wr=0x%x requested\n",
1120                        __func__, dev->id, attrs->cap.max_send_wr);
1121                 pr_err("%s(%d) supported send_wr=0x%x\n",
1122                        __func__, dev->id, dev->attr.max_wqe);
1123                 return -EINVAL;
1124         }
1125         if (!attrs->srq && (attrs->cap.max_recv_wr > dev->attr.max_rqe)) {
1126                 pr_err("%s(%d) unsupported recv_wr=0x%x requested\n",
1127                        __func__, dev->id, attrs->cap.max_recv_wr);
1128                 pr_err("%s(%d) supported recv_wr=0x%x\n",
1129                        __func__, dev->id, dev->attr.max_rqe);
1130                 return -EINVAL;
1131         }
1132         if (attrs->cap.max_inline_data > dev->attr.max_inline_data) {
1133                 pr_err("%s(%d) unsupported inline data size=0x%x requested\n",
1134                        __func__, dev->id, attrs->cap.max_inline_data);
1135                 pr_err("%s(%d) supported inline data size=0x%x\n",
1136                        __func__, dev->id, dev->attr.max_inline_data);
1137                 return -EINVAL;
1138         }
1139         if (attrs->cap.max_send_sge > dev->attr.max_send_sge) {
1140                 pr_err("%s(%d) unsupported send_sge=0x%x requested\n",
1141                        __func__, dev->id, attrs->cap.max_send_sge);
1142                 pr_err("%s(%d) supported send_sge=0x%x\n",
1143                        __func__, dev->id, dev->attr.max_send_sge);
1144                 return -EINVAL;
1145         }
1146         if (attrs->cap.max_recv_sge > dev->attr.max_recv_sge) {
1147                 pr_err("%s(%d) unsupported recv_sge=0x%x requested\n",
1148                        __func__, dev->id, attrs->cap.max_recv_sge);
1149                 pr_err("%s(%d) supported recv_sge=0x%x\n",
1150                        __func__, dev->id, dev->attr.max_recv_sge);
1151                 return -EINVAL;
1152         }
1153         /* unprivileged user space cannot create special QP */
1154         if (udata && attrs->qp_type == IB_QPT_GSI) {
1155                 pr_err
1156                     ("%s(%d) Userspace can't create special QPs of type=0x%x\n",
1157                      __func__, dev->id, attrs->qp_type);
1158                 return -EINVAL;
1159         }
1160         /* allow creating only one GSI type of QP */
1161         if (attrs->qp_type == IB_QPT_GSI && dev->gsi_qp_created) {
1162                 pr_err("%s(%d) GSI special QPs already created.\n",
1163                        __func__, dev->id);
1164                 return -EINVAL;
1165         }
1166         /* verify consumer QPs are not trying to use GSI QP's CQ */
1167         if ((attrs->qp_type != IB_QPT_GSI) && (dev->gsi_qp_created)) {
1168                 if ((dev->gsi_sqcq == get_ocrdma_cq(attrs->send_cq)) ||
1169                         (dev->gsi_rqcq == get_ocrdma_cq(attrs->recv_cq))) {
1170                         pr_err("%s(%d) Consumer QP cannot use GSI CQs.\n",
1171                                 __func__, dev->id);
1172                         return -EINVAL;
1173                 }
1174         }
1175         return 0;
1176 }
1177
1178 static int ocrdma_copy_qp_uresp(struct ocrdma_qp *qp,
1179                                 struct ib_udata *udata, int dpp_offset,
1180                                 int dpp_credit_lmt, int srq)
1181 {
1182         int status;
1183         u64 usr_db;
1184         struct ocrdma_create_qp_uresp uresp;
1185         struct ocrdma_pd *pd = qp->pd;
1186         struct ocrdma_dev *dev = get_ocrdma_dev(pd->ibpd.device);
1187
1188         memset(&uresp, 0, sizeof(uresp));
1189         usr_db = dev->nic_info.unmapped_db +
1190                         (pd->id * dev->nic_info.db_page_size);
1191         uresp.qp_id = qp->id;
1192         uresp.sq_dbid = qp->sq.dbid;
1193         uresp.num_sq_pages = 1;
1194         uresp.sq_page_size = PAGE_ALIGN(qp->sq.len);
1195         uresp.sq_page_addr[0] = virt_to_phys(qp->sq.va);
1196         uresp.num_wqe_allocated = qp->sq.max_cnt;
1197         if (!srq) {
1198                 uresp.rq_dbid = qp->rq.dbid;
1199                 uresp.num_rq_pages = 1;
1200                 uresp.rq_page_size = PAGE_ALIGN(qp->rq.len);
1201                 uresp.rq_page_addr[0] = virt_to_phys(qp->rq.va);
1202                 uresp.num_rqe_allocated = qp->rq.max_cnt;
1203         }
1204         uresp.db_page_addr = usr_db;
1205         uresp.db_page_size = dev->nic_info.db_page_size;
1206         uresp.db_sq_offset = OCRDMA_DB_GEN2_SQ_OFFSET;
1207         uresp.db_rq_offset = OCRDMA_DB_GEN2_RQ_OFFSET;
1208         uresp.db_shift = OCRDMA_DB_RQ_SHIFT;
1209
1210         if (qp->dpp_enabled) {
1211                 uresp.dpp_credit = dpp_credit_lmt;
1212                 uresp.dpp_offset = dpp_offset;
1213         }
1214         status = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1215         if (status) {
1216                 pr_err("%s(%d) user copy error.\n", __func__, dev->id);
1217                 goto err;
1218         }
1219         status = ocrdma_add_mmap(pd->uctx, uresp.sq_page_addr[0],
1220                                  uresp.sq_page_size);
1221         if (status)
1222                 goto err;
1223
1224         if (!srq) {
1225                 status = ocrdma_add_mmap(pd->uctx, uresp.rq_page_addr[0],
1226                                          uresp.rq_page_size);
1227                 if (status)
1228                         goto rq_map_err;
1229         }
1230         return status;
1231 rq_map_err:
1232         ocrdma_del_mmap(pd->uctx, uresp.sq_page_addr[0], uresp.sq_page_size);
1233 err:
1234         return status;
1235 }
1236
1237 static void ocrdma_set_qp_db(struct ocrdma_dev *dev, struct ocrdma_qp *qp,
1238                              struct ocrdma_pd *pd)
1239 {
1240         if (ocrdma_get_asic_type(dev) == OCRDMA_ASIC_GEN_SKH_R) {
1241                 qp->sq_db = dev->nic_info.db +
1242                         (pd->id * dev->nic_info.db_page_size) +
1243                         OCRDMA_DB_GEN2_SQ_OFFSET;
1244                 qp->rq_db = dev->nic_info.db +
1245                         (pd->id * dev->nic_info.db_page_size) +
1246                         OCRDMA_DB_GEN2_RQ_OFFSET;
1247         } else {
1248                 qp->sq_db = dev->nic_info.db +
1249                         (pd->id * dev->nic_info.db_page_size) +
1250                         OCRDMA_DB_SQ_OFFSET;
1251                 qp->rq_db = dev->nic_info.db +
1252                         (pd->id * dev->nic_info.db_page_size) +
1253                         OCRDMA_DB_RQ_OFFSET;
1254         }
1255 }
1256
1257 static int ocrdma_alloc_wr_id_tbl(struct ocrdma_qp *qp)
1258 {
1259         qp->wqe_wr_id_tbl =
1260             kcalloc(qp->sq.max_cnt, sizeof(*(qp->wqe_wr_id_tbl)),
1261                     GFP_KERNEL);
1262         if (qp->wqe_wr_id_tbl == NULL)
1263                 return -ENOMEM;
1264         qp->rqe_wr_id_tbl =
1265             kcalloc(qp->rq.max_cnt, sizeof(u64), GFP_KERNEL);
1266         if (qp->rqe_wr_id_tbl == NULL)
1267                 return -ENOMEM;
1268
1269         return 0;
1270 }
1271
1272 static void ocrdma_set_qp_init_params(struct ocrdma_qp *qp,
1273                                       struct ocrdma_pd *pd,
1274                                       struct ib_qp_init_attr *attrs)
1275 {
1276         qp->pd = pd;
1277         spin_lock_init(&qp->q_lock);
1278         INIT_LIST_HEAD(&qp->sq_entry);
1279         INIT_LIST_HEAD(&qp->rq_entry);
1280
1281         qp->qp_type = attrs->qp_type;
1282         qp->cap_flags = OCRDMA_QP_INB_RD | OCRDMA_QP_INB_WR;
1283         qp->max_inline_data = attrs->cap.max_inline_data;
1284         qp->sq.max_sges = attrs->cap.max_send_sge;
1285         qp->rq.max_sges = attrs->cap.max_recv_sge;
1286         qp->state = OCRDMA_QPS_RST;
1287         qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false;
1288 }
1289
1290 static void ocrdma_store_gsi_qp_cq(struct ocrdma_dev *dev,
1291                                    struct ib_qp_init_attr *attrs)
1292 {
1293         if (attrs->qp_type == IB_QPT_GSI) {
1294                 dev->gsi_qp_created = 1;
1295                 dev->gsi_sqcq = get_ocrdma_cq(attrs->send_cq);
1296                 dev->gsi_rqcq = get_ocrdma_cq(attrs->recv_cq);
1297         }
1298 }
1299
1300 struct ib_qp *ocrdma_create_qp(struct ib_pd *ibpd,
1301                                struct ib_qp_init_attr *attrs,
1302                                struct ib_udata *udata)
1303 {
1304         int status;
1305         struct ocrdma_pd *pd = get_ocrdma_pd(ibpd);
1306         struct ocrdma_qp *qp;
1307         struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device);
1308         struct ocrdma_create_qp_ureq ureq;
1309         u16 dpp_credit_lmt, dpp_offset;
1310
1311         status = ocrdma_check_qp_params(ibpd, dev, attrs, udata);
1312         if (status)
1313                 goto gen_err;
1314
1315         memset(&ureq, 0, sizeof(ureq));
1316         if (udata) {
1317                 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq)))
1318                         return ERR_PTR(-EFAULT);
1319         }
1320         qp = kzalloc(sizeof(*qp), GFP_KERNEL);
1321         if (!qp) {
1322                 status = -ENOMEM;
1323                 goto gen_err;
1324         }
1325         ocrdma_set_qp_init_params(qp, pd, attrs);
1326         if (udata == NULL)
1327                 qp->cap_flags |= (OCRDMA_QP_MW_BIND | OCRDMA_QP_LKEY0 |
1328                                         OCRDMA_QP_FAST_REG);
1329
1330         mutex_lock(&dev->dev_lock);
1331         status = ocrdma_mbx_create_qp(qp, attrs, ureq.enable_dpp_cq,
1332                                         ureq.dpp_cq_id,
1333                                         &dpp_offset, &dpp_credit_lmt);
1334         if (status)
1335                 goto mbx_err;
1336
1337         /* user space QP's wr_id table are managed in library */
1338         if (udata == NULL) {
1339                 status = ocrdma_alloc_wr_id_tbl(qp);
1340                 if (status)
1341                         goto map_err;
1342         }
1343
1344         status = ocrdma_add_qpn_map(dev, qp);
1345         if (status)
1346                 goto map_err;
1347         ocrdma_set_qp_db(dev, qp, pd);
1348         if (udata) {
1349                 status = ocrdma_copy_qp_uresp(qp, udata, dpp_offset,
1350                                               dpp_credit_lmt,
1351                                               (attrs->srq != NULL));
1352                 if (status)
1353                         goto cpy_err;
1354         }
1355         ocrdma_store_gsi_qp_cq(dev, attrs);
1356         qp->ibqp.qp_num = qp->id;
1357         mutex_unlock(&dev->dev_lock);
1358         return &qp->ibqp;
1359
1360 cpy_err:
1361         ocrdma_del_qpn_map(dev, qp);
1362 map_err:
1363         ocrdma_mbx_destroy_qp(dev, qp);
1364 mbx_err:
1365         mutex_unlock(&dev->dev_lock);
1366         kfree(qp->wqe_wr_id_tbl);
1367         kfree(qp->rqe_wr_id_tbl);
1368         kfree(qp);
1369         pr_err("%s(%d) error=%d\n", __func__, dev->id, status);
1370 gen_err:
1371         return ERR_PTR(status);
1372 }
1373
1374 int _ocrdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
1375                       int attr_mask)
1376 {
1377         int status = 0;
1378         struct ocrdma_qp *qp;
1379         struct ocrdma_dev *dev;
1380         enum ib_qp_state old_qps;
1381
1382         qp = get_ocrdma_qp(ibqp);
1383         dev = get_ocrdma_dev(ibqp->device);
1384         if (attr_mask & IB_QP_STATE)
1385                 status = ocrdma_qp_state_change(qp, attr->qp_state, &old_qps);
1386         /* if new and previous states are same hw doesn't need to
1387          * know about it.
1388          */
1389         if (status < 0)
1390                 return status;
1391         return ocrdma_mbx_modify_qp(dev, qp, attr, attr_mask);
1392 }
1393
1394 int ocrdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
1395                      int attr_mask, struct ib_udata *udata)
1396 {
1397         unsigned long flags;
1398         int status = -EINVAL;
1399         struct ocrdma_qp *qp;
1400         struct ocrdma_dev *dev;
1401         enum ib_qp_state old_qps, new_qps;
1402
1403         qp = get_ocrdma_qp(ibqp);
1404         dev = get_ocrdma_dev(ibqp->device);
1405
1406         /* syncronize with multiple context trying to change, retrive qps */
1407         mutex_lock(&dev->dev_lock);
1408         /* syncronize with wqe, rqe posting and cqe processing contexts */
1409         spin_lock_irqsave(&qp->q_lock, flags);
1410         old_qps = get_ibqp_state(qp->state);
1411         if (attr_mask & IB_QP_STATE)
1412                 new_qps = attr->qp_state;
1413         else
1414                 new_qps = old_qps;
1415         spin_unlock_irqrestore(&qp->q_lock, flags);
1416
1417         if (!ib_modify_qp_is_ok(old_qps, new_qps, ibqp->qp_type, attr_mask)) {
1418                 pr_err("%s(%d) invalid attribute mask=0x%x specified for\n"
1419                        "qpn=0x%x of type=0x%x old_qps=0x%x, new_qps=0x%x\n",
1420                        __func__, dev->id, attr_mask, qp->id, ibqp->qp_type,
1421                        old_qps, new_qps);
1422                 goto param_err;
1423         }
1424
1425         status = _ocrdma_modify_qp(ibqp, attr, attr_mask);
1426         if (status > 0)
1427                 status = 0;
1428 param_err:
1429         mutex_unlock(&dev->dev_lock);
1430         return status;
1431 }
1432
1433 static enum ib_mtu ocrdma_mtu_int_to_enum(u16 mtu)
1434 {
1435         switch (mtu) {
1436         case 256:
1437                 return IB_MTU_256;
1438         case 512:
1439                 return IB_MTU_512;
1440         case 1024:
1441                 return IB_MTU_1024;
1442         case 2048:
1443                 return IB_MTU_2048;
1444         case 4096:
1445                 return IB_MTU_4096;
1446         default:
1447                 return IB_MTU_1024;
1448         }
1449 }
1450
1451 static int ocrdma_to_ib_qp_acc_flags(int qp_cap_flags)
1452 {
1453         int ib_qp_acc_flags = 0;
1454
1455         if (qp_cap_flags & OCRDMA_QP_INB_WR)
1456                 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE;
1457         if (qp_cap_flags & OCRDMA_QP_INB_RD)
1458                 ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE;
1459         return ib_qp_acc_flags;
1460 }
1461
1462 int ocrdma_query_qp(struct ib_qp *ibqp,
1463                     struct ib_qp_attr *qp_attr,
1464                     int attr_mask, struct ib_qp_init_attr *qp_init_attr)
1465 {
1466         int status;
1467         u32 qp_state;
1468         struct ocrdma_qp_params params;
1469         struct ocrdma_qp *qp = get_ocrdma_qp(ibqp);
1470         struct ocrdma_dev *dev = get_ocrdma_dev(ibqp->device);
1471
1472         memset(&params, 0, sizeof(params));
1473         mutex_lock(&dev->dev_lock);
1474         status = ocrdma_mbx_query_qp(dev, qp, &params);
1475         mutex_unlock(&dev->dev_lock);
1476         if (status)
1477                 goto mbx_err;
1478         if (qp->qp_type == IB_QPT_UD)
1479                 qp_attr->qkey = params.qkey;
1480         qp_attr->path_mtu =
1481                 ocrdma_mtu_int_to_enum(params.path_mtu_pkey_indx &
1482                                 OCRDMA_QP_PARAMS_PATH_MTU_MASK) >>
1483                                 OCRDMA_QP_PARAMS_PATH_MTU_SHIFT;
1484         qp_attr->path_mig_state = IB_MIG_MIGRATED;
1485         qp_attr->rq_psn = params.hop_lmt_rq_psn & OCRDMA_QP_PARAMS_RQ_PSN_MASK;
1486         qp_attr->sq_psn = params.tclass_sq_psn & OCRDMA_QP_PARAMS_SQ_PSN_MASK;
1487         qp_attr->dest_qp_num =
1488             params.ack_to_rnr_rtc_dest_qpn & OCRDMA_QP_PARAMS_DEST_QPN_MASK;
1489
1490         qp_attr->qp_access_flags = ocrdma_to_ib_qp_acc_flags(qp->cap_flags);
1491         qp_attr->cap.max_send_wr = qp->sq.max_cnt - 1;
1492         qp_attr->cap.max_recv_wr = qp->rq.max_cnt - 1;
1493         qp_attr->cap.max_send_sge = qp->sq.max_sges;
1494         qp_attr->cap.max_recv_sge = qp->rq.max_sges;
1495         qp_attr->cap.max_inline_data = qp->max_inline_data;
1496         qp_init_attr->cap = qp_attr->cap;
1497         qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
1498
1499         rdma_ah_set_grh(&qp_attr->ah_attr, NULL,
1500                         params.rnt_rc_sl_fl &
1501                           OCRDMA_QP_PARAMS_FLOW_LABEL_MASK,
1502                         qp->sgid_idx,
1503                         (params.hop_lmt_rq_psn &
1504                          OCRDMA_QP_PARAMS_HOP_LMT_MASK) >>
1505                          OCRDMA_QP_PARAMS_HOP_LMT_SHIFT,
1506                         (params.tclass_sq_psn &
1507                          OCRDMA_QP_PARAMS_TCLASS_MASK) >>
1508                          OCRDMA_QP_PARAMS_TCLASS_SHIFT);
1509         rdma_ah_set_dgid_raw(&qp_attr->ah_attr, &params.dgid[0]);
1510
1511         rdma_ah_set_port_num(&qp_attr->ah_attr, 1);
1512         rdma_ah_set_sl(&qp_attr->ah_attr, (params.rnt_rc_sl_fl &
1513                                            OCRDMA_QP_PARAMS_SL_MASK) >>
1514                                            OCRDMA_QP_PARAMS_SL_SHIFT);
1515         qp_attr->timeout = (params.ack_to_rnr_rtc_dest_qpn &
1516                             OCRDMA_QP_PARAMS_ACK_TIMEOUT_MASK) >>
1517                                 OCRDMA_QP_PARAMS_ACK_TIMEOUT_SHIFT;
1518         qp_attr->rnr_retry = (params.ack_to_rnr_rtc_dest_qpn &
1519                               OCRDMA_QP_PARAMS_RNR_RETRY_CNT_MASK) >>
1520                                 OCRDMA_QP_PARAMS_RNR_RETRY_CNT_SHIFT;
1521         qp_attr->retry_cnt =
1522             (params.rnt_rc_sl_fl & OCRDMA_QP_PARAMS_RETRY_CNT_MASK) >>
1523                 OCRDMA_QP_PARAMS_RETRY_CNT_SHIFT;
1524         qp_attr->min_rnr_timer = 0;
1525         qp_attr->pkey_index = 0;
1526         qp_attr->port_num = 1;
1527         rdma_ah_set_path_bits(&qp_attr->ah_attr, 0);
1528         rdma_ah_set_static_rate(&qp_attr->ah_attr, 0);
1529         qp_attr->alt_pkey_index = 0;
1530         qp_attr->alt_port_num = 0;
1531         qp_attr->alt_timeout = 0;
1532         memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr));
1533         qp_state = (params.max_sge_recv_flags & OCRDMA_QP_PARAMS_STATE_MASK) >>
1534                     OCRDMA_QP_PARAMS_STATE_SHIFT;
1535         qp_attr->qp_state = get_ibqp_state(qp_state);
1536         qp_attr->cur_qp_state = qp_attr->qp_state;
1537         qp_attr->sq_draining = (qp_state == OCRDMA_QPS_SQ_DRAINING) ? 1 : 0;
1538         qp_attr->max_dest_rd_atomic =
1539             params.max_ord_ird >> OCRDMA_QP_PARAMS_MAX_ORD_SHIFT;
1540         qp_attr->max_rd_atomic =
1541             params.max_ord_ird & OCRDMA_QP_PARAMS_MAX_IRD_MASK;
1542         qp_attr->en_sqd_async_notify = (params.max_sge_recv_flags &
1543                                 OCRDMA_QP_PARAMS_FLAGS_SQD_ASYNC) ? 1 : 0;
1544         /* Sync driver QP state with FW */
1545         ocrdma_qp_state_change(qp, qp_attr->qp_state, NULL);
1546 mbx_err:
1547         return status;
1548 }
1549
1550 static void ocrdma_srq_toggle_bit(struct ocrdma_srq *srq, unsigned int idx)
1551 {
1552         unsigned int i = idx / 32;
1553         u32 mask = (1U << (idx % 32));
1554
1555         srq->idx_bit_fields[i] ^= mask;
1556 }
1557
1558 static int ocrdma_hwq_free_cnt(struct ocrdma_qp_hwq_info *q)
1559 {
1560         return ((q->max_wqe_idx - q->head) + q->tail) % q->max_cnt;
1561 }
1562
1563 static int is_hw_sq_empty(struct ocrdma_qp *qp)
1564 {
1565         return (qp->sq.tail == qp->sq.head);
1566 }
1567
1568 static int is_hw_rq_empty(struct ocrdma_qp *qp)
1569 {
1570         return (qp->rq.tail == qp->rq.head);
1571 }
1572
1573 static void *ocrdma_hwq_head(struct ocrdma_qp_hwq_info *q)
1574 {
1575         return q->va + (q->head * q->entry_size);
1576 }
1577
1578 static void *ocrdma_hwq_head_from_idx(struct ocrdma_qp_hwq_info *q,
1579                                       u32 idx)
1580 {
1581         return q->va + (idx * q->entry_size);
1582 }
1583
1584 static void ocrdma_hwq_inc_head(struct ocrdma_qp_hwq_info *q)
1585 {
1586         q->head = (q->head + 1) & q->max_wqe_idx;
1587 }
1588
1589 static void ocrdma_hwq_inc_tail(struct ocrdma_qp_hwq_info *q)
1590 {
1591         q->tail = (q->tail + 1) & q->max_wqe_idx;
1592 }
1593
1594 /* discard the cqe for a given QP */
1595 static void ocrdma_discard_cqes(struct ocrdma_qp *qp, struct ocrdma_cq *cq)
1596 {
1597         unsigned long cq_flags;
1598         unsigned long flags;
1599         int discard_cnt = 0;
1600         u32 cur_getp, stop_getp;
1601         struct ocrdma_cqe *cqe;
1602         u32 qpn = 0, wqe_idx = 0;
1603
1604         spin_lock_irqsave(&cq->cq_lock, cq_flags);
1605
1606         /* traverse through the CQEs in the hw CQ,
1607          * find the matching CQE for a given qp,
1608          * mark the matching one discarded by clearing qpn.
1609          * ring the doorbell in the poll_cq() as
1610          * we don't complete out of order cqe.
1611          */
1612
1613         cur_getp = cq->getp;
1614         /* find upto when do we reap the cq. */
1615         stop_getp = cur_getp;
1616         do {
1617                 if (is_hw_sq_empty(qp) && (!qp->srq && is_hw_rq_empty(qp)))
1618                         break;
1619
1620                 cqe = cq->va + cur_getp;
1621                 /* if (a) done reaping whole hw cq, or
1622                  *    (b) qp_xq becomes empty.
1623                  * then exit
1624                  */
1625                 qpn = cqe->cmn.qpn & OCRDMA_CQE_QPN_MASK;
1626                 /* if previously discarded cqe found, skip that too. */
1627                 /* check for matching qp */
1628                 if (qpn == 0 || qpn != qp->id)
1629                         goto skip_cqe;
1630
1631                 if (is_cqe_for_sq(cqe)) {
1632                         ocrdma_hwq_inc_tail(&qp->sq);
1633                 } else {
1634                         if (qp->srq) {
1635                                 wqe_idx = (le32_to_cpu(cqe->rq.buftag_qpn) >>
1636                                         OCRDMA_CQE_BUFTAG_SHIFT) &
1637                                         qp->srq->rq.max_wqe_idx;
1638                                 BUG_ON(wqe_idx < 1);
1639                                 spin_lock_irqsave(&qp->srq->q_lock, flags);
1640                                 ocrdma_hwq_inc_tail(&qp->srq->rq);
1641                                 ocrdma_srq_toggle_bit(qp->srq, wqe_idx - 1);
1642                                 spin_unlock_irqrestore(&qp->srq->q_lock, flags);
1643
1644                         } else {
1645                                 ocrdma_hwq_inc_tail(&qp->rq);
1646                         }
1647                 }
1648                 /* mark cqe discarded so that it is not picked up later
1649                  * in the poll_cq().
1650                  */
1651                 discard_cnt += 1;
1652                 cqe->cmn.qpn = 0;
1653 skip_cqe:
1654                 cur_getp = (cur_getp + 1) % cq->max_hw_cqe;
1655         } while (cur_getp != stop_getp);
1656         spin_unlock_irqrestore(&cq->cq_lock, cq_flags);
1657 }
1658
1659 void ocrdma_del_flush_qp(struct ocrdma_qp *qp)
1660 {
1661         int found = false;
1662         unsigned long flags;
1663         struct ocrdma_dev *dev = get_ocrdma_dev(qp->ibqp.device);
1664         /* sync with any active CQ poll */
1665
1666         spin_lock_irqsave(&dev->flush_q_lock, flags);
1667         found = ocrdma_is_qp_in_sq_flushlist(qp->sq_cq, qp);
1668         if (found)
1669                 list_del(&qp->sq_entry);
1670         if (!qp->srq) {
1671                 found = ocrdma_is_qp_in_rq_flushlist(qp->rq_cq, qp);
1672                 if (found)
1673                         list_del(&qp->rq_entry);
1674         }
1675         spin_unlock_irqrestore(&dev->flush_q_lock, flags);
1676 }
1677
1678 int ocrdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
1679 {
1680         struct ocrdma_pd *pd;
1681         struct ocrdma_qp *qp;
1682         struct ocrdma_dev *dev;
1683         struct ib_qp_attr attrs;
1684         int attr_mask;
1685         unsigned long flags;
1686
1687         qp = get_ocrdma_qp(ibqp);
1688         dev = get_ocrdma_dev(ibqp->device);
1689
1690         pd = qp->pd;
1691
1692         /* change the QP state to ERROR */
1693         if (qp->state != OCRDMA_QPS_RST) {
1694                 attrs.qp_state = IB_QPS_ERR;
1695                 attr_mask = IB_QP_STATE;
1696                 _ocrdma_modify_qp(ibqp, &attrs, attr_mask);
1697         }
1698         /* ensure that CQEs for newly created QP (whose id may be same with
1699          * one which just getting destroyed are same), dont get
1700          * discarded until the old CQEs are discarded.
1701          */
1702         mutex_lock(&dev->dev_lock);
1703         (void) ocrdma_mbx_destroy_qp(dev, qp);
1704
1705         /*
1706          * acquire CQ lock while destroy is in progress, in order to
1707          * protect against proessing in-flight CQEs for this QP.
1708          */
1709         spin_lock_irqsave(&qp->sq_cq->cq_lock, flags);
1710         if (qp->rq_cq && (qp->rq_cq != qp->sq_cq)) {
1711                 spin_lock(&qp->rq_cq->cq_lock);
1712                 ocrdma_del_qpn_map(dev, qp);
1713                 spin_unlock(&qp->rq_cq->cq_lock);
1714         } else {
1715                 ocrdma_del_qpn_map(dev, qp);
1716         }
1717         spin_unlock_irqrestore(&qp->sq_cq->cq_lock, flags);
1718
1719         if (!pd->uctx) {
1720                 ocrdma_discard_cqes(qp, qp->sq_cq);
1721                 ocrdma_discard_cqes(qp, qp->rq_cq);
1722         }
1723         mutex_unlock(&dev->dev_lock);
1724
1725         if (pd->uctx) {
1726                 ocrdma_del_mmap(pd->uctx, (u64) qp->sq.pa,
1727                                 PAGE_ALIGN(qp->sq.len));
1728                 if (!qp->srq)
1729                         ocrdma_del_mmap(pd->uctx, (u64) qp->rq.pa,
1730                                         PAGE_ALIGN(qp->rq.len));
1731         }
1732
1733         ocrdma_del_flush_qp(qp);
1734
1735         kfree(qp->wqe_wr_id_tbl);
1736         kfree(qp->rqe_wr_id_tbl);
1737         kfree(qp);
1738         return 0;
1739 }
1740
1741 static int ocrdma_copy_srq_uresp(struct ocrdma_dev *dev, struct ocrdma_srq *srq,
1742                                 struct ib_udata *udata)
1743 {
1744         int status;
1745         struct ocrdma_create_srq_uresp uresp;
1746
1747         memset(&uresp, 0, sizeof(uresp));
1748         uresp.rq_dbid = srq->rq.dbid;
1749         uresp.num_rq_pages = 1;
1750         uresp.rq_page_addr[0] = virt_to_phys(srq->rq.va);
1751         uresp.rq_page_size = srq->rq.len;
1752         uresp.db_page_addr = dev->nic_info.unmapped_db +
1753             (srq->pd->id * dev->nic_info.db_page_size);
1754         uresp.db_page_size = dev->nic_info.db_page_size;
1755         uresp.num_rqe_allocated = srq->rq.max_cnt;
1756         if (ocrdma_get_asic_type(dev) == OCRDMA_ASIC_GEN_SKH_R) {
1757                 uresp.db_rq_offset = OCRDMA_DB_GEN2_RQ_OFFSET;
1758                 uresp.db_shift = 24;
1759         } else {
1760                 uresp.db_rq_offset = OCRDMA_DB_RQ_OFFSET;
1761                 uresp.db_shift = 16;
1762         }
1763
1764         status = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1765         if (status)
1766                 return status;
1767         status = ocrdma_add_mmap(srq->pd->uctx, uresp.rq_page_addr[0],
1768                                  uresp.rq_page_size);
1769         if (status)
1770                 return status;
1771         return status;
1772 }
1773
1774 int ocrdma_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr,
1775                       struct ib_udata *udata)
1776 {
1777         int status;
1778         struct ocrdma_pd *pd = get_ocrdma_pd(ibsrq->pd);
1779         struct ocrdma_dev *dev = get_ocrdma_dev(ibsrq->device);
1780         struct ocrdma_srq *srq = get_ocrdma_srq(ibsrq);
1781
1782         if (init_attr->attr.max_sge > dev->attr.max_recv_sge)
1783                 return -EINVAL;
1784         if (init_attr->attr.max_wr > dev->attr.max_rqe)
1785                 return -EINVAL;
1786
1787         spin_lock_init(&srq->q_lock);
1788         srq->pd = pd;
1789         srq->db = dev->nic_info.db + (pd->id * dev->nic_info.db_page_size);
1790         status = ocrdma_mbx_create_srq(dev, srq, init_attr, pd);
1791         if (status)
1792                 return status;
1793
1794         if (!udata) {
1795                 srq->rqe_wr_id_tbl = kcalloc(srq->rq.max_cnt, sizeof(u64),
1796                                              GFP_KERNEL);
1797                 if (!srq->rqe_wr_id_tbl) {
1798                         status = -ENOMEM;
1799                         goto arm_err;
1800                 }
1801
1802                 srq->bit_fields_len = (srq->rq.max_cnt / 32) +
1803                     (srq->rq.max_cnt % 32 ? 1 : 0);
1804                 srq->idx_bit_fields =
1805                     kmalloc_array(srq->bit_fields_len, sizeof(u32),
1806                                   GFP_KERNEL);
1807                 if (!srq->idx_bit_fields) {
1808                         status = -ENOMEM;
1809                         goto arm_err;
1810                 }
1811                 memset(srq->idx_bit_fields, 0xff,
1812                        srq->bit_fields_len * sizeof(u32));
1813         }
1814
1815         if (init_attr->attr.srq_limit) {
1816                 status = ocrdma_mbx_modify_srq(srq, &init_attr->attr);
1817                 if (status)
1818                         goto arm_err;
1819         }
1820
1821         if (udata) {
1822                 status = ocrdma_copy_srq_uresp(dev, srq, udata);
1823                 if (status)
1824                         goto arm_err;
1825         }
1826
1827         return 0;
1828
1829 arm_err:
1830         ocrdma_mbx_destroy_srq(dev, srq);
1831         kfree(srq->rqe_wr_id_tbl);
1832         kfree(srq->idx_bit_fields);
1833         return status;
1834 }
1835
1836 int ocrdma_modify_srq(struct ib_srq *ibsrq,
1837                       struct ib_srq_attr *srq_attr,
1838                       enum ib_srq_attr_mask srq_attr_mask,
1839                       struct ib_udata *udata)
1840 {
1841         int status;
1842         struct ocrdma_srq *srq;
1843
1844         srq = get_ocrdma_srq(ibsrq);
1845         if (srq_attr_mask & IB_SRQ_MAX_WR)
1846                 status = -EINVAL;
1847         else
1848                 status = ocrdma_mbx_modify_srq(srq, srq_attr);
1849         return status;
1850 }
1851
1852 int ocrdma_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr)
1853 {
1854         int status;
1855         struct ocrdma_srq *srq;
1856
1857         srq = get_ocrdma_srq(ibsrq);
1858         status = ocrdma_mbx_query_srq(srq, srq_attr);
1859         return status;
1860 }
1861
1862 void ocrdma_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata)
1863 {
1864         struct ocrdma_srq *srq;
1865         struct ocrdma_dev *dev = get_ocrdma_dev(ibsrq->device);
1866
1867         srq = get_ocrdma_srq(ibsrq);
1868
1869         ocrdma_mbx_destroy_srq(dev, srq);
1870
1871         if (srq->pd->uctx)
1872                 ocrdma_del_mmap(srq->pd->uctx, (u64) srq->rq.pa,
1873                                 PAGE_ALIGN(srq->rq.len));
1874
1875         kfree(srq->idx_bit_fields);
1876         kfree(srq->rqe_wr_id_tbl);
1877 }
1878
1879 /* unprivileged verbs and their support functions. */
1880 static void ocrdma_build_ud_hdr(struct ocrdma_qp *qp,
1881                                 struct ocrdma_hdr_wqe *hdr,
1882                                 const struct ib_send_wr *wr)
1883 {
1884         struct ocrdma_ewqe_ud_hdr *ud_hdr =
1885                 (struct ocrdma_ewqe_ud_hdr *)(hdr + 1);
1886         struct ocrdma_ah *ah = get_ocrdma_ah(ud_wr(wr)->ah);
1887
1888         ud_hdr->rsvd_dest_qpn = ud_wr(wr)->remote_qpn;
1889         if (qp->qp_type == IB_QPT_GSI)
1890                 ud_hdr->qkey = qp->qkey;
1891         else
1892                 ud_hdr->qkey = ud_wr(wr)->remote_qkey;
1893         ud_hdr->rsvd_ahid = ah->id;
1894         ud_hdr->hdr_type = ah->hdr_type;
1895         if (ah->av->valid & OCRDMA_AV_VLAN_VALID)
1896                 hdr->cw |= (OCRDMA_FLAG_AH_VLAN_PR << OCRDMA_WQE_FLAGS_SHIFT);
1897 }
1898
1899 static void ocrdma_build_sges(struct ocrdma_hdr_wqe *hdr,
1900                               struct ocrdma_sge *sge, int num_sge,
1901                               struct ib_sge *sg_list)
1902 {
1903         int i;
1904
1905         for (i = 0; i < num_sge; i++) {
1906                 sge[i].lrkey = sg_list[i].lkey;
1907                 sge[i].addr_lo = sg_list[i].addr;
1908                 sge[i].addr_hi = upper_32_bits(sg_list[i].addr);
1909                 sge[i].len = sg_list[i].length;
1910                 hdr->total_len += sg_list[i].length;
1911         }
1912         if (num_sge == 0)
1913                 memset(sge, 0, sizeof(*sge));
1914 }
1915
1916 static inline uint32_t ocrdma_sglist_len(struct ib_sge *sg_list, int num_sge)
1917 {
1918         uint32_t total_len = 0, i;
1919
1920         for (i = 0; i < num_sge; i++)
1921                 total_len += sg_list[i].length;
1922         return total_len;
1923 }
1924
1925
1926 static int ocrdma_build_inline_sges(struct ocrdma_qp *qp,
1927                                     struct ocrdma_hdr_wqe *hdr,
1928                                     struct ocrdma_sge *sge,
1929                                     const struct ib_send_wr *wr, u32 wqe_size)
1930 {
1931         int i;
1932         char *dpp_addr;
1933
1934         if (wr->send_flags & IB_SEND_INLINE && qp->qp_type != IB_QPT_UD) {
1935                 hdr->total_len = ocrdma_sglist_len(wr->sg_list, wr->num_sge);
1936                 if (unlikely(hdr->total_len > qp->max_inline_data)) {
1937                         pr_err("%s() supported_len=0x%x,\n"
1938                                " unsupported len req=0x%x\n", __func__,
1939                                 qp->max_inline_data, hdr->total_len);
1940                         return -EINVAL;
1941                 }
1942                 dpp_addr = (char *)sge;
1943                 for (i = 0; i < wr->num_sge; i++) {
1944                         memcpy(dpp_addr,
1945                                (void *)(unsigned long)wr->sg_list[i].addr,
1946                                wr->sg_list[i].length);
1947                         dpp_addr += wr->sg_list[i].length;
1948                 }
1949
1950                 wqe_size += roundup(hdr->total_len, OCRDMA_WQE_ALIGN_BYTES);
1951                 if (0 == hdr->total_len)
1952                         wqe_size += sizeof(struct ocrdma_sge);
1953                 hdr->cw |= (OCRDMA_TYPE_INLINE << OCRDMA_WQE_TYPE_SHIFT);
1954         } else {
1955                 ocrdma_build_sges(hdr, sge, wr->num_sge, wr->sg_list);
1956                 if (wr->num_sge)
1957                         wqe_size += (wr->num_sge * sizeof(struct ocrdma_sge));
1958                 else
1959                         wqe_size += sizeof(struct ocrdma_sge);
1960                 hdr->cw |= (OCRDMA_TYPE_LKEY << OCRDMA_WQE_TYPE_SHIFT);
1961         }
1962         hdr->cw |= ((wqe_size / OCRDMA_WQE_STRIDE) << OCRDMA_WQE_SIZE_SHIFT);
1963         return 0;
1964 }
1965
1966 static int ocrdma_build_send(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr,
1967                              const struct ib_send_wr *wr)
1968 {
1969         int status;
1970         struct ocrdma_sge *sge;
1971         u32 wqe_size = sizeof(*hdr);
1972
1973         if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) {
1974                 ocrdma_build_ud_hdr(qp, hdr, wr);
1975                 sge = (struct ocrdma_sge *)(hdr + 2);
1976                 wqe_size += sizeof(struct ocrdma_ewqe_ud_hdr);
1977         } else {
1978                 sge = (struct ocrdma_sge *)(hdr + 1);
1979         }
1980
1981         status = ocrdma_build_inline_sges(qp, hdr, sge, wr, wqe_size);
1982         return status;
1983 }
1984
1985 static int ocrdma_build_write(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr,
1986                               const struct ib_send_wr *wr)
1987 {
1988         int status;
1989         struct ocrdma_sge *ext_rw = (struct ocrdma_sge *)(hdr + 1);
1990         struct ocrdma_sge *sge = ext_rw + 1;
1991         u32 wqe_size = sizeof(*hdr) + sizeof(*ext_rw);
1992
1993         status = ocrdma_build_inline_sges(qp, hdr, sge, wr, wqe_size);
1994         if (status)
1995                 return status;
1996         ext_rw->addr_lo = rdma_wr(wr)->remote_addr;
1997         ext_rw->addr_hi = upper_32_bits(rdma_wr(wr)->remote_addr);
1998         ext_rw->lrkey = rdma_wr(wr)->rkey;
1999         ext_rw->len = hdr->total_len;
2000         return 0;
2001 }
2002
2003 static void ocrdma_build_read(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr,
2004                               const struct ib_send_wr *wr)
2005 {
2006         struct ocrdma_sge *ext_rw = (struct ocrdma_sge *)(hdr + 1);
2007         struct ocrdma_sge *sge = ext_rw + 1;
2008         u32 wqe_size = ((wr->num_sge + 1) * sizeof(struct ocrdma_sge)) +
2009             sizeof(struct ocrdma_hdr_wqe);
2010
2011         ocrdma_build_sges(hdr, sge, wr->num_sge, wr->sg_list);
2012         hdr->cw |= ((wqe_size / OCRDMA_WQE_STRIDE) << OCRDMA_WQE_SIZE_SHIFT);
2013         hdr->cw |= (OCRDMA_READ << OCRDMA_WQE_OPCODE_SHIFT);
2014         hdr->cw |= (OCRDMA_TYPE_LKEY << OCRDMA_WQE_TYPE_SHIFT);
2015
2016         ext_rw->addr_lo = rdma_wr(wr)->remote_addr;
2017         ext_rw->addr_hi = upper_32_bits(rdma_wr(wr)->remote_addr);
2018         ext_rw->lrkey = rdma_wr(wr)->rkey;
2019         ext_rw->len = hdr->total_len;
2020 }
2021
2022 static int get_encoded_page_size(int pg_sz)
2023 {
2024         /* Max size is 256M 4096 << 16 */
2025         int i = 0;
2026         for (; i < 17; i++)
2027                 if (pg_sz == (4096 << i))
2028                         break;
2029         return i;
2030 }
2031
2032 static int ocrdma_build_reg(struct ocrdma_qp *qp,
2033                             struct ocrdma_hdr_wqe *hdr,
2034                             const struct ib_reg_wr *wr)
2035 {
2036         u64 fbo;
2037         struct ocrdma_ewqe_fr *fast_reg = (struct ocrdma_ewqe_fr *)(hdr + 1);
2038         struct ocrdma_mr *mr = get_ocrdma_mr(wr->mr);
2039         struct ocrdma_pbl *pbl_tbl = mr->hwmr.pbl_table;
2040         struct ocrdma_pbe *pbe;
2041         u32 wqe_size = sizeof(*fast_reg) + sizeof(*hdr);
2042         int num_pbes = 0, i;
2043
2044         wqe_size = roundup(wqe_size, OCRDMA_WQE_ALIGN_BYTES);
2045
2046         hdr->cw |= (OCRDMA_FR_MR << OCRDMA_WQE_OPCODE_SHIFT);
2047         hdr->cw |= ((wqe_size / OCRDMA_WQE_STRIDE) << OCRDMA_WQE_SIZE_SHIFT);
2048
2049         if (wr->access & IB_ACCESS_LOCAL_WRITE)
2050                 hdr->rsvd_lkey_flags |= OCRDMA_LKEY_FLAG_LOCAL_WR;
2051         if (wr->access & IB_ACCESS_REMOTE_WRITE)
2052                 hdr->rsvd_lkey_flags |= OCRDMA_LKEY_FLAG_REMOTE_WR;
2053         if (wr->access & IB_ACCESS_REMOTE_READ)
2054                 hdr->rsvd_lkey_flags |= OCRDMA_LKEY_FLAG_REMOTE_RD;
2055         hdr->lkey = wr->key;
2056         hdr->total_len = mr->ibmr.length;
2057
2058         fbo = mr->ibmr.iova - mr->pages[0];
2059
2060         fast_reg->va_hi = upper_32_bits(mr->ibmr.iova);
2061         fast_reg->va_lo = (u32) (mr->ibmr.iova & 0xffffffff);
2062         fast_reg->fbo_hi = upper_32_bits(fbo);
2063         fast_reg->fbo_lo = (u32) fbo & 0xffffffff;
2064         fast_reg->num_sges = mr->npages;
2065         fast_reg->size_sge = get_encoded_page_size(mr->ibmr.page_size);
2066
2067         pbe = pbl_tbl->va;
2068         for (i = 0; i < mr->npages; i++) {
2069                 u64 buf_addr = mr->pages[i];
2070
2071                 pbe->pa_lo = cpu_to_le32((u32) (buf_addr & PAGE_MASK));
2072                 pbe->pa_hi = cpu_to_le32((u32) upper_32_bits(buf_addr));
2073                 num_pbes += 1;
2074                 pbe++;
2075
2076                 /* if the pbl is full storing the pbes,
2077                  * move to next pbl.
2078                 */
2079                 if (num_pbes == (mr->hwmr.pbl_size/sizeof(u64))) {
2080                         pbl_tbl++;
2081                         pbe = (struct ocrdma_pbe *)pbl_tbl->va;
2082                 }
2083         }
2084
2085         return 0;
2086 }
2087
2088 static void ocrdma_ring_sq_db(struct ocrdma_qp *qp)
2089 {
2090         u32 val = qp->sq.dbid | (1 << OCRDMA_DB_SQ_SHIFT);
2091
2092         iowrite32(val, qp->sq_db);
2093 }
2094
2095 int ocrdma_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
2096                      const struct ib_send_wr **bad_wr)
2097 {
2098         int status = 0;
2099         struct ocrdma_qp *qp = get_ocrdma_qp(ibqp);
2100         struct ocrdma_hdr_wqe *hdr;
2101         unsigned long flags;
2102
2103         spin_lock_irqsave(&qp->q_lock, flags);
2104         if (qp->state != OCRDMA_QPS_RTS && qp->state != OCRDMA_QPS_SQD) {
2105                 spin_unlock_irqrestore(&qp->q_lock, flags);
2106                 *bad_wr = wr;
2107                 return -EINVAL;
2108         }
2109
2110         while (wr) {
2111                 if (qp->qp_type == IB_QPT_UD &&
2112                     (wr->opcode != IB_WR_SEND &&
2113                      wr->opcode != IB_WR_SEND_WITH_IMM)) {
2114                         *bad_wr = wr;
2115                         status = -EINVAL;
2116                         break;
2117                 }
2118                 if (ocrdma_hwq_free_cnt(&qp->sq) == 0 ||
2119                     wr->num_sge > qp->sq.max_sges) {
2120                         *bad_wr = wr;
2121                         status = -ENOMEM;
2122                         break;
2123                 }
2124                 hdr = ocrdma_hwq_head(&qp->sq);
2125                 hdr->cw = 0;
2126                 if (wr->send_flags & IB_SEND_SIGNALED || qp->signaled)
2127                         hdr->cw |= (OCRDMA_FLAG_SIG << OCRDMA_WQE_FLAGS_SHIFT);
2128                 if (wr->send_flags & IB_SEND_FENCE)
2129                         hdr->cw |=
2130                             (OCRDMA_FLAG_FENCE_L << OCRDMA_WQE_FLAGS_SHIFT);
2131                 if (wr->send_flags & IB_SEND_SOLICITED)
2132                         hdr->cw |=
2133                             (OCRDMA_FLAG_SOLICIT << OCRDMA_WQE_FLAGS_SHIFT);
2134                 hdr->total_len = 0;
2135                 switch (wr->opcode) {
2136                 case IB_WR_SEND_WITH_IMM:
2137                         hdr->cw |= (OCRDMA_FLAG_IMM << OCRDMA_WQE_FLAGS_SHIFT);
2138                         hdr->immdt = ntohl(wr->ex.imm_data);
2139                         /* fall through */
2140                 case IB_WR_SEND:
2141                         hdr->cw |= (OCRDMA_SEND << OCRDMA_WQE_OPCODE_SHIFT);
2142                         ocrdma_build_send(qp, hdr, wr);
2143                         break;
2144                 case IB_WR_SEND_WITH_INV:
2145                         hdr->cw |= (OCRDMA_FLAG_INV << OCRDMA_WQE_FLAGS_SHIFT);
2146                         hdr->cw |= (OCRDMA_SEND << OCRDMA_WQE_OPCODE_SHIFT);
2147                         hdr->lkey = wr->ex.invalidate_rkey;
2148                         status = ocrdma_build_send(qp, hdr, wr);
2149                         break;
2150                 case IB_WR_RDMA_WRITE_WITH_IMM:
2151                         hdr->cw |= (OCRDMA_FLAG_IMM << OCRDMA_WQE_FLAGS_SHIFT);
2152                         hdr->immdt = ntohl(wr->ex.imm_data);
2153                         /* fall through */
2154                 case IB_WR_RDMA_WRITE:
2155                         hdr->cw |= (OCRDMA_WRITE << OCRDMA_WQE_OPCODE_SHIFT);
2156                         status = ocrdma_build_write(qp, hdr, wr);
2157                         break;
2158                 case IB_WR_RDMA_READ:
2159                         ocrdma_build_read(qp, hdr, wr);
2160                         break;
2161                 case IB_WR_LOCAL_INV:
2162                         hdr->cw |=
2163                             (OCRDMA_LKEY_INV << OCRDMA_WQE_OPCODE_SHIFT);
2164                         hdr->cw |= ((sizeof(struct ocrdma_hdr_wqe) +
2165                                         sizeof(struct ocrdma_sge)) /
2166                                 OCRDMA_WQE_STRIDE) << OCRDMA_WQE_SIZE_SHIFT;
2167                         hdr->lkey = wr->ex.invalidate_rkey;
2168                         break;
2169                 case IB_WR_REG_MR:
2170                         status = ocrdma_build_reg(qp, hdr, reg_wr(wr));
2171                         break;
2172                 default:
2173                         status = -EINVAL;
2174                         break;
2175                 }
2176                 if (status) {
2177                         *bad_wr = wr;
2178                         break;
2179                 }
2180                 if (wr->send_flags & IB_SEND_SIGNALED || qp->signaled)
2181                         qp->wqe_wr_id_tbl[qp->sq.head].signaled = 1;
2182                 else
2183                         qp->wqe_wr_id_tbl[qp->sq.head].signaled = 0;
2184                 qp->wqe_wr_id_tbl[qp->sq.head].wrid = wr->wr_id;
2185                 ocrdma_cpu_to_le32(hdr, ((hdr->cw >> OCRDMA_WQE_SIZE_SHIFT) &
2186                                    OCRDMA_WQE_SIZE_MASK) * OCRDMA_WQE_STRIDE);
2187                 /* make sure wqe is written before adapter can access it */
2188                 wmb();
2189                 /* inform hw to start processing it */
2190                 ocrdma_ring_sq_db(qp);
2191
2192                 /* update pointer, counter for next wr */
2193                 ocrdma_hwq_inc_head(&qp->sq);
2194                 wr = wr->next;
2195         }
2196         spin_unlock_irqrestore(&qp->q_lock, flags);
2197         return status;
2198 }
2199
2200 static void ocrdma_ring_rq_db(struct ocrdma_qp *qp)
2201 {
2202         u32 val = qp->rq.dbid | (1 << OCRDMA_DB_RQ_SHIFT);
2203
2204         iowrite32(val, qp->rq_db);
2205 }
2206
2207 static void ocrdma_build_rqe(struct ocrdma_hdr_wqe *rqe,
2208                              const struct ib_recv_wr *wr, u16 tag)
2209 {
2210         u32 wqe_size = 0;
2211         struct ocrdma_sge *sge;
2212         if (wr->num_sge)
2213                 wqe_size = (wr->num_sge * sizeof(*sge)) + sizeof(*rqe);
2214         else
2215                 wqe_size = sizeof(*sge) + sizeof(*rqe);
2216
2217         rqe->cw = ((wqe_size / OCRDMA_WQE_STRIDE) <<
2218                                 OCRDMA_WQE_SIZE_SHIFT);
2219         rqe->cw |= (OCRDMA_FLAG_SIG << OCRDMA_WQE_FLAGS_SHIFT);
2220         rqe->cw |= (OCRDMA_TYPE_LKEY << OCRDMA_WQE_TYPE_SHIFT);
2221         rqe->total_len = 0;
2222         rqe->rsvd_tag = tag;
2223         sge = (struct ocrdma_sge *)(rqe + 1);
2224         ocrdma_build_sges(rqe, sge, wr->num_sge, wr->sg_list);
2225         ocrdma_cpu_to_le32(rqe, wqe_size);
2226 }
2227
2228 int ocrdma_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
2229                      const struct ib_recv_wr **bad_wr)
2230 {
2231         int status = 0;
2232         unsigned long flags;
2233         struct ocrdma_qp *qp = get_ocrdma_qp(ibqp);
2234         struct ocrdma_hdr_wqe *rqe;
2235
2236         spin_lock_irqsave(&qp->q_lock, flags);
2237         if (qp->state == OCRDMA_QPS_RST || qp->state == OCRDMA_QPS_ERR) {
2238                 spin_unlock_irqrestore(&qp->q_lock, flags);
2239                 *bad_wr = wr;
2240                 return -EINVAL;
2241         }
2242         while (wr) {
2243                 if (ocrdma_hwq_free_cnt(&qp->rq) == 0 ||
2244                     wr->num_sge > qp->rq.max_sges) {
2245                         *bad_wr = wr;
2246                         status = -ENOMEM;
2247                         break;
2248                 }
2249                 rqe = ocrdma_hwq_head(&qp->rq);
2250                 ocrdma_build_rqe(rqe, wr, 0);
2251
2252                 qp->rqe_wr_id_tbl[qp->rq.head] = wr->wr_id;
2253                 /* make sure rqe is written before adapter can access it */
2254                 wmb();
2255
2256                 /* inform hw to start processing it */
2257                 ocrdma_ring_rq_db(qp);
2258
2259                 /* update pointer, counter for next wr */
2260                 ocrdma_hwq_inc_head(&qp->rq);
2261                 wr = wr->next;
2262         }
2263         spin_unlock_irqrestore(&qp->q_lock, flags);
2264         return status;
2265 }
2266
2267 /* cqe for srq's rqe can potentially arrive out of order.
2268  * index gives the entry in the shadow table where to store
2269  * the wr_id. tag/index is returned in cqe to reference back
2270  * for a given rqe.
2271  */
2272 static int ocrdma_srq_get_idx(struct ocrdma_srq *srq)
2273 {
2274         int row = 0;
2275         int indx = 0;
2276
2277         for (row = 0; row < srq->bit_fields_len; row++) {
2278                 if (srq->idx_bit_fields[row]) {
2279                         indx = ffs(srq->idx_bit_fields[row]);
2280                         indx = (row * 32) + (indx - 1);
2281                         BUG_ON(indx >= srq->rq.max_cnt);
2282                         ocrdma_srq_toggle_bit(srq, indx);
2283                         break;
2284                 }
2285         }
2286
2287         BUG_ON(row == srq->bit_fields_len);
2288         return indx + 1; /* Use from index 1 */
2289 }
2290
2291 static void ocrdma_ring_srq_db(struct ocrdma_srq *srq)
2292 {
2293         u32 val = srq->rq.dbid | (1 << 16);
2294
2295         iowrite32(val, srq->db + OCRDMA_DB_GEN2_SRQ_OFFSET);
2296 }
2297
2298 int ocrdma_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,
2299                          const struct ib_recv_wr **bad_wr)
2300 {
2301         int status = 0;
2302         unsigned long flags;
2303         struct ocrdma_srq *srq;
2304         struct ocrdma_hdr_wqe *rqe;
2305         u16 tag;
2306
2307         srq = get_ocrdma_srq(ibsrq);
2308
2309         spin_lock_irqsave(&srq->q_lock, flags);
2310         while (wr) {
2311                 if (ocrdma_hwq_free_cnt(&srq->rq) == 0 ||
2312                     wr->num_sge > srq->rq.max_sges) {
2313                         status = -ENOMEM;
2314                         *bad_wr = wr;
2315                         break;
2316                 }
2317                 tag = ocrdma_srq_get_idx(srq);
2318                 rqe = ocrdma_hwq_head(&srq->rq);
2319                 ocrdma_build_rqe(rqe, wr, tag);
2320
2321                 srq->rqe_wr_id_tbl[tag] = wr->wr_id;
2322                 /* make sure rqe is written before adapter can perform DMA */
2323                 wmb();
2324                 /* inform hw to start processing it */
2325                 ocrdma_ring_srq_db(srq);
2326                 /* update pointer, counter for next wr */
2327                 ocrdma_hwq_inc_head(&srq->rq);
2328                 wr = wr->next;
2329         }
2330         spin_unlock_irqrestore(&srq->q_lock, flags);
2331         return status;
2332 }
2333
2334 static enum ib_wc_status ocrdma_to_ibwc_err(u16 status)
2335 {
2336         enum ib_wc_status ibwc_status;
2337
2338         switch (status) {
2339         case OCRDMA_CQE_GENERAL_ERR:
2340                 ibwc_status = IB_WC_GENERAL_ERR;
2341                 break;
2342         case OCRDMA_CQE_LOC_LEN_ERR:
2343                 ibwc_status = IB_WC_LOC_LEN_ERR;
2344                 break;
2345         case OCRDMA_CQE_LOC_QP_OP_ERR:
2346                 ibwc_status = IB_WC_LOC_QP_OP_ERR;
2347                 break;
2348         case OCRDMA_CQE_LOC_EEC_OP_ERR:
2349                 ibwc_status = IB_WC_LOC_EEC_OP_ERR;
2350                 break;
2351         case OCRDMA_CQE_LOC_PROT_ERR:
2352                 ibwc_status = IB_WC_LOC_PROT_ERR;
2353                 break;
2354         case OCRDMA_CQE_WR_FLUSH_ERR:
2355                 ibwc_status = IB_WC_WR_FLUSH_ERR;
2356                 break;
2357         case OCRDMA_CQE_MW_BIND_ERR:
2358                 ibwc_status = IB_WC_MW_BIND_ERR;
2359                 break;
2360         case OCRDMA_CQE_BAD_RESP_ERR:
2361                 ibwc_status = IB_WC_BAD_RESP_ERR;
2362                 break;
2363         case OCRDMA_CQE_LOC_ACCESS_ERR:
2364                 ibwc_status = IB_WC_LOC_ACCESS_ERR;
2365                 break;
2366         case OCRDMA_CQE_REM_INV_REQ_ERR:
2367                 ibwc_status = IB_WC_REM_INV_REQ_ERR;
2368                 break;
2369         case OCRDMA_CQE_REM_ACCESS_ERR:
2370                 ibwc_status = IB_WC_REM_ACCESS_ERR;
2371                 break;
2372         case OCRDMA_CQE_REM_OP_ERR:
2373                 ibwc_status = IB_WC_REM_OP_ERR;
2374                 break;
2375         case OCRDMA_CQE_RETRY_EXC_ERR:
2376                 ibwc_status = IB_WC_RETRY_EXC_ERR;
2377                 break;
2378         case OCRDMA_CQE_RNR_RETRY_EXC_ERR:
2379                 ibwc_status = IB_WC_RNR_RETRY_EXC_ERR;
2380                 break;
2381         case OCRDMA_CQE_LOC_RDD_VIOL_ERR:
2382                 ibwc_status = IB_WC_LOC_RDD_VIOL_ERR;
2383                 break;
2384         case OCRDMA_CQE_REM_INV_RD_REQ_ERR:
2385                 ibwc_status = IB_WC_REM_INV_RD_REQ_ERR;
2386                 break;
2387         case OCRDMA_CQE_REM_ABORT_ERR:
2388                 ibwc_status = IB_WC_REM_ABORT_ERR;
2389                 break;
2390         case OCRDMA_CQE_INV_EECN_ERR:
2391                 ibwc_status = IB_WC_INV_EECN_ERR;
2392                 break;
2393         case OCRDMA_CQE_INV_EEC_STATE_ERR:
2394                 ibwc_status = IB_WC_INV_EEC_STATE_ERR;
2395                 break;
2396         case OCRDMA_CQE_FATAL_ERR:
2397                 ibwc_status = IB_WC_FATAL_ERR;
2398                 break;
2399         case OCRDMA_CQE_RESP_TIMEOUT_ERR:
2400                 ibwc_status = IB_WC_RESP_TIMEOUT_ERR;
2401                 break;
2402         default:
2403                 ibwc_status = IB_WC_GENERAL_ERR;
2404                 break;
2405         }
2406         return ibwc_status;
2407 }
2408
2409 static void ocrdma_update_wc(struct ocrdma_qp *qp, struct ib_wc *ibwc,
2410                       u32 wqe_idx)
2411 {
2412         struct ocrdma_hdr_wqe *hdr;
2413         struct ocrdma_sge *rw;
2414         int opcode;
2415
2416         hdr = ocrdma_hwq_head_from_idx(&qp->sq, wqe_idx);
2417
2418         ibwc->wr_id = qp->wqe_wr_id_tbl[wqe_idx].wrid;
2419         /* Undo the hdr->cw swap */
2420         opcode = le32_to_cpu(hdr->cw) & OCRDMA_WQE_OPCODE_MASK;
2421         switch (opcode) {
2422         case OCRDMA_WRITE:
2423                 ibwc->opcode = IB_WC_RDMA_WRITE;
2424                 break;
2425         case OCRDMA_READ:
2426                 rw = (struct ocrdma_sge *)(hdr + 1);
2427                 ibwc->opcode = IB_WC_RDMA_READ;
2428                 ibwc->byte_len = rw->len;
2429                 break;
2430         case OCRDMA_SEND:
2431                 ibwc->opcode = IB_WC_SEND;
2432                 break;
2433         case OCRDMA_FR_MR:
2434                 ibwc->opcode = IB_WC_REG_MR;
2435                 break;
2436         case OCRDMA_LKEY_INV:
2437                 ibwc->opcode = IB_WC_LOCAL_INV;
2438                 break;
2439         default:
2440                 ibwc->status = IB_WC_GENERAL_ERR;
2441                 pr_err("%s() invalid opcode received = 0x%x\n",
2442                        __func__, hdr->cw & OCRDMA_WQE_OPCODE_MASK);
2443                 break;
2444         }
2445 }
2446
2447 static void ocrdma_set_cqe_status_flushed(struct ocrdma_qp *qp,
2448                                                 struct ocrdma_cqe *cqe)
2449 {
2450         if (is_cqe_for_sq(cqe)) {
2451                 cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu(
2452                                 cqe->flags_status_srcqpn) &
2453                                         ~OCRDMA_CQE_STATUS_MASK);
2454                 cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu(
2455                                 cqe->flags_status_srcqpn) |
2456                                 (OCRDMA_CQE_WR_FLUSH_ERR <<
2457                                         OCRDMA_CQE_STATUS_SHIFT));
2458         } else {
2459                 if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) {
2460                         cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu(
2461                                         cqe->flags_status_srcqpn) &
2462                                                 ~OCRDMA_CQE_UD_STATUS_MASK);
2463                         cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu(
2464                                         cqe->flags_status_srcqpn) |
2465                                         (OCRDMA_CQE_WR_FLUSH_ERR <<
2466                                                 OCRDMA_CQE_UD_STATUS_SHIFT));
2467                 } else {
2468                         cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu(
2469                                         cqe->flags_status_srcqpn) &
2470                                                 ~OCRDMA_CQE_STATUS_MASK);
2471                         cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu(
2472                                         cqe->flags_status_srcqpn) |
2473                                         (OCRDMA_CQE_WR_FLUSH_ERR <<
2474                                                 OCRDMA_CQE_STATUS_SHIFT));
2475                 }
2476         }
2477 }
2478
2479 static bool ocrdma_update_err_cqe(struct ib_wc *ibwc, struct ocrdma_cqe *cqe,
2480                                   struct ocrdma_qp *qp, int status)
2481 {
2482         bool expand = false;
2483
2484         ibwc->byte_len = 0;
2485         ibwc->qp = &qp->ibqp;
2486         ibwc->status = ocrdma_to_ibwc_err(status);
2487
2488         ocrdma_flush_qp(qp);
2489         ocrdma_qp_state_change(qp, IB_QPS_ERR, NULL);
2490
2491         /* if wqe/rqe pending for which cqe needs to be returned,
2492          * trigger inflating it.
2493          */
2494         if (!is_hw_rq_empty(qp) || !is_hw_sq_empty(qp)) {
2495                 expand = true;
2496                 ocrdma_set_cqe_status_flushed(qp, cqe);
2497         }
2498         return expand;
2499 }
2500
2501 static int ocrdma_update_err_rcqe(struct ib_wc *ibwc, struct ocrdma_cqe *cqe,
2502                                   struct ocrdma_qp *qp, int status)
2503 {
2504         ibwc->opcode = IB_WC_RECV;
2505         ibwc->wr_id = qp->rqe_wr_id_tbl[qp->rq.tail];
2506         ocrdma_hwq_inc_tail(&qp->rq);
2507
2508         return ocrdma_update_err_cqe(ibwc, cqe, qp, status);
2509 }
2510
2511 static int ocrdma_update_err_scqe(struct ib_wc *ibwc, struct ocrdma_cqe *cqe,
2512                                   struct ocrdma_qp *qp, int status)
2513 {
2514         ocrdma_update_wc(qp, ibwc, qp->sq.tail);
2515         ocrdma_hwq_inc_tail(&qp->sq);
2516
2517         return ocrdma_update_err_cqe(ibwc, cqe, qp, status);
2518 }
2519
2520
2521 static bool ocrdma_poll_err_scqe(struct ocrdma_qp *qp,
2522                                  struct ocrdma_cqe *cqe, struct ib_wc *ibwc,
2523                                  bool *polled, bool *stop)
2524 {
2525         bool expand;
2526         struct ocrdma_dev *dev = get_ocrdma_dev(qp->ibqp.device);
2527         int status = (le32_to_cpu(cqe->flags_status_srcqpn) &
2528                 OCRDMA_CQE_STATUS_MASK) >> OCRDMA_CQE_STATUS_SHIFT;
2529         if (status < OCRDMA_MAX_CQE_ERR)
2530                 atomic_inc(&dev->cqe_err_stats[status]);
2531
2532         /* when hw sq is empty, but rq is not empty, so we continue
2533          * to keep the cqe in order to get the cq event again.
2534          */
2535         if (is_hw_sq_empty(qp) && !is_hw_rq_empty(qp)) {
2536                 /* when cq for rq and sq is same, it is safe to return
2537                  * flush cqe for RQEs.
2538                  */
2539                 if (!qp->srq && (qp->sq_cq == qp->rq_cq)) {
2540                         *polled = true;
2541                         status = OCRDMA_CQE_WR_FLUSH_ERR;
2542                         expand = ocrdma_update_err_rcqe(ibwc, cqe, qp, status);
2543                 } else {
2544                         /* stop processing further cqe as this cqe is used for
2545                          * triggering cq event on buddy cq of RQ.
2546                          * When QP is destroyed, this cqe will be removed
2547                          * from the cq's hardware q.
2548                          */
2549                         *polled = false;
2550                         *stop = true;
2551                         expand = false;
2552                 }
2553         } else if (is_hw_sq_empty(qp)) {
2554                 /* Do nothing */
2555                 expand = false;
2556                 *polled = false;
2557                 *stop = false;
2558         } else {
2559                 *polled = true;
2560                 expand = ocrdma_update_err_scqe(ibwc, cqe, qp, status);
2561         }
2562         return expand;
2563 }
2564
2565 static bool ocrdma_poll_success_scqe(struct ocrdma_qp *qp,
2566                                      struct ocrdma_cqe *cqe,
2567                                      struct ib_wc *ibwc, bool *polled)
2568 {
2569         bool expand = false;
2570         int tail = qp->sq.tail;
2571         u32 wqe_idx;
2572
2573         if (!qp->wqe_wr_id_tbl[tail].signaled) {
2574                 *polled = false;    /* WC cannot be consumed yet */
2575         } else {
2576                 ibwc->status = IB_WC_SUCCESS;
2577                 ibwc->wc_flags = 0;
2578                 ibwc->qp = &qp->ibqp;
2579                 ocrdma_update_wc(qp, ibwc, tail);
2580                 *polled = true;
2581         }
2582         wqe_idx = (le32_to_cpu(cqe->wq.wqeidx) &
2583                         OCRDMA_CQE_WQEIDX_MASK) & qp->sq.max_wqe_idx;
2584         if (tail != wqe_idx)
2585                 expand = true; /* Coalesced CQE can't be consumed yet */
2586
2587         ocrdma_hwq_inc_tail(&qp->sq);
2588         return expand;
2589 }
2590
2591 static bool ocrdma_poll_scqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe,
2592                              struct ib_wc *ibwc, bool *polled, bool *stop)
2593 {
2594         int status;
2595         bool expand;
2596
2597         status = (le32_to_cpu(cqe->flags_status_srcqpn) &
2598                 OCRDMA_CQE_STATUS_MASK) >> OCRDMA_CQE_STATUS_SHIFT;
2599
2600         if (status == OCRDMA_CQE_SUCCESS)
2601                 expand = ocrdma_poll_success_scqe(qp, cqe, ibwc, polled);
2602         else
2603                 expand = ocrdma_poll_err_scqe(qp, cqe, ibwc, polled, stop);
2604         return expand;
2605 }
2606
2607 static int ocrdma_update_ud_rcqe(struct ocrdma_dev *dev, struct ib_wc *ibwc,
2608                                  struct ocrdma_cqe *cqe)
2609 {
2610         int status;
2611         u16 hdr_type = 0;
2612
2613         status = (le32_to_cpu(cqe->flags_status_srcqpn) &
2614                 OCRDMA_CQE_UD_STATUS_MASK) >> OCRDMA_CQE_UD_STATUS_SHIFT;
2615         ibwc->src_qp = le32_to_cpu(cqe->flags_status_srcqpn) &
2616                                                 OCRDMA_CQE_SRCQP_MASK;
2617         ibwc->pkey_index = 0;
2618         ibwc->wc_flags = IB_WC_GRH;
2619         ibwc->byte_len = (le32_to_cpu(cqe->ud.rxlen_pkey) >>
2620                           OCRDMA_CQE_UD_XFER_LEN_SHIFT) &
2621                           OCRDMA_CQE_UD_XFER_LEN_MASK;
2622
2623         if (ocrdma_is_udp_encap_supported(dev)) {
2624                 hdr_type = (le32_to_cpu(cqe->ud.rxlen_pkey) >>
2625                             OCRDMA_CQE_UD_L3TYPE_SHIFT) &
2626                             OCRDMA_CQE_UD_L3TYPE_MASK;
2627                 ibwc->wc_flags |= IB_WC_WITH_NETWORK_HDR_TYPE;
2628                 ibwc->network_hdr_type = hdr_type;
2629         }
2630
2631         return status;
2632 }
2633
2634 static void ocrdma_update_free_srq_cqe(struct ib_wc *ibwc,
2635                                        struct ocrdma_cqe *cqe,
2636                                        struct ocrdma_qp *qp)
2637 {
2638         unsigned long flags;
2639         struct ocrdma_srq *srq;
2640         u32 wqe_idx;
2641
2642         srq = get_ocrdma_srq(qp->ibqp.srq);
2643         wqe_idx = (le32_to_cpu(cqe->rq.buftag_qpn) >>
2644                 OCRDMA_CQE_BUFTAG_SHIFT) & srq->rq.max_wqe_idx;
2645         BUG_ON(wqe_idx < 1);
2646
2647         ibwc->wr_id = srq->rqe_wr_id_tbl[wqe_idx];
2648         spin_lock_irqsave(&srq->q_lock, flags);
2649         ocrdma_srq_toggle_bit(srq, wqe_idx - 1);
2650         spin_unlock_irqrestore(&srq->q_lock, flags);
2651         ocrdma_hwq_inc_tail(&srq->rq);
2652 }
2653
2654 static bool ocrdma_poll_err_rcqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe,
2655                                 struct ib_wc *ibwc, bool *polled, bool *stop,
2656                                 int status)
2657 {
2658         bool expand;
2659         struct ocrdma_dev *dev = get_ocrdma_dev(qp->ibqp.device);
2660
2661         if (status < OCRDMA_MAX_CQE_ERR)
2662                 atomic_inc(&dev->cqe_err_stats[status]);
2663
2664         /* when hw_rq is empty, but wq is not empty, so continue
2665          * to keep the cqe to get the cq event again.
2666          */
2667         if (is_hw_rq_empty(qp) && !is_hw_sq_empty(qp)) {
2668                 if (!qp->srq && (qp->sq_cq == qp->rq_cq)) {
2669                         *polled = true;
2670                         status = OCRDMA_CQE_WR_FLUSH_ERR;
2671                         expand = ocrdma_update_err_scqe(ibwc, cqe, qp, status);
2672                 } else {
2673                         *polled = false;
2674                         *stop = true;
2675                         expand = false;
2676                 }
2677         } else if (is_hw_rq_empty(qp)) {
2678                 /* Do nothing */
2679                 expand = false;
2680                 *polled = false;
2681                 *stop = false;
2682         } else {
2683                 *polled = true;
2684                 expand = ocrdma_update_err_rcqe(ibwc, cqe, qp, status);
2685         }
2686         return expand;
2687 }
2688
2689 static void ocrdma_poll_success_rcqe(struct ocrdma_qp *qp,
2690                                      struct ocrdma_cqe *cqe, struct ib_wc *ibwc)
2691 {
2692         struct ocrdma_dev *dev;
2693
2694         dev = get_ocrdma_dev(qp->ibqp.device);
2695         ibwc->opcode = IB_WC_RECV;
2696         ibwc->qp = &qp->ibqp;
2697         ibwc->status = IB_WC_SUCCESS;
2698
2699         if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI)
2700                 ocrdma_update_ud_rcqe(dev, ibwc, cqe);
2701         else
2702                 ibwc->byte_len = le32_to_cpu(cqe->rq.rxlen);
2703
2704         if (is_cqe_imm(cqe)) {
2705                 ibwc->ex.imm_data = htonl(le32_to_cpu(cqe->rq.lkey_immdt));
2706                 ibwc->wc_flags |= IB_WC_WITH_IMM;
2707         } else if (is_cqe_wr_imm(cqe)) {
2708                 ibwc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
2709                 ibwc->ex.imm_data = htonl(le32_to_cpu(cqe->rq.lkey_immdt));
2710                 ibwc->wc_flags |= IB_WC_WITH_IMM;
2711         } else if (is_cqe_invalidated(cqe)) {
2712                 ibwc->ex.invalidate_rkey = le32_to_cpu(cqe->rq.lkey_immdt);
2713                 ibwc->wc_flags |= IB_WC_WITH_INVALIDATE;
2714         }
2715         if (qp->ibqp.srq) {
2716                 ocrdma_update_free_srq_cqe(ibwc, cqe, qp);
2717         } else {
2718                 ibwc->wr_id = qp->rqe_wr_id_tbl[qp->rq.tail];
2719                 ocrdma_hwq_inc_tail(&qp->rq);
2720         }
2721 }
2722
2723 static bool ocrdma_poll_rcqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe,
2724                              struct ib_wc *ibwc, bool *polled, bool *stop)
2725 {
2726         int status;
2727         bool expand = false;
2728
2729         ibwc->wc_flags = 0;
2730         if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) {
2731                 status = (le32_to_cpu(cqe->flags_status_srcqpn) &
2732                                         OCRDMA_CQE_UD_STATUS_MASK) >>
2733                                         OCRDMA_CQE_UD_STATUS_SHIFT;
2734         } else {
2735                 status = (le32_to_cpu(cqe->flags_status_srcqpn) &
2736                              OCRDMA_CQE_STATUS_MASK) >> OCRDMA_CQE_STATUS_SHIFT;
2737         }
2738
2739         if (status == OCRDMA_CQE_SUCCESS) {
2740                 *polled = true;
2741                 ocrdma_poll_success_rcqe(qp, cqe, ibwc);
2742         } else {
2743                 expand = ocrdma_poll_err_rcqe(qp, cqe, ibwc, polled, stop,
2744                                               status);
2745         }
2746         return expand;
2747 }
2748
2749 static void ocrdma_change_cq_phase(struct ocrdma_cq *cq, struct ocrdma_cqe *cqe,
2750                                    u16 cur_getp)
2751 {
2752         if (cq->phase_change) {
2753                 if (cur_getp == 0)
2754                         cq->phase = (~cq->phase & OCRDMA_CQE_VALID);
2755         } else {
2756                 /* clear valid bit */
2757                 cqe->flags_status_srcqpn = 0;
2758         }
2759 }
2760
2761 static int ocrdma_poll_hwcq(struct ocrdma_cq *cq, int num_entries,
2762                             struct ib_wc *ibwc)
2763 {
2764         u16 qpn = 0;
2765         int i = 0;
2766         bool expand = false;
2767         int polled_hw_cqes = 0;
2768         struct ocrdma_qp *qp = NULL;
2769         struct ocrdma_dev *dev = get_ocrdma_dev(cq->ibcq.device);
2770         struct ocrdma_cqe *cqe;
2771         u16 cur_getp; bool polled = false; bool stop = false;
2772
2773         cur_getp = cq->getp;
2774         while (num_entries) {
2775                 cqe = cq->va + cur_getp;
2776                 /* check whether valid cqe or not */
2777                 if (!is_cqe_valid(cq, cqe))
2778                         break;
2779                 qpn = (le32_to_cpu(cqe->cmn.qpn) & OCRDMA_CQE_QPN_MASK);
2780                 /* ignore discarded cqe */
2781                 if (qpn == 0)
2782                         goto skip_cqe;
2783                 qp = dev->qp_tbl[qpn];
2784                 BUG_ON(qp == NULL);
2785
2786                 if (is_cqe_for_sq(cqe)) {
2787                         expand = ocrdma_poll_scqe(qp, cqe, ibwc, &polled,
2788                                                   &stop);
2789                 } else {
2790                         expand = ocrdma_poll_rcqe(qp, cqe, ibwc, &polled,
2791                                                   &stop);
2792                 }
2793                 if (expand)
2794                         goto expand_cqe;
2795                 if (stop)
2796                         goto stop_cqe;
2797                 /* clear qpn to avoid duplicate processing by discard_cqe() */
2798                 cqe->cmn.qpn = 0;
2799 skip_cqe:
2800                 polled_hw_cqes += 1;
2801                 cur_getp = (cur_getp + 1) % cq->max_hw_cqe;
2802                 ocrdma_change_cq_phase(cq, cqe, cur_getp);
2803 expand_cqe:
2804                 if (polled) {
2805                         num_entries -= 1;
2806                         i += 1;
2807                         ibwc = ibwc + 1;
2808                         polled = false;
2809                 }
2810         }
2811 stop_cqe:
2812         cq->getp = cur_getp;
2813
2814         if (polled_hw_cqes)
2815                 ocrdma_ring_cq_db(dev, cq->id, false, false, polled_hw_cqes);
2816
2817         return i;
2818 }
2819
2820 /* insert error cqe if the QP's SQ or RQ's CQ matches the CQ under poll. */
2821 static int ocrdma_add_err_cqe(struct ocrdma_cq *cq, int num_entries,
2822                               struct ocrdma_qp *qp, struct ib_wc *ibwc)
2823 {
2824         int err_cqes = 0;
2825
2826         while (num_entries) {
2827                 if (is_hw_sq_empty(qp) && is_hw_rq_empty(qp))
2828                         break;
2829                 if (!is_hw_sq_empty(qp) && qp->sq_cq == cq) {
2830                         ocrdma_update_wc(qp, ibwc, qp->sq.tail);
2831                         ocrdma_hwq_inc_tail(&qp->sq);
2832                 } else if (!is_hw_rq_empty(qp) && qp->rq_cq == cq) {
2833                         ibwc->wr_id = qp->rqe_wr_id_tbl[qp->rq.tail];
2834                         ocrdma_hwq_inc_tail(&qp->rq);
2835                 } else {
2836                         return err_cqes;
2837                 }
2838                 ibwc->byte_len = 0;
2839                 ibwc->status = IB_WC_WR_FLUSH_ERR;
2840                 ibwc = ibwc + 1;
2841                 err_cqes += 1;
2842                 num_entries -= 1;
2843         }
2844         return err_cqes;
2845 }
2846
2847 int ocrdma_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
2848 {
2849         int cqes_to_poll = num_entries;
2850         struct ocrdma_cq *cq = get_ocrdma_cq(ibcq);
2851         struct ocrdma_dev *dev = get_ocrdma_dev(ibcq->device);
2852         int num_os_cqe = 0, err_cqes = 0;
2853         struct ocrdma_qp *qp;
2854         unsigned long flags;
2855
2856         /* poll cqes from adapter CQ */
2857         spin_lock_irqsave(&cq->cq_lock, flags);
2858         num_os_cqe = ocrdma_poll_hwcq(cq, cqes_to_poll, wc);
2859         spin_unlock_irqrestore(&cq->cq_lock, flags);
2860         cqes_to_poll -= num_os_cqe;
2861
2862         if (cqes_to_poll) {
2863                 wc = wc + num_os_cqe;
2864                 /* adapter returns single error cqe when qp moves to
2865                  * error state. So insert error cqes with wc_status as
2866                  * FLUSHED for pending WQEs and RQEs of QP's SQ and RQ
2867                  * respectively which uses this CQ.
2868                  */
2869                 spin_lock_irqsave(&dev->flush_q_lock, flags);
2870                 list_for_each_entry(qp, &cq->sq_head, sq_entry) {
2871                         if (cqes_to_poll == 0)
2872                                 break;
2873                         err_cqes = ocrdma_add_err_cqe(cq, cqes_to_poll, qp, wc);
2874                         cqes_to_poll -= err_cqes;
2875                         num_os_cqe += err_cqes;
2876                         wc = wc + err_cqes;
2877                 }
2878                 spin_unlock_irqrestore(&dev->flush_q_lock, flags);
2879         }
2880         return num_os_cqe;
2881 }
2882
2883 int ocrdma_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags cq_flags)
2884 {
2885         struct ocrdma_cq *cq = get_ocrdma_cq(ibcq);
2886         struct ocrdma_dev *dev = get_ocrdma_dev(ibcq->device);
2887         u16 cq_id;
2888         unsigned long flags;
2889         bool arm_needed = false, sol_needed = false;
2890
2891         cq_id = cq->id;
2892
2893         spin_lock_irqsave(&cq->cq_lock, flags);
2894         if (cq_flags & IB_CQ_NEXT_COMP || cq_flags & IB_CQ_SOLICITED)
2895                 arm_needed = true;
2896         if (cq_flags & IB_CQ_SOLICITED)
2897                 sol_needed = true;
2898
2899         ocrdma_ring_cq_db(dev, cq_id, arm_needed, sol_needed, 0);
2900         spin_unlock_irqrestore(&cq->cq_lock, flags);
2901
2902         return 0;
2903 }
2904
2905 struct ib_mr *ocrdma_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
2906                               u32 max_num_sg, struct ib_udata *udata)
2907 {
2908         int status;
2909         struct ocrdma_mr *mr;
2910         struct ocrdma_pd *pd = get_ocrdma_pd(ibpd);
2911         struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device);
2912
2913         if (mr_type != IB_MR_TYPE_MEM_REG)
2914                 return ERR_PTR(-EINVAL);
2915
2916         if (max_num_sg > dev->attr.max_pages_per_frmr)
2917                 return ERR_PTR(-EINVAL);
2918
2919         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2920         if (!mr)
2921                 return ERR_PTR(-ENOMEM);
2922
2923         mr->pages = kcalloc(max_num_sg, sizeof(u64), GFP_KERNEL);
2924         if (!mr->pages) {
2925                 status = -ENOMEM;
2926                 goto pl_err;
2927         }
2928
2929         status = ocrdma_get_pbl_info(dev, mr, max_num_sg);
2930         if (status)
2931                 goto pbl_err;
2932         mr->hwmr.fr_mr = 1;
2933         mr->hwmr.remote_rd = 0;
2934         mr->hwmr.remote_wr = 0;
2935         mr->hwmr.local_rd = 0;
2936         mr->hwmr.local_wr = 0;
2937         mr->hwmr.mw_bind = 0;
2938         status = ocrdma_build_pbl_tbl(dev, &mr->hwmr);
2939         if (status)
2940                 goto pbl_err;
2941         status = ocrdma_reg_mr(dev, &mr->hwmr, pd->id, 0);
2942         if (status)
2943                 goto mbx_err;
2944         mr->ibmr.rkey = mr->hwmr.lkey;
2945         mr->ibmr.lkey = mr->hwmr.lkey;
2946         dev->stag_arr[(mr->hwmr.lkey >> 8) & (OCRDMA_MAX_STAG - 1)] =
2947                 (unsigned long) mr;
2948         return &mr->ibmr;
2949 mbx_err:
2950         ocrdma_free_mr_pbl_tbl(dev, &mr->hwmr);
2951 pbl_err:
2952         kfree(mr->pages);
2953 pl_err:
2954         kfree(mr);
2955         return ERR_PTR(-ENOMEM);
2956 }
2957
2958 static int ocrdma_set_page(struct ib_mr *ibmr, u64 addr)
2959 {
2960         struct ocrdma_mr *mr = get_ocrdma_mr(ibmr);
2961
2962         if (unlikely(mr->npages == mr->hwmr.num_pbes))
2963                 return -ENOMEM;
2964
2965         mr->pages[mr->npages++] = addr;
2966
2967         return 0;
2968 }
2969
2970 int ocrdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
2971                      unsigned int *sg_offset)
2972 {
2973         struct ocrdma_mr *mr = get_ocrdma_mr(ibmr);
2974
2975         mr->npages = 0;
2976
2977         return ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, ocrdma_set_page);
2978 }