drivers/infiniband/hw/hns/hns_roce_hw_v2.c

   1 /*
   2  * Copyright (c) 2016-2017 Hisilicon Limited.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32
  33 #include <linux/acpi.h>
  34 #include <linux/etherdevice.h>
  35 #include <linux/interrupt.h>
  36 #include <linux/kernel.h>
  37 #include <linux/types.h>
  38 #include <net/addrconf.h>
  39 #include <rdma/ib_addr.h>
  40 #include <rdma/ib_umem.h>
  41
  42 #include "hnae3.h"
  43 #include "hns_roce_common.h"
  44 #include "hns_roce_device.h"
  45 #include "hns_roce_cmd.h"
  46 #include "hns_roce_hem.h"
  47 #include "hns_roce_hw_v2.h"
  48
  49 static void set_data_seg_v2(struct hns_roce_v2_wqe_data_seg *dseg,
  50                             struct ib_sge *sg)
  51 {
  52         dseg->lkey = cpu_to_le32(sg->lkey);
  53         dseg->addr = cpu_to_le64(sg->addr);
  54         dseg->len  = cpu_to_le32(sg->length);
  55 }
  56
  57 static void set_frmr_seg(struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,
  58                          struct hns_roce_wqe_frmr_seg *fseg,
  59                          const struct ib_reg_wr *wr)
  60 {
  61         struct hns_roce_mr *mr = to_hr_mr(wr->mr);
  62
  63         /* use ib_access_flags */
  64         roce_set_bit(rc_sq_wqe->byte_4,
  65                      V2_RC_FRMR_WQE_BYTE_4_BIND_EN_S,
  66                      wr->access & IB_ACCESS_MW_BIND ? 1 : 0);
  67         roce_set_bit(rc_sq_wqe->byte_4,
  68                      V2_RC_FRMR_WQE_BYTE_4_ATOMIC_S,
  69                      wr->access & IB_ACCESS_REMOTE_ATOMIC ? 1 : 0);
  70         roce_set_bit(rc_sq_wqe->byte_4,
  71                      V2_RC_FRMR_WQE_BYTE_4_RR_S,
  72                      wr->access & IB_ACCESS_REMOTE_READ ? 1 : 0);
  73         roce_set_bit(rc_sq_wqe->byte_4,
  74                      V2_RC_FRMR_WQE_BYTE_4_RW_S,
  75                      wr->access & IB_ACCESS_REMOTE_WRITE ? 1 : 0);
  76         roce_set_bit(rc_sq_wqe->byte_4,
  77                      V2_RC_FRMR_WQE_BYTE_4_LW_S,
  78                      wr->access & IB_ACCESS_LOCAL_WRITE ? 1 : 0);
  79
  80         /* Data structure reuse may lead to confusion */
  81         rc_sq_wqe->msg_len = cpu_to_le32(mr->pbl_ba & 0xffffffff);
  82         rc_sq_wqe->inv_key = cpu_to_le32(mr->pbl_ba >> 32);
  83
  84         rc_sq_wqe->byte_16 = cpu_to_le32(wr->mr->length & 0xffffffff);
  85         rc_sq_wqe->byte_20 = cpu_to_le32(wr->mr->length >> 32);
  86         rc_sq_wqe->rkey = cpu_to_le32(wr->key);
  87         rc_sq_wqe->va = cpu_to_le64(wr->mr->iova);
  88
  89         fseg->pbl_size = cpu_to_le32(mr->pbl_size);
  90         roce_set_field(fseg->mode_buf_pg_sz,
  91                        V2_RC_FRMR_WQE_BYTE_40_PBL_BUF_PG_SZ_M,
  92                        V2_RC_FRMR_WQE_BYTE_40_PBL_BUF_PG_SZ_S,
  93                        mr->pbl_buf_pg_sz + PG_SHIFT_OFFSET);
  94         roce_set_bit(fseg->mode_buf_pg_sz,
  95                      V2_RC_FRMR_WQE_BYTE_40_BLK_MODE_S, 0);
  96 }
  97
  98 static void set_atomic_seg(struct hns_roce_wqe_atomic_seg *aseg,
  99                            const struct ib_atomic_wr *wr)
 100 {
 101         if (wr->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP) {
 102                 aseg->fetchadd_swap_data = cpu_to_le64(wr->swap);
 103                 aseg->cmp_data  = cpu_to_le64(wr->compare_add);
 104         } else {
 105                 aseg->fetchadd_swap_data = cpu_to_le64(wr->compare_add);
 106                 aseg->cmp_data  = 0;
 107         }
 108 }
 109
 110 static void set_extend_sge(struct hns_roce_qp *qp, const struct ib_send_wr *wr,
 111                            unsigned int *sge_ind)
 112 {
 113         struct hns_roce_v2_wqe_data_seg *dseg;
 114         struct ib_sge *sg;
 115         int num_in_wqe = 0;
 116         int extend_sge_num;
 117         int fi_sge_num;
 118         int se_sge_num;
 119         int shift;
 120         int i;
 121
 122         if (qp->ibqp.qp_type == IB_QPT_RC || qp->ibqp.qp_type == IB_QPT_UC)
 123                 num_in_wqe = HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE;
 124         extend_sge_num = wr->num_sge - num_in_wqe;
 125         sg = wr->sg_list + num_in_wqe;
 126         shift = qp->hr_buf.page_shift;
 127
 128         /*
 129          * Check whether wr->num_sge sges are in the same page. If not, we
 130          * should calculate how many sges in the first page and the second
 131          * page.
 132          */
 133         dseg = get_send_extend_sge(qp, (*sge_ind) & (qp->sge.sge_cnt - 1));
 134         fi_sge_num = (round_up((uintptr_t)dseg, 1 << shift) -
 135                       (uintptr_t)dseg) /
 136                       sizeof(struct hns_roce_v2_wqe_data_seg);
 137         if (extend_sge_num > fi_sge_num) {
 138                 se_sge_num = extend_sge_num - fi_sge_num;
 139                 for (i = 0; i < fi_sge_num; i++) {
 140                         set_data_seg_v2(dseg++, sg + i);
 141                         (*sge_ind)++;
 142                 }
 143                 dseg = get_send_extend_sge(qp,
 144                                            (*sge_ind) & (qp->sge.sge_cnt - 1));
 145                 for (i = 0; i < se_sge_num; i++) {
 146                         set_data_seg_v2(dseg++, sg + fi_sge_num + i);
 147                         (*sge_ind)++;
 148                 }
 149         } else {
 150                 for (i = 0; i < extend_sge_num; i++) {
 151                         set_data_seg_v2(dseg++, sg + i);
 152                         (*sge_ind)++;
 153                 }
 154         }
 155 }
 156
 157 static int set_rwqe_data_seg(struct ib_qp *ibqp, const struct ib_send_wr *wr,
 158                              struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,
 159                              void *wqe, unsigned int *sge_ind,
 160                              const struct ib_send_wr **bad_wr)
 161 {
 162         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
 163         struct hns_roce_v2_wqe_data_seg *dseg = wqe;
 164         struct hns_roce_qp *qp = to_hr_qp(ibqp);
 165         int i;
 166
 167         if (wr->send_flags & IB_SEND_INLINE && wr->num_sge) {
 168                 if (le32_to_cpu(rc_sq_wqe->msg_len) >
 169                     hr_dev->caps.max_sq_inline) {
 170                         *bad_wr = wr;
 171                         dev_err(hr_dev->dev, "inline len(1-%d)=%d, illegal",
 172                                 rc_sq_wqe->msg_len, hr_dev->caps.max_sq_inline);
 173                         return -EINVAL;
 174                 }
 175
 176                 if (wr->opcode == IB_WR_RDMA_READ) {
 177                         *bad_wr =  wr;
 178                         dev_err(hr_dev->dev, "Not support inline data!\n");
 179                         return -EINVAL;
 180                 }
 181
 182                 for (i = 0; i < wr->num_sge; i++) {
 183                         memcpy(wqe, ((void *)wr->sg_list[i].addr),
 184                                wr->sg_list[i].length);
 185                         wqe += wr->sg_list[i].length;
 186                 }
 187
 188                 roce_set_bit(rc_sq_wqe->byte_4, V2_RC_SEND_WQE_BYTE_4_INLINE_S,
 189                              1);
 190         } else {
 191                 if (wr->num_sge <= HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE) {
 192                         for (i = 0; i < wr->num_sge; i++) {
 193                                 if (likely(wr->sg_list[i].length)) {
 194                                         set_data_seg_v2(dseg, wr->sg_list + i);
 195                                         dseg++;
 196                                 }
 197                         }
 198                 } else {
 199                         roce_set_field(rc_sq_wqe->byte_20,
 200                                      V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_M,
 201                                      V2_RC_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_S,
 202                                      (*sge_ind) & (qp->sge.sge_cnt - 1));
 203
 204                         for (i = 0; i < HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE; i++) {
 205                                 if (likely(wr->sg_list[i].length)) {
 206                                         set_data_seg_v2(dseg, wr->sg_list + i);
 207                                         dseg++;
 208                                 }
 209                         }
 210
 211                         set_extend_sge(qp, wr, sge_ind);
 212                 }
 213
 214                 roce_set_field(rc_sq_wqe->byte_16,
 215                                V2_RC_SEND_WQE_BYTE_16_SGE_NUM_M,
 216                                V2_RC_SEND_WQE_BYTE_16_SGE_NUM_S, wr->num_sge);
 217         }
 218
 219         return 0;
 220 }
 221
 222 static int hns_roce_v2_modify_qp(struct ib_qp *ibqp,
 223                                  const struct ib_qp_attr *attr,
 224                                  int attr_mask, enum ib_qp_state cur_state,
 225                                  enum ib_qp_state new_state);
 226
 227 static int hns_roce_v2_post_send(struct ib_qp *ibqp,
 228                                  const struct ib_send_wr *wr,
 229                                  const struct ib_send_wr **bad_wr)
 230 {
 231         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
 232         struct hns_roce_ah *ah = to_hr_ah(ud_wr(wr)->ah);
 233         struct hns_roce_v2_ud_send_wqe *ud_sq_wqe;
 234         struct hns_roce_v2_rc_send_wqe *rc_sq_wqe;
 235         struct hns_roce_qp *qp = to_hr_qp(ibqp);
 236         struct hns_roce_wqe_frmr_seg *fseg;
 237         struct device *dev = hr_dev->dev;
 238         struct hns_roce_v2_db sq_db;
 239         struct ib_qp_attr attr;
 240         unsigned int sge_ind = 0;
 241         unsigned int owner_bit;
 242         unsigned long flags;
 243         unsigned int ind;
 244         void *wqe = NULL;
 245         bool loopback;
 246         int attr_mask;
 247         u32 tmp_len;
 248         int ret = 0;
 249         u32 hr_op;
 250         u8 *smac;
 251         int nreq;
 252         int i;
 253
 254         if (unlikely(ibqp->qp_type != IB_QPT_RC &&
 255                      ibqp->qp_type != IB_QPT_GSI &&
 256                      ibqp->qp_type != IB_QPT_UD)) {
 257                 dev_err(dev, "Not supported QP(0x%x)type!\n", ibqp->qp_type);
 258                 *bad_wr = wr;
 259                 return -EOPNOTSUPP;
 260         }
 261
 262         if (unlikely(qp->state == IB_QPS_RESET || qp->state == IB_QPS_INIT ||
 263                      qp->state == IB_QPS_RTR)) {
 264                 dev_err(dev, "Post WQE fail, QP state %d err!\n", qp->state);
 265                 *bad_wr = wr;
 266                 return -EINVAL;
 267         }
 268
 269         spin_lock_irqsave(&qp->sq.lock, flags);
 270         ind = qp->sq_next_wqe;
 271         sge_ind = qp->next_sge;
 272
 273         for (nreq = 0; wr; ++nreq, wr = wr->next) {
 274                 if (hns_roce_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) {
 275                         ret = -ENOMEM;
 276                         *bad_wr = wr;
 277                         goto out;
 278                 }
 279
 280                 if (unlikely(wr->num_sge > qp->sq.max_gs)) {
 281                         dev_err(dev, "num_sge=%d > qp->sq.max_gs=%d\n",
 282                                 wr->num_sge, qp->sq.max_gs);
 283                         ret = -EINVAL;
 284                         *bad_wr = wr;
 285                         goto out;
 286                 }
 287
 288                 wqe = get_send_wqe(qp, ind & (qp->sq.wqe_cnt - 1));
 289                 qp->sq.wrid[(qp->sq.head + nreq) & (qp->sq.wqe_cnt - 1)] =
 290                                                                       wr->wr_id;
 291
 292                 owner_bit =
 293                        ~(((qp->sq.head + nreq) >> ilog2(qp->sq.wqe_cnt)) & 0x1);
 294                 tmp_len = 0;
 295
 296                 /* Corresponding to the QP type, wqe process separately */
 297                 if (ibqp->qp_type == IB_QPT_GSI) {
 298                         ud_sq_wqe = wqe;
 299                         memset(ud_sq_wqe, 0, sizeof(*ud_sq_wqe));
 300
 301                         roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_0_M,
 302                                        V2_UD_SEND_WQE_DMAC_0_S, ah->av.mac[0]);
 303                         roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_1_M,
 304                                        V2_UD_SEND_WQE_DMAC_1_S, ah->av.mac[1]);
 305                         roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_2_M,
 306                                        V2_UD_SEND_WQE_DMAC_2_S, ah->av.mac[2]);
 307                         roce_set_field(ud_sq_wqe->dmac, V2_UD_SEND_WQE_DMAC_3_M,
 308                                        V2_UD_SEND_WQE_DMAC_3_S, ah->av.mac[3]);
 309                         roce_set_field(ud_sq_wqe->byte_48,
 310                                        V2_UD_SEND_WQE_BYTE_48_DMAC_4_M,
 311                                        V2_UD_SEND_WQE_BYTE_48_DMAC_4_S,
 312                                        ah->av.mac[4]);
 313                         roce_set_field(ud_sq_wqe->byte_48,
 314                                        V2_UD_SEND_WQE_BYTE_48_DMAC_5_M,
 315                                        V2_UD_SEND_WQE_BYTE_48_DMAC_5_S,
 316                                        ah->av.mac[5]);
 317
 318                         /* MAC loopback */
 319                         smac = (u8 *)hr_dev->dev_addr[qp->port];
 320                         loopback = ether_addr_equal_unaligned(ah->av.mac,
 321                                                               smac) ? 1 : 0;
 322
 323                         roce_set_bit(ud_sq_wqe->byte_40,
 324                                      V2_UD_SEND_WQE_BYTE_40_LBI_S, loopback);
 325
 326                         roce_set_field(ud_sq_wqe->byte_4,
 327                                        V2_UD_SEND_WQE_BYTE_4_OPCODE_M,
 328                                        V2_UD_SEND_WQE_BYTE_4_OPCODE_S,
 329                                        HNS_ROCE_V2_WQE_OP_SEND);
 330
 331                         for (i = 0; i < wr->num_sge; i++)
 332                                 tmp_len += wr->sg_list[i].length;
 333
 334                         ud_sq_wqe->msg_len =
 335                          cpu_to_le32(le32_to_cpu(ud_sq_wqe->msg_len) + tmp_len);
 336
 337                         switch (wr->opcode) {
 338                         case IB_WR_SEND_WITH_IMM:
 339                         case IB_WR_RDMA_WRITE_WITH_IMM:
 340                                 ud_sq_wqe->immtdata =
 341                                       cpu_to_le32(be32_to_cpu(wr->ex.imm_data));
 342                                 break;
 343                         default:
 344                                 ud_sq_wqe->immtdata = 0;
 345                                 break;
 346                         }
 347
 348                         /* Set sig attr */
 349                         roce_set_bit(ud_sq_wqe->byte_4,
 350                                    V2_UD_SEND_WQE_BYTE_4_CQE_S,
 351                                    (wr->send_flags & IB_SEND_SIGNALED) ? 1 : 0);
 352
 353                         /* Set se attr */
 354                         roce_set_bit(ud_sq_wqe->byte_4,
 355                                   V2_UD_SEND_WQE_BYTE_4_SE_S,
 356                                   (wr->send_flags & IB_SEND_SOLICITED) ? 1 : 0);
 357
 358                         roce_set_bit(ud_sq_wqe->byte_4,
 359                                      V2_UD_SEND_WQE_BYTE_4_OWNER_S, owner_bit);
 360
 361                         roce_set_field(ud_sq_wqe->byte_16,
 362                                        V2_UD_SEND_WQE_BYTE_16_PD_M,
 363                                        V2_UD_SEND_WQE_BYTE_16_PD_S,
 364                                        to_hr_pd(ibqp->pd)->pdn);
 365
 366                         roce_set_field(ud_sq_wqe->byte_16,
 367                                        V2_UD_SEND_WQE_BYTE_16_SGE_NUM_M,
 368                                        V2_UD_SEND_WQE_BYTE_16_SGE_NUM_S,
 369                                        wr->num_sge);
 370
 371                         roce_set_field(ud_sq_wqe->byte_20,
 372                                      V2_UD_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_M,
 373                                      V2_UD_SEND_WQE_BYTE_20_MSG_START_SGE_IDX_S,
 374                                      sge_ind & (qp->sge.sge_cnt - 1));
 375
 376                         roce_set_field(ud_sq_wqe->byte_24,
 377                                        V2_UD_SEND_WQE_BYTE_24_UDPSPN_M,
 378                                        V2_UD_SEND_WQE_BYTE_24_UDPSPN_S, 0);
 379                         ud_sq_wqe->qkey =
 380                              cpu_to_le32(ud_wr(wr)->remote_qkey & 0x80000000 ?
 381                              qp->qkey : ud_wr(wr)->remote_qkey);
 382                         roce_set_field(ud_sq_wqe->byte_32,
 383                                        V2_UD_SEND_WQE_BYTE_32_DQPN_M,
 384                                        V2_UD_SEND_WQE_BYTE_32_DQPN_S,
 385                                        ud_wr(wr)->remote_qpn);
 386
 387                         roce_set_field(ud_sq_wqe->byte_36,
 388                                        V2_UD_SEND_WQE_BYTE_36_VLAN_M,
 389                                        V2_UD_SEND_WQE_BYTE_36_VLAN_S,
 390                                        le16_to_cpu(ah->av.vlan));
 391                         roce_set_field(ud_sq_wqe->byte_36,
 392                                        V2_UD_SEND_WQE_BYTE_36_HOPLIMIT_M,
 393                                        V2_UD_SEND_WQE_BYTE_36_HOPLIMIT_S,
 394                                        ah->av.hop_limit);
 395                         roce_set_field(ud_sq_wqe->byte_36,
 396                                        V2_UD_SEND_WQE_BYTE_36_TCLASS_M,
 397                                        V2_UD_SEND_WQE_BYTE_36_TCLASS_S,
 398                                        ah->av.sl_tclass_flowlabel >>
 399                                        HNS_ROCE_TCLASS_SHIFT);
 400                         roce_set_field(ud_sq_wqe->byte_40,
 401                                        V2_UD_SEND_WQE_BYTE_40_FLOW_LABEL_M,
 402                                        V2_UD_SEND_WQE_BYTE_40_FLOW_LABEL_S,
 403                                        ah->av.sl_tclass_flowlabel &
 404                                        HNS_ROCE_FLOW_LABEL_MASK);
 405                         roce_set_field(ud_sq_wqe->byte_40,
 406                                        V2_UD_SEND_WQE_BYTE_40_SL_M,
 407                                        V2_UD_SEND_WQE_BYTE_40_SL_S,
 408                                       le32_to_cpu(ah->av.sl_tclass_flowlabel) >>
 409                                       HNS_ROCE_SL_SHIFT);
 410                         roce_set_field(ud_sq_wqe->byte_40,
 411                                        V2_UD_SEND_WQE_BYTE_40_PORTN_M,
 412                                        V2_UD_SEND_WQE_BYTE_40_PORTN_S,
 413                                        qp->port);
 414
 415                         roce_set_bit(ud_sq_wqe->byte_40,
 416                                      V2_UD_SEND_WQE_BYTE_40_UD_VLAN_EN_S,
 417                                      ah->av.vlan_en ? 1 : 0);
 418                         roce_set_field(ud_sq_wqe->byte_48,
 419                                        V2_UD_SEND_WQE_BYTE_48_SGID_INDX_M,
 420                                        V2_UD_SEND_WQE_BYTE_48_SGID_INDX_S,
 421                                        hns_get_gid_index(hr_dev, qp->phy_port,
 422                                                          ah->av.gid_index));
 423
 424                         memcpy(&ud_sq_wqe->dgid[0], &ah->av.dgid[0],
 425                                GID_LEN_V2);
 426
 427                         set_extend_sge(qp, wr, &sge_ind);
 428                         ind++;
 429                 } else if (ibqp->qp_type == IB_QPT_RC) {
 430                         rc_sq_wqe = wqe;
 431                         memset(rc_sq_wqe, 0, sizeof(*rc_sq_wqe));
 432                         for (i = 0; i < wr->num_sge; i++)
 433                                 tmp_len += wr->sg_list[i].length;
 434
 435                         rc_sq_wqe->msg_len =
 436                          cpu_to_le32(le32_to_cpu(rc_sq_wqe->msg_len) + tmp_len);
 437
 438                         switch (wr->opcode) {
 439                         case IB_WR_SEND_WITH_IMM:
 440                         case IB_WR_RDMA_WRITE_WITH_IMM:
 441                                 rc_sq_wqe->immtdata =
 442                                       cpu_to_le32(be32_to_cpu(wr->ex.imm_data));
 443                                 break;
 444                         case IB_WR_SEND_WITH_INV:
 445                                 rc_sq_wqe->inv_key =
 446                                         cpu_to_le32(wr->ex.invalidate_rkey);
 447                                 break;
 448                         default:
 449                                 rc_sq_wqe->immtdata = 0;
 450                                 break;
 451                         }
 452
 453                         roce_set_bit(rc_sq_wqe->byte_4,
 454                                      V2_RC_SEND_WQE_BYTE_4_FENCE_S,
 455                                      (wr->send_flags & IB_SEND_FENCE) ? 1 : 0);
 456
 457                         roce_set_bit(rc_sq_wqe->byte_4,
 458                                   V2_RC_SEND_WQE_BYTE_4_SE_S,
 459                                   (wr->send_flags & IB_SEND_SOLICITED) ? 1 : 0);
 460
 461                         roce_set_bit(rc_sq_wqe->byte_4,
 462                                    V2_RC_SEND_WQE_BYTE_4_CQE_S,
 463                                    (wr->send_flags & IB_SEND_SIGNALED) ? 1 : 0);
 464
 465                         roce_set_bit(rc_sq_wqe->byte_4,
 466                                      V2_RC_SEND_WQE_BYTE_4_OWNER_S, owner_bit);
 467
 468                         wqe += sizeof(struct hns_roce_v2_rc_send_wqe);
 469                         switch (wr->opcode) {
 470                         case IB_WR_RDMA_READ:
 471                                 hr_op = HNS_ROCE_V2_WQE_OP_RDMA_READ;
 472                                 rc_sq_wqe->rkey =
 473                                         cpu_to_le32(rdma_wr(wr)->rkey);
 474                                 rc_sq_wqe->va =
 475                                         cpu_to_le64(rdma_wr(wr)->remote_addr);
 476                                 break;
 477                         case IB_WR_RDMA_WRITE:
 478                                 hr_op = HNS_ROCE_V2_WQE_OP_RDMA_WRITE;
 479                                 rc_sq_wqe->rkey =
 480                                         cpu_to_le32(rdma_wr(wr)->rkey);
 481                                 rc_sq_wqe->va =
 482                                         cpu_to_le64(rdma_wr(wr)->remote_addr);
 483                                 break;
 484                         case IB_WR_RDMA_WRITE_WITH_IMM:
 485                                 hr_op = HNS_ROCE_V2_WQE_OP_RDMA_WRITE_WITH_IMM;
 486                                 rc_sq_wqe->rkey =
 487                                         cpu_to_le32(rdma_wr(wr)->rkey);
 488                                 rc_sq_wqe->va =
 489                                         cpu_to_le64(rdma_wr(wr)->remote_addr);
 490                                 break;
 491                         case IB_WR_SEND:
 492                                 hr_op = HNS_ROCE_V2_WQE_OP_SEND;
 493                                 break;
 494                         case IB_WR_SEND_WITH_INV:
 495                                 hr_op = HNS_ROCE_V2_WQE_OP_SEND_WITH_INV;
 496                                 break;
 497                         case IB_WR_SEND_WITH_IMM:
 498                                 hr_op = HNS_ROCE_V2_WQE_OP_SEND_WITH_IMM;
 499                                 break;
 500                         case IB_WR_LOCAL_INV:
 501                                 hr_op = HNS_ROCE_V2_WQE_OP_LOCAL_INV;
 502                                 roce_set_bit(rc_sq_wqe->byte_4,
 503                                                V2_RC_SEND_WQE_BYTE_4_SO_S, 1);
 504                                 rc_sq_wqe->inv_key =
 505                                             cpu_to_le32(wr->ex.invalidate_rkey);
 506                                 break;
 507                         case IB_WR_REG_MR:
 508                                 hr_op = HNS_ROCE_V2_WQE_OP_FAST_REG_PMR;
 509                                 fseg = wqe;
 510                                 set_frmr_seg(rc_sq_wqe, fseg, reg_wr(wr));
 511                                 break;
 512                         case IB_WR_ATOMIC_CMP_AND_SWP:
 513                                 hr_op = HNS_ROCE_V2_WQE_OP_ATOM_CMP_AND_SWAP;
 514                                 rc_sq_wqe->rkey =
 515                                         cpu_to_le32(atomic_wr(wr)->rkey);
 516                                 rc_sq_wqe->va =
 517                                         cpu_to_le64(atomic_wr(wr)->remote_addr);
 518                                 break;
 519                         case IB_WR_ATOMIC_FETCH_AND_ADD:
 520                                 hr_op = HNS_ROCE_V2_WQE_OP_ATOM_FETCH_AND_ADD;
 521                                 rc_sq_wqe->rkey =
 522                                         cpu_to_le32(atomic_wr(wr)->rkey);
 523                                 rc_sq_wqe->va =
 524                                         cpu_to_le64(atomic_wr(wr)->remote_addr);
 525                                 break;
 526                         case IB_WR_MASKED_ATOMIC_CMP_AND_SWP:
 527                                 hr_op =
 528                                        HNS_ROCE_V2_WQE_OP_ATOM_MSK_CMP_AND_SWAP;
 529                                 break;
 530                         case IB_WR_MASKED_ATOMIC_FETCH_AND_ADD:
 531                                 hr_op =
 532                                       HNS_ROCE_V2_WQE_OP_ATOM_MSK_FETCH_AND_ADD;
 533                                 break;
 534                         default:
 535                                 hr_op = HNS_ROCE_V2_WQE_OP_MASK;
 536                                 break;
 537                         }
 538
 539                         roce_set_field(rc_sq_wqe->byte_4,
 540                                        V2_RC_SEND_WQE_BYTE_4_OPCODE_M,
 541                                        V2_RC_SEND_WQE_BYTE_4_OPCODE_S, hr_op);
 542
 543                         if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP ||
 544                             wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
 545                                 struct hns_roce_v2_wqe_data_seg *dseg;
 546
 547                                 dseg = wqe;
 548                                 set_data_seg_v2(dseg, wr->sg_list);
 549                                 wqe += sizeof(struct hns_roce_v2_wqe_data_seg);
 550                                 set_atomic_seg(wqe, atomic_wr(wr));
 551                                 roce_set_field(rc_sq_wqe->byte_16,
 552                                                V2_RC_SEND_WQE_BYTE_16_SGE_NUM_M,
 553                                                V2_RC_SEND_WQE_BYTE_16_SGE_NUM_S,
 554                                                wr->num_sge);
 555                         } else if (wr->opcode != IB_WR_REG_MR) {
 556                                 ret = set_rwqe_data_seg(ibqp, wr, rc_sq_wqe,
 557                                                         wqe, &sge_ind, bad_wr);
 558                                 if (ret)
 559                                         goto out;
 560                         }
 561
 562                         ind++;
 563                 } else {
 564                         dev_err(dev, "Illegal qp_type(0x%x)\n", ibqp->qp_type);
 565                         spin_unlock_irqrestore(&qp->sq.lock, flags);
 566                         *bad_wr = wr;
 567                         return -EOPNOTSUPP;
 568                 }
 569         }
 570
 571 out:
 572         if (likely(nreq)) {
 573                 qp->sq.head += nreq;
 574                 /* Memory barrier */
 575                 wmb();
 576
 577                 sq_db.byte_4 = 0;
 578                 sq_db.parameter = 0;
 579
 580                 roce_set_field(sq_db.byte_4, V2_DB_BYTE_4_TAG_M,
 581                                V2_DB_BYTE_4_TAG_S, qp->doorbell_qpn);
 582                 roce_set_field(sq_db.byte_4, V2_DB_BYTE_4_CMD_M,
 583                                V2_DB_BYTE_4_CMD_S, HNS_ROCE_V2_SQ_DB);
 584                 roce_set_field(sq_db.parameter, V2_DB_PARAMETER_IDX_M,
 585                                V2_DB_PARAMETER_IDX_S,
 586                                qp->sq.head & ((qp->sq.wqe_cnt << 1) - 1));
 587                 roce_set_field(sq_db.parameter, V2_DB_PARAMETER_SL_M,
 588                                V2_DB_PARAMETER_SL_S, qp->sl);
 589
 590                 hns_roce_write64_k((__le32 *)&sq_db, qp->sq.db_reg_l);
 591
 592                 qp->sq_next_wqe = ind;
 593                 qp->next_sge = sge_ind;
 594
 595                 if (qp->state == IB_QPS_ERR) {
 596                         attr_mask = IB_QP_STATE;
 597                         attr.qp_state = IB_QPS_ERR;
 598
 599                         ret = hns_roce_v2_modify_qp(&qp->ibqp, &attr, attr_mask,
 600                                                     qp->state, IB_QPS_ERR);
 601                         if (ret) {
 602                                 spin_unlock_irqrestore(&qp->sq.lock, flags);
 603                                 *bad_wr = wr;
 604                                 return ret;
 605                         }
 606                 }
 607         }
 608
 609         spin_unlock_irqrestore(&qp->sq.lock, flags);
 610
 611         return ret;
 612 }
 613
 614 static int hns_roce_v2_post_recv(struct ib_qp *ibqp,
 615                                  const struct ib_recv_wr *wr,
 616                                  const struct ib_recv_wr **bad_wr)
 617 {
 618         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
 619         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
 620         struct hns_roce_v2_wqe_data_seg *dseg;
 621         struct hns_roce_rinl_sge *sge_list;
 622         struct device *dev = hr_dev->dev;
 623         struct ib_qp_attr attr;
 624         unsigned long flags;
 625         void *wqe = NULL;
 626         int attr_mask;
 627         int ret = 0;
 628         int nreq;
 629         int ind;
 630         int i;
 631
 632         spin_lock_irqsave(&hr_qp->rq.lock, flags);
 633         ind = hr_qp->rq.head & (hr_qp->rq.wqe_cnt - 1);
 634
 635         if (hr_qp->state == IB_QPS_RESET) {
 636                 spin_unlock_irqrestore(&hr_qp->rq.lock, flags);
 637                 *bad_wr = wr;
 638                 return -EINVAL;
 639         }
 640
 641         for (nreq = 0; wr; ++nreq, wr = wr->next) {
 642                 if (hns_roce_wq_overflow(&hr_qp->rq, nreq,
 643                         hr_qp->ibqp.recv_cq)) {
 644                         ret = -ENOMEM;
 645                         *bad_wr = wr;
 646                         goto out;
 647                 }
 648
 649                 if (unlikely(wr->num_sge > hr_qp->rq.max_gs)) {
 650                         dev_err(dev, "rq:num_sge=%d > qp->sq.max_gs=%d\n",
 651                                 wr->num_sge, hr_qp->rq.max_gs);
 652                         ret = -EINVAL;
 653                         *bad_wr = wr;
 654                         goto out;
 655                 }
 656
 657                 wqe = get_recv_wqe(hr_qp, ind);
 658                 dseg = (struct hns_roce_v2_wqe_data_seg *)wqe;
 659                 for (i = 0; i < wr->num_sge; i++) {
 660                         if (!wr->sg_list[i].length)
 661                                 continue;
 662                         set_data_seg_v2(dseg, wr->sg_list + i);
 663                         dseg++;
 664                 }
 665
 666                 if (i < hr_qp->rq.max_gs) {
 667                         dseg->lkey = cpu_to_le32(HNS_ROCE_INVALID_LKEY);
 668                         dseg->addr = 0;
 669                 }
 670
 671                 /* rq support inline data */
 672                 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) {
 673                         sge_list = hr_qp->rq_inl_buf.wqe_list[ind].sg_list;
 674                         hr_qp->rq_inl_buf.wqe_list[ind].sge_cnt =
 675                                                                (u32)wr->num_sge;
 676                         for (i = 0; i < wr->num_sge; i++) {
 677                                 sge_list[i].addr =
 678                                                (void *)(u64)wr->sg_list[i].addr;
 679                                 sge_list[i].len = wr->sg_list[i].length;
 680                         }
 681                 }
 682
 683                 hr_qp->rq.wrid[ind] = wr->wr_id;
 684
 685                 ind = (ind + 1) & (hr_qp->rq.wqe_cnt - 1);
 686         }
 687
 688 out:
 689         if (likely(nreq)) {
 690                 hr_qp->rq.head += nreq;
 691                 /* Memory barrier */
 692                 wmb();
 693
 694                 *hr_qp->rdb.db_record = hr_qp->rq.head & 0xffff;
 695
 696                 if (hr_qp->state == IB_QPS_ERR) {
 697                         attr_mask = IB_QP_STATE;
 698                         attr.qp_state = IB_QPS_ERR;
 699
 700                         ret = hns_roce_v2_modify_qp(&hr_qp->ibqp, &attr,
 701                                                     attr_mask, hr_qp->state,
 702                                                     IB_QPS_ERR);
 703                         if (ret) {
 704                                 spin_unlock_irqrestore(&hr_qp->rq.lock, flags);
 705                                 *bad_wr = wr;
 706                                 return ret;
 707                         }
 708                 }
 709         }
 710         spin_unlock_irqrestore(&hr_qp->rq.lock, flags);
 711
 712         return ret;
 713 }
 714
 715 static int hns_roce_cmq_space(struct hns_roce_v2_cmq_ring *ring)
 716 {
 717         int ntu = ring->next_to_use;
 718         int ntc = ring->next_to_clean;
 719         int used = (ntu - ntc + ring->desc_num) % ring->desc_num;
 720
 721         return ring->desc_num - used - 1;
 722 }
 723
 724 static int hns_roce_alloc_cmq_desc(struct hns_roce_dev *hr_dev,
 725                                    struct hns_roce_v2_cmq_ring *ring)
 726 {
 727         int size = ring->desc_num * sizeof(struct hns_roce_cmq_desc);
 728
 729         ring->desc = kzalloc(size, GFP_KERNEL);
 730         if (!ring->desc)
 731                 return -ENOMEM;
 732
 733         ring->desc_dma_addr = dma_map_single(hr_dev->dev, ring->desc, size,
 734                                              DMA_BIDIRECTIONAL);
 735         if (dma_mapping_error(hr_dev->dev, ring->desc_dma_addr)) {
 736                 ring->desc_dma_addr = 0;
 737                 kfree(ring->desc);
 738                 ring->desc = NULL;
 739                 return -ENOMEM;
 740         }
 741
 742         return 0;
 743 }
 744
 745 static void hns_roce_free_cmq_desc(struct hns_roce_dev *hr_dev,
 746                                    struct hns_roce_v2_cmq_ring *ring)
 747 {
 748         dma_unmap_single(hr_dev->dev, ring->desc_dma_addr,
 749                          ring->desc_num * sizeof(struct hns_roce_cmq_desc),
 750                          DMA_BIDIRECTIONAL);
 751
 752         ring->desc_dma_addr = 0;
 753         kfree(ring->desc);
 754 }
 755
 756 static int hns_roce_init_cmq_ring(struct hns_roce_dev *hr_dev, bool ring_type)
 757 {
 758         struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
 759         struct hns_roce_v2_cmq_ring *ring = (ring_type == TYPE_CSQ) ?
 760                                             &priv->cmq.csq : &priv->cmq.crq;
 761
 762         ring->flag = ring_type;
 763         ring->next_to_clean = 0;
 764         ring->next_to_use = 0;
 765
 766         return hns_roce_alloc_cmq_desc(hr_dev, ring);
 767 }
 768
 769 static void hns_roce_cmq_init_regs(struct hns_roce_dev *hr_dev, bool ring_type)
 770 {
 771         struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
 772         struct hns_roce_v2_cmq_ring *ring = (ring_type == TYPE_CSQ) ?
 773                                             &priv->cmq.csq : &priv->cmq.crq;
 774         dma_addr_t dma = ring->desc_dma_addr;
 775
 776         if (ring_type == TYPE_CSQ) {
 777                 roce_write(hr_dev, ROCEE_TX_CMQ_BASEADDR_L_REG, (u32)dma);
 778                 roce_write(hr_dev, ROCEE_TX_CMQ_BASEADDR_H_REG,
 779                            upper_32_bits(dma));
 780                 roce_write(hr_dev, ROCEE_TX_CMQ_DEPTH_REG,
 781                           (ring->desc_num >> HNS_ROCE_CMQ_DESC_NUM_S) |
 782                            HNS_ROCE_CMQ_ENABLE);
 783                 roce_write(hr_dev, ROCEE_TX_CMQ_HEAD_REG, 0);
 784                 roce_write(hr_dev, ROCEE_TX_CMQ_TAIL_REG, 0);
 785         } else {
 786                 roce_write(hr_dev, ROCEE_RX_CMQ_BASEADDR_L_REG, (u32)dma);
 787                 roce_write(hr_dev, ROCEE_RX_CMQ_BASEADDR_H_REG,
 788                            upper_32_bits(dma));
 789                 roce_write(hr_dev, ROCEE_RX_CMQ_DEPTH_REG,
 790                           (ring->desc_num >> HNS_ROCE_CMQ_DESC_NUM_S) |
 791                            HNS_ROCE_CMQ_ENABLE);
 792                 roce_write(hr_dev, ROCEE_RX_CMQ_HEAD_REG, 0);
 793                 roce_write(hr_dev, ROCEE_RX_CMQ_TAIL_REG, 0);
 794         }
 795 }
 796
 797 static int hns_roce_v2_cmq_init(struct hns_roce_dev *hr_dev)
 798 {
 799         struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
 800         int ret;
 801
 802         /* Setup the queue entries for command queue */
 803         priv->cmq.csq.desc_num = CMD_CSQ_DESC_NUM;
 804         priv->cmq.crq.desc_num = CMD_CRQ_DESC_NUM;
 805
 806         /* Setup the lock for command queue */
 807         spin_lock_init(&priv->cmq.csq.lock);
 808         spin_lock_init(&priv->cmq.crq.lock);
 809
 810         /* Setup Tx write back timeout */
 811         priv->cmq.tx_timeout = HNS_ROCE_CMQ_TX_TIMEOUT;
 812
 813         /* Init CSQ */
 814         ret = hns_roce_init_cmq_ring(hr_dev, TYPE_CSQ);
 815         if (ret) {
 816                 dev_err(hr_dev->dev, "Init CSQ error, ret = %d.\n", ret);
 817                 return ret;
 818         }
 819
 820         /* Init CRQ */
 821         ret = hns_roce_init_cmq_ring(hr_dev, TYPE_CRQ);
 822         if (ret) {
 823                 dev_err(hr_dev->dev, "Init CRQ error, ret = %d.\n", ret);
 824                 goto err_crq;
 825         }
 826
 827         /* Init CSQ REG */
 828         hns_roce_cmq_init_regs(hr_dev, TYPE_CSQ);
 829
 830         /* Init CRQ REG */
 831         hns_roce_cmq_init_regs(hr_dev, TYPE_CRQ);
 832
 833         return 0;
 834
 835 err_crq:
 836         hns_roce_free_cmq_desc(hr_dev, &priv->cmq.csq);
 837
 838         return ret;
 839 }
 840
 841 static void hns_roce_v2_cmq_exit(struct hns_roce_dev *hr_dev)
 842 {
 843         struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
 844
 845         hns_roce_free_cmq_desc(hr_dev, &priv->cmq.csq);
 846         hns_roce_free_cmq_desc(hr_dev, &priv->cmq.crq);
 847 }
 848
 849 static void hns_roce_cmq_setup_basic_desc(struct hns_roce_cmq_desc *desc,
 850                                           enum hns_roce_opcode_type opcode,
 851                                           bool is_read)
 852 {
 853         memset((void *)desc, 0, sizeof(struct hns_roce_cmq_desc));
 854         desc->opcode = cpu_to_le16(opcode);
 855         desc->flag =
 856                 cpu_to_le16(HNS_ROCE_CMD_FLAG_NO_INTR | HNS_ROCE_CMD_FLAG_IN);
 857         if (is_read)
 858                 desc->flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_WR);
 859         else
 860                 desc->flag &= cpu_to_le16(~HNS_ROCE_CMD_FLAG_WR);
 861 }
 862
 863 static int hns_roce_cmq_csq_done(struct hns_roce_dev *hr_dev)
 864 {
 865         struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
 866         u32 head = roce_read(hr_dev, ROCEE_TX_CMQ_HEAD_REG);
 867
 868         return head == priv->cmq.csq.next_to_use;
 869 }
 870
 871 static int hns_roce_cmq_csq_clean(struct hns_roce_dev *hr_dev)
 872 {
 873         struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
 874         struct hns_roce_v2_cmq_ring *csq = &priv->cmq.csq;
 875         struct hns_roce_cmq_desc *desc;
 876         u16 ntc = csq->next_to_clean;
 877         u32 head;
 878         int clean = 0;
 879
 880         desc = &csq->desc[ntc];
 881         head = roce_read(hr_dev, ROCEE_TX_CMQ_HEAD_REG);
 882         while (head != ntc) {
 883                 memset(desc, 0, sizeof(*desc));
 884                 ntc++;
 885                 if (ntc == csq->desc_num)
 886                         ntc = 0;
 887                 desc = &csq->desc[ntc];
 888                 clean++;
 889         }
 890         csq->next_to_clean = ntc;
 891
 892         return clean;
 893 }
 894
 895 static int hns_roce_cmq_send(struct hns_roce_dev *hr_dev,
 896                              struct hns_roce_cmq_desc *desc, int num)
 897 {
 898         struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
 899         struct hns_roce_v2_cmq_ring *csq = &priv->cmq.csq;
 900         struct hns_roce_cmq_desc *desc_to_use;
 901         bool complete = false;
 902         u32 timeout = 0;
 903         int handle = 0;
 904         u16 desc_ret;
 905         int ret = 0;
 906         int ntc;
 907
 908         if (hr_dev->is_reset)
 909                 return 0;
 910
 911         spin_lock_bh(&csq->lock);
 912
 913         if (num > hns_roce_cmq_space(csq)) {
 914                 spin_unlock_bh(&csq->lock);
 915                 return -EBUSY;
 916         }
 917
 918         /*
 919          * Record the location of desc in the cmq for this time
 920          * which will be use for hardware to write back
 921          */
 922         ntc = csq->next_to_use;
 923
 924         while (handle < num) {
 925                 desc_to_use = &csq->desc[csq->next_to_use];
 926                 *desc_to_use = desc[handle];
 927                 dev_dbg(hr_dev->dev, "set cmq desc:\n");
 928                 csq->next_to_use++;
 929                 if (csq->next_to_use == csq->desc_num)
 930                         csq->next_to_use = 0;
 931                 handle++;
 932         }
 933
 934         /* Write to hardware */
 935         roce_write(hr_dev, ROCEE_TX_CMQ_TAIL_REG, csq->next_to_use);
 936
 937         /*
 938          * If the command is sync, wait for the firmware to write back,
 939          * if multi descriptors to be sent, use the first one to check
 940          */
 941         if ((desc->flag) & HNS_ROCE_CMD_FLAG_NO_INTR) {
 942                 do {
 943                         if (hns_roce_cmq_csq_done(hr_dev))
 944                                 break;
 945                         udelay(1);
 946                         timeout++;
 947                 } while (timeout < priv->cmq.tx_timeout);
 948         }
 949
 950         if (hns_roce_cmq_csq_done(hr_dev)) {
 951                 complete = true;
 952                 handle = 0;
 953                 while (handle < num) {
 954                         /* get the result of hardware write back */
 955                         desc_to_use = &csq->desc[ntc];
 956                         desc[handle] = *desc_to_use;
 957                         dev_dbg(hr_dev->dev, "Get cmq desc:\n");
 958                         desc_ret = desc[handle].retval;
 959                         if (desc_ret == CMD_EXEC_SUCCESS)
 960                                 ret = 0;
 961                         else
 962                                 ret = -EIO;
 963                         priv->cmq.last_status = desc_ret;
 964                         ntc++;
 965                         handle++;
 966                         if (ntc == csq->desc_num)
 967                                 ntc = 0;
 968                 }
 969         }
 970
 971         if (!complete)
 972                 ret = -EAGAIN;
 973
 974         /* clean the command send queue */
 975         handle = hns_roce_cmq_csq_clean(hr_dev);
 976         if (handle != num)
 977                 dev_warn(hr_dev->dev, "Cleaned %d, need to clean %d\n",
 978                          handle, num);
 979
 980         spin_unlock_bh(&csq->lock);
 981
 982         return ret;
 983 }
 984
 985 static int hns_roce_cmq_query_hw_info(struct hns_roce_dev *hr_dev)
 986 {
 987         struct hns_roce_query_version *resp;
 988         struct hns_roce_cmq_desc desc;
 989         int ret;
 990
 991         hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_QUERY_HW_VER, true);
 992         ret = hns_roce_cmq_send(hr_dev, &desc, 1);
 993         if (ret)
 994                 return ret;
 995
 996         resp = (struct hns_roce_query_version *)desc.data;
 997         hr_dev->hw_rev = le32_to_cpu(resp->rocee_hw_version);
 998         hr_dev->vendor_id = hr_dev->pci_dev->vendor;
 999
1000         return 0;
1001 }
1002
1003 static int hns_roce_query_fw_ver(struct hns_roce_dev *hr_dev)
1004 {
1005         struct hns_roce_query_fw_info *resp;
1006         struct hns_roce_cmq_desc desc;
1007         int ret;
1008
1009         hns_roce_cmq_setup_basic_desc(&desc, HNS_QUERY_FW_VER, true);
1010         ret = hns_roce_cmq_send(hr_dev, &desc, 1);
1011         if (ret)
1012                 return ret;
1013
1014         resp = (struct hns_roce_query_fw_info *)desc.data;
1015         hr_dev->caps.fw_ver = (u64)(le32_to_cpu(resp->fw_ver));
1016
1017         return 0;
1018 }
1019
1020 static int hns_roce_config_global_param(struct hns_roce_dev *hr_dev)
1021 {
1022         struct hns_roce_cfg_global_param *req;
1023         struct hns_roce_cmq_desc desc;
1024
1025         hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_GLOBAL_PARAM,
1026                                       false);
1027
1028         req = (struct hns_roce_cfg_global_param *)desc.data;
1029         memset(req, 0, sizeof(*req));
1030         roce_set_field(req->time_cfg_udp_port,
1031                        CFG_GLOBAL_PARAM_DATA_0_ROCEE_TIME_1US_CFG_M,
1032                        CFG_GLOBAL_PARAM_DATA_0_ROCEE_TIME_1US_CFG_S, 0x3e8);
1033         roce_set_field(req->time_cfg_udp_port,
1034                        CFG_GLOBAL_PARAM_DATA_0_ROCEE_UDP_PORT_M,
1035                        CFG_GLOBAL_PARAM_DATA_0_ROCEE_UDP_PORT_S, 0x12b7);
1036
1037         return hns_roce_cmq_send(hr_dev, &desc, 1);
1038 }
1039
1040 static int hns_roce_query_pf_resource(struct hns_roce_dev *hr_dev)
1041 {
1042         struct hns_roce_cmq_desc desc[2];
1043         struct hns_roce_pf_res_a *req_a;
1044         struct hns_roce_pf_res_b *req_b;
1045         int ret;
1046         int i;
1047
1048         for (i = 0; i < 2; i++) {
1049                 hns_roce_cmq_setup_basic_desc(&desc[i],
1050                                               HNS_ROCE_OPC_QUERY_PF_RES, true);
1051
1052                 if (i == 0)
1053                         desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1054                 else
1055                         desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1056         }
1057
1058         ret = hns_roce_cmq_send(hr_dev, desc, 2);
1059         if (ret)
1060                 return ret;
1061
1062         req_a = (struct hns_roce_pf_res_a *)desc[0].data;
1063         req_b = (struct hns_roce_pf_res_b *)desc[1].data;
1064
1065         hr_dev->caps.qpc_bt_num = roce_get_field(req_a->qpc_bt_idx_num,
1066                                                  PF_RES_DATA_1_PF_QPC_BT_NUM_M,
1067                                                  PF_RES_DATA_1_PF_QPC_BT_NUM_S);
1068         hr_dev->caps.srqc_bt_num = roce_get_field(req_a->srqc_bt_idx_num,
1069                                                 PF_RES_DATA_2_PF_SRQC_BT_NUM_M,
1070                                                 PF_RES_DATA_2_PF_SRQC_BT_NUM_S);
1071         hr_dev->caps.cqc_bt_num = roce_get_field(req_a->cqc_bt_idx_num,
1072                                                  PF_RES_DATA_3_PF_CQC_BT_NUM_M,
1073                                                  PF_RES_DATA_3_PF_CQC_BT_NUM_S);
1074         hr_dev->caps.mpt_bt_num = roce_get_field(req_a->mpt_bt_idx_num,
1075                                                  PF_RES_DATA_4_PF_MPT_BT_NUM_M,
1076                                                  PF_RES_DATA_4_PF_MPT_BT_NUM_S);
1077
1078         hr_dev->caps.sl_num = roce_get_field(req_b->qid_idx_sl_num,
1079                                              PF_RES_DATA_3_PF_SL_NUM_M,
1080                                              PF_RES_DATA_3_PF_SL_NUM_S);
1081
1082         return 0;
1083 }
1084
1085 static int hns_roce_alloc_vf_resource(struct hns_roce_dev *hr_dev)
1086 {
1087         struct hns_roce_cmq_desc desc[2];
1088         struct hns_roce_vf_res_a *req_a;
1089         struct hns_roce_vf_res_b *req_b;
1090         int i;
1091
1092         req_a = (struct hns_roce_vf_res_a *)desc[0].data;
1093         req_b = (struct hns_roce_vf_res_b *)desc[1].data;
1094         memset(req_a, 0, sizeof(*req_a));
1095         memset(req_b, 0, sizeof(*req_b));
1096         for (i = 0; i < 2; i++) {
1097                 hns_roce_cmq_setup_basic_desc(&desc[i],
1098                                               HNS_ROCE_OPC_ALLOC_VF_RES, false);
1099
1100                 if (i == 0)
1101                         desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1102                 else
1103                         desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1104
1105                 if (i == 0) {
1106                         roce_set_field(req_a->vf_qpc_bt_idx_num,
1107                                        VF_RES_A_DATA_1_VF_QPC_BT_IDX_M,
1108                                        VF_RES_A_DATA_1_VF_QPC_BT_IDX_S, 0);
1109                         roce_set_field(req_a->vf_qpc_bt_idx_num,
1110                                        VF_RES_A_DATA_1_VF_QPC_BT_NUM_M,
1111                                        VF_RES_A_DATA_1_VF_QPC_BT_NUM_S,
1112                                        HNS_ROCE_VF_QPC_BT_NUM);
1113
1114                         roce_set_field(req_a->vf_srqc_bt_idx_num,
1115                                        VF_RES_A_DATA_2_VF_SRQC_BT_IDX_M,
1116                                        VF_RES_A_DATA_2_VF_SRQC_BT_IDX_S, 0);
1117                         roce_set_field(req_a->vf_srqc_bt_idx_num,
1118                                        VF_RES_A_DATA_2_VF_SRQC_BT_NUM_M,
1119                                        VF_RES_A_DATA_2_VF_SRQC_BT_NUM_S,
1120                                        HNS_ROCE_VF_SRQC_BT_NUM);
1121
1122                         roce_set_field(req_a->vf_cqc_bt_idx_num,
1123                                        VF_RES_A_DATA_3_VF_CQC_BT_IDX_M,
1124                                        VF_RES_A_DATA_3_VF_CQC_BT_IDX_S, 0);
1125                         roce_set_field(req_a->vf_cqc_bt_idx_num,
1126                                        VF_RES_A_DATA_3_VF_CQC_BT_NUM_M,
1127                                        VF_RES_A_DATA_3_VF_CQC_BT_NUM_S,
1128                                        HNS_ROCE_VF_CQC_BT_NUM);
1129
1130                         roce_set_field(req_a->vf_mpt_bt_idx_num,
1131                                        VF_RES_A_DATA_4_VF_MPT_BT_IDX_M,
1132                                        VF_RES_A_DATA_4_VF_MPT_BT_IDX_S, 0);
1133                         roce_set_field(req_a->vf_mpt_bt_idx_num,
1134                                        VF_RES_A_DATA_4_VF_MPT_BT_NUM_M,
1135                                        VF_RES_A_DATA_4_VF_MPT_BT_NUM_S,
1136                                        HNS_ROCE_VF_MPT_BT_NUM);
1137
1138                         roce_set_field(req_a->vf_eqc_bt_idx_num,
1139                                        VF_RES_A_DATA_5_VF_EQC_IDX_M,
1140                                        VF_RES_A_DATA_5_VF_EQC_IDX_S, 0);
1141                         roce_set_field(req_a->vf_eqc_bt_idx_num,
1142                                        VF_RES_A_DATA_5_VF_EQC_NUM_M,
1143                                        VF_RES_A_DATA_5_VF_EQC_NUM_S,
1144                                        HNS_ROCE_VF_EQC_NUM);
1145                 } else {
1146                         roce_set_field(req_b->vf_smac_idx_num,
1147                                        VF_RES_B_DATA_1_VF_SMAC_IDX_M,
1148                                        VF_RES_B_DATA_1_VF_SMAC_IDX_S, 0);
1149                         roce_set_field(req_b->vf_smac_idx_num,
1150                                        VF_RES_B_DATA_1_VF_SMAC_NUM_M,
1151                                        VF_RES_B_DATA_1_VF_SMAC_NUM_S,
1152                                        HNS_ROCE_VF_SMAC_NUM);
1153
1154                         roce_set_field(req_b->vf_sgid_idx_num,
1155                                        VF_RES_B_DATA_2_VF_SGID_IDX_M,
1156                                        VF_RES_B_DATA_2_VF_SGID_IDX_S, 0);
1157                         roce_set_field(req_b->vf_sgid_idx_num,
1158                                        VF_RES_B_DATA_2_VF_SGID_NUM_M,
1159                                        VF_RES_B_DATA_2_VF_SGID_NUM_S,
1160                                        HNS_ROCE_VF_SGID_NUM);
1161
1162                         roce_set_field(req_b->vf_qid_idx_sl_num,
1163                                        VF_RES_B_DATA_3_VF_QID_IDX_M,
1164                                        VF_RES_B_DATA_3_VF_QID_IDX_S, 0);
1165                         roce_set_field(req_b->vf_qid_idx_sl_num,
1166                                        VF_RES_B_DATA_3_VF_SL_NUM_M,
1167                                        VF_RES_B_DATA_3_VF_SL_NUM_S,
1168                                        HNS_ROCE_VF_SL_NUM);
1169                 }
1170         }
1171
1172         return hns_roce_cmq_send(hr_dev, desc, 2);
1173 }
1174
1175 static int hns_roce_v2_set_bt(struct hns_roce_dev *hr_dev)
1176 {
1177         u8 srqc_hop_num = hr_dev->caps.srqc_hop_num;
1178         u8 qpc_hop_num = hr_dev->caps.qpc_hop_num;
1179         u8 cqc_hop_num = hr_dev->caps.cqc_hop_num;
1180         u8 mpt_hop_num = hr_dev->caps.mpt_hop_num;
1181         struct hns_roce_cfg_bt_attr *req;
1182         struct hns_roce_cmq_desc desc;
1183
1184         hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_BT_ATTR, false);
1185         req = (struct hns_roce_cfg_bt_attr *)desc.data;
1186         memset(req, 0, sizeof(*req));
1187
1188         roce_set_field(req->vf_qpc_cfg, CFG_BT_ATTR_DATA_0_VF_QPC_BA_PGSZ_M,
1189                        CFG_BT_ATTR_DATA_0_VF_QPC_BA_PGSZ_S,
1190                        hr_dev->caps.qpc_ba_pg_sz + PG_SHIFT_OFFSET);
1191         roce_set_field(req->vf_qpc_cfg, CFG_BT_ATTR_DATA_0_VF_QPC_BUF_PGSZ_M,
1192                        CFG_BT_ATTR_DATA_0_VF_QPC_BUF_PGSZ_S,
1193                        hr_dev->caps.qpc_buf_pg_sz + PG_SHIFT_OFFSET);
1194         roce_set_field(req->vf_qpc_cfg, CFG_BT_ATTR_DATA_0_VF_QPC_HOPNUM_M,
1195                        CFG_BT_ATTR_DATA_0_VF_QPC_HOPNUM_S,
1196                        qpc_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : qpc_hop_num);
1197
1198         roce_set_field(req->vf_srqc_cfg, CFG_BT_ATTR_DATA_1_VF_SRQC_BA_PGSZ_M,
1199                        CFG_BT_ATTR_DATA_1_VF_SRQC_BA_PGSZ_S,
1200                        hr_dev->caps.srqc_ba_pg_sz + PG_SHIFT_OFFSET);
1201         roce_set_field(req->vf_srqc_cfg, CFG_BT_ATTR_DATA_1_VF_SRQC_BUF_PGSZ_M,
1202                        CFG_BT_ATTR_DATA_1_VF_SRQC_BUF_PGSZ_S,
1203                        hr_dev->caps.srqc_buf_pg_sz + PG_SHIFT_OFFSET);
1204         roce_set_field(req->vf_srqc_cfg, CFG_BT_ATTR_DATA_1_VF_SRQC_HOPNUM_M,
1205                        CFG_BT_ATTR_DATA_1_VF_SRQC_HOPNUM_S,
1206                        srqc_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : srqc_hop_num);
1207
1208         roce_set_field(req->vf_cqc_cfg, CFG_BT_ATTR_DATA_2_VF_CQC_BA_PGSZ_M,
1209                        CFG_BT_ATTR_DATA_2_VF_CQC_BA_PGSZ_S,
1210                        hr_dev->caps.cqc_ba_pg_sz + PG_SHIFT_OFFSET);
1211         roce_set_field(req->vf_cqc_cfg, CFG_BT_ATTR_DATA_2_VF_CQC_BUF_PGSZ_M,
1212                        CFG_BT_ATTR_DATA_2_VF_CQC_BUF_PGSZ_S,
1213                        hr_dev->caps.cqc_buf_pg_sz + PG_SHIFT_OFFSET);
1214         roce_set_field(req->vf_cqc_cfg, CFG_BT_ATTR_DATA_2_VF_CQC_HOPNUM_M,
1215                        CFG_BT_ATTR_DATA_2_VF_CQC_HOPNUM_S,
1216                        cqc_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : cqc_hop_num);
1217
1218         roce_set_field(req->vf_mpt_cfg, CFG_BT_ATTR_DATA_3_VF_MPT_BA_PGSZ_M,
1219                        CFG_BT_ATTR_DATA_3_VF_MPT_BA_PGSZ_S,
1220                        hr_dev->caps.mpt_ba_pg_sz + PG_SHIFT_OFFSET);
1221         roce_set_field(req->vf_mpt_cfg, CFG_BT_ATTR_DATA_3_VF_MPT_BUF_PGSZ_M,
1222                        CFG_BT_ATTR_DATA_3_VF_MPT_BUF_PGSZ_S,
1223                        hr_dev->caps.mpt_buf_pg_sz + PG_SHIFT_OFFSET);
1224         roce_set_field(req->vf_mpt_cfg, CFG_BT_ATTR_DATA_3_VF_MPT_HOPNUM_M,
1225                        CFG_BT_ATTR_DATA_3_VF_MPT_HOPNUM_S,
1226                        mpt_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : mpt_hop_num);
1227
1228         return hns_roce_cmq_send(hr_dev, &desc, 1);
1229 }
1230
1231 static int hns_roce_v2_profile(struct hns_roce_dev *hr_dev)
1232 {
1233         struct hns_roce_caps *caps = &hr_dev->caps;
1234         int ret;
1235
1236         ret = hns_roce_cmq_query_hw_info(hr_dev);
1237         if (ret) {
1238                 dev_err(hr_dev->dev, "Query hardware version fail, ret = %d.\n",
1239                         ret);
1240                 return ret;
1241         }
1242
1243         ret = hns_roce_query_fw_ver(hr_dev);
1244         if (ret) {
1245                 dev_err(hr_dev->dev, "Query firmware version fail, ret = %d.\n",
1246                         ret);
1247                 return ret;
1248         }
1249
1250         ret = hns_roce_config_global_param(hr_dev);
1251         if (ret) {
1252                 dev_err(hr_dev->dev, "Configure global param fail, ret = %d.\n",
1253                         ret);
1254                 return ret;
1255         }
1256
1257         /* Get pf resource owned by every pf */
1258         ret = hns_roce_query_pf_resource(hr_dev);
1259         if (ret) {
1260                 dev_err(hr_dev->dev, "Query pf resource fail, ret = %d.\n",
1261                         ret);
1262                 return ret;
1263         }
1264
1265         ret = hns_roce_alloc_vf_resource(hr_dev);
1266         if (ret) {
1267                 dev_err(hr_dev->dev, "Allocate vf resource fail, ret = %d.\n",
1268                         ret);
1269                 return ret;
1270         }
1271
1272
1273         hr_dev->vendor_part_id = hr_dev->pci_dev->device;
1274         hr_dev->sys_image_guid = be64_to_cpu(hr_dev->ib_dev.node_guid);
1275
1276         caps->num_qps           = HNS_ROCE_V2_MAX_QP_NUM;
1277         caps->max_wqes          = HNS_ROCE_V2_MAX_WQE_NUM;
1278         caps->num_cqs           = HNS_ROCE_V2_MAX_CQ_NUM;
1279         caps->max_cqes          = HNS_ROCE_V2_MAX_CQE_NUM;
1280         caps->max_sq_sg         = HNS_ROCE_V2_MAX_SQ_SGE_NUM;
1281         caps->max_extend_sg     = HNS_ROCE_V2_MAX_EXTEND_SGE_NUM;
1282         caps->max_rq_sg         = HNS_ROCE_V2_MAX_RQ_SGE_NUM;
1283         caps->max_sq_inline     = HNS_ROCE_V2_MAX_SQ_INLINE;
1284         caps->num_uars          = HNS_ROCE_V2_UAR_NUM;
1285         caps->phy_num_uars      = HNS_ROCE_V2_PHY_UAR_NUM;
1286         caps->num_aeq_vectors   = HNS_ROCE_V2_AEQE_VEC_NUM;
1287         caps->num_comp_vectors  = HNS_ROCE_V2_COMP_VEC_NUM;
1288         caps->num_other_vectors = HNS_ROCE_V2_ABNORMAL_VEC_NUM;
1289         caps->num_mtpts         = HNS_ROCE_V2_MAX_MTPT_NUM;
1290         caps->num_mtt_segs      = HNS_ROCE_V2_MAX_MTT_SEGS;
1291         caps->num_cqe_segs      = HNS_ROCE_V2_MAX_CQE_SEGS;
1292         caps->num_pds           = HNS_ROCE_V2_MAX_PD_NUM;
1293         caps->max_qp_init_rdma  = HNS_ROCE_V2_MAX_QP_INIT_RDMA;
1294         caps->max_qp_dest_rdma  = HNS_ROCE_V2_MAX_QP_DEST_RDMA;
1295         caps->max_sq_desc_sz    = HNS_ROCE_V2_MAX_SQ_DESC_SZ;
1296         caps->max_rq_desc_sz    = HNS_ROCE_V2_MAX_RQ_DESC_SZ;
1297         caps->max_srq_desc_sz   = HNS_ROCE_V2_MAX_SRQ_DESC_SZ;
1298         caps->qpc_entry_sz      = HNS_ROCE_V2_QPC_ENTRY_SZ;
1299         caps->irrl_entry_sz     = HNS_ROCE_V2_IRRL_ENTRY_SZ;
1300         caps->trrl_entry_sz     = HNS_ROCE_V2_TRRL_ENTRY_SZ;
1301         caps->cqc_entry_sz      = HNS_ROCE_V2_CQC_ENTRY_SZ;
1302         caps->mtpt_entry_sz     = HNS_ROCE_V2_MTPT_ENTRY_SZ;
1303         caps->mtt_entry_sz      = HNS_ROCE_V2_MTT_ENTRY_SZ;
1304         caps->cq_entry_sz       = HNS_ROCE_V2_CQE_ENTRY_SIZE;
1305         caps->page_size_cap     = HNS_ROCE_V2_PAGE_SIZE_SUPPORTED;
1306         caps->reserved_lkey     = 0;
1307         caps->reserved_pds      = 0;
1308         caps->reserved_mrws     = 1;
1309         caps->reserved_uars     = 0;
1310         caps->reserved_cqs      = 0;
1311         caps->reserved_qps      = HNS_ROCE_V2_RSV_QPS;
1312
1313         caps->qpc_ba_pg_sz      = 0;
1314         caps->qpc_buf_pg_sz     = 0;
1315         caps->qpc_hop_num       = HNS_ROCE_CONTEXT_HOP_NUM;
1316         caps->srqc_ba_pg_sz     = 0;
1317         caps->srqc_buf_pg_sz    = 0;
1318         caps->srqc_hop_num      = HNS_ROCE_HOP_NUM_0;
1319         caps->cqc_ba_pg_sz      = 0;
1320         caps->cqc_buf_pg_sz     = 0;
1321         caps->cqc_hop_num       = HNS_ROCE_CONTEXT_HOP_NUM;
1322         caps->mpt_ba_pg_sz      = 0;
1323         caps->mpt_buf_pg_sz     = 0;
1324         caps->mpt_hop_num       = HNS_ROCE_CONTEXT_HOP_NUM;
1325         caps->pbl_ba_pg_sz      = 0;
1326         caps->pbl_buf_pg_sz     = 0;
1327         caps->pbl_hop_num       = HNS_ROCE_PBL_HOP_NUM;
1328         caps->mtt_ba_pg_sz      = 0;
1329         caps->mtt_buf_pg_sz     = 0;
1330         caps->mtt_hop_num       = HNS_ROCE_MTT_HOP_NUM;
1331         caps->cqe_ba_pg_sz      = 0;
1332         caps->cqe_buf_pg_sz     = 0;
1333         caps->cqe_hop_num       = HNS_ROCE_CQE_HOP_NUM;
1334         caps->eqe_ba_pg_sz      = 0;
1335         caps->eqe_buf_pg_sz     = 0;
1336         caps->eqe_hop_num       = HNS_ROCE_EQE_HOP_NUM;
1337         caps->tsq_buf_pg_sz     = 0;
1338         caps->chunk_sz          = HNS_ROCE_V2_TABLE_CHUNK_SIZE;
1339
1340         caps->flags             = HNS_ROCE_CAP_FLAG_REREG_MR |
1341                                   HNS_ROCE_CAP_FLAG_ROCE_V1_V2 |
1342                                   HNS_ROCE_CAP_FLAG_RQ_INLINE |
1343                                   HNS_ROCE_CAP_FLAG_RECORD_DB |
1344                                   HNS_ROCE_CAP_FLAG_SQ_RECORD_DB;
1345
1346         if (hr_dev->pci_dev->revision == 0x21)
1347                 caps->flags |= HNS_ROCE_CAP_FLAG_MW |
1348                                HNS_ROCE_CAP_FLAG_FRMR;
1349
1350         caps->pkey_table_len[0] = 1;
1351         caps->gid_table_len[0] = HNS_ROCE_V2_GID_INDEX_NUM;
1352         caps->ceqe_depth        = HNS_ROCE_V2_COMP_EQE_NUM;
1353         caps->aeqe_depth        = HNS_ROCE_V2_ASYNC_EQE_NUM;
1354         caps->local_ca_ack_delay = 0;
1355         caps->max_mtu = IB_MTU_4096;
1356
1357         if (hr_dev->pci_dev->revision == 0x21)
1358                 caps->flags |= HNS_ROCE_CAP_FLAG_ATOMIC;
1359
1360         ret = hns_roce_v2_set_bt(hr_dev);
1361         if (ret)
1362                 dev_err(hr_dev->dev, "Configure bt attribute fail, ret = %d.\n",
1363                         ret);
1364
1365         return ret;
1366 }
1367
1368 static int hns_roce_config_link_table(struct hns_roce_dev *hr_dev,
1369                                       enum hns_roce_link_table_type type)
1370 {
1371         struct hns_roce_cmq_desc desc[2];
1372         struct hns_roce_cfg_llm_a *req_a =
1373                                 (struct hns_roce_cfg_llm_a *)desc[0].data;
1374         struct hns_roce_cfg_llm_b *req_b =
1375                                 (struct hns_roce_cfg_llm_b *)desc[1].data;
1376         struct hns_roce_v2_priv *priv = hr_dev->priv;
1377         struct hns_roce_link_table *link_tbl;
1378         struct hns_roce_link_table_entry *entry;
1379         enum hns_roce_opcode_type opcode;
1380         u32 page_num;
1381         int i;
1382
1383         switch (type) {
1384         case TSQ_LINK_TABLE:
1385                 link_tbl = &priv->tsq;
1386                 opcode = HNS_ROCE_OPC_CFG_EXT_LLM;
1387                 break;
1388         case TPQ_LINK_TABLE:
1389                 link_tbl = &priv->tpq;
1390                 opcode = HNS_ROCE_OPC_CFG_TMOUT_LLM;
1391                 break;
1392         default:
1393                 return -EINVAL;
1394         }
1395
1396         page_num = link_tbl->npages;
1397         entry = link_tbl->table.buf;
1398         memset(req_a, 0, sizeof(*req_a));
1399         memset(req_b, 0, sizeof(*req_b));
1400
1401         for (i = 0; i < 2; i++) {
1402                 hns_roce_cmq_setup_basic_desc(&desc[i], opcode, false);
1403
1404                 if (i == 0)
1405                         desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1406                 else
1407                         desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
1408
1409                 if (i == 0) {
1410                         req_a->base_addr_l = link_tbl->table.map & 0xffffffff;
1411                         req_a->base_addr_h = (link_tbl->table.map >> 32) &
1412                                              0xffffffff;
1413                         roce_set_field(req_a->depth_pgsz_init_en,
1414                                        CFG_LLM_QUE_DEPTH_M,
1415                                        CFG_LLM_QUE_DEPTH_S,
1416                                        link_tbl->npages);
1417                         roce_set_field(req_a->depth_pgsz_init_en,
1418                                        CFG_LLM_QUE_PGSZ_M,
1419                                        CFG_LLM_QUE_PGSZ_S,
1420                                        link_tbl->pg_sz);
1421                         req_a->head_ba_l = entry[0].blk_ba0;
1422                         req_a->head_ba_h_nxtptr = entry[0].blk_ba1_nxt_ptr;
1423                         roce_set_field(req_a->head_ptr,
1424                                        CFG_LLM_HEAD_PTR_M,
1425                                        CFG_LLM_HEAD_PTR_S, 0);
1426                 } else {
1427                         req_b->tail_ba_l = entry[page_num - 1].blk_ba0;
1428                         roce_set_field(req_b->tail_ba_h,
1429                                        CFG_LLM_TAIL_BA_H_M,
1430                                        CFG_LLM_TAIL_BA_H_S,
1431                                        entry[page_num - 1].blk_ba1_nxt_ptr &
1432                                        HNS_ROCE_LINK_TABLE_BA1_M);
1433                         roce_set_field(req_b->tail_ptr,
1434                                        CFG_LLM_TAIL_PTR_M,
1435                                        CFG_LLM_TAIL_PTR_S,
1436                                        (entry[page_num - 2].blk_ba1_nxt_ptr &
1437                                        HNS_ROCE_LINK_TABLE_NXT_PTR_M) >>
1438                                        HNS_ROCE_LINK_TABLE_NXT_PTR_S);
1439                 }
1440         }
1441         roce_set_field(req_a->depth_pgsz_init_en,
1442                        CFG_LLM_INIT_EN_M, CFG_LLM_INIT_EN_S, 1);
1443
1444         return hns_roce_cmq_send(hr_dev, desc, 2);
1445 }
1446
1447 static int hns_roce_init_link_table(struct hns_roce_dev *hr_dev,
1448                                     enum hns_roce_link_table_type type)
1449 {
1450         struct hns_roce_v2_priv *priv = hr_dev->priv;
1451         struct hns_roce_link_table *link_tbl;
1452         struct hns_roce_link_table_entry *entry;
1453         struct device *dev = hr_dev->dev;
1454         u32 buf_chk_sz;
1455         dma_addr_t t;
1456         int func_num = 1;
1457         int pg_num_a;
1458         int pg_num_b;
1459         int pg_num;
1460         int size;
1461         int i;
1462
1463         switch (type) {
1464         case TSQ_LINK_TABLE:
1465                 link_tbl = &priv->tsq;
1466                 buf_chk_sz = 1 << (hr_dev->caps.tsq_buf_pg_sz + PAGE_SHIFT);
1467                 pg_num_a = hr_dev->caps.num_qps * 8 / buf_chk_sz;
1468                 pg_num_b = hr_dev->caps.sl_num * 4 + 2;
1469                 break;
1470         case TPQ_LINK_TABLE:
1471                 link_tbl = &priv->tpq;
1472                 buf_chk_sz = 1 << (hr_dev->caps.tpq_buf_pg_sz + PAGE_SHIFT);
1473                 pg_num_a = hr_dev->caps.num_cqs * 4 / buf_chk_sz;
1474                 pg_num_b = 2 * 4 * func_num + 2;
1475                 break;
1476         default:
1477                 return -EINVAL;
1478         }
1479
1480         pg_num = max(pg_num_a, pg_num_b);
1481         size = pg_num * sizeof(struct hns_roce_link_table_entry);
1482
1483         link_tbl->table.buf = dma_alloc_coherent(dev, size,
1484                                                  &link_tbl->table.map,
1485                                                  GFP_KERNEL);
1486         if (!link_tbl->table.buf)
1487                 goto out;
1488
1489         link_tbl->pg_list = kcalloc(pg_num, sizeof(*link_tbl->pg_list),
1490                                     GFP_KERNEL);
1491         if (!link_tbl->pg_list)
1492                 goto err_kcalloc_failed;
1493
1494         entry = link_tbl->table.buf;
1495         for (i = 0; i < pg_num; ++i) {
1496                 link_tbl->pg_list[i].buf = dma_alloc_coherent(dev, buf_chk_sz,
1497                                                               &t, GFP_KERNEL);
1498                 if (!link_tbl->pg_list[i].buf)
1499                         goto err_alloc_buf_failed;
1500
1501                 link_tbl->pg_list[i].map = t;
1502                 memset(link_tbl->pg_list[i].buf, 0, buf_chk_sz);
1503
1504                 entry[i].blk_ba0 = (t >> 12) & 0xffffffff;
1505                 roce_set_field(entry[i].blk_ba1_nxt_ptr,
1506                                HNS_ROCE_LINK_TABLE_BA1_M,
1507                                HNS_ROCE_LINK_TABLE_BA1_S,
1508                                t >> 44);
1509
1510                 if (i < (pg_num - 1))
1511                         roce_set_field(entry[i].blk_ba1_nxt_ptr,
1512                                        HNS_ROCE_LINK_TABLE_NXT_PTR_M,
1513                                        HNS_ROCE_LINK_TABLE_NXT_PTR_S,
1514                                        i + 1);
1515         }
1516         link_tbl->npages = pg_num;
1517         link_tbl->pg_sz = buf_chk_sz;
1518
1519         return hns_roce_config_link_table(hr_dev, type);
1520
1521 err_alloc_buf_failed:
1522         for (i -= 1; i >= 0; i--)
1523                 dma_free_coherent(dev, buf_chk_sz,
1524                                   link_tbl->pg_list[i].buf,
1525                                   link_tbl->pg_list[i].map);
1526         kfree(link_tbl->pg_list);
1527
1528 err_kcalloc_failed:
1529         dma_free_coherent(dev, size, link_tbl->table.buf,
1530                           link_tbl->table.map);
1531
1532 out:
1533         return -ENOMEM;
1534 }
1535
1536 static void hns_roce_free_link_table(struct hns_roce_dev *hr_dev,
1537                                      struct hns_roce_link_table *link_tbl)
1538 {
1539         struct device *dev = hr_dev->dev;
1540         int size;
1541         int i;
1542
1543         size = link_tbl->npages * sizeof(struct hns_roce_link_table_entry);
1544
1545         for (i = 0; i < link_tbl->npages; ++i)
1546                 if (link_tbl->pg_list[i].buf)
1547                         dma_free_coherent(dev, link_tbl->pg_sz,
1548                                           link_tbl->pg_list[i].buf,
1549                                           link_tbl->pg_list[i].map);
1550         kfree(link_tbl->pg_list);
1551
1552         dma_free_coherent(dev, size, link_tbl->table.buf,
1553                           link_tbl->table.map);
1554 }
1555
1556 static int hns_roce_v2_init(struct hns_roce_dev *hr_dev)
1557 {
1558         struct hns_roce_v2_priv *priv = hr_dev->priv;
1559         int ret;
1560
1561         /* TSQ includes SQ doorbell and ack doorbell */
1562         ret = hns_roce_init_link_table(hr_dev, TSQ_LINK_TABLE);
1563         if (ret) {
1564                 dev_err(hr_dev->dev, "TSQ init failed, ret = %d.\n", ret);
1565                 return ret;
1566         }
1567
1568         ret = hns_roce_init_link_table(hr_dev, TPQ_LINK_TABLE);
1569         if (ret) {
1570                 dev_err(hr_dev->dev, "TPQ init failed, ret = %d.\n", ret);
1571                 goto err_tpq_init_failed;
1572         }
1573
1574         return 0;
1575
1576 err_tpq_init_failed:
1577         hns_roce_free_link_table(hr_dev, &priv->tsq);
1578
1579         return ret;
1580 }
1581
1582 static void hns_roce_v2_exit(struct hns_roce_dev *hr_dev)
1583 {
1584         struct hns_roce_v2_priv *priv = hr_dev->priv;
1585
1586         hns_roce_free_link_table(hr_dev, &priv->tpq);
1587         hns_roce_free_link_table(hr_dev, &priv->tsq);
1588 }
1589
1590 static int hns_roce_v2_cmd_pending(struct hns_roce_dev *hr_dev)
1591 {
1592         u32 status = readl(hr_dev->reg_base + ROCEE_VF_MB_STATUS_REG);
1593
1594         return status >> HNS_ROCE_HW_RUN_BIT_SHIFT;
1595 }
1596
1597 static int hns_roce_v2_cmd_complete(struct hns_roce_dev *hr_dev)
1598 {
1599         u32 status = readl(hr_dev->reg_base + ROCEE_VF_MB_STATUS_REG);
1600
1601         return status & HNS_ROCE_HW_MB_STATUS_MASK;
1602 }
1603
1604 static int hns_roce_v2_post_mbox(struct hns_roce_dev *hr_dev, u64 in_param,
1605                                  u64 out_param, u32 in_modifier, u8 op_modifier,
1606                                  u16 op, u16 token, int event)
1607 {
1608         struct device *dev = hr_dev->dev;
1609         u32 __iomem *hcr = (u32 __iomem *)(hr_dev->reg_base +
1610                                            ROCEE_VF_MB_CFG0_REG);
1611         unsigned long end;
1612         u32 val0 = 0;
1613         u32 val1 = 0;
1614
1615         end = msecs_to_jiffies(HNS_ROCE_V2_GO_BIT_TIMEOUT_MSECS) + jiffies;
1616         while (hns_roce_v2_cmd_pending(hr_dev)) {
1617                 if (time_after(jiffies, end)) {
1618                         dev_dbg(dev, "jiffies=%d end=%d\n", (int)jiffies,
1619                                 (int)end);
1620                         return -EAGAIN;
1621                 }
1622                 cond_resched();
1623         }
1624
1625         roce_set_field(val0, HNS_ROCE_VF_MB4_TAG_MASK,
1626                        HNS_ROCE_VF_MB4_TAG_SHIFT, in_modifier);
1627         roce_set_field(val0, HNS_ROCE_VF_MB4_CMD_MASK,
1628                        HNS_ROCE_VF_MB4_CMD_SHIFT, op);
1629         roce_set_field(val1, HNS_ROCE_VF_MB5_EVENT_MASK,
1630                        HNS_ROCE_VF_MB5_EVENT_SHIFT, event);
1631         roce_set_field(val1, HNS_ROCE_VF_MB5_TOKEN_MASK,
1632                        HNS_ROCE_VF_MB5_TOKEN_SHIFT, token);
1633
1634         writeq(in_param, hcr + 0);
1635         writeq(out_param, hcr + 2);
1636
1637         /* Memory barrier */
1638         wmb();
1639
1640         writel(val0, hcr + 4);
1641         writel(val1, hcr + 5);
1642
1643         mmiowb();
1644
1645         return 0;
1646 }
1647
1648 static int hns_roce_v2_chk_mbox(struct hns_roce_dev *hr_dev,
1649                                 unsigned long timeout)
1650 {
1651         struct device *dev = hr_dev->dev;
1652         unsigned long end = 0;
1653         u32 status;
1654
1655         end = msecs_to_jiffies(timeout) + jiffies;
1656         while (hns_roce_v2_cmd_pending(hr_dev) && time_before(jiffies, end))
1657                 cond_resched();
1658
1659         if (hns_roce_v2_cmd_pending(hr_dev)) {
1660                 dev_err(dev, "[cmd_poll]hw run cmd TIMEDOUT!\n");
1661                 return -ETIMEDOUT;
1662         }
1663
1664         status = hns_roce_v2_cmd_complete(hr_dev);
1665         if (status != 0x1) {
1666                 dev_err(dev, "mailbox status 0x%x!\n", status);
1667                 return -EBUSY;
1668         }
1669
1670         return 0;
1671 }
1672
1673 static int hns_roce_config_sgid_table(struct hns_roce_dev *hr_dev,
1674                                       int gid_index, const union ib_gid *gid,
1675                                       enum hns_roce_sgid_type sgid_type)
1676 {
1677         struct hns_roce_cmq_desc desc;
1678         struct hns_roce_cfg_sgid_tb *sgid_tb =
1679                                     (struct hns_roce_cfg_sgid_tb *)desc.data;
1680         u32 *p;
1681
1682         hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_SGID_TB, false);
1683
1684         roce_set_field(sgid_tb->table_idx_rsv,
1685                        CFG_SGID_TB_TABLE_IDX_M,
1686                        CFG_SGID_TB_TABLE_IDX_S, gid_index);
1687         roce_set_field(sgid_tb->vf_sgid_type_rsv,
1688                        CFG_SGID_TB_VF_SGID_TYPE_M,
1689                        CFG_SGID_TB_VF_SGID_TYPE_S, sgid_type);
1690
1691         p = (u32 *)&gid->raw[0];
1692         sgid_tb->vf_sgid_l = cpu_to_le32(*p);
1693
1694         p = (u32 *)&gid->raw[4];
1695         sgid_tb->vf_sgid_ml = cpu_to_le32(*p);
1696
1697         p = (u32 *)&gid->raw[8];
1698         sgid_tb->vf_sgid_mh = cpu_to_le32(*p);
1699
1700         p = (u32 *)&gid->raw[0xc];
1701         sgid_tb->vf_sgid_h = cpu_to_le32(*p);
1702
1703         return hns_roce_cmq_send(hr_dev, &desc, 1);
1704 }
1705
1706 static int hns_roce_v2_set_gid(struct hns_roce_dev *hr_dev, u8 port,
1707                                int gid_index, const union ib_gid *gid,
1708                                const struct ib_gid_attr *attr)
1709 {
1710         enum hns_roce_sgid_type sgid_type = GID_TYPE_FLAG_ROCE_V1;
1711         int ret;
1712
1713         if (!gid || !attr)
1714                 return -EINVAL;
1715
1716         if (attr->gid_type == IB_GID_TYPE_ROCE)
1717                 sgid_type = GID_TYPE_FLAG_ROCE_V1;
1718
1719         if (attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) {
1720                 if (ipv6_addr_v4mapped((void *)gid))
1721                         sgid_type = GID_TYPE_FLAG_ROCE_V2_IPV4;
1722                 else
1723                         sgid_type = GID_TYPE_FLAG_ROCE_V2_IPV6;
1724         }
1725
1726         ret = hns_roce_config_sgid_table(hr_dev, gid_index, gid, sgid_type);
1727         if (ret)
1728                 dev_err(hr_dev->dev, "Configure sgid table failed(%d)!\n", ret);
1729
1730         return ret;
1731 }
1732
1733 static int hns_roce_v2_set_mac(struct hns_roce_dev *hr_dev, u8 phy_port,
1734                                u8 *addr)
1735 {
1736         struct hns_roce_cmq_desc desc;
1737         struct hns_roce_cfg_smac_tb *smac_tb =
1738                                     (struct hns_roce_cfg_smac_tb *)desc.data;
1739         u16 reg_smac_h;
1740         u32 reg_smac_l;
1741
1742         hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_CFG_SMAC_TB, false);
1743
1744         reg_smac_l = *(u32 *)(&addr[0]);
1745         reg_smac_h = *(u16 *)(&addr[4]);
1746
1747         memset(smac_tb, 0, sizeof(*smac_tb));
1748         roce_set_field(smac_tb->tb_idx_rsv,
1749                        CFG_SMAC_TB_IDX_M,
1750                        CFG_SMAC_TB_IDX_S, phy_port);
1751         roce_set_field(smac_tb->vf_smac_h_rsv,
1752                        CFG_SMAC_TB_VF_SMAC_H_M,
1753                        CFG_SMAC_TB_VF_SMAC_H_S, reg_smac_h);
1754         smac_tb->vf_smac_l = reg_smac_l;
1755
1756         return hns_roce_cmq_send(hr_dev, &desc, 1);
1757 }
1758
1759 static int set_mtpt_pbl(struct hns_roce_v2_mpt_entry *mpt_entry,
1760                         struct hns_roce_mr *mr)
1761 {
1762         struct scatterlist *sg;
1763         u64 page_addr;
1764         u64 *pages;
1765         int i, j;
1766         int len;
1767         int entry;
1768
1769         mpt_entry->pbl_size = cpu_to_le32(mr->pbl_size);
1770         mpt_entry->pbl_ba_l = cpu_to_le32(lower_32_bits(mr->pbl_ba >> 3));
1771         roce_set_field(mpt_entry->byte_48_mode_ba,
1772                        V2_MPT_BYTE_48_PBL_BA_H_M, V2_MPT_BYTE_48_PBL_BA_H_S,
1773                        upper_32_bits(mr->pbl_ba >> 3));
1774
1775         pages = (u64 *)__get_free_page(GFP_KERNEL);
1776         if (!pages)
1777                 return -ENOMEM;
1778
1779         i = 0;
1780         for_each_sg(mr->umem->sg_head.sgl, sg, mr->umem->nmap, entry) {
1781                 len = sg_dma_len(sg) >> PAGE_SHIFT;
1782                 for (j = 0; j < len; ++j) {
1783                         page_addr = sg_dma_address(sg) +
1784                                 (j << mr->umem->page_shift);
1785                         pages[i] = page_addr >> 6;
1786                         /* Record the first 2 entry directly to MTPT table */
1787                         if (i >= HNS_ROCE_V2_MAX_INNER_MTPT_NUM - 1)
1788                                 goto found;
1789                         i++;
1790                 }
1791         }
1792 found:
1793         mpt_entry->pa0_l = cpu_to_le32(lower_32_bits(pages[0]));
1794         roce_set_field(mpt_entry->byte_56_pa0_h, V2_MPT_BYTE_56_PA0_H_M,
1795                        V2_MPT_BYTE_56_PA0_H_S, upper_32_bits(pages[0]));
1796
1797         mpt_entry->pa1_l = cpu_to_le32(lower_32_bits(pages[1]));
1798         roce_set_field(mpt_entry->byte_64_buf_pa1, V2_MPT_BYTE_64_PA1_H_M,
1799                        V2_MPT_BYTE_64_PA1_H_S, upper_32_bits(pages[1]));
1800         roce_set_field(mpt_entry->byte_64_buf_pa1,
1801                        V2_MPT_BYTE_64_PBL_BUF_PG_SZ_M,
1802                        V2_MPT_BYTE_64_PBL_BUF_PG_SZ_S,
1803                        mr->pbl_buf_pg_sz + PG_SHIFT_OFFSET);
1804
1805         free_page((unsigned long)pages);
1806
1807         return 0;
1808 }
1809
1810 static int hns_roce_v2_write_mtpt(void *mb_buf, struct hns_roce_mr *mr,
1811                                   unsigned long mtpt_idx)
1812 {
1813         struct hns_roce_v2_mpt_entry *mpt_entry;
1814         int ret;
1815
1816         mpt_entry = mb_buf;
1817         memset(mpt_entry, 0, sizeof(*mpt_entry));
1818
1819         roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_MPT_ST_M,
1820                        V2_MPT_BYTE_4_MPT_ST_S, V2_MPT_ST_VALID);
1821         roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PBL_HOP_NUM_M,
1822                        V2_MPT_BYTE_4_PBL_HOP_NUM_S, mr->pbl_hop_num ==
1823                        HNS_ROCE_HOP_NUM_0 ? 0 : mr->pbl_hop_num);
1824         roce_set_field(mpt_entry->byte_4_pd_hop_st,
1825                        V2_MPT_BYTE_4_PBL_BA_PG_SZ_M,
1826                        V2_MPT_BYTE_4_PBL_BA_PG_SZ_S,
1827                        mr->pbl_ba_pg_sz + PG_SHIFT_OFFSET);
1828         roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M,
1829                        V2_MPT_BYTE_4_PD_S, mr->pd);
1830
1831         roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RA_EN_S, 0);
1832         roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_R_INV_EN_S, 1);
1833         roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_L_INV_EN_S, 1);
1834         roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_BIND_EN_S,
1835                      (mr->access & IB_ACCESS_MW_BIND ? 1 : 0));
1836         roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_ATOMIC_EN_S,
1837                      mr->access & IB_ACCESS_REMOTE_ATOMIC ? 1 : 0);
1838         roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RR_EN_S,
1839                      (mr->access & IB_ACCESS_REMOTE_READ ? 1 : 0));
1840         roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RW_EN_S,
1841                      (mr->access & IB_ACCESS_REMOTE_WRITE ? 1 : 0));
1842         roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_LW_EN_S,
1843                      (mr->access & IB_ACCESS_LOCAL_WRITE ? 1 : 0));
1844
1845         roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_PA_S,
1846                      mr->type == MR_TYPE_MR ? 0 : 1);
1847         roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_INNER_PA_VLD_S,
1848                      1);
1849
1850         mpt_entry->len_l = cpu_to_le32(lower_32_bits(mr->size));
1851         mpt_entry->len_h = cpu_to_le32(upper_32_bits(mr->size));
1852         mpt_entry->lkey = cpu_to_le32(mr->key);
1853         mpt_entry->va_l = cpu_to_le32(lower_32_bits(mr->iova));
1854         mpt_entry->va_h = cpu_to_le32(upper_32_bits(mr->iova));
1855
1856         if (mr->type == MR_TYPE_DMA)
1857                 return 0;
1858
1859         ret = set_mtpt_pbl(mpt_entry, mr);
1860
1861         return ret;
1862 }
1863
1864 static int hns_roce_v2_rereg_write_mtpt(struct hns_roce_dev *hr_dev,
1865                                         struct hns_roce_mr *mr, int flags,
1866                                         u32 pdn, int mr_access_flags, u64 iova,
1867                                         u64 size, void *mb_buf)
1868 {
1869         struct hns_roce_v2_mpt_entry *mpt_entry = mb_buf;
1870         int ret = 0;
1871
1872         if (flags & IB_MR_REREG_PD) {
1873                 roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M,
1874                                V2_MPT_BYTE_4_PD_S, pdn);
1875                 mr->pd = pdn;
1876         }
1877
1878         if (flags & IB_MR_REREG_ACCESS) {
1879                 roce_set_bit(mpt_entry->byte_8_mw_cnt_en,
1880                              V2_MPT_BYTE_8_BIND_EN_S,
1881                              (mr_access_flags & IB_ACCESS_MW_BIND ? 1 : 0));
1882                 roce_set_bit(mpt_entry->byte_8_mw_cnt_en,
1883                              V2_MPT_BYTE_8_ATOMIC_EN_S,
1884                              mr_access_flags & IB_ACCESS_REMOTE_ATOMIC ? 1 : 0);
1885                 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RR_EN_S,
1886                              mr_access_flags & IB_ACCESS_REMOTE_READ ? 1 : 0);
1887                 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RW_EN_S,
1888                              mr_access_flags & IB_ACCESS_REMOTE_WRITE ? 1 : 0);
1889                 roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_LW_EN_S,
1890                              mr_access_flags & IB_ACCESS_LOCAL_WRITE ? 1 : 0);
1891         }
1892
1893         if (flags & IB_MR_REREG_TRANS) {
1894                 mpt_entry->va_l = cpu_to_le32(lower_32_bits(iova));
1895                 mpt_entry->va_h = cpu_to_le32(upper_32_bits(iova));
1896                 mpt_entry->len_l = cpu_to_le32(lower_32_bits(size));
1897                 mpt_entry->len_h = cpu_to_le32(upper_32_bits(size));
1898
1899                 mr->iova = iova;
1900                 mr->size = size;
1901
1902                 ret = set_mtpt_pbl(mpt_entry, mr);
1903         }
1904
1905         return ret;
1906 }
1907
1908 static int hns_roce_v2_frmr_write_mtpt(void *mb_buf, struct hns_roce_mr *mr)
1909 {
1910         struct hns_roce_v2_mpt_entry *mpt_entry;
1911
1912         mpt_entry = mb_buf;
1913         memset(mpt_entry, 0, sizeof(*mpt_entry));
1914
1915         roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_MPT_ST_M,
1916                        V2_MPT_BYTE_4_MPT_ST_S, V2_MPT_ST_FREE);
1917         roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PBL_HOP_NUM_M,
1918                        V2_MPT_BYTE_4_PBL_HOP_NUM_S, 1);
1919         roce_set_field(mpt_entry->byte_4_pd_hop_st,
1920                        V2_MPT_BYTE_4_PBL_BA_PG_SZ_M,
1921                        V2_MPT_BYTE_4_PBL_BA_PG_SZ_S,
1922                        mr->pbl_ba_pg_sz + PG_SHIFT_OFFSET);
1923         roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M,
1924                        V2_MPT_BYTE_4_PD_S, mr->pd);
1925
1926         roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_RA_EN_S, 1);
1927         roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_R_INV_EN_S, 1);
1928         roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_L_INV_EN_S, 1);
1929
1930         roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_FRE_S, 1);
1931         roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_PA_S, 0);
1932         roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_MR_MW_S, 0);
1933         roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_BPD_S, 1);
1934
1935         mpt_entry->pbl_size = cpu_to_le32(mr->pbl_size);
1936
1937         mpt_entry->pbl_ba_l = cpu_to_le32(lower_32_bits(mr->pbl_ba >> 3));
1938         roce_set_field(mpt_entry->byte_48_mode_ba, V2_MPT_BYTE_48_PBL_BA_H_M,
1939                        V2_MPT_BYTE_48_PBL_BA_H_S,
1940                        upper_32_bits(mr->pbl_ba >> 3));
1941
1942         roce_set_field(mpt_entry->byte_64_buf_pa1,
1943                        V2_MPT_BYTE_64_PBL_BUF_PG_SZ_M,
1944                        V2_MPT_BYTE_64_PBL_BUF_PG_SZ_S,
1945                        mr->pbl_buf_pg_sz + PG_SHIFT_OFFSET);
1946
1947         return 0;
1948 }
1949
1950 static int hns_roce_v2_mw_write_mtpt(void *mb_buf, struct hns_roce_mw *mw)
1951 {
1952         struct hns_roce_v2_mpt_entry *mpt_entry;
1953
1954         mpt_entry = mb_buf;
1955         memset(mpt_entry, 0, sizeof(*mpt_entry));
1956
1957         roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_MPT_ST_M,
1958                        V2_MPT_BYTE_4_MPT_ST_S, V2_MPT_ST_FREE);
1959         roce_set_field(mpt_entry->byte_4_pd_hop_st, V2_MPT_BYTE_4_PD_M,
1960                        V2_MPT_BYTE_4_PD_S, mw->pdn);
1961         roce_set_field(mpt_entry->byte_4_pd_hop_st,
1962                        V2_MPT_BYTE_4_PBL_HOP_NUM_M,
1963                        V2_MPT_BYTE_4_PBL_HOP_NUM_S,
1964                        mw->pbl_hop_num == HNS_ROCE_HOP_NUM_0 ?
1965                        0 : mw->pbl_hop_num);
1966         roce_set_field(mpt_entry->byte_4_pd_hop_st,
1967                        V2_MPT_BYTE_4_PBL_BA_PG_SZ_M,
1968                        V2_MPT_BYTE_4_PBL_BA_PG_SZ_S,
1969                        mw->pbl_ba_pg_sz + PG_SHIFT_OFFSET);
1970
1971         roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_R_INV_EN_S, 1);
1972         roce_set_bit(mpt_entry->byte_8_mw_cnt_en, V2_MPT_BYTE_8_L_INV_EN_S, 1);
1973
1974         roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_PA_S, 0);
1975         roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_MR_MW_S, 1);
1976         roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_BPD_S, 1);
1977         roce_set_bit(mpt_entry->byte_12_mw_pa, V2_MPT_BYTE_12_BQP_S,
1978                      mw->ibmw.type == IB_MW_TYPE_1 ? 0 : 1);
1979
1980         roce_set_field(mpt_entry->byte_64_buf_pa1,
1981                        V2_MPT_BYTE_64_PBL_BUF_PG_SZ_M,
1982                        V2_MPT_BYTE_64_PBL_BUF_PG_SZ_S,
1983                        mw->pbl_buf_pg_sz + PG_SHIFT_OFFSET);
1984
1985         mpt_entry->lkey = cpu_to_le32(mw->rkey);
1986
1987         return 0;
1988 }
1989
1990 static void *get_cqe_v2(struct hns_roce_cq *hr_cq, int n)
1991 {
1992         return hns_roce_buf_offset(&hr_cq->hr_buf.hr_buf,
1993                                    n * HNS_ROCE_V2_CQE_ENTRY_SIZE);
1994 }
1995
1996 static void *get_sw_cqe_v2(struct hns_roce_cq *hr_cq, int n)
1997 {
1998         struct hns_roce_v2_cqe *cqe = get_cqe_v2(hr_cq, n & hr_cq->ib_cq.cqe);
1999
2000         /* Get cqe when Owner bit is Conversely with the MSB of cons_idx */
2001         return (roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_OWNER_S) ^
2002                 !!(n & (hr_cq->ib_cq.cqe + 1))) ? cqe : NULL;
2003 }
2004
2005 static struct hns_roce_v2_cqe *next_cqe_sw_v2(struct hns_roce_cq *hr_cq)
2006 {
2007         return get_sw_cqe_v2(hr_cq, hr_cq->cons_index);
2008 }
2009
2010 static void hns_roce_v2_cq_set_ci(struct hns_roce_cq *hr_cq, u32 cons_index)
2011 {
2012         *hr_cq->set_ci_db = cons_index & 0xffffff;
2013 }
2014
2015 static void __hns_roce_v2_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn,
2016                                    struct hns_roce_srq *srq)
2017 {
2018         struct hns_roce_v2_cqe *cqe, *dest;
2019         u32 prod_index;
2020         int nfreed = 0;
2021         u8 owner_bit;
2022
2023         for (prod_index = hr_cq->cons_index; get_sw_cqe_v2(hr_cq, prod_index);
2024              ++prod_index) {
2025                 if (prod_index == hr_cq->cons_index + hr_cq->ib_cq.cqe)
2026                         break;
2027         }
2028
2029         /*
2030          * Now backwards through the CQ, removing CQ entries
2031          * that match our QP by overwriting them with next entries.
2032          */
2033         while ((int) --prod_index - (int) hr_cq->cons_index >= 0) {
2034                 cqe = get_cqe_v2(hr_cq, prod_index & hr_cq->ib_cq.cqe);
2035                 if ((roce_get_field(cqe->byte_16, V2_CQE_BYTE_16_LCL_QPN_M,
2036                                     V2_CQE_BYTE_16_LCL_QPN_S) &
2037                                     HNS_ROCE_V2_CQE_QPN_MASK) == qpn) {
2038                         /* In v1 engine, not support SRQ */
2039                         ++nfreed;
2040                 } else if (nfreed) {
2041                         dest = get_cqe_v2(hr_cq, (prod_index + nfreed) &
2042                                           hr_cq->ib_cq.cqe);
2043                         owner_bit = roce_get_bit(dest->byte_4,
2044                                                  V2_CQE_BYTE_4_OWNER_S);
2045                         memcpy(dest, cqe, sizeof(*cqe));
2046                         roce_set_bit(dest->byte_4, V2_CQE_BYTE_4_OWNER_S,
2047                                      owner_bit);
2048                 }
2049         }
2050
2051         if (nfreed) {
2052                 hr_cq->cons_index += nfreed;
2053                 /*
2054                  * Make sure update of buffer contents is done before
2055                  * updating consumer index.
2056                  */
2057                 wmb();
2058                 hns_roce_v2_cq_set_ci(hr_cq, hr_cq->cons_index);
2059         }
2060 }
2061
2062 static void hns_roce_v2_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn,
2063                                  struct hns_roce_srq *srq)
2064 {
2065         spin_lock_irq(&hr_cq->lock);
2066         __hns_roce_v2_cq_clean(hr_cq, qpn, srq);
2067         spin_unlock_irq(&hr_cq->lock);
2068 }
2069
2070 static void hns_roce_v2_write_cqc(struct hns_roce_dev *hr_dev,
2071                                   struct hns_roce_cq *hr_cq, void *mb_buf,
2072                                   u64 *mtts, dma_addr_t dma_handle, int nent,
2073                                   u32 vector)
2074 {
2075         struct hns_roce_v2_cq_context *cq_context;
2076
2077         cq_context = mb_buf;
2078         memset(cq_context, 0, sizeof(*cq_context));
2079
2080         roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_CQ_ST_M,
2081                        V2_CQC_BYTE_4_CQ_ST_S, V2_CQ_STATE_VALID);
2082         roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_ARM_ST_M,
2083                        V2_CQC_BYTE_4_ARM_ST_S, REG_NXT_CEQE);
2084         roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_SHIFT_M,
2085                        V2_CQC_BYTE_4_SHIFT_S, ilog2((unsigned int)nent));
2086         roce_set_field(cq_context->byte_4_pg_ceqn, V2_CQC_BYTE_4_CEQN_M,
2087                        V2_CQC_BYTE_4_CEQN_S, vector);
2088         cq_context->byte_4_pg_ceqn = cpu_to_le32(cq_context->byte_4_pg_ceqn);
2089
2090         roce_set_field(cq_context->byte_8_cqn, V2_CQC_BYTE_8_CQN_M,
2091                        V2_CQC_BYTE_8_CQN_S, hr_cq->cqn);
2092
2093         cq_context->cqe_cur_blk_addr = (u32)(mtts[0] >> PAGE_ADDR_SHIFT);
2094         cq_context->cqe_cur_blk_addr =
2095                                 cpu_to_le32(cq_context->cqe_cur_blk_addr);
2096
2097         roce_set_field(cq_context->byte_16_hop_addr,
2098                        V2_CQC_BYTE_16_CQE_CUR_BLK_ADDR_M,
2099                        V2_CQC_BYTE_16_CQE_CUR_BLK_ADDR_S,
2100                        cpu_to_le32((mtts[0]) >> (32 + PAGE_ADDR_SHIFT)));
2101         roce_set_field(cq_context->byte_16_hop_addr,
2102                        V2_CQC_BYTE_16_CQE_HOP_NUM_M,
2103                        V2_CQC_BYTE_16_CQE_HOP_NUM_S, hr_dev->caps.cqe_hop_num ==
2104                        HNS_ROCE_HOP_NUM_0 ? 0 : hr_dev->caps.cqe_hop_num);
2105
2106         cq_context->cqe_nxt_blk_addr = (u32)(mtts[1] >> PAGE_ADDR_SHIFT);
2107         roce_set_field(cq_context->byte_24_pgsz_addr,
2108                        V2_CQC_BYTE_24_CQE_NXT_BLK_ADDR_M,
2109                        V2_CQC_BYTE_24_CQE_NXT_BLK_ADDR_S,
2110                        cpu_to_le32((mtts[1]) >> (32 + PAGE_ADDR_SHIFT)));
2111         roce_set_field(cq_context->byte_24_pgsz_addr,
2112                        V2_CQC_BYTE_24_CQE_BA_PG_SZ_M,
2113                        V2_CQC_BYTE_24_CQE_BA_PG_SZ_S,
2114                        hr_dev->caps.cqe_ba_pg_sz + PG_SHIFT_OFFSET);
2115         roce_set_field(cq_context->byte_24_pgsz_addr,
2116                        V2_CQC_BYTE_24_CQE_BUF_PG_SZ_M,
2117                        V2_CQC_BYTE_24_CQE_BUF_PG_SZ_S,
2118                        hr_dev->caps.cqe_buf_pg_sz + PG_SHIFT_OFFSET);
2119
2120         cq_context->cqe_ba = (u32)(dma_handle >> 3);
2121
2122         roce_set_field(cq_context->byte_40_cqe_ba, V2_CQC_BYTE_40_CQE_BA_M,
2123                        V2_CQC_BYTE_40_CQE_BA_S, (dma_handle >> (32 + 3)));
2124
2125         if (hr_cq->db_en)
2126                 roce_set_bit(cq_context->byte_44_db_record,
2127                              V2_CQC_BYTE_44_DB_RECORD_EN_S, 1);
2128
2129         roce_set_field(cq_context->byte_44_db_record,
2130                        V2_CQC_BYTE_44_DB_RECORD_ADDR_M,
2131                        V2_CQC_BYTE_44_DB_RECORD_ADDR_S,
2132                        ((u32)hr_cq->db.dma) >> 1);
2133         cq_context->db_record_addr = hr_cq->db.dma >> 32;
2134
2135         roce_set_field(cq_context->byte_56_cqe_period_maxcnt,
2136                        V2_CQC_BYTE_56_CQ_MAX_CNT_M,
2137                        V2_CQC_BYTE_56_CQ_MAX_CNT_S,
2138                        HNS_ROCE_V2_CQ_DEFAULT_BURST_NUM);
2139         roce_set_field(cq_context->byte_56_cqe_period_maxcnt,
2140                        V2_CQC_BYTE_56_CQ_PERIOD_M,
2141                        V2_CQC_BYTE_56_CQ_PERIOD_S,
2142                        HNS_ROCE_V2_CQ_DEFAULT_INTERVAL);
2143 }
2144
2145 static int hns_roce_v2_req_notify_cq(struct ib_cq *ibcq,
2146                                      enum ib_cq_notify_flags flags)
2147 {
2148         struct hns_roce_cq *hr_cq = to_hr_cq(ibcq);
2149         u32 notification_flag;
2150         u32 doorbell[2];
2151
2152         doorbell[0] = 0;
2153         doorbell[1] = 0;
2154
2155         notification_flag = (flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED ?
2156                              V2_CQ_DB_REQ_NOT : V2_CQ_DB_REQ_NOT_SOL;
2157         /*
2158          * flags = 0; Notification Flag = 1, next
2159          * flags = 1; Notification Flag = 0, solocited
2160          */
2161         roce_set_field(doorbell[0], V2_CQ_DB_BYTE_4_TAG_M, V2_DB_BYTE_4_TAG_S,
2162                        hr_cq->cqn);
2163         roce_set_field(doorbell[0], V2_CQ_DB_BYTE_4_CMD_M, V2_DB_BYTE_4_CMD_S,
2164                        HNS_ROCE_V2_CQ_DB_NTR);
2165         roce_set_field(doorbell[1], V2_CQ_DB_PARAMETER_CONS_IDX_M,
2166                        V2_CQ_DB_PARAMETER_CONS_IDX_S,
2167                        hr_cq->cons_index & ((hr_cq->cq_depth << 1) - 1));
2168         roce_set_field(doorbell[1], V2_CQ_DB_PARAMETER_CMD_SN_M,
2169                        V2_CQ_DB_PARAMETER_CMD_SN_S, hr_cq->arm_sn & 0x3);
2170         roce_set_bit(doorbell[1], V2_CQ_DB_PARAMETER_NOTIFY_S,
2171                      notification_flag);
2172
2173         hns_roce_write64_k(doorbell, hr_cq->cq_db_l);
2174
2175         return 0;
2176 }
2177
2178 static int hns_roce_handle_recv_inl_wqe(struct hns_roce_v2_cqe *cqe,
2179                                                     struct hns_roce_qp **cur_qp,
2180                                                     struct ib_wc *wc)
2181 {
2182         struct hns_roce_rinl_sge *sge_list;
2183         u32 wr_num, wr_cnt, sge_num;
2184         u32 sge_cnt, data_len, size;
2185         void *wqe_buf;
2186
2187         wr_num = roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_WQE_INDX_M,
2188                                 V2_CQE_BYTE_4_WQE_INDX_S) & 0xffff;
2189         wr_cnt = wr_num & ((*cur_qp)->rq.wqe_cnt - 1);
2190
2191         sge_list = (*cur_qp)->rq_inl_buf.wqe_list[wr_cnt].sg_list;
2192         sge_num = (*cur_qp)->rq_inl_buf.wqe_list[wr_cnt].sge_cnt;
2193         wqe_buf = get_recv_wqe(*cur_qp, wr_cnt);
2194         data_len = wc->byte_len;
2195
2196         for (sge_cnt = 0; (sge_cnt < sge_num) && (data_len); sge_cnt++) {
2197                 size = min(sge_list[sge_cnt].len, data_len);
2198                 memcpy((void *)sge_list[sge_cnt].addr, wqe_buf, size);
2199
2200                 data_len -= size;
2201                 wqe_buf += size;
2202         }
2203
2204         if (data_len) {
2205                 wc->status = IB_WC_LOC_LEN_ERR;
2206                 return -EAGAIN;
2207         }
2208
2209         return 0;
2210 }
2211
2212 static int hns_roce_v2_poll_one(struct hns_roce_cq *hr_cq,
2213                                 struct hns_roce_qp **cur_qp, struct ib_wc *wc)
2214 {
2215         struct hns_roce_dev *hr_dev;
2216         struct hns_roce_v2_cqe *cqe;
2217         struct hns_roce_qp *hr_qp;
2218         struct hns_roce_wq *wq;
2219         struct ib_qp_attr attr;
2220         int attr_mask;
2221         int is_send;
2222         u16 wqe_ctr;
2223         u32 opcode;
2224         u32 status;
2225         int qpn;
2226         int ret;
2227
2228         /* Find cqe according to consumer index */
2229         cqe = next_cqe_sw_v2(hr_cq);
2230         if (!cqe)
2231                 return -EAGAIN;
2232
2233         ++hr_cq->cons_index;
2234         /* Memory barrier */
2235         rmb();
2236
2237         /* 0->SQ, 1->RQ */
2238         is_send = !roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_S_R_S);
2239
2240         qpn = roce_get_field(cqe->byte_16, V2_CQE_BYTE_16_LCL_QPN_M,
2241                                 V2_CQE_BYTE_16_LCL_QPN_S);
2242
2243         if (!*cur_qp || (qpn & HNS_ROCE_V2_CQE_QPN_MASK) != (*cur_qp)->qpn) {
2244                 hr_dev = to_hr_dev(hr_cq->ib_cq.device);
2245                 hr_qp = __hns_roce_qp_lookup(hr_dev, qpn);
2246                 if (unlikely(!hr_qp)) {
2247                         dev_err(hr_dev->dev, "CQ %06lx with entry for unknown QPN %06x\n",
2248                                 hr_cq->cqn, (qpn & HNS_ROCE_V2_CQE_QPN_MASK));
2249                         return -EINVAL;
2250                 }
2251                 *cur_qp = hr_qp;
2252         }
2253
2254         wc->qp = &(*cur_qp)->ibqp;
2255         wc->vendor_err = 0;
2256
2257         status = roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_STATUS_M,
2258                                 V2_CQE_BYTE_4_STATUS_S);
2259         switch (status & HNS_ROCE_V2_CQE_STATUS_MASK) {
2260         case HNS_ROCE_CQE_V2_SUCCESS:
2261                 wc->status = IB_WC_SUCCESS;
2262                 break;
2263         case HNS_ROCE_CQE_V2_LOCAL_LENGTH_ERR:
2264                 wc->status = IB_WC_LOC_LEN_ERR;
2265                 break;
2266         case HNS_ROCE_CQE_V2_LOCAL_QP_OP_ERR:
2267                 wc->status = IB_WC_LOC_QP_OP_ERR;
2268                 break;
2269         case HNS_ROCE_CQE_V2_LOCAL_PROT_ERR:
2270                 wc->status = IB_WC_LOC_PROT_ERR;
2271                 break;
2272         case HNS_ROCE_CQE_V2_WR_FLUSH_ERR:
2273                 wc->status = IB_WC_WR_FLUSH_ERR;
2274                 break;
2275         case HNS_ROCE_CQE_V2_MW_BIND_ERR:
2276                 wc->status = IB_WC_MW_BIND_ERR;
2277                 break;
2278         case HNS_ROCE_CQE_V2_BAD_RESP_ERR:
2279                 wc->status = IB_WC_BAD_RESP_ERR;
2280                 break;
2281         case HNS_ROCE_CQE_V2_LOCAL_ACCESS_ERR:
2282                 wc->status = IB_WC_LOC_ACCESS_ERR;
2283                 break;
2284         case HNS_ROCE_CQE_V2_REMOTE_INVAL_REQ_ERR:
2285                 wc->status = IB_WC_REM_INV_REQ_ERR;
2286                 break;
2287         case HNS_ROCE_CQE_V2_REMOTE_ACCESS_ERR:
2288                 wc->status = IB_WC_REM_ACCESS_ERR;
2289                 break;
2290         case HNS_ROCE_CQE_V2_REMOTE_OP_ERR:
2291                 wc->status = IB_WC_REM_OP_ERR;
2292                 break;
2293         case HNS_ROCE_CQE_V2_TRANSPORT_RETRY_EXC_ERR:
2294                 wc->status = IB_WC_RETRY_EXC_ERR;
2295                 break;
2296         case HNS_ROCE_CQE_V2_RNR_RETRY_EXC_ERR:
2297                 wc->status = IB_WC_RNR_RETRY_EXC_ERR;
2298                 break;
2299         case HNS_ROCE_CQE_V2_REMOTE_ABORT_ERR:
2300                 wc->status = IB_WC_REM_ABORT_ERR;
2301                 break;
2302         default:
2303                 wc->status = IB_WC_GENERAL_ERR;
2304                 break;
2305         }
2306
2307         /* flush cqe if wc status is error, excluding flush error */
2308         if ((wc->status != IB_WC_SUCCESS) &&
2309             (wc->status != IB_WC_WR_FLUSH_ERR)) {
2310                 attr_mask = IB_QP_STATE;
2311                 attr.qp_state = IB_QPS_ERR;
2312                 return hns_roce_v2_modify_qp(&(*cur_qp)->ibqp,
2313                                              &attr, attr_mask,
2314                                              (*cur_qp)->state, IB_QPS_ERR);
2315         }
2316
2317         if (wc->status == IB_WC_WR_FLUSH_ERR)
2318                 return 0;
2319
2320         if (is_send) {
2321                 wc->wc_flags = 0;
2322                 /* SQ corresponding to CQE */
2323                 switch (roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_OPCODE_M,
2324                                        V2_CQE_BYTE_4_OPCODE_S) & 0x1f) {
2325                 case HNS_ROCE_SQ_OPCODE_SEND:
2326                         wc->opcode = IB_WC_SEND;
2327                         break;
2328                 case HNS_ROCE_SQ_OPCODE_SEND_WITH_INV:
2329                         wc->opcode = IB_WC_SEND;
2330                         break;
2331                 case HNS_ROCE_SQ_OPCODE_SEND_WITH_IMM:
2332                         wc->opcode = IB_WC_SEND;
2333                         wc->wc_flags |= IB_WC_WITH_IMM;
2334                         break;
2335                 case HNS_ROCE_SQ_OPCODE_RDMA_READ:
2336                         wc->opcode = IB_WC_RDMA_READ;
2337                         wc->byte_len = le32_to_cpu(cqe->byte_cnt);
2338                         break;
2339                 case HNS_ROCE_SQ_OPCODE_RDMA_WRITE:
2340                         wc->opcode = IB_WC_RDMA_WRITE;
2341                         break;
2342                 case HNS_ROCE_SQ_OPCODE_RDMA_WRITE_WITH_IMM:
2343                         wc->opcode = IB_WC_RDMA_WRITE;
2344                         wc->wc_flags |= IB_WC_WITH_IMM;
2345                         break;
2346                 case HNS_ROCE_SQ_OPCODE_LOCAL_INV:
2347                         wc->opcode = IB_WC_LOCAL_INV;
2348                         wc->wc_flags |= IB_WC_WITH_INVALIDATE;
2349                         break;
2350                 case HNS_ROCE_SQ_OPCODE_ATOMIC_COMP_AND_SWAP:
2351                         wc->opcode = IB_WC_COMP_SWAP;
2352                         wc->byte_len  = 8;
2353                         break;
2354                 case HNS_ROCE_SQ_OPCODE_ATOMIC_FETCH_AND_ADD:
2355                         wc->opcode = IB_WC_FETCH_ADD;
2356                         wc->byte_len  = 8;
2357                         break;
2358                 case HNS_ROCE_SQ_OPCODE_ATOMIC_MASK_COMP_AND_SWAP:
2359                         wc->opcode = IB_WC_MASKED_COMP_SWAP;
2360                         wc->byte_len  = 8;
2361                         break;
2362                 case HNS_ROCE_SQ_OPCODE_ATOMIC_MASK_FETCH_AND_ADD:
2363                         wc->opcode = IB_WC_MASKED_FETCH_ADD;
2364                         wc->byte_len  = 8;
2365                         break;
2366                 case HNS_ROCE_SQ_OPCODE_FAST_REG_WR:
2367                         wc->opcode = IB_WC_REG_MR;
2368                         break;
2369                 case HNS_ROCE_SQ_OPCODE_BIND_MW:
2370                         wc->opcode = IB_WC_REG_MR;
2371                         break;
2372                 default:
2373                         wc->status = IB_WC_GENERAL_ERR;
2374                         break;
2375                 }
2376
2377                 wq = &(*cur_qp)->sq;
2378                 if ((*cur_qp)->sq_signal_bits) {
2379                         /*
2380                          * If sg_signal_bit is 1,
2381                          * firstly tail pointer updated to wqe
2382                          * which current cqe correspond to
2383                          */
2384                         wqe_ctr = (u16)roce_get_field(cqe->byte_4,
2385                                                       V2_CQE_BYTE_4_WQE_INDX_M,
2386                                                       V2_CQE_BYTE_4_WQE_INDX_S);
2387                         wq->tail += (wqe_ctr - (u16)wq->tail) &
2388                                     (wq->wqe_cnt - 1);
2389                 }
2390
2391                 wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
2392                 ++wq->tail;
2393         } else {
2394                 /* RQ correspond to CQE */
2395                 wc->byte_len = le32_to_cpu(cqe->byte_cnt);
2396
2397                 opcode = roce_get_field(cqe->byte_4, V2_CQE_BYTE_4_OPCODE_M,
2398                                         V2_CQE_BYTE_4_OPCODE_S);
2399                 switch (opcode & 0x1f) {
2400                 case HNS_ROCE_V2_OPCODE_RDMA_WRITE_IMM:
2401                         wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
2402                         wc->wc_flags = IB_WC_WITH_IMM;
2403                         wc->ex.imm_data =
2404                                 cpu_to_be32(le32_to_cpu(cqe->immtdata));
2405                         break;
2406                 case HNS_ROCE_V2_OPCODE_SEND:
2407                         wc->opcode = IB_WC_RECV;
2408                         wc->wc_flags = 0;
2409                         break;
2410                 case HNS_ROCE_V2_OPCODE_SEND_WITH_IMM:
2411                         wc->opcode = IB_WC_RECV;
2412                         wc->wc_flags = IB_WC_WITH_IMM;
2413                         wc->ex.imm_data =
2414                                 cpu_to_be32(le32_to_cpu(cqe->immtdata));
2415                         break;
2416                 case HNS_ROCE_V2_OPCODE_SEND_WITH_INV:
2417                         wc->opcode = IB_WC_RECV;
2418                         wc->wc_flags = IB_WC_WITH_INVALIDATE;
2419                         wc->ex.invalidate_rkey = le32_to_cpu(cqe->rkey);
2420                         break;
2421                 default:
2422                         wc->status = IB_WC_GENERAL_ERR;
2423                         break;
2424                 }
2425
2426                 if ((wc->qp->qp_type == IB_QPT_RC ||
2427                      wc->qp->qp_type == IB_QPT_UC) &&
2428                     (opcode == HNS_ROCE_V2_OPCODE_SEND ||
2429                     opcode == HNS_ROCE_V2_OPCODE_SEND_WITH_IMM ||
2430                     opcode == HNS_ROCE_V2_OPCODE_SEND_WITH_INV) &&
2431                     (roce_get_bit(cqe->byte_4, V2_CQE_BYTE_4_RQ_INLINE_S))) {
2432                         ret = hns_roce_handle_recv_inl_wqe(cqe, cur_qp, wc);
2433                         if (ret)
2434                                 return -EAGAIN;
2435                 }
2436
2437                 /* Update tail pointer, record wr_id */
2438                 wq = &(*cur_qp)->rq;
2439                 wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
2440                 ++wq->tail;
2441
2442                 wc->sl = (u8)roce_get_field(cqe->byte_32, V2_CQE_BYTE_32_SL_M,
2443                                             V2_CQE_BYTE_32_SL_S);
2444                 wc->src_qp = (u8)roce_get_field(cqe->byte_32,
2445                                                 V2_CQE_BYTE_32_RMT_QPN_M,
2446                                                 V2_CQE_BYTE_32_RMT_QPN_S);
2447                 wc->slid = 0;
2448                 wc->wc_flags |= (roce_get_bit(cqe->byte_32,
2449                                               V2_CQE_BYTE_32_GRH_S) ?
2450                                               IB_WC_GRH : 0);
2451                 wc->port_num = roce_get_field(cqe->byte_32,
2452                                 V2_CQE_BYTE_32_PORTN_M, V2_CQE_BYTE_32_PORTN_S);
2453                 wc->pkey_index = 0;
2454                 memcpy(wc->smac, cqe->smac, 4);
2455                 wc->smac[4] = roce_get_field(cqe->byte_28,
2456                                              V2_CQE_BYTE_28_SMAC_4_M,
2457                                              V2_CQE_BYTE_28_SMAC_4_S);
2458                 wc->smac[5] = roce_get_field(cqe->byte_28,
2459                                              V2_CQE_BYTE_28_SMAC_5_M,
2460                                              V2_CQE_BYTE_28_SMAC_5_S);
2461                 if (roce_get_bit(cqe->byte_28, V2_CQE_BYTE_28_VID_VLD_S)) {
2462                         wc->vlan_id = (u16)roce_get_field(cqe->byte_28,
2463                                                           V2_CQE_BYTE_28_VID_M,
2464                                                           V2_CQE_BYTE_28_VID_S);
2465                 } else {
2466                         wc->vlan_id = 0xffff;
2467                 }
2468
2469                 wc->wc_flags |= (IB_WC_WITH_VLAN | IB_WC_WITH_SMAC);
2470                 wc->network_hdr_type = roce_get_field(cqe->byte_28,
2471                                                     V2_CQE_BYTE_28_PORT_TYPE_M,
2472                                                     V2_CQE_BYTE_28_PORT_TYPE_S);
2473         }
2474
2475         return 0;
2476 }
2477
2478 static int hns_roce_v2_poll_cq(struct ib_cq *ibcq, int num_entries,
2479                                struct ib_wc *wc)
2480 {
2481         struct hns_roce_cq *hr_cq = to_hr_cq(ibcq);
2482         struct hns_roce_qp *cur_qp = NULL;
2483         unsigned long flags;
2484         int npolled;
2485
2486         spin_lock_irqsave(&hr_cq->lock, flags);
2487
2488         for (npolled = 0; npolled < num_entries; ++npolled) {
2489                 if (hns_roce_v2_poll_one(hr_cq, &cur_qp, wc + npolled))
2490                         break;
2491         }
2492
2493         if (npolled) {
2494                 /* Memory barrier */
2495                 wmb();
2496                 hns_roce_v2_cq_set_ci(hr_cq, hr_cq->cons_index);
2497         }
2498
2499         spin_unlock_irqrestore(&hr_cq->lock, flags);
2500
2501         return npolled;
2502 }
2503
2504 static int hns_roce_v2_set_hem(struct hns_roce_dev *hr_dev,
2505                                struct hns_roce_hem_table *table, int obj,
2506                                int step_idx)
2507 {
2508         struct device *dev = hr_dev->dev;
2509         struct hns_roce_cmd_mailbox *mailbox;
2510         struct hns_roce_hem_iter iter;
2511         struct hns_roce_hem_mhop mhop;
2512         struct hns_roce_hem *hem;
2513         unsigned long mhop_obj = obj;
2514         int i, j, k;
2515         int ret = 0;
2516         u64 hem_idx = 0;
2517         u64 l1_idx = 0;
2518         u64 bt_ba = 0;
2519         u32 chunk_ba_num;
2520         u32 hop_num;
2521         u16 op = 0xff;
2522
2523         if (!hns_roce_check_whether_mhop(hr_dev, table->type))
2524                 return 0;
2525
2526         hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop);
2527         i = mhop.l0_idx;
2528         j = mhop.l1_idx;
2529         k = mhop.l2_idx;
2530         hop_num = mhop.hop_num;
2531         chunk_ba_num = mhop.bt_chunk_size / 8;
2532
2533         if (hop_num == 2) {
2534                 hem_idx = i * chunk_ba_num * chunk_ba_num + j * chunk_ba_num +
2535                           k;
2536                 l1_idx = i * chunk_ba_num + j;
2537         } else if (hop_num == 1) {
2538                 hem_idx = i * chunk_ba_num + j;
2539         } else if (hop_num == HNS_ROCE_HOP_NUM_0) {
2540                 hem_idx = i;
2541         }
2542
2543         switch (table->type) {
2544         case HEM_TYPE_QPC:
2545                 op = HNS_ROCE_CMD_WRITE_QPC_BT0;
2546                 break;
2547         case HEM_TYPE_MTPT:
2548                 op = HNS_ROCE_CMD_WRITE_MPT_BT0;
2549                 break;
2550         case HEM_TYPE_CQC:
2551                 op = HNS_ROCE_CMD_WRITE_CQC_BT0;
2552                 break;
2553         case HEM_TYPE_SRQC:
2554                 op = HNS_ROCE_CMD_WRITE_SRQC_BT0;
2555                 break;
2556         default:
2557                 dev_warn(dev, "Table %d not to be written by mailbox!\n",
2558                          table->type);
2559                 return 0;
2560         }
2561         op += step_idx;
2562
2563         mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
2564         if (IS_ERR(mailbox))
2565                 return PTR_ERR(mailbox);
2566
2567         if (check_whether_last_step(hop_num, step_idx)) {
2568                 hem = table->hem[hem_idx];
2569                 for (hns_roce_hem_first(hem, &iter);
2570                      !hns_roce_hem_last(&iter); hns_roce_hem_next(&iter)) {
2571                         bt_ba = hns_roce_hem_addr(&iter);
2572
2573                         /* configure the ba, tag, and op */
2574                         ret = hns_roce_cmd_mbox(hr_dev, bt_ba, mailbox->dma,
2575                                                 obj, 0, op,
2576                                                 HNS_ROCE_CMD_TIMEOUT_MSECS);
2577                 }
2578         } else {
2579                 if (step_idx == 0)
2580                         bt_ba = table->bt_l0_dma_addr[i];
2581                 else if (step_idx == 1 && hop_num == 2)
2582                         bt_ba = table->bt_l1_dma_addr[l1_idx];
2583
2584                 /* configure the ba, tag, and op */
2585                 ret = hns_roce_cmd_mbox(hr_dev, bt_ba, mailbox->dma, obj,
2586                                         0, op, HNS_ROCE_CMD_TIMEOUT_MSECS);
2587         }
2588
2589         hns_roce_free_cmd_mailbox(hr_dev, mailbox);
2590         return ret;
2591 }
2592
2593 static int hns_roce_v2_clear_hem(struct hns_roce_dev *hr_dev,
2594                                  struct hns_roce_hem_table *table, int obj,
2595                                  int step_idx)
2596 {
2597         struct device *dev = hr_dev->dev;
2598         struct hns_roce_cmd_mailbox *mailbox;
2599         int ret = 0;
2600         u16 op = 0xff;
2601
2602         if (!hns_roce_check_whether_mhop(hr_dev, table->type))
2603                 return 0;
2604
2605         switch (table->type) {
2606         case HEM_TYPE_QPC:
2607                 op = HNS_ROCE_CMD_DESTROY_QPC_BT0;
2608                 break;
2609         case HEM_TYPE_MTPT:
2610                 op = HNS_ROCE_CMD_DESTROY_MPT_BT0;
2611                 break;
2612         case HEM_TYPE_CQC:
2613                 op = HNS_ROCE_CMD_DESTROY_CQC_BT0;
2614                 break;
2615         case HEM_TYPE_SRQC:
2616                 op = HNS_ROCE_CMD_DESTROY_SRQC_BT0;
2617                 break;
2618         default:
2619                 dev_warn(dev, "Table %d not to be destroyed by mailbox!\n",
2620                          table->type);
2621                 return 0;
2622         }
2623         op += step_idx;
2624
2625         mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
2626         if (IS_ERR(mailbox))
2627                 return PTR_ERR(mailbox);
2628
2629         /* configure the tag and op */
2630         ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, obj, 0, op,
2631                                 HNS_ROCE_CMD_TIMEOUT_MSECS);
2632
2633         hns_roce_free_cmd_mailbox(hr_dev, mailbox);
2634         return ret;
2635 }
2636
2637 static int hns_roce_v2_qp_modify(struct hns_roce_dev *hr_dev,
2638                                  struct hns_roce_mtt *mtt,
2639                                  enum ib_qp_state cur_state,
2640                                  enum ib_qp_state new_state,
2641                                  struct hns_roce_v2_qp_context *context,
2642                                  struct hns_roce_qp *hr_qp)
2643 {
2644         struct hns_roce_cmd_mailbox *mailbox;
2645         int ret;
2646
2647         mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
2648         if (IS_ERR(mailbox))
2649                 return PTR_ERR(mailbox);
2650
2651         memcpy(mailbox->buf, context, sizeof(*context) * 2);
2652
2653         ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, hr_qp->qpn, 0,
2654                                 HNS_ROCE_CMD_MODIFY_QPC,
2655                                 HNS_ROCE_CMD_TIMEOUT_MSECS);
2656
2657         hns_roce_free_cmd_mailbox(hr_dev, mailbox);
2658
2659         return ret;
2660 }
2661
2662 static void set_access_flags(struct hns_roce_qp *hr_qp,
2663                              struct hns_roce_v2_qp_context *context,
2664                              struct hns_roce_v2_qp_context *qpc_mask,
2665                              const struct ib_qp_attr *attr, int attr_mask)
2666 {
2667         u8 dest_rd_atomic;
2668         u32 access_flags;
2669
2670         dest_rd_atomic = (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) ?
2671                          attr->max_dest_rd_atomic : hr_qp->resp_depth;
2672
2673         access_flags = (attr_mask & IB_QP_ACCESS_FLAGS) ?
2674                        attr->qp_access_flags : hr_qp->atomic_rd_en;
2675
2676         if (!dest_rd_atomic)
2677                 access_flags &= IB_ACCESS_REMOTE_WRITE;
2678
2679         roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S,
2680                      !!(access_flags & IB_ACCESS_REMOTE_READ));
2681         roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S, 0);
2682
2683         roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S,
2684                      !!(access_flags & IB_ACCESS_REMOTE_WRITE));
2685         roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S, 0);
2686
2687         roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S,
2688                      !!(access_flags & IB_ACCESS_REMOTE_ATOMIC));
2689         roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S, 0);
2690 }
2691
2692 static void modify_qp_reset_to_init(struct ib_qp *ibqp,
2693                                     const struct ib_qp_attr *attr,
2694                                     int attr_mask,
2695                                     struct hns_roce_v2_qp_context *context,
2696                                     struct hns_roce_v2_qp_context *qpc_mask)
2697 {
2698         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
2699         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
2700
2701         /*
2702          * In v2 engine, software pass context and context mask to hardware
2703          * when modifying qp. If software need modify some fields in context,
2704          * we should set all bits of the relevant fields in context mask to
2705          * 0 at the same time, else set them to 0x1.
2706          */
2707         roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M,
2708                        V2_QPC_BYTE_4_TST_S, to_hr_qp_type(hr_qp->ibqp.qp_type));
2709         roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M,
2710                        V2_QPC_BYTE_4_TST_S, 0);
2711
2712         if (ibqp->qp_type == IB_QPT_GSI)
2713                 roce_set_field(context->byte_4_sqpn_tst,
2714                                V2_QPC_BYTE_4_SGE_SHIFT_M,
2715                                V2_QPC_BYTE_4_SGE_SHIFT_S,
2716                                ilog2((unsigned int)hr_qp->sge.sge_cnt));
2717         else
2718                 roce_set_field(context->byte_4_sqpn_tst,
2719                                V2_QPC_BYTE_4_SGE_SHIFT_M,
2720                                V2_QPC_BYTE_4_SGE_SHIFT_S,
2721                                hr_qp->sq.max_gs > 2 ?
2722                                ilog2((unsigned int)hr_qp->sge.sge_cnt) : 0);
2723
2724         roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_SGE_SHIFT_M,
2725                        V2_QPC_BYTE_4_SGE_SHIFT_S, 0);
2726
2727         roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M,
2728                        V2_QPC_BYTE_4_SQPN_S, hr_qp->qpn);
2729         roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M,
2730                        V2_QPC_BYTE_4_SQPN_S, 0);
2731
2732         roce_set_field(context->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M,
2733                        V2_QPC_BYTE_16_PD_S, to_hr_pd(ibqp->pd)->pdn);
2734         roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M,
2735                        V2_QPC_BYTE_16_PD_S, 0);
2736
2737         roce_set_field(context->byte_20_smac_sgid_idx, V2_QPC_BYTE_20_RQWS_M,
2738                        V2_QPC_BYTE_20_RQWS_S, ilog2(hr_qp->rq.max_gs));
2739         roce_set_field(qpc_mask->byte_20_smac_sgid_idx, V2_QPC_BYTE_20_RQWS_M,
2740                        V2_QPC_BYTE_20_RQWS_S, 0);
2741
2742         roce_set_field(context->byte_20_smac_sgid_idx,
2743                        V2_QPC_BYTE_20_SQ_SHIFT_M, V2_QPC_BYTE_20_SQ_SHIFT_S,
2744                        ilog2((unsigned int)hr_qp->sq.wqe_cnt));
2745         roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
2746                        V2_QPC_BYTE_20_SQ_SHIFT_M, V2_QPC_BYTE_20_SQ_SHIFT_S, 0);
2747
2748         roce_set_field(context->byte_20_smac_sgid_idx,
2749                        V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S,
2750                        ilog2((unsigned int)hr_qp->rq.wqe_cnt));
2751         roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
2752                        V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S, 0);
2753
2754         /* No VLAN need to set 0xFFF */
2755         roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_VLAN_ID_M,
2756                        V2_QPC_BYTE_24_VLAN_ID_S, 0xfff);
2757         roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_VLAN_ID_M,
2758                        V2_QPC_BYTE_24_VLAN_ID_S, 0);
2759
2760         /*
2761          * Set some fields in context to zero, Because the default values
2762          * of all fields in context are zero, we need not set them to 0 again.
2763          * but we should set the relevant fields of context mask to 0.
2764          */
2765         roce_set_bit(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_SQ_TX_ERR_S, 0);
2766         roce_set_bit(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_SQ_RX_ERR_S, 0);
2767         roce_set_bit(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_RQ_TX_ERR_S, 0);
2768         roce_set_bit(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_RQ_RX_ERR_S, 0);
2769
2770         roce_set_field(qpc_mask->byte_60_qpst_tempid, V2_QPC_BYTE_60_TEMPID_M,
2771                        V2_QPC_BYTE_60_TEMPID_S, 0);
2772
2773         roce_set_field(qpc_mask->byte_60_qpst_tempid,
2774                        V2_QPC_BYTE_60_SCC_TOKEN_M, V2_QPC_BYTE_60_SCC_TOKEN_S,
2775                        0);
2776         roce_set_bit(qpc_mask->byte_60_qpst_tempid,
2777                      V2_QPC_BYTE_60_SQ_DB_DOING_S, 0);
2778         roce_set_bit(qpc_mask->byte_60_qpst_tempid,
2779                      V2_QPC_BYTE_60_RQ_DB_DOING_S, 0);
2780         roce_set_bit(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_CNP_TX_FLAG_S, 0);
2781         roce_set_bit(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_CE_FLAG_S, 0);
2782
2783         if (attr_mask & IB_QP_QKEY) {
2784                 context->qkey_xrcd = attr->qkey;
2785                 qpc_mask->qkey_xrcd = 0;
2786                 hr_qp->qkey = attr->qkey;
2787         }
2788
2789         if (hr_qp->rdb_en) {
2790                 roce_set_bit(context->byte_68_rq_db,
2791                              V2_QPC_BYTE_68_RQ_RECORD_EN_S, 1);
2792                 roce_set_bit(qpc_mask->byte_68_rq_db,
2793                              V2_QPC_BYTE_68_RQ_RECORD_EN_S, 0);
2794         }
2795
2796         roce_set_field(context->byte_68_rq_db,
2797                        V2_QPC_BYTE_68_RQ_DB_RECORD_ADDR_M,
2798                        V2_QPC_BYTE_68_RQ_DB_RECORD_ADDR_S,
2799                        ((u32)hr_qp->rdb.dma) >> 1);
2800         roce_set_field(qpc_mask->byte_68_rq_db,
2801                        V2_QPC_BYTE_68_RQ_DB_RECORD_ADDR_M,
2802                        V2_QPC_BYTE_68_RQ_DB_RECORD_ADDR_S, 0);
2803         context->rq_db_record_addr = hr_qp->rdb.dma >> 32;
2804         qpc_mask->rq_db_record_addr = 0;
2805
2806         roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RQIE_S,
2807                     (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) ? 1 : 0);
2808         roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RQIE_S, 0);
2809
2810         roce_set_field(context->byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_RX_CQN_M,
2811                        V2_QPC_BYTE_80_RX_CQN_S, to_hr_cq(ibqp->recv_cq)->cqn);
2812         roce_set_field(qpc_mask->byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_RX_CQN_M,
2813                        V2_QPC_BYTE_80_RX_CQN_S, 0);
2814         if (ibqp->srq) {
2815                 roce_set_field(context->byte_76_srqn_op_en,
2816                                V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S,
2817                                to_hr_srq(ibqp->srq)->srqn);
2818                 roce_set_field(qpc_mask->byte_76_srqn_op_en,
2819                                V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S, 0);
2820                 roce_set_bit(context->byte_76_srqn_op_en,
2821                              V2_QPC_BYTE_76_SRQ_EN_S, 1);
2822                 roce_set_bit(qpc_mask->byte_76_srqn_op_en,
2823                              V2_QPC_BYTE_76_SRQ_EN_S, 0);
2824         }
2825
2826         roce_set_field(qpc_mask->byte_84_rq_ci_pi,
2827                        V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
2828                        V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, 0);
2829         roce_set_field(qpc_mask->byte_84_rq_ci_pi,
2830                        V2_QPC_BYTE_84_RQ_CONSUMER_IDX_M,
2831                        V2_QPC_BYTE_84_RQ_CONSUMER_IDX_S, 0);
2832
2833         roce_set_field(qpc_mask->byte_92_srq_info, V2_QPC_BYTE_92_SRQ_INFO_M,
2834                        V2_QPC_BYTE_92_SRQ_INFO_S, 0);
2835
2836         roce_set_field(qpc_mask->byte_96_rx_reqmsn, V2_QPC_BYTE_96_RX_REQ_MSN_M,
2837                        V2_QPC_BYTE_96_RX_REQ_MSN_S, 0);
2838
2839         roce_set_field(qpc_mask->byte_104_rq_sge,
2840                        V2_QPC_BYTE_104_RQ_CUR_WQE_SGE_NUM_M,
2841                        V2_QPC_BYTE_104_RQ_CUR_WQE_SGE_NUM_S, 0);
2842
2843         roce_set_bit(qpc_mask->byte_108_rx_reqepsn,
2844                      V2_QPC_BYTE_108_RX_REQ_PSN_ERR_S, 0);
2845         roce_set_field(qpc_mask->byte_108_rx_reqepsn,
2846                        V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_M,
2847                        V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_S, 0);
2848         roce_set_bit(qpc_mask->byte_108_rx_reqepsn,
2849                      V2_QPC_BYTE_108_RX_REQ_RNR_S, 0);
2850
2851         qpc_mask->rq_rnr_timer = 0;
2852         qpc_mask->rx_msg_len = 0;
2853         qpc_mask->rx_rkey_pkt_info = 0;
2854         qpc_mask->rx_va = 0;
2855
2856         roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_HEAD_MAX_M,
2857                        V2_QPC_BYTE_132_TRRL_HEAD_MAX_S, 0);
2858         roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_TAIL_MAX_M,
2859                        V2_QPC_BYTE_132_TRRL_TAIL_MAX_S, 0);
2860
2861         roce_set_bit(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_RQ_RTY_WAIT_DO_S,
2862                      0);
2863         roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_RAQ_TRRL_HEAD_M,
2864                        V2_QPC_BYTE_140_RAQ_TRRL_HEAD_S, 0);
2865         roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_RAQ_TRRL_TAIL_M,
2866                        V2_QPC_BYTE_140_RAQ_TRRL_TAIL_S, 0);
2867
2868         roce_set_field(qpc_mask->byte_144_raq,
2869                        V2_QPC_BYTE_144_RAQ_RTY_INI_PSN_M,
2870                        V2_QPC_BYTE_144_RAQ_RTY_INI_PSN_S, 0);
2871         roce_set_field(qpc_mask->byte_144_raq, V2_QPC_BYTE_144_RAQ_CREDIT_M,
2872                        V2_QPC_BYTE_144_RAQ_CREDIT_S, 0);
2873         roce_set_bit(qpc_mask->byte_144_raq, V2_QPC_BYTE_144_RESP_RTY_FLG_S, 0);
2874
2875         roce_set_field(qpc_mask->byte_148_raq, V2_QPC_BYTE_148_RQ_MSN_M,
2876                        V2_QPC_BYTE_148_RQ_MSN_S, 0);
2877         roce_set_field(qpc_mask->byte_148_raq, V2_QPC_BYTE_148_RAQ_SYNDROME_M,
2878                        V2_QPC_BYTE_148_RAQ_SYNDROME_S, 0);
2879
2880         roce_set_field(qpc_mask->byte_152_raq, V2_QPC_BYTE_152_RAQ_PSN_M,
2881                        V2_QPC_BYTE_152_RAQ_PSN_S, 0);
2882         roce_set_field(qpc_mask->byte_152_raq,
2883                        V2_QPC_BYTE_152_RAQ_TRRL_RTY_HEAD_M,
2884                        V2_QPC_BYTE_152_RAQ_TRRL_RTY_HEAD_S, 0);
2885
2886         roce_set_field(qpc_mask->byte_156_raq, V2_QPC_BYTE_156_RAQ_USE_PKTN_M,
2887                        V2_QPC_BYTE_156_RAQ_USE_PKTN_S, 0);
2888
2889         roce_set_field(qpc_mask->byte_160_sq_ci_pi,
2890                        V2_QPC_BYTE_160_SQ_PRODUCER_IDX_M,
2891                        V2_QPC_BYTE_160_SQ_PRODUCER_IDX_S, 0);
2892         roce_set_field(qpc_mask->byte_160_sq_ci_pi,
2893                        V2_QPC_BYTE_160_SQ_CONSUMER_IDX_M,
2894                        V2_QPC_BYTE_160_SQ_CONSUMER_IDX_S, 0);
2895
2896         roce_set_bit(qpc_mask->byte_168_irrl_idx,
2897                      V2_QPC_BYTE_168_POLL_DB_WAIT_DO_S, 0);
2898         roce_set_bit(qpc_mask->byte_168_irrl_idx,
2899                      V2_QPC_BYTE_168_SCC_TOKEN_FORBID_SQ_DEQ_S, 0);
2900         roce_set_bit(qpc_mask->byte_168_irrl_idx,
2901                      V2_QPC_BYTE_168_WAIT_ACK_TIMEOUT_S, 0);
2902         roce_set_bit(qpc_mask->byte_168_irrl_idx,
2903                      V2_QPC_BYTE_168_MSG_RTY_LP_FLG_S, 0);
2904         roce_set_bit(qpc_mask->byte_168_irrl_idx,
2905                      V2_QPC_BYTE_168_SQ_INVLD_FLG_S, 0);
2906         roce_set_field(qpc_mask->byte_168_irrl_idx,
2907                        V2_QPC_BYTE_168_IRRL_IDX_LSB_M,
2908                        V2_QPC_BYTE_168_IRRL_IDX_LSB_S, 0);
2909
2910         roce_set_field(context->byte_172_sq_psn, V2_QPC_BYTE_172_ACK_REQ_FREQ_M,
2911                        V2_QPC_BYTE_172_ACK_REQ_FREQ_S, 4);
2912         roce_set_field(qpc_mask->byte_172_sq_psn,
2913                        V2_QPC_BYTE_172_ACK_REQ_FREQ_M,
2914                        V2_QPC_BYTE_172_ACK_REQ_FREQ_S, 0);
2915
2916         roce_set_bit(qpc_mask->byte_172_sq_psn, V2_QPC_BYTE_172_MSG_RNR_FLG_S,
2917                      0);
2918
2919         roce_set_bit(context->byte_172_sq_psn, V2_QPC_BYTE_172_FRE_S, 1);
2920         roce_set_bit(qpc_mask->byte_172_sq_psn, V2_QPC_BYTE_172_FRE_S, 0);
2921
2922         roce_set_field(qpc_mask->byte_176_msg_pktn,
2923                        V2_QPC_BYTE_176_MSG_USE_PKTN_M,
2924                        V2_QPC_BYTE_176_MSG_USE_PKTN_S, 0);
2925         roce_set_field(qpc_mask->byte_176_msg_pktn,
2926                        V2_QPC_BYTE_176_IRRL_HEAD_PRE_M,
2927                        V2_QPC_BYTE_176_IRRL_HEAD_PRE_S, 0);
2928
2929         roce_set_field(qpc_mask->byte_184_irrl_idx,
2930                        V2_QPC_BYTE_184_IRRL_IDX_MSB_M,
2931                        V2_QPC_BYTE_184_IRRL_IDX_MSB_S, 0);
2932
2933         qpc_mask->cur_sge_offset = 0;
2934
2935         roce_set_field(qpc_mask->byte_192_ext_sge,
2936                        V2_QPC_BYTE_192_CUR_SGE_IDX_M,
2937                        V2_QPC_BYTE_192_CUR_SGE_IDX_S, 0);
2938         roce_set_field(qpc_mask->byte_192_ext_sge,
2939                        V2_QPC_BYTE_192_EXT_SGE_NUM_LEFT_M,
2940                        V2_QPC_BYTE_192_EXT_SGE_NUM_LEFT_S, 0);
2941
2942         roce_set_field(qpc_mask->byte_196_sq_psn, V2_QPC_BYTE_196_IRRL_HEAD_M,
2943                        V2_QPC_BYTE_196_IRRL_HEAD_S, 0);
2944
2945         roce_set_field(qpc_mask->byte_200_sq_max, V2_QPC_BYTE_200_SQ_MAX_IDX_M,
2946                        V2_QPC_BYTE_200_SQ_MAX_IDX_S, 0);
2947         roce_set_field(qpc_mask->byte_200_sq_max,
2948                        V2_QPC_BYTE_200_LCL_OPERATED_CNT_M,
2949                        V2_QPC_BYTE_200_LCL_OPERATED_CNT_S, 0);
2950
2951         roce_set_bit(qpc_mask->byte_208_irrl, V2_QPC_BYTE_208_PKT_RNR_FLG_S, 0);
2952         roce_set_bit(qpc_mask->byte_208_irrl, V2_QPC_BYTE_208_PKT_RTY_FLG_S, 0);
2953
2954         roce_set_field(qpc_mask->byte_212_lsn, V2_QPC_BYTE_212_CHECK_FLG_M,
2955                        V2_QPC_BYTE_212_CHECK_FLG_S, 0);
2956
2957         qpc_mask->sq_timer = 0;
2958
2959         roce_set_field(qpc_mask->byte_220_retry_psn_msn,
2960                        V2_QPC_BYTE_220_RETRY_MSG_MSN_M,
2961                        V2_QPC_BYTE_220_RETRY_MSG_MSN_S, 0);
2962         roce_set_field(qpc_mask->byte_232_irrl_sge,
2963                        V2_QPC_BYTE_232_IRRL_SGE_IDX_M,
2964                        V2_QPC_BYTE_232_IRRL_SGE_IDX_S, 0);
2965
2966         roce_set_bit(qpc_mask->byte_232_irrl_sge, V2_QPC_BYTE_232_SO_LP_VLD_S,
2967                      0);
2968         roce_set_bit(qpc_mask->byte_232_irrl_sge,
2969                      V2_QPC_BYTE_232_FENCE_LP_VLD_S, 0);
2970         roce_set_bit(qpc_mask->byte_232_irrl_sge, V2_QPC_BYTE_232_IRRL_LP_VLD_S,
2971                      0);
2972
2973         qpc_mask->irrl_cur_sge_offset = 0;
2974
2975         roce_set_field(qpc_mask->byte_240_irrl_tail,
2976                        V2_QPC_BYTE_240_IRRL_TAIL_REAL_M,
2977                        V2_QPC_BYTE_240_IRRL_TAIL_REAL_S, 0);
2978         roce_set_field(qpc_mask->byte_240_irrl_tail,
2979                        V2_QPC_BYTE_240_IRRL_TAIL_RD_M,
2980                        V2_QPC_BYTE_240_IRRL_TAIL_RD_S, 0);
2981         roce_set_field(qpc_mask->byte_240_irrl_tail,
2982                        V2_QPC_BYTE_240_RX_ACK_MSN_M,
2983                        V2_QPC_BYTE_240_RX_ACK_MSN_S, 0);
2984
2985         roce_set_field(qpc_mask->byte_248_ack_psn, V2_QPC_BYTE_248_IRRL_PSN_M,
2986                        V2_QPC_BYTE_248_IRRL_PSN_S, 0);
2987         roce_set_bit(qpc_mask->byte_248_ack_psn, V2_QPC_BYTE_248_ACK_PSN_ERR_S,
2988                      0);
2989         roce_set_field(qpc_mask->byte_248_ack_psn,
2990                        V2_QPC_BYTE_248_ACK_LAST_OPTYPE_M,
2991                        V2_QPC_BYTE_248_ACK_LAST_OPTYPE_S, 0);
2992         roce_set_bit(qpc_mask->byte_248_ack_psn, V2_QPC_BYTE_248_IRRL_PSN_VLD_S,
2993                      0);
2994         roce_set_bit(qpc_mask->byte_248_ack_psn,
2995                      V2_QPC_BYTE_248_RNR_RETRY_FLAG_S, 0);
2996         roce_set_bit(qpc_mask->byte_248_ack_psn, V2_QPC_BYTE_248_CQ_ERR_IND_S,
2997                      0);
2998
2999         hr_qp->access_flags = attr->qp_access_flags;
3000         hr_qp->pkey_index = attr->pkey_index;
3001         roce_set_field(context->byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M,
3002                        V2_QPC_BYTE_252_TX_CQN_S, to_hr_cq(ibqp->send_cq)->cqn);
3003         roce_set_field(qpc_mask->byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M,
3004                        V2_QPC_BYTE_252_TX_CQN_S, 0);
3005
3006         roce_set_field(qpc_mask->byte_252_err_txcqn, V2_QPC_BYTE_252_ERR_TYPE_M,
3007                        V2_QPC_BYTE_252_ERR_TYPE_S, 0);
3008
3009         roce_set_field(qpc_mask->byte_256_sqflush_rqcqe,
3010                        V2_QPC_BYTE_256_RQ_CQE_IDX_M,
3011                        V2_QPC_BYTE_256_RQ_CQE_IDX_S, 0);
3012         roce_set_field(qpc_mask->byte_256_sqflush_rqcqe,
3013                        V2_QPC_BYTE_256_SQ_FLUSH_IDX_M,
3014                        V2_QPC_BYTE_256_SQ_FLUSH_IDX_S, 0);
3015 }
3016
3017 static void modify_qp_init_to_init(struct ib_qp *ibqp,
3018                                    const struct ib_qp_attr *attr, int attr_mask,
3019                                    struct hns_roce_v2_qp_context *context,
3020                                    struct hns_roce_v2_qp_context *qpc_mask)
3021 {
3022         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
3023
3024         /*
3025          * In v2 engine, software pass context and context mask to hardware
3026          * when modifying qp. If software need modify some fields in context,
3027          * we should set all bits of the relevant fields in context mask to
3028          * 0 at the same time, else set them to 0x1.
3029          */
3030         roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M,
3031                        V2_QPC_BYTE_4_TST_S, to_hr_qp_type(hr_qp->ibqp.qp_type));
3032         roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_TST_M,
3033                        V2_QPC_BYTE_4_TST_S, 0);
3034
3035         if (ibqp->qp_type == IB_QPT_GSI)
3036                 roce_set_field(context->byte_4_sqpn_tst,
3037                                V2_QPC_BYTE_4_SGE_SHIFT_M,
3038                                V2_QPC_BYTE_4_SGE_SHIFT_S,
3039                                ilog2((unsigned int)hr_qp->sge.sge_cnt));
3040         else
3041                 roce_set_field(context->byte_4_sqpn_tst,
3042                                V2_QPC_BYTE_4_SGE_SHIFT_M,
3043                                V2_QPC_BYTE_4_SGE_SHIFT_S, hr_qp->sq.max_gs > 2 ?
3044                                ilog2((unsigned int)hr_qp->sge.sge_cnt) : 0);
3045
3046         roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_SGE_SHIFT_M,
3047                        V2_QPC_BYTE_4_SGE_SHIFT_S, 0);
3048
3049         if (attr_mask & IB_QP_ACCESS_FLAGS) {
3050                 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S,
3051                              !!(attr->qp_access_flags & IB_ACCESS_REMOTE_READ));
3052                 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S,
3053                              0);
3054
3055                 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S,
3056                              !!(attr->qp_access_flags &
3057                              IB_ACCESS_REMOTE_WRITE));
3058                 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S,
3059                              0);
3060
3061                 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S,
3062                              !!(attr->qp_access_flags &
3063                              IB_ACCESS_REMOTE_ATOMIC));
3064                 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S,
3065                              0);
3066         } else {
3067                 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S,
3068                              !!(hr_qp->access_flags & IB_ACCESS_REMOTE_READ));
3069                 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RRE_S,
3070                              0);
3071
3072                 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S,
3073                              !!(hr_qp->access_flags & IB_ACCESS_REMOTE_WRITE));
3074                 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_RWE_S,
3075                              0);
3076
3077                 roce_set_bit(context->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S,
3078                              !!(hr_qp->access_flags & IB_ACCESS_REMOTE_ATOMIC));
3079                 roce_set_bit(qpc_mask->byte_76_srqn_op_en, V2_QPC_BYTE_76_ATE_S,
3080                              0);
3081         }
3082
3083         roce_set_field(context->byte_20_smac_sgid_idx,
3084                        V2_QPC_BYTE_20_SQ_SHIFT_M, V2_QPC_BYTE_20_SQ_SHIFT_S,
3085                        ilog2((unsigned int)hr_qp->sq.wqe_cnt));
3086         roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
3087                        V2_QPC_BYTE_20_SQ_SHIFT_M, V2_QPC_BYTE_20_SQ_SHIFT_S, 0);
3088
3089         roce_set_field(context->byte_20_smac_sgid_idx,
3090                        V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S,
3091                        ilog2((unsigned int)hr_qp->rq.wqe_cnt));
3092         roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
3093                        V2_QPC_BYTE_20_RQ_SHIFT_M, V2_QPC_BYTE_20_RQ_SHIFT_S, 0);
3094
3095         roce_set_field(context->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M,
3096                        V2_QPC_BYTE_16_PD_S, to_hr_pd(ibqp->pd)->pdn);
3097         roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_PD_M,
3098                        V2_QPC_BYTE_16_PD_S, 0);
3099
3100         roce_set_field(context->byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_RX_CQN_M,
3101                        V2_QPC_BYTE_80_RX_CQN_S, to_hr_cq(ibqp->recv_cq)->cqn);
3102         roce_set_field(qpc_mask->byte_80_rnr_rx_cqn, V2_QPC_BYTE_80_RX_CQN_M,
3103                        V2_QPC_BYTE_80_RX_CQN_S, 0);
3104
3105         roce_set_field(context->byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M,
3106                        V2_QPC_BYTE_252_TX_CQN_S, to_hr_cq(ibqp->send_cq)->cqn);
3107         roce_set_field(qpc_mask->byte_252_err_txcqn, V2_QPC_BYTE_252_TX_CQN_M,
3108                        V2_QPC_BYTE_252_TX_CQN_S, 0);
3109
3110         if (ibqp->srq) {
3111                 roce_set_bit(context->byte_76_srqn_op_en,
3112                              V2_QPC_BYTE_76_SRQ_EN_S, 1);
3113                 roce_set_bit(qpc_mask->byte_76_srqn_op_en,
3114                              V2_QPC_BYTE_76_SRQ_EN_S, 0);
3115                 roce_set_field(context->byte_76_srqn_op_en,
3116                                V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S,
3117                                to_hr_srq(ibqp->srq)->srqn);
3118                 roce_set_field(qpc_mask->byte_76_srqn_op_en,
3119                                V2_QPC_BYTE_76_SRQN_M, V2_QPC_BYTE_76_SRQN_S, 0);
3120         }
3121
3122         if (attr_mask & IB_QP_QKEY) {
3123                 context->qkey_xrcd = attr->qkey;
3124                 qpc_mask->qkey_xrcd = 0;
3125         }
3126
3127         roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M,
3128                        V2_QPC_BYTE_4_SQPN_S, hr_qp->qpn);
3129         roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_SQPN_M,
3130                        V2_QPC_BYTE_4_SQPN_S, 0);
3131
3132         if (attr_mask & IB_QP_DEST_QPN) {
3133                 roce_set_field(context->byte_56_dqpn_err, V2_QPC_BYTE_56_DQPN_M,
3134                                V2_QPC_BYTE_56_DQPN_S, hr_qp->qpn);
3135                 roce_set_field(qpc_mask->byte_56_dqpn_err,
3136                                V2_QPC_BYTE_56_DQPN_M, V2_QPC_BYTE_56_DQPN_S, 0);
3137         }
3138 }
3139
3140 static int modify_qp_init_to_rtr(struct ib_qp *ibqp,
3141                                  const struct ib_qp_attr *attr, int attr_mask,
3142                                  struct hns_roce_v2_qp_context *context,
3143                                  struct hns_roce_v2_qp_context *qpc_mask)
3144 {
3145         const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
3146         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
3147         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
3148         struct device *dev = hr_dev->dev;
3149         dma_addr_t dma_handle_3;
3150         dma_addr_t dma_handle_2;
3151         dma_addr_t dma_handle;
3152         u32 page_size;
3153         u8 port_num;
3154         u64 *mtts_3;
3155         u64 *mtts_2;
3156         u64 *mtts;
3157         u8 *dmac;
3158         u8 *smac;
3159         int port;
3160
3161         /* Search qp buf's mtts */
3162         mtts = hns_roce_table_find(hr_dev, &hr_dev->mr_table.mtt_table,
3163                                    hr_qp->mtt.first_seg, &dma_handle);
3164         if (!mtts) {
3165                 dev_err(dev, "qp buf pa find failed\n");
3166                 return -EINVAL;
3167         }
3168
3169         /* Search IRRL's mtts */
3170         mtts_2 = hns_roce_table_find(hr_dev, &hr_dev->qp_table.irrl_table,
3171                                      hr_qp->qpn, &dma_handle_2);
3172         if (!mtts_2) {
3173                 dev_err(dev, "qp irrl_table find failed\n");
3174                 return -EINVAL;
3175         }
3176
3177         /* Search TRRL's mtts */
3178         mtts_3 = hns_roce_table_find(hr_dev, &hr_dev->qp_table.trrl_table,
3179                                      hr_qp->qpn, &dma_handle_3);
3180         if (!mtts_3) {
3181                 dev_err(dev, "qp trrl_table find failed\n");
3182                 return -EINVAL;
3183         }
3184
3185         if (attr_mask & IB_QP_ALT_PATH) {
3186                 dev_err(dev, "INIT2RTR attr_mask (0x%x) error\n", attr_mask);
3187                 return -EINVAL;
3188         }
3189
3190         dmac = (u8 *)attr->ah_attr.roce.dmac;
3191         context->wqe_sge_ba = (u32)(dma_handle >> 3);
3192         qpc_mask->wqe_sge_ba = 0;
3193
3194         /*
3195          * In v2 engine, software pass context and context mask to hardware
3196          * when modifying qp. If software need modify some fields in context,
3197          * we should set all bits of the relevant fields in context mask to
3198          * 0 at the same time, else set them to 0x1.
3199          */
3200         roce_set_field(context->byte_12_sq_hop, V2_QPC_BYTE_12_WQE_SGE_BA_M,
3201                        V2_QPC_BYTE_12_WQE_SGE_BA_S, dma_handle >> (32 + 3));
3202         roce_set_field(qpc_mask->byte_12_sq_hop, V2_QPC_BYTE_12_WQE_SGE_BA_M,
3203                        V2_QPC_BYTE_12_WQE_SGE_BA_S, 0);
3204
3205         roce_set_field(context->byte_12_sq_hop, V2_QPC_BYTE_12_SQ_HOP_NUM_M,
3206                        V2_QPC_BYTE_12_SQ_HOP_NUM_S,
3207                        hr_dev->caps.mtt_hop_num == HNS_ROCE_HOP_NUM_0 ?
3208                        0 : hr_dev->caps.mtt_hop_num);
3209         roce_set_field(qpc_mask->byte_12_sq_hop, V2_QPC_BYTE_12_SQ_HOP_NUM_M,
3210                        V2_QPC_BYTE_12_SQ_HOP_NUM_S, 0);
3211
3212         roce_set_field(context->byte_20_smac_sgid_idx,
3213                        V2_QPC_BYTE_20_SGE_HOP_NUM_M,
3214                        V2_QPC_BYTE_20_SGE_HOP_NUM_S,
3215                        ((ibqp->qp_type == IB_QPT_GSI) || hr_qp->sq.max_gs > 2) ?
3216                        hr_dev->caps.mtt_hop_num : 0);
3217         roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
3218                        V2_QPC_BYTE_20_SGE_HOP_NUM_M,
3219                        V2_QPC_BYTE_20_SGE_HOP_NUM_S, 0);
3220
3221         roce_set_field(context->byte_20_smac_sgid_idx,
3222                        V2_QPC_BYTE_20_RQ_HOP_NUM_M,
3223                        V2_QPC_BYTE_20_RQ_HOP_NUM_S,
3224                        hr_dev->caps.mtt_hop_num == HNS_ROCE_HOP_NUM_0 ?
3225                        0 : hr_dev->caps.mtt_hop_num);
3226         roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
3227                        V2_QPC_BYTE_20_RQ_HOP_NUM_M,
3228                        V2_QPC_BYTE_20_RQ_HOP_NUM_S, 0);
3229
3230         roce_set_field(context->byte_16_buf_ba_pg_sz,
3231                        V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_M,
3232                        V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_S,
3233                        hr_dev->caps.mtt_ba_pg_sz + PG_SHIFT_OFFSET);
3234         roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz,
3235                        V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_M,
3236                        V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_S, 0);
3237
3238         roce_set_field(context->byte_16_buf_ba_pg_sz,
3239                        V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_M,
3240                        V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_S,
3241                        hr_dev->caps.mtt_buf_pg_sz + PG_SHIFT_OFFSET);
3242         roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz,
3243                        V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_M,
3244                        V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_S, 0);
3245
3246         roce_set_field(context->byte_80_rnr_rx_cqn,
3247                        V2_QPC_BYTE_80_MIN_RNR_TIME_M,
3248                        V2_QPC_BYTE_80_MIN_RNR_TIME_S, attr->min_rnr_timer);
3249         roce_set_field(qpc_mask->byte_80_rnr_rx_cqn,
3250                        V2_QPC_BYTE_80_MIN_RNR_TIME_M,
3251                        V2_QPC_BYTE_80_MIN_RNR_TIME_S, 0);
3252
3253         page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT);
3254         context->rq_cur_blk_addr = (u32)(mtts[hr_qp->rq.offset / page_size]
3255                                     >> PAGE_ADDR_SHIFT);
3256         qpc_mask->rq_cur_blk_addr = 0;
3257
3258         roce_set_field(context->byte_92_srq_info,
3259                        V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_M,
3260                        V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_S,
3261                        mtts[hr_qp->rq.offset / page_size]
3262                        >> (32 + PAGE_ADDR_SHIFT));
3263         roce_set_field(qpc_mask->byte_92_srq_info,
3264                        V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_M,
3265                        V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_S, 0);
3266
3267         context->rq_nxt_blk_addr = (u32)(mtts[hr_qp->rq.offset / page_size + 1]
3268                                     >> PAGE_ADDR_SHIFT);
3269         qpc_mask->rq_nxt_blk_addr = 0;
3270
3271         roce_set_field(context->byte_104_rq_sge,
3272                        V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_M,
3273                        V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_S,
3274                        mtts[hr_qp->rq.offset / page_size + 1]
3275                        >> (32 + PAGE_ADDR_SHIFT));
3276         roce_set_field(qpc_mask->byte_104_rq_sge,
3277                        V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_M,
3278                        V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_S, 0);
3279
3280         roce_set_field(context->byte_108_rx_reqepsn,
3281                        V2_QPC_BYTE_108_RX_REQ_EPSN_M,
3282                        V2_QPC_BYTE_108_RX_REQ_EPSN_S, attr->rq_psn);
3283         roce_set_field(qpc_mask->byte_108_rx_reqepsn,
3284                        V2_QPC_BYTE_108_RX_REQ_EPSN_M,
3285                        V2_QPC_BYTE_108_RX_REQ_EPSN_S, 0);
3286
3287         roce_set_field(context->byte_132_trrl, V2_QPC_BYTE_132_TRRL_BA_M,
3288                        V2_QPC_BYTE_132_TRRL_BA_S, dma_handle_3 >> 4);
3289         roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_BA_M,
3290                        V2_QPC_BYTE_132_TRRL_BA_S, 0);
3291         context->trrl_ba = (u32)(dma_handle_3 >> (16 + 4));
3292         qpc_mask->trrl_ba = 0;
3293         roce_set_field(context->byte_140_raq, V2_QPC_BYTE_140_TRRL_BA_M,
3294                        V2_QPC_BYTE_140_TRRL_BA_S,
3295                        (u32)(dma_handle_3 >> (32 + 16 + 4)));
3296         roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_TRRL_BA_M,
3297                        V2_QPC_BYTE_140_TRRL_BA_S, 0);
3298
3299         context->irrl_ba = (u32)(dma_handle_2 >> 6);
3300         qpc_mask->irrl_ba = 0;
3301         roce_set_field(context->byte_208_irrl, V2_QPC_BYTE_208_IRRL_BA_M,
3302                        V2_QPC_BYTE_208_IRRL_BA_S,
3303                        dma_handle_2 >> (32 + 6));
3304         roce_set_field(qpc_mask->byte_208_irrl, V2_QPC_BYTE_208_IRRL_BA_M,
3305                        V2_QPC_BYTE_208_IRRL_BA_S, 0);
3306
3307         roce_set_bit(context->byte_208_irrl, V2_QPC_BYTE_208_RMT_E2E_S, 1);
3308         roce_set_bit(qpc_mask->byte_208_irrl, V2_QPC_BYTE_208_RMT_E2E_S, 0);
3309
3310         roce_set_bit(context->byte_252_err_txcqn, V2_QPC_BYTE_252_SIG_TYPE_S,
3311                      hr_qp->sq_signal_bits);
3312         roce_set_bit(qpc_mask->byte_252_err_txcqn, V2_QPC_BYTE_252_SIG_TYPE_S,
3313                      0);
3314
3315         port = (attr_mask & IB_QP_PORT) ? (attr->port_num - 1) : hr_qp->port;
3316
3317         smac = (u8 *)hr_dev->dev_addr[port];
3318         /* when dmac equals smac or loop_idc is 1, it should loopback */
3319         if (ether_addr_equal_unaligned(dmac, smac) ||
3320             hr_dev->loop_idc == 0x1) {
3321                 roce_set_bit(context->byte_28_at_fl, V2_QPC_BYTE_28_LBI_S, 1);
3322                 roce_set_bit(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_LBI_S, 0);
3323         }
3324
3325         if ((attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) &&
3326              attr->max_dest_rd_atomic) {
3327                 roce_set_field(context->byte_140_raq, V2_QPC_BYTE_140_RR_MAX_M,
3328                                V2_QPC_BYTE_140_RR_MAX_S,
3329                                fls(attr->max_dest_rd_atomic - 1));
3330                 roce_set_field(qpc_mask->byte_140_raq, V2_QPC_BYTE_140_RR_MAX_M,
3331                                V2_QPC_BYTE_140_RR_MAX_S, 0);
3332         }
3333
3334         if (attr_mask & IB_QP_DEST_QPN) {
3335                 roce_set_field(context->byte_56_dqpn_err, V2_QPC_BYTE_56_DQPN_M,
3336                                V2_QPC_BYTE_56_DQPN_S, attr->dest_qp_num);
3337                 roce_set_field(qpc_mask->byte_56_dqpn_err,
3338                                V2_QPC_BYTE_56_DQPN_M, V2_QPC_BYTE_56_DQPN_S, 0);
3339         }
3340
3341         /* Configure GID index */
3342         port_num = rdma_ah_get_port_num(&attr->ah_attr);
3343         roce_set_field(context->byte_20_smac_sgid_idx,
3344                        V2_QPC_BYTE_20_SGID_IDX_M,
3345                        V2_QPC_BYTE_20_SGID_IDX_S,
3346                        hns_get_gid_index(hr_dev, port_num - 1,
3347                                          grh->sgid_index));
3348         roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
3349                        V2_QPC_BYTE_20_SGID_IDX_M,
3350                        V2_QPC_BYTE_20_SGID_IDX_S, 0);
3351         memcpy(&(context->dmac), dmac, 4);
3352         roce_set_field(context->byte_52_udpspn_dmac, V2_QPC_BYTE_52_DMAC_M,
3353                        V2_QPC_BYTE_52_DMAC_S, *((u16 *)(&dmac[4])));
3354         qpc_mask->dmac = 0;
3355         roce_set_field(qpc_mask->byte_52_udpspn_dmac, V2_QPC_BYTE_52_DMAC_M,
3356                        V2_QPC_BYTE_52_DMAC_S, 0);
3357
3358         roce_set_field(context->byte_56_dqpn_err, V2_QPC_BYTE_56_LP_PKTN_INI_M,
3359                        V2_QPC_BYTE_56_LP_PKTN_INI_S, 4);
3360         roce_set_field(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_LP_PKTN_INI_M,
3361                        V2_QPC_BYTE_56_LP_PKTN_INI_S, 0);
3362
3363         if (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_UD)
3364                 roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_MTU_M,
3365                                V2_QPC_BYTE_24_MTU_S, IB_MTU_4096);
3366         else if (attr_mask & IB_QP_PATH_MTU)
3367                 roce_set_field(context->byte_24_mtu_tc, V2_QPC_BYTE_24_MTU_M,
3368                                V2_QPC_BYTE_24_MTU_S, attr->path_mtu);
3369
3370         roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_MTU_M,
3371                        V2_QPC_BYTE_24_MTU_S, 0);
3372
3373         roce_set_field(context->byte_84_rq_ci_pi,
3374                        V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
3375                        V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, hr_qp->rq.head);
3376         roce_set_field(qpc_mask->byte_84_rq_ci_pi,
3377                        V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
3378                        V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, 0);
3379
3380         roce_set_field(qpc_mask->byte_84_rq_ci_pi,
3381                        V2_QPC_BYTE_84_RQ_CONSUMER_IDX_M,
3382                        V2_QPC_BYTE_84_RQ_CONSUMER_IDX_S, 0);
3383         roce_set_bit(qpc_mask->byte_108_rx_reqepsn,
3384                      V2_QPC_BYTE_108_RX_REQ_PSN_ERR_S, 0);
3385         roce_set_field(qpc_mask->byte_96_rx_reqmsn, V2_QPC_BYTE_96_RX_REQ_MSN_M,
3386                        V2_QPC_BYTE_96_RX_REQ_MSN_S, 0);
3387         roce_set_field(qpc_mask->byte_108_rx_reqepsn,
3388                        V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_M,
3389                        V2_QPC_BYTE_108_RX_REQ_LAST_OPTYPE_S, 0);
3390
3391         context->rq_rnr_timer = 0;
3392         qpc_mask->rq_rnr_timer = 0;
3393
3394         roce_set_field(context->byte_152_raq, V2_QPC_BYTE_152_RAQ_PSN_M,
3395                        V2_QPC_BYTE_152_RAQ_PSN_S, attr->rq_psn - 1);
3396         roce_set_field(qpc_mask->byte_152_raq, V2_QPC_BYTE_152_RAQ_PSN_M,
3397                        V2_QPC_BYTE_152_RAQ_PSN_S, 0);
3398
3399         roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_HEAD_MAX_M,
3400                        V2_QPC_BYTE_132_TRRL_HEAD_MAX_S, 0);
3401         roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_TAIL_MAX_M,
3402                        V2_QPC_BYTE_132_TRRL_TAIL_MAX_S, 0);
3403
3404         roce_set_field(context->byte_168_irrl_idx,
3405                        V2_QPC_BYTE_168_LP_SGEN_INI_M,
3406                        V2_QPC_BYTE_168_LP_SGEN_INI_S, 3);
3407         roce_set_field(qpc_mask->byte_168_irrl_idx,
3408                        V2_QPC_BYTE_168_LP_SGEN_INI_M,
3409                        V2_QPC_BYTE_168_LP_SGEN_INI_S, 0);
3410
3411         return 0;
3412 }
3413
3414 static int modify_qp_rtr_to_rts(struct ib_qp *ibqp,
3415                                 const struct ib_qp_attr *attr, int attr_mask,
3416                                 struct hns_roce_v2_qp_context *context,
3417                                 struct hns_roce_v2_qp_context *qpc_mask)
3418 {
3419         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
3420         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
3421         struct device *dev = hr_dev->dev;
3422         dma_addr_t dma_handle;
3423         u32 page_size;
3424         u64 *mtts;
3425
3426         /* Search qp buf's mtts */
3427         mtts = hns_roce_table_find(hr_dev, &hr_dev->mr_table.mtt_table,
3428                                    hr_qp->mtt.first_seg, &dma_handle);
3429         if (!mtts) {
3430                 dev_err(dev, "qp buf pa find failed\n");
3431                 return -EINVAL;
3432         }
3433
3434         /* Not support alternate path and path migration */
3435         if ((attr_mask & IB_QP_ALT_PATH) ||
3436             (attr_mask & IB_QP_PATH_MIG_STATE)) {
3437                 dev_err(dev, "RTR2RTS attr_mask (0x%x)error\n", attr_mask);
3438                 return -EINVAL;
3439         }
3440
3441         /*
3442          * In v2 engine, software pass context and context mask to hardware
3443          * when modifying qp. If software need modify some fields in context,
3444          * we should set all bits of the relevant fields in context mask to
3445          * 0 at the same time, else set them to 0x1.
3446          */
3447         context->sq_cur_blk_addr = (u32)(mtts[0] >> PAGE_ADDR_SHIFT);
3448         roce_set_field(context->byte_168_irrl_idx,
3449                        V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_M,
3450                        V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_S,
3451                        mtts[0] >> (32 + PAGE_ADDR_SHIFT));
3452         qpc_mask->sq_cur_blk_addr = 0;
3453         roce_set_field(qpc_mask->byte_168_irrl_idx,
3454                        V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_M,
3455                        V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_S, 0);
3456
3457         page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT);
3458         context->sq_cur_sge_blk_addr =
3459                        ((ibqp->qp_type == IB_QPT_GSI) || hr_qp->sq.max_gs > 2) ?
3460                                       ((u32)(mtts[hr_qp->sge.offset / page_size]
3461                                       >> PAGE_ADDR_SHIFT)) : 0;
3462         roce_set_field(context->byte_184_irrl_idx,
3463                        V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_M,
3464                        V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_S,
3465                        ((ibqp->qp_type == IB_QPT_GSI) || hr_qp->sq.max_gs > 2) ?
3466                        (mtts[hr_qp->sge.offset / page_size] >>
3467                        (32 + PAGE_ADDR_SHIFT)) : 0);
3468         qpc_mask->sq_cur_sge_blk_addr = 0;
3469         roce_set_field(qpc_mask->byte_184_irrl_idx,
3470                        V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_M,
3471                        V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_S, 0);
3472
3473         context->rx_sq_cur_blk_addr = (u32)(mtts[0] >> PAGE_ADDR_SHIFT);
3474         roce_set_field(context->byte_232_irrl_sge,
3475                        V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_M,
3476                        V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_S,
3477                        mtts[0] >> (32 + PAGE_ADDR_SHIFT));
3478         qpc_mask->rx_sq_cur_blk_addr = 0;
3479         roce_set_field(qpc_mask->byte_232_irrl_sge,
3480                        V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_M,
3481                        V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_S, 0);
3482
3483         /*
3484          * Set some fields in context to zero, Because the default values
3485          * of all fields in context are zero, we need not set them to 0 again.
3486          * but we should set the relevant fields of context mask to 0.
3487          */
3488         roce_set_field(qpc_mask->byte_232_irrl_sge,
3489                        V2_QPC_BYTE_232_IRRL_SGE_IDX_M,
3490                        V2_QPC_BYTE_232_IRRL_SGE_IDX_S, 0);
3491
3492         roce_set_field(qpc_mask->byte_240_irrl_tail,
3493                        V2_QPC_BYTE_240_RX_ACK_MSN_M,
3494                        V2_QPC_BYTE_240_RX_ACK_MSN_S, 0);
3495
3496         roce_set_field(context->byte_244_rnr_rxack,
3497                        V2_QPC_BYTE_244_RX_ACK_EPSN_M,
3498                        V2_QPC_BYTE_244_RX_ACK_EPSN_S, attr->sq_psn);
3499         roce_set_field(qpc_mask->byte_244_rnr_rxack,
3500                        V2_QPC_BYTE_244_RX_ACK_EPSN_M,
3501                        V2_QPC_BYTE_244_RX_ACK_EPSN_S, 0);
3502
3503         roce_set_field(qpc_mask->byte_248_ack_psn,
3504                        V2_QPC_BYTE_248_ACK_LAST_OPTYPE_M,
3505                        V2_QPC_BYTE_248_ACK_LAST_OPTYPE_S, 0);
3506         roce_set_bit(qpc_mask->byte_248_ack_psn,
3507                      V2_QPC_BYTE_248_IRRL_PSN_VLD_S, 0);
3508         roce_set_field(qpc_mask->byte_248_ack_psn,
3509                        V2_QPC_BYTE_248_IRRL_PSN_M,
3510                        V2_QPC_BYTE_248_IRRL_PSN_S, 0);
3511
3512         roce_set_field(qpc_mask->byte_240_irrl_tail,
3513                        V2_QPC_BYTE_240_IRRL_TAIL_REAL_M,
3514                        V2_QPC_BYTE_240_IRRL_TAIL_REAL_S, 0);
3515
3516         roce_set_field(context->byte_220_retry_psn_msn,
3517                        V2_QPC_BYTE_220_RETRY_MSG_PSN_M,
3518                        V2_QPC_BYTE_220_RETRY_MSG_PSN_S, attr->sq_psn);
3519         roce_set_field(qpc_mask->byte_220_retry_psn_msn,
3520                        V2_QPC_BYTE_220_RETRY_MSG_PSN_M,
3521                        V2_QPC_BYTE_220_RETRY_MSG_PSN_S, 0);
3522
3523         roce_set_field(context->byte_224_retry_msg,
3524                        V2_QPC_BYTE_224_RETRY_MSG_PSN_M,
3525                        V2_QPC_BYTE_224_RETRY_MSG_PSN_S, attr->sq_psn >> 16);
3526         roce_set_field(qpc_mask->byte_224_retry_msg,
3527                        V2_QPC_BYTE_224_RETRY_MSG_PSN_M,
3528                        V2_QPC_BYTE_224_RETRY_MSG_PSN_S, 0);
3529
3530         roce_set_field(context->byte_224_retry_msg,
3531                        V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_M,
3532                        V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_S, attr->sq_psn);
3533         roce_set_field(qpc_mask->byte_224_retry_msg,
3534                        V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_M,
3535                        V2_QPC_BYTE_224_RETRY_MSG_FPKT_PSN_S, 0);
3536
3537         roce_set_field(qpc_mask->byte_220_retry_psn_msn,
3538                        V2_QPC_BYTE_220_RETRY_MSG_MSN_M,
3539                        V2_QPC_BYTE_220_RETRY_MSG_MSN_S, 0);
3540
3541         roce_set_bit(qpc_mask->byte_248_ack_psn,
3542                      V2_QPC_BYTE_248_RNR_RETRY_FLAG_S, 0);
3543
3544         roce_set_field(qpc_mask->byte_212_lsn, V2_QPC_BYTE_212_CHECK_FLG_M,
3545                        V2_QPC_BYTE_212_CHECK_FLG_S, 0);
3546
3547         roce_set_field(context->byte_212_lsn, V2_QPC_BYTE_212_RETRY_CNT_M,
3548                        V2_QPC_BYTE_212_RETRY_CNT_S, attr->retry_cnt);
3549         roce_set_field(qpc_mask->byte_212_lsn, V2_QPC_BYTE_212_RETRY_CNT_M,
3550                        V2_QPC_BYTE_212_RETRY_CNT_S, 0);
3551
3552         roce_set_field(context->byte_212_lsn, V2_QPC_BYTE_212_RETRY_NUM_INIT_M,
3553                        V2_QPC_BYTE_212_RETRY_NUM_INIT_S, attr->retry_cnt);
3554         roce_set_field(qpc_mask->byte_212_lsn, V2_QPC_BYTE_212_RETRY_NUM_INIT_M,
3555                        V2_QPC_BYTE_212_RETRY_NUM_INIT_S, 0);
3556
3557         roce_set_field(context->byte_244_rnr_rxack,
3558                        V2_QPC_BYTE_244_RNR_NUM_INIT_M,
3559                        V2_QPC_BYTE_244_RNR_NUM_INIT_S, attr->rnr_retry);
3560         roce_set_field(qpc_mask->byte_244_rnr_rxack,
3561                        V2_QPC_BYTE_244_RNR_NUM_INIT_M,
3562                        V2_QPC_BYTE_244_RNR_NUM_INIT_S, 0);
3563
3564         roce_set_field(context->byte_244_rnr_rxack, V2_QPC_BYTE_244_RNR_CNT_M,
3565                        V2_QPC_BYTE_244_RNR_CNT_S, attr->rnr_retry);
3566         roce_set_field(qpc_mask->byte_244_rnr_rxack, V2_QPC_BYTE_244_RNR_CNT_M,
3567                        V2_QPC_BYTE_244_RNR_CNT_S, 0);
3568
3569         roce_set_field(context->byte_212_lsn, V2_QPC_BYTE_212_LSN_M,
3570                        V2_QPC_BYTE_212_LSN_S, 0x100);
3571         roce_set_field(qpc_mask->byte_212_lsn, V2_QPC_BYTE_212_LSN_M,
3572                        V2_QPC_BYTE_212_LSN_S, 0);
3573
3574         if (attr_mask & IB_QP_TIMEOUT) {
3575                 roce_set_field(context->byte_28_at_fl, V2_QPC_BYTE_28_AT_M,
3576                                V2_QPC_BYTE_28_AT_S, attr->timeout);
3577                 roce_set_field(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_AT_M,
3578                               V2_QPC_BYTE_28_AT_S, 0);
3579         }
3580
3581         roce_set_field(context->byte_172_sq_psn, V2_QPC_BYTE_172_SQ_CUR_PSN_M,
3582                        V2_QPC_BYTE_172_SQ_CUR_PSN_S, attr->sq_psn);
3583         roce_set_field(qpc_mask->byte_172_sq_psn, V2_QPC_BYTE_172_SQ_CUR_PSN_M,
3584                        V2_QPC_BYTE_172_SQ_CUR_PSN_S, 0);
3585
3586         roce_set_field(qpc_mask->byte_196_sq_psn, V2_QPC_BYTE_196_IRRL_HEAD_M,
3587                        V2_QPC_BYTE_196_IRRL_HEAD_S, 0);
3588         roce_set_field(context->byte_196_sq_psn, V2_QPC_BYTE_196_SQ_MAX_PSN_M,
3589                        V2_QPC_BYTE_196_SQ_MAX_PSN_S, attr->sq_psn);
3590         roce_set_field(qpc_mask->byte_196_sq_psn, V2_QPC_BYTE_196_SQ_MAX_PSN_M,
3591                        V2_QPC_BYTE_196_SQ_MAX_PSN_S, 0);
3592
3593         if ((attr_mask & IB_QP_MAX_QP_RD_ATOMIC) && attr->max_rd_atomic) {
3594                 roce_set_field(context->byte_208_irrl, V2_QPC_BYTE_208_SR_MAX_M,
3595                                V2_QPC_BYTE_208_SR_MAX_S,
3596                                fls(attr->max_rd_atomic - 1));
3597                 roce_set_field(qpc_mask->byte_208_irrl,
3598                                V2_QPC_BYTE_208_SR_MAX_M,
3599                                V2_QPC_BYTE_208_SR_MAX_S, 0);
3600         }
3601         return 0;
3602 }
3603
3604 static int hns_roce_v2_modify_qp(struct ib_qp *ibqp,
3605                                  const struct ib_qp_attr *attr,
3606                                  int attr_mask, enum ib_qp_state cur_state,
3607                                  enum ib_qp_state new_state)
3608 {
3609         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
3610         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
3611         struct hns_roce_v2_qp_context *context;
3612         struct hns_roce_v2_qp_context *qpc_mask;
3613         struct device *dev = hr_dev->dev;
3614         int ret = -EINVAL;
3615
3616         context = kcalloc(2, sizeof(*context), GFP_KERNEL);
3617         if (!context)
3618                 return -ENOMEM;
3619
3620         qpc_mask = context + 1;
3621         /*
3622          * In v2 engine, software pass context and context mask to hardware
3623          * when modifying qp. If software need modify some fields in context,
3624          * we should set all bits of the relevant fields in context mask to
3625          * 0 at the same time, else set them to 0x1.
3626          */
3627         memset(qpc_mask, 0xff, sizeof(*qpc_mask));
3628         if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
3629                 modify_qp_reset_to_init(ibqp, attr, attr_mask, context,
3630                                         qpc_mask);
3631         } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_INIT) {
3632                 modify_qp_init_to_init(ibqp, attr, attr_mask, context,
3633                                        qpc_mask);
3634         } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) {
3635                 ret = modify_qp_init_to_rtr(ibqp, attr, attr_mask, context,
3636                                             qpc_mask);
3637                 if (ret)
3638                         goto out;
3639         } else if (cur_state == IB_QPS_RTR && new_state == IB_QPS_RTS) {
3640                 ret = modify_qp_rtr_to_rts(ibqp, attr, attr_mask, context,
3641                                            qpc_mask);
3642                 if (ret)
3643                         goto out;
3644         } else if ((cur_state == IB_QPS_RTS && new_state == IB_QPS_RTS) ||
3645                    (cur_state == IB_QPS_SQE && new_state == IB_QPS_RTS) ||
3646                    (cur_state == IB_QPS_RTS && new_state == IB_QPS_SQD) ||
3647                    (cur_state == IB_QPS_SQD && new_state == IB_QPS_SQD) ||
3648                    (cur_state == IB_QPS_SQD && new_state == IB_QPS_RTS) ||
3649                    (cur_state == IB_QPS_INIT && new_state == IB_QPS_RESET) ||
3650                    (cur_state == IB_QPS_RTR && new_state == IB_QPS_RESET) ||
3651                    (cur_state == IB_QPS_RTS && new_state == IB_QPS_RESET) ||
3652                    (cur_state == IB_QPS_ERR && new_state == IB_QPS_RESET) ||
3653                    (cur_state == IB_QPS_INIT && new_state == IB_QPS_ERR) ||
3654                    (cur_state == IB_QPS_RTR && new_state == IB_QPS_ERR) ||
3655                    (cur_state == IB_QPS_RTS && new_state == IB_QPS_ERR) ||
3656                    (cur_state == IB_QPS_SQD && new_state == IB_QPS_ERR) ||
3657                    (cur_state == IB_QPS_SQE && new_state == IB_QPS_ERR) ||
3658                    (cur_state == IB_QPS_ERR && new_state == IB_QPS_ERR)) {
3659                 /* Nothing */
3660                 ;
3661         } else {
3662                 dev_err(dev, "Illegal state for QP!\n");
3663                 ret = -EINVAL;
3664                 goto out;
3665         }
3666
3667         /* When QP state is err, SQ and RQ WQE should be flushed */
3668         if (new_state == IB_QPS_ERR) {
3669                 roce_set_field(context->byte_160_sq_ci_pi,
3670                                V2_QPC_BYTE_160_SQ_PRODUCER_IDX_M,
3671                                V2_QPC_BYTE_160_SQ_PRODUCER_IDX_S,
3672                                hr_qp->sq.head);
3673                 roce_set_field(qpc_mask->byte_160_sq_ci_pi,
3674                                V2_QPC_BYTE_160_SQ_PRODUCER_IDX_M,
3675                                V2_QPC_BYTE_160_SQ_PRODUCER_IDX_S, 0);
3676                 roce_set_field(context->byte_84_rq_ci_pi,
3677                                V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
3678                                V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S,
3679                                hr_qp->rq.head);
3680                 roce_set_field(qpc_mask->byte_84_rq_ci_pi,
3681                                V2_QPC_BYTE_84_RQ_PRODUCER_IDX_M,
3682                                V2_QPC_BYTE_84_RQ_PRODUCER_IDX_S, 0);
3683         }
3684
3685         if (attr_mask & IB_QP_AV) {
3686                 const struct ib_global_route *grh =
3687                                             rdma_ah_read_grh(&attr->ah_attr);
3688                 const struct ib_gid_attr *gid_attr = NULL;
3689                 u8 src_mac[ETH_ALEN];
3690                 int is_roce_protocol;
3691                 u16 vlan = 0xffff;
3692                 u8 ib_port;
3693                 u8 hr_port;
3694
3695                 ib_port = (attr_mask & IB_QP_PORT) ? attr->port_num :
3696                            hr_qp->port + 1;
3697                 hr_port = ib_port - 1;
3698                 is_roce_protocol = rdma_cap_eth_ah(&hr_dev->ib_dev, ib_port) &&
3699                                rdma_ah_get_ah_flags(&attr->ah_attr) & IB_AH_GRH;
3700
3701                 if (is_roce_protocol) {
3702                         gid_attr = attr->ah_attr.grh.sgid_attr;
3703                         vlan = rdma_vlan_dev_vlan_id(gid_attr->ndev);
3704                         memcpy(src_mac, gid_attr->ndev->dev_addr, ETH_ALEN);
3705                 }
3706
3707                 if (is_vlan_dev(gid_attr->ndev)) {
3708                         roce_set_bit(context->byte_76_srqn_op_en,
3709                                      V2_QPC_BYTE_76_RQ_VLAN_EN_S, 1);
3710                         roce_set_bit(qpc_mask->byte_76_srqn_op_en,
3711                                      V2_QPC_BYTE_76_RQ_VLAN_EN_S, 0);
3712                         roce_set_bit(context->byte_168_irrl_idx,
3713                                      V2_QPC_BYTE_168_SQ_VLAN_EN_S, 1);
3714                         roce_set_bit(qpc_mask->byte_168_irrl_idx,
3715                                      V2_QPC_BYTE_168_SQ_VLAN_EN_S, 0);
3716                 }
3717
3718                 roce_set_field(context->byte_24_mtu_tc,
3719                                V2_QPC_BYTE_24_VLAN_ID_M,
3720                                V2_QPC_BYTE_24_VLAN_ID_S, vlan);
3721                 roce_set_field(qpc_mask->byte_24_mtu_tc,
3722                                V2_QPC_BYTE_24_VLAN_ID_M,
3723                                V2_QPC_BYTE_24_VLAN_ID_S, 0);
3724
3725                 if (grh->sgid_index >= hr_dev->caps.gid_table_len[hr_port]) {
3726                         dev_err(hr_dev->dev,
3727                                 "sgid_index(%u) too large. max is %d\n",
3728                                 grh->sgid_index,
3729                                 hr_dev->caps.gid_table_len[hr_port]);
3730                         ret = -EINVAL;
3731                         goto out;
3732                 }
3733
3734                 if (attr->ah_attr.type != RDMA_AH_ATTR_TYPE_ROCE) {
3735                         dev_err(hr_dev->dev, "ah attr is not RDMA roce type\n");
3736                         ret = -EINVAL;
3737                         goto out;
3738                 }
3739
3740                 roce_set_field(context->byte_52_udpspn_dmac,
3741                            V2_QPC_BYTE_52_UDPSPN_M, V2_QPC_BYTE_52_UDPSPN_S,
3742                            (gid_attr->gid_type != IB_GID_TYPE_ROCE_UDP_ENCAP) ?
3743                            0 : 0x12b7);
3744
3745                 roce_set_field(qpc_mask->byte_52_udpspn_dmac,
3746                                V2_QPC_BYTE_52_UDPSPN_M,
3747                                V2_QPC_BYTE_52_UDPSPN_S, 0);
3748
3749                 roce_set_field(context->byte_20_smac_sgid_idx,
3750                                V2_QPC_BYTE_20_SGID_IDX_M,
3751                                V2_QPC_BYTE_20_SGID_IDX_S, grh->sgid_index);
3752
3753                 roce_set_field(qpc_mask->byte_20_smac_sgid_idx,
3754                                V2_QPC_BYTE_20_SGID_IDX_M,
3755                                V2_QPC_BYTE_20_SGID_IDX_S, 0);
3756
3757                 roce_set_field(context->byte_24_mtu_tc,
3758                                V2_QPC_BYTE_24_HOP_LIMIT_M,
3759                                V2_QPC_BYTE_24_HOP_LIMIT_S, grh->hop_limit);
3760                 roce_set_field(qpc_mask->byte_24_mtu_tc,
3761                                V2_QPC_BYTE_24_HOP_LIMIT_M,
3762                                V2_QPC_BYTE_24_HOP_LIMIT_S, 0);
3763
3764                 if (hr_dev->pci_dev->revision == 0x21 &&
3765                     gid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP)
3766                         roce_set_field(context->byte_24_mtu_tc,
3767                                        V2_QPC_BYTE_24_TC_M, V2_QPC_BYTE_24_TC_S,
3768                                        grh->traffic_class >> 2);
3769                 else
3770                         roce_set_field(context->byte_24_mtu_tc,
3771                                        V2_QPC_BYTE_24_TC_M, V2_QPC_BYTE_24_TC_S,
3772                                        grh->traffic_class);
3773                 roce_set_field(qpc_mask->byte_24_mtu_tc, V2_QPC_BYTE_24_TC_M,
3774                                V2_QPC_BYTE_24_TC_S, 0);
3775                 roce_set_field(context->byte_28_at_fl, V2_QPC_BYTE_28_FL_M,
3776                                V2_QPC_BYTE_28_FL_S, grh->flow_label);
3777                 roce_set_field(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_FL_M,
3778                                V2_QPC_BYTE_28_FL_S, 0);
3779                 memcpy(context->dgid, grh->dgid.raw, sizeof(grh->dgid.raw));
3780                 memset(qpc_mask->dgid, 0, sizeof(grh->dgid.raw));
3781                 roce_set_field(context->byte_28_at_fl, V2_QPC_BYTE_28_SL_M,
3782                                V2_QPC_BYTE_28_SL_S,
3783                                rdma_ah_get_sl(&attr->ah_attr));
3784                 roce_set_field(qpc_mask->byte_28_at_fl, V2_QPC_BYTE_28_SL_M,
3785                                V2_QPC_BYTE_28_SL_S, 0);
3786                 hr_qp->sl = rdma_ah_get_sl(&attr->ah_attr);
3787         }
3788
3789         if (attr_mask & (IB_QP_ACCESS_FLAGS | IB_QP_MAX_DEST_RD_ATOMIC))
3790                 set_access_flags(hr_qp, context, qpc_mask, attr, attr_mask);
3791
3792         /* Every status migrate must change state */
3793         roce_set_field(context->byte_60_qpst_tempid, V2_QPC_BYTE_60_QP_ST_M,
3794                        V2_QPC_BYTE_60_QP_ST_S, new_state);
3795         roce_set_field(qpc_mask->byte_60_qpst_tempid, V2_QPC_BYTE_60_QP_ST_M,
3796                        V2_QPC_BYTE_60_QP_ST_S, 0);
3797
3798         /* SW pass context to HW */
3799         ret = hns_roce_v2_qp_modify(hr_dev, &hr_qp->mtt, cur_state, new_state,
3800                                     context, hr_qp);
3801         if (ret) {
3802                 dev_err(dev, "hns_roce_qp_modify failed(%d)\n", ret);
3803                 goto out;
3804         }
3805
3806         hr_qp->state = new_state;
3807
3808         if (attr_mask & IB_QP_ACCESS_FLAGS)
3809                 hr_qp->atomic_rd_en = attr->qp_access_flags;
3810
3811         if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
3812                 hr_qp->resp_depth = attr->max_dest_rd_atomic;
3813         if (attr_mask & IB_QP_PORT) {
3814                 hr_qp->port = attr->port_num - 1;
3815                 hr_qp->phy_port = hr_dev->iboe.phy_port[hr_qp->port];
3816         }
3817
3818         if (new_state == IB_QPS_RESET && !ibqp->uobject) {
3819                 hns_roce_v2_cq_clean(to_hr_cq(ibqp->recv_cq), hr_qp->qpn,
3820                                      ibqp->srq ? to_hr_srq(ibqp->srq) : NULL);
3821                 if (ibqp->send_cq != ibqp->recv_cq)
3822                         hns_roce_v2_cq_clean(to_hr_cq(ibqp->send_cq),
3823                                              hr_qp->qpn, NULL);
3824
3825                 hr_qp->rq.head = 0;
3826                 hr_qp->rq.tail = 0;
3827                 hr_qp->sq.head = 0;
3828                 hr_qp->sq.tail = 0;
3829                 hr_qp->sq_next_wqe = 0;
3830                 hr_qp->next_sge = 0;
3831                 if (hr_qp->rq.wqe_cnt)
3832                         *hr_qp->rdb.db_record = 0;
3833         }
3834
3835 out:
3836         kfree(context);
3837         return ret;
3838 }
3839
3840 static inline enum ib_qp_state to_ib_qp_st(enum hns_roce_v2_qp_state state)
3841 {
3842         switch (state) {
3843         case HNS_ROCE_QP_ST_RST:        return IB_QPS_RESET;
3844         case HNS_ROCE_QP_ST_INIT:       return IB_QPS_INIT;
3845         case HNS_ROCE_QP_ST_RTR:        return IB_QPS_RTR;
3846         case HNS_ROCE_QP_ST_RTS:        return IB_QPS_RTS;
3847         case HNS_ROCE_QP_ST_SQ_DRAINING:
3848         case HNS_ROCE_QP_ST_SQD:        return IB_QPS_SQD;
3849         case HNS_ROCE_QP_ST_SQER:       return IB_QPS_SQE;
3850         case HNS_ROCE_QP_ST_ERR:        return IB_QPS_ERR;
3851         default:                        return -1;
3852         }
3853 }
3854
3855 static int hns_roce_v2_query_qpc(struct hns_roce_dev *hr_dev,
3856                                  struct hns_roce_qp *hr_qp,
3857                                  struct hns_roce_v2_qp_context *hr_context)
3858 {
3859         struct hns_roce_cmd_mailbox *mailbox;
3860         int ret;
3861
3862         mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
3863         if (IS_ERR(mailbox))
3864                 return PTR_ERR(mailbox);
3865
3866         ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, hr_qp->qpn, 0,
3867                                 HNS_ROCE_CMD_QUERY_QPC,
3868                                 HNS_ROCE_CMD_TIMEOUT_MSECS);
3869         if (ret) {
3870                 dev_err(hr_dev->dev, "QUERY QP cmd process error\n");
3871                 goto out;
3872         }
3873
3874         memcpy(hr_context, mailbox->buf, sizeof(*hr_context));
3875
3876 out:
3877         hns_roce_free_cmd_mailbox(hr_dev, mailbox);
3878         return ret;
3879 }
3880
3881 static int hns_roce_v2_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
3882                                 int qp_attr_mask,
3883                                 struct ib_qp_init_attr *qp_init_attr)
3884 {
3885         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
3886         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
3887         struct hns_roce_v2_qp_context *context;
3888         struct device *dev = hr_dev->dev;
3889         int tmp_qp_state;
3890         int state;
3891         int ret;
3892
3893         context = kzalloc(sizeof(*context), GFP_KERNEL);
3894         if (!context)
3895                 return -ENOMEM;
3896
3897         memset(qp_attr, 0, sizeof(*qp_attr));
3898         memset(qp_init_attr, 0, sizeof(*qp_init_attr));
3899
3900         mutex_lock(&hr_qp->mutex);
3901
3902         if (hr_qp->state == IB_QPS_RESET) {
3903                 qp_attr->qp_state = IB_QPS_RESET;
3904                 ret = 0;
3905                 goto done;
3906         }
3907
3908         ret = hns_roce_v2_query_qpc(hr_dev, hr_qp, context);
3909         if (ret) {
3910                 dev_err(dev, "query qpc error\n");
3911                 ret = -EINVAL;
3912                 goto out;
3913         }
3914
3915         state = roce_get_field(context->byte_60_qpst_tempid,
3916                                V2_QPC_BYTE_60_QP_ST_M, V2_QPC_BYTE_60_QP_ST_S);
3917         tmp_qp_state = to_ib_qp_st((enum hns_roce_v2_qp_state)state);
3918         if (tmp_qp_state == -1) {
3919                 dev_err(dev, "Illegal ib_qp_state\n");
3920                 ret = -EINVAL;
3921                 goto out;
3922         }
3923         hr_qp->state = (u8)tmp_qp_state;
3924         qp_attr->qp_state = (enum ib_qp_state)hr_qp->state;
3925         qp_attr->path_mtu = (enum ib_mtu)roce_get_field(context->byte_24_mtu_tc,
3926                                                         V2_QPC_BYTE_24_MTU_M,
3927                                                         V2_QPC_BYTE_24_MTU_S);
3928         qp_attr->path_mig_state = IB_MIG_ARMED;
3929         qp_attr->ah_attr.type   = RDMA_AH_ATTR_TYPE_ROCE;
3930         if (hr_qp->ibqp.qp_type == IB_QPT_UD)
3931                 qp_attr->qkey = V2_QKEY_VAL;
3932
3933         qp_attr->rq_psn = roce_get_field(context->byte_108_rx_reqepsn,
3934                                          V2_QPC_BYTE_108_RX_REQ_EPSN_M,
3935                                          V2_QPC_BYTE_108_RX_REQ_EPSN_S);
3936         qp_attr->sq_psn = (u32)roce_get_field(context->byte_172_sq_psn,
3937                                               V2_QPC_BYTE_172_SQ_CUR_PSN_M,
3938                                               V2_QPC_BYTE_172_SQ_CUR_PSN_S);
3939         qp_attr->dest_qp_num = (u8)roce_get_field(context->byte_56_dqpn_err,
3940                                                   V2_QPC_BYTE_56_DQPN_M,
3941                                                   V2_QPC_BYTE_56_DQPN_S);
3942         qp_attr->qp_access_flags = ((roce_get_bit(context->byte_76_srqn_op_en,
3943                                                   V2_QPC_BYTE_76_RRE_S)) << 2) |
3944                                    ((roce_get_bit(context->byte_76_srqn_op_en,
3945                                                   V2_QPC_BYTE_76_RWE_S)) << 1) |
3946                                    ((roce_get_bit(context->byte_76_srqn_op_en,
3947                                                   V2_QPC_BYTE_76_ATE_S)) << 3);
3948         if (hr_qp->ibqp.qp_type == IB_QPT_RC ||
3949             hr_qp->ibqp.qp_type == IB_QPT_UC) {
3950                 struct ib_global_route *grh =
3951                                 rdma_ah_retrieve_grh(&qp_attr->ah_attr);
3952
3953                 rdma_ah_set_sl(&qp_attr->ah_attr,
3954                                roce_get_field(context->byte_28_at_fl,
3955                                               V2_QPC_BYTE_28_SL_M,
3956                                               V2_QPC_BYTE_28_SL_S));
3957                 grh->flow_label = roce_get_field(context->byte_28_at_fl,
3958                                                  V2_QPC_BYTE_28_FL_M,
3959                                                  V2_QPC_BYTE_28_FL_S);
3960                 grh->sgid_index = roce_get_field(context->byte_20_smac_sgid_idx,
3961                                                  V2_QPC_BYTE_20_SGID_IDX_M,
3962                                                  V2_QPC_BYTE_20_SGID_IDX_S);
3963                 grh->hop_limit = roce_get_field(context->byte_24_mtu_tc,
3964                                                 V2_QPC_BYTE_24_HOP_LIMIT_M,
3965                                                 V2_QPC_BYTE_24_HOP_LIMIT_S);
3966                 grh->traffic_class = roce_get_field(context->byte_24_mtu_tc,
3967                                                     V2_QPC_BYTE_24_TC_M,
3968                                                     V2_QPC_BYTE_24_TC_S);
3969
3970                 memcpy(grh->dgid.raw, context->dgid, sizeof(grh->dgid.raw));
3971         }
3972
3973         qp_attr->port_num = hr_qp->port + 1;
3974         qp_attr->sq_draining = 0;
3975         qp_attr->max_rd_atomic = 1 << roce_get_field(context->byte_208_irrl,
3976                                                      V2_QPC_BYTE_208_SR_MAX_M,
3977                                                      V2_QPC_BYTE_208_SR_MAX_S);
3978         qp_attr->max_dest_rd_atomic = 1 << roce_get_field(context->byte_140_raq,
3979                                                      V2_QPC_BYTE_140_RR_MAX_M,
3980                                                      V2_QPC_BYTE_140_RR_MAX_S);
3981         qp_attr->min_rnr_timer = (u8)roce_get_field(context->byte_80_rnr_rx_cqn,
3982                                                  V2_QPC_BYTE_80_MIN_RNR_TIME_M,
3983                                                  V2_QPC_BYTE_80_MIN_RNR_TIME_S);
3984         qp_attr->timeout = (u8)roce_get_field(context->byte_28_at_fl,
3985                                               V2_QPC_BYTE_28_AT_M,
3986                                               V2_QPC_BYTE_28_AT_S);
3987         qp_attr->retry_cnt = roce_get_field(context->byte_212_lsn,
3988                                             V2_QPC_BYTE_212_RETRY_CNT_M,
3989                                             V2_QPC_BYTE_212_RETRY_CNT_S);
3990         qp_attr->rnr_retry = context->rq_rnr_timer;
3991
3992 done:
3993         qp_attr->cur_qp_state = qp_attr->qp_state;
3994         qp_attr->cap.max_recv_wr = hr_qp->rq.wqe_cnt;
3995         qp_attr->cap.max_recv_sge = hr_qp->rq.max_gs;
3996
3997         if (!ibqp->uobject) {
3998                 qp_attr->cap.max_send_wr = hr_qp->sq.wqe_cnt;
3999                 qp_attr->cap.max_send_sge = hr_qp->sq.max_gs;
4000         } else {
4001                 qp_attr->cap.max_send_wr = 0;
4002                 qp_attr->cap.max_send_sge = 0;
4003         }
4004
4005         qp_init_attr->cap = qp_attr->cap;
4006
4007 out:
4008         mutex_unlock(&hr_qp->mutex);
4009         kfree(context);
4010         return ret;
4011 }
4012
4013 static int hns_roce_v2_destroy_qp_common(struct hns_roce_dev *hr_dev,
4014                                          struct hns_roce_qp *hr_qp,
4015                                          int is_user)
4016 {
4017         struct hns_roce_cq *send_cq, *recv_cq;
4018         struct device *dev = hr_dev->dev;
4019         int ret;
4020
4021         if (hr_qp->ibqp.qp_type == IB_QPT_RC && hr_qp->state != IB_QPS_RESET) {
4022                 /* Modify qp to reset before destroying qp */
4023                 ret = hns_roce_v2_modify_qp(&hr_qp->ibqp, NULL, 0,
4024                                             hr_qp->state, IB_QPS_RESET);
4025                 if (ret) {
4026                         dev_err(dev, "modify QP %06lx to ERR failed.\n",
4027                                 hr_qp->qpn);
4028                         return ret;
4029                 }
4030         }
4031
4032         send_cq = to_hr_cq(hr_qp->ibqp.send_cq);
4033         recv_cq = to_hr_cq(hr_qp->ibqp.recv_cq);
4034
4035         hns_roce_lock_cqs(send_cq, recv_cq);
4036
4037         if (!is_user) {
4038                 __hns_roce_v2_cq_clean(recv_cq, hr_qp->qpn, hr_qp->ibqp.srq ?
4039                                        to_hr_srq(hr_qp->ibqp.srq) : NULL);
4040                 if (send_cq != recv_cq)
4041                         __hns_roce_v2_cq_clean(send_cq, hr_qp->qpn, NULL);
4042         }
4043
4044         hns_roce_qp_remove(hr_dev, hr_qp);
4045
4046         hns_roce_unlock_cqs(send_cq, recv_cq);
4047
4048         hns_roce_qp_free(hr_dev, hr_qp);
4049
4050         /* Not special_QP, free their QPN */
4051         if ((hr_qp->ibqp.qp_type == IB_QPT_RC) ||
4052             (hr_qp->ibqp.qp_type == IB_QPT_UC) ||
4053             (hr_qp->ibqp.qp_type == IB_QPT_UD))
4054                 hns_roce_release_range_qp(hr_dev, hr_qp->qpn, 1);
4055
4056         hns_roce_mtt_cleanup(hr_dev, &hr_qp->mtt);
4057
4058         if (is_user) {
4059                 if (hr_qp->sq.wqe_cnt && (hr_qp->sdb_en == 1))
4060                         hns_roce_db_unmap_user(
4061                                 to_hr_ucontext(hr_qp->ibqp.uobject->context),
4062                                 &hr_qp->sdb);
4063
4064                 if (hr_qp->rq.wqe_cnt && (hr_qp->rdb_en == 1))
4065                         hns_roce_db_unmap_user(
4066                                 to_hr_ucontext(hr_qp->ibqp.uobject->context),
4067                                 &hr_qp->rdb);
4068                 ib_umem_release(hr_qp->umem);
4069         } else {
4070                 kfree(hr_qp->sq.wrid);
4071                 kfree(hr_qp->rq.wrid);
4072                 hns_roce_buf_free(hr_dev, hr_qp->buff_size, &hr_qp->hr_buf);
4073                 if (hr_qp->rq.wqe_cnt)
4074                         hns_roce_free_db(hr_dev, &hr_qp->rdb);
4075         }
4076
4077         if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) {
4078                 kfree(hr_qp->rq_inl_buf.wqe_list[0].sg_list);
4079                 kfree(hr_qp->rq_inl_buf.wqe_list);
4080         }
4081
4082         return 0;
4083 }
4084
4085 static int hns_roce_v2_destroy_qp(struct ib_qp *ibqp)
4086 {
4087         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
4088         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
4089         int ret;
4090
4091         ret = hns_roce_v2_destroy_qp_common(hr_dev, hr_qp, !!ibqp->pd->uobject);
4092         if (ret) {
4093                 dev_err(hr_dev->dev, "Destroy qp failed(%d)\n", ret);
4094                 return ret;
4095         }
4096
4097         if (hr_qp->ibqp.qp_type == IB_QPT_GSI)
4098                 kfree(hr_to_hr_sqp(hr_qp));
4099         else
4100                 kfree(hr_qp);
4101
4102         return 0;
4103 }
4104
4105 static int hns_roce_v2_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period)
4106 {
4107         struct hns_roce_dev *hr_dev = to_hr_dev(cq->device);
4108         struct hns_roce_v2_cq_context *cq_context;
4109         struct hns_roce_cq *hr_cq = to_hr_cq(cq);
4110         struct hns_roce_v2_cq_context *cqc_mask;
4111         struct hns_roce_cmd_mailbox *mailbox;
4112         int ret;
4113
4114         mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
4115         if (IS_ERR(mailbox))
4116                 return PTR_ERR(mailbox);
4117
4118         cq_context = mailbox->buf;
4119         cqc_mask = (struct hns_roce_v2_cq_context *)mailbox->buf + 1;
4120
4121         memset(cqc_mask, 0xff, sizeof(*cqc_mask));
4122
4123         roce_set_field(cq_context->byte_56_cqe_period_maxcnt,
4124                        V2_CQC_BYTE_56_CQ_MAX_CNT_M, V2_CQC_BYTE_56_CQ_MAX_CNT_S,
4125                        cq_count);
4126         roce_set_field(cqc_mask->byte_56_cqe_period_maxcnt,
4127                        V2_CQC_BYTE_56_CQ_MAX_CNT_M, V2_CQC_BYTE_56_CQ_MAX_CNT_S,
4128                        0);
4129         roce_set_field(cq_context->byte_56_cqe_period_maxcnt,
4130                        V2_CQC_BYTE_56_CQ_PERIOD_M, V2_CQC_BYTE_56_CQ_PERIOD_S,
4131                        cq_period);
4132         roce_set_field(cqc_mask->byte_56_cqe_period_maxcnt,
4133                        V2_CQC_BYTE_56_CQ_PERIOD_M, V2_CQC_BYTE_56_CQ_PERIOD_S,
4134                        0);
4135
4136         ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, hr_cq->cqn, 1,
4137                                 HNS_ROCE_CMD_MODIFY_CQC,
4138                                 HNS_ROCE_CMD_TIMEOUT_MSECS);
4139         hns_roce_free_cmd_mailbox(hr_dev, mailbox);
4140         if (ret)
4141                 dev_err(hr_dev->dev, "MODIFY CQ Failed to cmd mailbox.\n");
4142
4143         return ret;
4144 }
4145
4146 static void hns_roce_set_qps_to_err(struct hns_roce_dev *hr_dev, u32 qpn)
4147 {
4148         struct hns_roce_qp *hr_qp;
4149         struct ib_qp_attr attr;
4150         int attr_mask;
4151         int ret;
4152
4153         hr_qp = __hns_roce_qp_lookup(hr_dev, qpn);
4154         if (!hr_qp) {
4155                 dev_warn(hr_dev->dev, "no hr_qp can be found!\n");
4156                 return;
4157         }
4158
4159         if (hr_qp->ibqp.uobject) {
4160                 if (hr_qp->sdb_en == 1) {
4161                         hr_qp->sq.head = *(int *)(hr_qp->sdb.virt_addr);
4162                         hr_qp->rq.head = *(int *)(hr_qp->rdb.virt_addr);
4163                 } else {
4164                         dev_warn(hr_dev->dev, "flush cqe is unsupported in userspace!\n");
4165                         return;
4166                 }
4167         }
4168
4169         attr_mask = IB_QP_STATE;
4170         attr.qp_state = IB_QPS_ERR;
4171         ret = hns_roce_v2_modify_qp(&hr_qp->ibqp, &attr, attr_mask,
4172                                     hr_qp->state, IB_QPS_ERR);
4173         if (ret)
4174                 dev_err(hr_dev->dev, "failed to modify qp %d to err state.\n",
4175                         qpn);
4176 }
4177
4178 static void hns_roce_irq_work_handle(struct work_struct *work)
4179 {
4180         struct hns_roce_work *irq_work =
4181                                 container_of(work, struct hns_roce_work, work);
4182         struct device *dev = irq_work->hr_dev->dev;
4183         u32 qpn = irq_work->qpn;
4184         u32 cqn = irq_work->cqn;
4185
4186         switch (irq_work->event_type) {
4187         case HNS_ROCE_EVENT_TYPE_PATH_MIG:
4188                 dev_info(dev, "Path migrated succeeded.\n");
4189                 break;
4190         case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED:
4191                 dev_warn(dev, "Path migration failed.\n");
4192                 break;
4193         case HNS_ROCE_EVENT_TYPE_COMM_EST:
4194                 dev_info(dev, "Communication established.\n");
4195                 break;
4196         case HNS_ROCE_EVENT_TYPE_SQ_DRAINED:
4197                 dev_warn(dev, "Send queue drained.\n");
4198                 break;
4199         case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR:
4200                 dev_err(dev, "Local work queue catastrophic error.\n");
4201                 hns_roce_set_qps_to_err(irq_work->hr_dev, qpn);
4202                 switch (irq_work->sub_type) {
4203                 case HNS_ROCE_LWQCE_QPC_ERROR:
4204                         dev_err(dev, "QP %d, QPC error.\n", qpn);
4205                         break;
4206                 case HNS_ROCE_LWQCE_MTU_ERROR:
4207                         dev_err(dev, "QP %d, MTU error.\n", qpn);
4208                         break;
4209                 case HNS_ROCE_LWQCE_WQE_BA_ADDR_ERROR:
4210                         dev_err(dev, "QP %d, WQE BA addr error.\n", qpn);
4211                         break;
4212                 case HNS_ROCE_LWQCE_WQE_ADDR_ERROR:
4213                         dev_err(dev, "QP %d, WQE addr error.\n", qpn);
4214                         break;
4215                 case HNS_ROCE_LWQCE_SQ_WQE_SHIFT_ERROR:
4216                         dev_err(dev, "QP %d, WQE shift error.\n", qpn);
4217                         break;
4218                 default:
4219                         dev_err(dev, "Unhandled sub_event type %d.\n",
4220                                 irq_work->sub_type);
4221                         break;
4222                 }
4223                 break;
4224         case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR:
4225                 dev_err(dev, "Invalid request local work queue error.\n");
4226                 hns_roce_set_qps_to_err(irq_work->hr_dev, qpn);
4227                 break;
4228         case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR:
4229                 dev_err(dev, "Local access violation work queue error.\n");
4230                 hns_roce_set_qps_to_err(irq_work->hr_dev, qpn);
4231                 switch (irq_work->sub_type) {
4232                 case HNS_ROCE_LAVWQE_R_KEY_VIOLATION:
4233                         dev_err(dev, "QP %d, R_key violation.\n", qpn);
4234                         break;
4235                 case HNS_ROCE_LAVWQE_LENGTH_ERROR:
4236                         dev_err(dev, "QP %d, length error.\n", qpn);
4237                         break;
4238                 case HNS_ROCE_LAVWQE_VA_ERROR:
4239                         dev_err(dev, "QP %d, VA error.\n", qpn);
4240                         break;
4241                 case HNS_ROCE_LAVWQE_PD_ERROR:
4242                         dev_err(dev, "QP %d, PD error.\n", qpn);
4243                         break;
4244                 case HNS_ROCE_LAVWQE_RW_ACC_ERROR:
4245                         dev_err(dev, "QP %d, rw acc error.\n", qpn);
4246                         break;
4247                 case HNS_ROCE_LAVWQE_KEY_STATE_ERROR:
4248                         dev_err(dev, "QP %d, key state error.\n", qpn);
4249                         break;
4250                 case HNS_ROCE_LAVWQE_MR_OPERATION_ERROR:
4251                         dev_err(dev, "QP %d, MR operation error.\n", qpn);
4252                         break;
4253                 default:
4254                         dev_err(dev, "Unhandled sub_event type %d.\n",
4255                                 irq_work->sub_type);
4256                         break;
4257                 }
4258                 break;
4259         case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH:
4260                 dev_warn(dev, "SRQ limit reach.\n");
4261                 break;
4262         case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH:
4263                 dev_warn(dev, "SRQ last wqe reach.\n");
4264                 break;
4265         case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR:
4266                 dev_err(dev, "SRQ catas error.\n");
4267                 break;
4268         case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR:
4269                 dev_err(dev, "CQ 0x%x access err.\n", cqn);
4270                 break;
4271         case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW:
4272                 dev_warn(dev, "CQ 0x%x overflow\n", cqn);
4273                 break;
4274         case HNS_ROCE_EVENT_TYPE_DB_OVERFLOW:
4275                 dev_warn(dev, "DB overflow.\n");
4276                 break;
4277         case HNS_ROCE_EVENT_TYPE_FLR:
4278                 dev_warn(dev, "Function level reset.\n");
4279                 break;
4280         default:
4281                 break;
4282         }
4283
4284         kfree(irq_work);
4285 }
4286
4287 static void hns_roce_v2_init_irq_work(struct hns_roce_dev *hr_dev,
4288                                       struct hns_roce_eq *eq,
4289                                       u32 qpn, u32 cqn)
4290 {
4291         struct hns_roce_work *irq_work;
4292
4293         irq_work = kzalloc(sizeof(struct hns_roce_work), GFP_ATOMIC);
4294         if (!irq_work)
4295                 return;
4296
4297         INIT_WORK(&(irq_work->work), hns_roce_irq_work_handle);
4298         irq_work->hr_dev = hr_dev;
4299         irq_work->qpn = qpn;
4300         irq_work->cqn = cqn;
4301         irq_work->event_type = eq->event_type;
4302         irq_work->sub_type = eq->sub_type;
4303         queue_work(hr_dev->irq_workq, &(irq_work->work));
4304 }
4305
4306 static void set_eq_cons_index_v2(struct hns_roce_eq *eq)
4307 {
4308         u32 doorbell[2];
4309
4310         doorbell[0] = 0;
4311         doorbell[1] = 0;
4312
4313         if (eq->type_flag == HNS_ROCE_AEQ) {
4314                 roce_set_field(doorbell[0], HNS_ROCE_V2_EQ_DB_CMD_M,
4315                                HNS_ROCE_V2_EQ_DB_CMD_S,
4316                                eq->arm_st == HNS_ROCE_V2_EQ_ALWAYS_ARMED ?
4317                                HNS_ROCE_EQ_DB_CMD_AEQ :
4318                                HNS_ROCE_EQ_DB_CMD_AEQ_ARMED);
4319         } else {
4320                 roce_set_field(doorbell[0], HNS_ROCE_V2_EQ_DB_TAG_M,
4321                                HNS_ROCE_V2_EQ_DB_TAG_S, eq->eqn);
4322
4323                 roce_set_field(doorbell[0], HNS_ROCE_V2_EQ_DB_CMD_M,
4324                                HNS_ROCE_V2_EQ_DB_CMD_S,
4325                                eq->arm_st == HNS_ROCE_V2_EQ_ALWAYS_ARMED ?
4326                                HNS_ROCE_EQ_DB_CMD_CEQ :
4327                                HNS_ROCE_EQ_DB_CMD_CEQ_ARMED);
4328         }
4329
4330         roce_set_field(doorbell[1], HNS_ROCE_V2_EQ_DB_PARA_M,
4331                        HNS_ROCE_V2_EQ_DB_PARA_S,
4332                        (eq->cons_index & HNS_ROCE_V2_CONS_IDX_M));
4333
4334         hns_roce_write64_k(doorbell, eq->doorbell);
4335 }
4336
4337 static struct hns_roce_aeqe *get_aeqe_v2(struct hns_roce_eq *eq, u32 entry)
4338 {
4339         u32 buf_chk_sz;
4340         unsigned long off;
4341
4342         buf_chk_sz = 1 << (eq->eqe_buf_pg_sz + PAGE_SHIFT);
4343         off = (entry & (eq->entries - 1)) * HNS_ROCE_AEQ_ENTRY_SIZE;
4344
4345         return (struct hns_roce_aeqe *)((char *)(eq->buf_list->buf) +
4346                 off % buf_chk_sz);
4347 }
4348
4349 static struct hns_roce_aeqe *mhop_get_aeqe(struct hns_roce_eq *eq, u32 entry)
4350 {
4351         u32 buf_chk_sz;
4352         unsigned long off;
4353
4354         buf_chk_sz = 1 << (eq->eqe_buf_pg_sz + PAGE_SHIFT);
4355
4356         off = (entry & (eq->entries - 1)) * HNS_ROCE_AEQ_ENTRY_SIZE;
4357
4358         if (eq->hop_num == HNS_ROCE_HOP_NUM_0)
4359                 return (struct hns_roce_aeqe *)((u8 *)(eq->bt_l0) +
4360                         off % buf_chk_sz);
4361         else
4362                 return (struct hns_roce_aeqe *)((u8 *)
4363                         (eq->buf[off / buf_chk_sz]) + off % buf_chk_sz);
4364 }
4365
4366 static struct hns_roce_aeqe *next_aeqe_sw_v2(struct hns_roce_eq *eq)
4367 {
4368         struct hns_roce_aeqe *aeqe;
4369
4370         if (!eq->hop_num)
4371                 aeqe = get_aeqe_v2(eq, eq->cons_index);
4372         else
4373                 aeqe = mhop_get_aeqe(eq, eq->cons_index);
4374
4375         return (roce_get_bit(aeqe->asyn, HNS_ROCE_V2_AEQ_AEQE_OWNER_S) ^
4376                 !!(eq->cons_index & eq->entries)) ? aeqe : NULL;
4377 }
4378
4379 static int hns_roce_v2_aeq_int(struct hns_roce_dev *hr_dev,
4380                                struct hns_roce_eq *eq)
4381 {
4382         struct device *dev = hr_dev->dev;
4383         struct hns_roce_aeqe *aeqe;
4384         int aeqe_found = 0;
4385         int event_type;
4386         int sub_type;
4387         u32 qpn;
4388         u32 cqn;
4389
4390         while ((aeqe = next_aeqe_sw_v2(eq))) {
4391
4392                 /* Make sure we read AEQ entry after we have checked the
4393                  * ownership bit
4394                  */
4395                 dma_rmb();
4396
4397                 event_type = roce_get_field(aeqe->asyn,
4398                                             HNS_ROCE_V2_AEQE_EVENT_TYPE_M,
4399                                             HNS_ROCE_V2_AEQE_EVENT_TYPE_S);
4400                 sub_type = roce_get_field(aeqe->asyn,
4401                                           HNS_ROCE_V2_AEQE_SUB_TYPE_M,
4402                                           HNS_ROCE_V2_AEQE_SUB_TYPE_S);
4403                 qpn = roce_get_field(aeqe->event.qp_event.qp,
4404                                      HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_M,
4405                                      HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_S);
4406                 cqn = roce_get_field(aeqe->event.cq_event.cq,
4407                                      HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_M,
4408                                      HNS_ROCE_V2_AEQE_EVENT_QUEUE_NUM_S);
4409
4410                 switch (event_type) {
4411                 case HNS_ROCE_EVENT_TYPE_PATH_MIG:
4412                 case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED:
4413                 case HNS_ROCE_EVENT_TYPE_COMM_EST:
4414                 case HNS_ROCE_EVENT_TYPE_SQ_DRAINED:
4415                 case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR:
4416                 case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR:
4417                 case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR:
4418                         hns_roce_qp_event(hr_dev, qpn, event_type);
4419                         break;
4420                 case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH:
4421                 case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH:
4422                 case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR:
4423                         break;
4424                 case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR:
4425                 case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW:
4426                         hns_roce_cq_event(hr_dev, cqn, event_type);
4427                         break;
4428                 case HNS_ROCE_EVENT_TYPE_DB_OVERFLOW:
4429                         break;
4430                 case HNS_ROCE_EVENT_TYPE_MB:
4431                         hns_roce_cmd_event(hr_dev,
4432                                         le16_to_cpu(aeqe->event.cmd.token),
4433                                         aeqe->event.cmd.status,
4434                                         le64_to_cpu(aeqe->event.cmd.out_param));
4435                         break;
4436                 case HNS_ROCE_EVENT_TYPE_CEQ_OVERFLOW:
4437                         break;
4438                 case HNS_ROCE_EVENT_TYPE_FLR:
4439                         break;
4440                 default:
4441                         dev_err(dev, "Unhandled event %d on EQ %d at idx %u.\n",
4442                                 event_type, eq->eqn, eq->cons_index);
4443                         break;
4444                 };
4445
4446                 eq->event_type = event_type;
4447                 eq->sub_type = sub_type;
4448                 ++eq->cons_index;
4449                 aeqe_found = 1;
4450
4451                 if (eq->cons_index > (2 * eq->entries - 1)) {
4452                         dev_warn(dev, "cons_index overflow, set back to 0.\n");
4453                         eq->cons_index = 0;
4454                 }
4455                 hns_roce_v2_init_irq_work(hr_dev, eq, qpn, cqn);
4456         }
4457
4458         set_eq_cons_index_v2(eq);
4459         return aeqe_found;
4460 }
4461
4462 static struct hns_roce_ceqe *get_ceqe_v2(struct hns_roce_eq *eq, u32 entry)
4463 {
4464         u32 buf_chk_sz;
4465         unsigned long off;
4466
4467         buf_chk_sz = 1 << (eq->eqe_buf_pg_sz + PAGE_SHIFT);
4468         off = (entry & (eq->entries - 1)) * HNS_ROCE_CEQ_ENTRY_SIZE;
4469
4470         return (struct hns_roce_ceqe *)((char *)(eq->buf_list->buf) +
4471                 off % buf_chk_sz);
4472 }
4473
4474 static struct hns_roce_ceqe *mhop_get_ceqe(struct hns_roce_eq *eq, u32 entry)
4475 {
4476         u32 buf_chk_sz;
4477         unsigned long off;
4478
4479         buf_chk_sz = 1 << (eq->eqe_buf_pg_sz + PAGE_SHIFT);
4480
4481         off = (entry & (eq->entries - 1)) * HNS_ROCE_CEQ_ENTRY_SIZE;
4482
4483         if (eq->hop_num == HNS_ROCE_HOP_NUM_0)
4484                 return (struct hns_roce_ceqe *)((u8 *)(eq->bt_l0) +
4485                         off % buf_chk_sz);
4486         else
4487                 return (struct hns_roce_ceqe *)((u8 *)(eq->buf[off /
4488                         buf_chk_sz]) + off % buf_chk_sz);
4489 }
4490
4491 static struct hns_roce_ceqe *next_ceqe_sw_v2(struct hns_roce_eq *eq)
4492 {
4493         struct hns_roce_ceqe *ceqe;
4494
4495         if (!eq->hop_num)
4496                 ceqe = get_ceqe_v2(eq, eq->cons_index);
4497         else
4498                 ceqe = mhop_get_ceqe(eq, eq->cons_index);
4499
4500         return (!!(roce_get_bit(ceqe->comp, HNS_ROCE_V2_CEQ_CEQE_OWNER_S))) ^
4501                 (!!(eq->cons_index & eq->entries)) ? ceqe : NULL;
4502 }
4503
4504 static int hns_roce_v2_ceq_int(struct hns_roce_dev *hr_dev,
4505                                struct hns_roce_eq *eq)
4506 {
4507         struct device *dev = hr_dev->dev;
4508         struct hns_roce_ceqe *ceqe;
4509         int ceqe_found = 0;
4510         u32 cqn;
4511
4512         while ((ceqe = next_ceqe_sw_v2(eq))) {
4513
4514                 /* Make sure we read CEQ entry after we have checked the
4515                  * ownership bit
4516                  */
4517                 dma_rmb();
4518
4519                 cqn = roce_get_field(ceqe->comp,
4520                                      HNS_ROCE_V2_CEQE_COMP_CQN_M,
4521                                      HNS_ROCE_V2_CEQE_COMP_CQN_S);
4522
4523                 hns_roce_cq_completion(hr_dev, cqn);
4524
4525                 ++eq->cons_index;
4526                 ceqe_found = 1;
4527
4528                 if (eq->cons_index > (2 * eq->entries - 1)) {
4529                         dev_warn(dev, "cons_index overflow, set back to 0.\n");
4530                         eq->cons_index = 0;
4531                 }
4532         }
4533
4534         set_eq_cons_index_v2(eq);
4535
4536         return ceqe_found;
4537 }
4538
4539 static irqreturn_t hns_roce_v2_msix_interrupt_eq(int irq, void *eq_ptr)
4540 {
4541         struct hns_roce_eq *eq = eq_ptr;
4542         struct hns_roce_dev *hr_dev = eq->hr_dev;
4543         int int_work = 0;
4544
4545         if (eq->type_flag == HNS_ROCE_CEQ)
4546                 /* Completion event interrupt */
4547                 int_work = hns_roce_v2_ceq_int(hr_dev, eq);
4548         else
4549                 /* Asychronous event interrupt */
4550                 int_work = hns_roce_v2_aeq_int(hr_dev, eq);
4551
4552         return IRQ_RETVAL(int_work);
4553 }
4554
4555 static irqreturn_t hns_roce_v2_msix_interrupt_abn(int irq, void *dev_id)
4556 {
4557         struct hns_roce_dev *hr_dev = dev_id;
4558         struct device *dev = hr_dev->dev;
4559         int int_work = 0;
4560         u32 int_st;
4561         u32 int_en;
4562
4563         /* Abnormal interrupt */
4564         int_st = roce_read(hr_dev, ROCEE_VF_ABN_INT_ST_REG);
4565         int_en = roce_read(hr_dev, ROCEE_VF_ABN_INT_EN_REG);
4566
4567         if (roce_get_bit(int_st, HNS_ROCE_V2_VF_INT_ST_AEQ_OVERFLOW_S)) {
4568                 dev_err(dev, "AEQ overflow!\n");
4569
4570                 roce_set_bit(int_st, HNS_ROCE_V2_VF_INT_ST_AEQ_OVERFLOW_S, 1);
4571                 roce_write(hr_dev, ROCEE_VF_ABN_INT_ST_REG, int_st);
4572
4573                 roce_set_bit(int_en, HNS_ROCE_V2_VF_ABN_INT_EN_S, 1);
4574                 roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, int_en);
4575
4576                 int_work = 1;
4577         } else if (roce_get_bit(int_st, HNS_ROCE_V2_VF_INT_ST_BUS_ERR_S)) {
4578                 dev_err(dev, "BUS ERR!\n");
4579
4580                 roce_set_bit(int_st, HNS_ROCE_V2_VF_INT_ST_BUS_ERR_S, 1);
4581                 roce_write(hr_dev, ROCEE_VF_ABN_INT_ST_REG, int_st);
4582
4583                 roce_set_bit(int_en, HNS_ROCE_V2_VF_ABN_INT_EN_S, 1);
4584                 roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, int_en);
4585
4586                 int_work = 1;
4587         } else if (roce_get_bit(int_st, HNS_ROCE_V2_VF_INT_ST_OTHER_ERR_S)) {
4588                 dev_err(dev, "OTHER ERR!\n");
4589
4590                 roce_set_bit(int_st, HNS_ROCE_V2_VF_INT_ST_OTHER_ERR_S, 1);
4591                 roce_write(hr_dev, ROCEE_VF_ABN_INT_ST_REG, int_st);
4592
4593                 roce_set_bit(int_en, HNS_ROCE_V2_VF_ABN_INT_EN_S, 1);
4594                 roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG, int_en);
4595
4596                 int_work = 1;
4597         } else
4598                 dev_err(dev, "There is no abnormal irq found!\n");
4599
4600         return IRQ_RETVAL(int_work);
4601 }
4602
4603 static void hns_roce_v2_int_mask_enable(struct hns_roce_dev *hr_dev,
4604                                         int eq_num, int enable_flag)
4605 {
4606         int i;
4607
4608         if (enable_flag == EQ_ENABLE) {
4609                 for (i = 0; i < eq_num; i++)
4610                         roce_write(hr_dev, ROCEE_VF_EVENT_INT_EN_REG +
4611                                    i * EQ_REG_OFFSET,
4612                                    HNS_ROCE_V2_VF_EVENT_INT_EN_M);
4613
4614                 roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG,
4615                            HNS_ROCE_V2_VF_ABN_INT_EN_M);
4616                 roce_write(hr_dev, ROCEE_VF_ABN_INT_CFG_REG,
4617                            HNS_ROCE_V2_VF_ABN_INT_CFG_M);
4618         } else {
4619                 for (i = 0; i < eq_num; i++)
4620                         roce_write(hr_dev, ROCEE_VF_EVENT_INT_EN_REG +
4621                                    i * EQ_REG_OFFSET,
4622                                    HNS_ROCE_V2_VF_EVENT_INT_EN_M & 0x0);
4623
4624                 roce_write(hr_dev, ROCEE_VF_ABN_INT_EN_REG,
4625                            HNS_ROCE_V2_VF_ABN_INT_EN_M & 0x0);
4626                 roce_write(hr_dev, ROCEE_VF_ABN_INT_CFG_REG,
4627                            HNS_ROCE_V2_VF_ABN_INT_CFG_M & 0x0);
4628         }
4629 }
4630
4631 static void hns_roce_v2_destroy_eqc(struct hns_roce_dev *hr_dev, int eqn)
4632 {
4633         struct device *dev = hr_dev->dev;
4634         int ret;
4635
4636         if (eqn < hr_dev->caps.num_comp_vectors)
4637                 ret = hns_roce_cmd_mbox(hr_dev, 0, 0, eqn & HNS_ROCE_V2_EQN_M,
4638                                         0, HNS_ROCE_CMD_DESTROY_CEQC,
4639                                         HNS_ROCE_CMD_TIMEOUT_MSECS);
4640         else
4641                 ret = hns_roce_cmd_mbox(hr_dev, 0, 0, eqn & HNS_ROCE_V2_EQN_M,
4642                                         0, HNS_ROCE_CMD_DESTROY_AEQC,
4643                                         HNS_ROCE_CMD_TIMEOUT_MSECS);
4644         if (ret)
4645                 dev_err(dev, "[mailbox cmd] destroy eqc(%d) failed.\n", eqn);
4646 }
4647
4648 static void hns_roce_mhop_free_eq(struct hns_roce_dev *hr_dev,
4649                                   struct hns_roce_eq *eq)
4650 {
4651         struct device *dev = hr_dev->dev;
4652         u64 idx;
4653         u64 size;
4654         u32 buf_chk_sz;
4655         u32 bt_chk_sz;
4656         u32 mhop_num;
4657         int eqe_alloc;
4658         int i = 0;
4659         int j = 0;
4660
4661         mhop_num = hr_dev->caps.eqe_hop_num;
4662         buf_chk_sz = 1 << (hr_dev->caps.eqe_buf_pg_sz + PAGE_SHIFT);
4663         bt_chk_sz = 1 << (hr_dev->caps.eqe_ba_pg_sz + PAGE_SHIFT);
4664
4665         /* hop_num = 0 */
4666         if (mhop_num == HNS_ROCE_HOP_NUM_0) {
4667                 dma_free_coherent(dev, (unsigned int)(eq->entries *
4668                                   eq->eqe_size), eq->bt_l0, eq->l0_dma);
4669                 return;
4670         }
4671
4672         /* hop_num = 1 or hop = 2 */
4673         dma_free_coherent(dev, bt_chk_sz, eq->bt_l0, eq->l0_dma);
4674         if (mhop_num == 1) {
4675                 for (i = 0; i < eq->l0_last_num; i++) {
4676                         if (i == eq->l0_last_num - 1) {
4677                                 eqe_alloc = i * (buf_chk_sz / eq->eqe_size);
4678                                 size = (eq->entries - eqe_alloc) * eq->eqe_size;
4679                                 dma_free_coherent(dev, size, eq->buf[i],
4680                                                   eq->buf_dma[i]);
4681                                 break;
4682                         }
4683                         dma_free_coherent(dev, buf_chk_sz, eq->buf[i],
4684                                           eq->buf_dma[i]);
4685                 }
4686         } else if (mhop_num == 2) {
4687                 for (i = 0; i < eq->l0_last_num; i++) {
4688                         dma_free_coherent(dev, bt_chk_sz, eq->bt_l1[i],
4689                                           eq->l1_dma[i]);
4690
4691                         for (j = 0; j < bt_chk_sz / 8; j++) {
4692                                 idx = i * (bt_chk_sz / 8) + j;
4693                                 if ((i == eq->l0_last_num - 1)
4694                                      && j == eq->l1_last_num - 1) {
4695                                         eqe_alloc = (buf_chk_sz / eq->eqe_size)
4696                                                     * idx;
4697                                         size = (eq->entries - eqe_alloc)
4698                                                 * eq->eqe_size;
4699                                         dma_free_coherent(dev, size,
4700                                                           eq->buf[idx],
4701                                                           eq->buf_dma[idx]);
4702                                         break;
4703                                 }
4704                                 dma_free_coherent(dev, buf_chk_sz, eq->buf[idx],
4705                                                   eq->buf_dma[idx]);
4706                         }
4707                 }
4708         }
4709         kfree(eq->buf_dma);
4710         kfree(eq->buf);
4711         kfree(eq->l1_dma);
4712         kfree(eq->bt_l1);
4713         eq->buf_dma = NULL;
4714         eq->buf = NULL;
4715         eq->l1_dma = NULL;
4716         eq->bt_l1 = NULL;
4717 }
4718
4719 static void hns_roce_v2_free_eq(struct hns_roce_dev *hr_dev,
4720                                 struct hns_roce_eq *eq)
4721 {
4722         u32 buf_chk_sz;
4723
4724         buf_chk_sz = 1 << (eq->eqe_buf_pg_sz + PAGE_SHIFT);
4725
4726         if (hr_dev->caps.eqe_hop_num) {
4727                 hns_roce_mhop_free_eq(hr_dev, eq);
4728                 return;
4729         }
4730
4731         if (eq->buf_list)
4732                 dma_free_coherent(hr_dev->dev, buf_chk_sz,
4733                                   eq->buf_list->buf, eq->buf_list->map);
4734 }
4735
4736 static void hns_roce_config_eqc(struct hns_roce_dev *hr_dev,
4737                                 struct hns_roce_eq *eq,
4738                                 void *mb_buf)
4739 {
4740         struct hns_roce_eq_context *eqc;
4741
4742         eqc = mb_buf;
4743         memset(eqc, 0, sizeof(struct hns_roce_eq_context));
4744
4745         /* init eqc */
4746         eq->doorbell = hr_dev->reg_base + ROCEE_VF_EQ_DB_CFG0_REG;
4747         eq->hop_num = hr_dev->caps.eqe_hop_num;
4748         eq->cons_index = 0;
4749         eq->over_ignore = HNS_ROCE_V2_EQ_OVER_IGNORE_0;
4750         eq->coalesce = HNS_ROCE_V2_EQ_COALESCE_0;
4751         eq->arm_st = HNS_ROCE_V2_EQ_ALWAYS_ARMED;
4752         eq->eqe_ba_pg_sz = hr_dev->caps.eqe_ba_pg_sz;
4753         eq->eqe_buf_pg_sz = hr_dev->caps.eqe_buf_pg_sz;
4754         eq->shift = ilog2((unsigned int)eq->entries);
4755
4756         if (!eq->hop_num)
4757                 eq->eqe_ba = eq->buf_list->map;
4758         else
4759                 eq->eqe_ba = eq->l0_dma;
4760
4761         /* set eqc state */
4762         roce_set_field(eqc->byte_4,
4763                        HNS_ROCE_EQC_EQ_ST_M,
4764                        HNS_ROCE_EQC_EQ_ST_S,
4765                        HNS_ROCE_V2_EQ_STATE_VALID);
4766
4767         /* set eqe hop num */
4768         roce_set_field(eqc->byte_4,
4769                        HNS_ROCE_EQC_HOP_NUM_M,
4770                        HNS_ROCE_EQC_HOP_NUM_S, eq->hop_num);
4771
4772         /* set eqc over_ignore */
4773         roce_set_field(eqc->byte_4,
4774                        HNS_ROCE_EQC_OVER_IGNORE_M,
4775                        HNS_ROCE_EQC_OVER_IGNORE_S, eq->over_ignore);
4776
4777         /* set eqc coalesce */
4778         roce_set_field(eqc->byte_4,
4779                        HNS_ROCE_EQC_COALESCE_M,
4780                        HNS_ROCE_EQC_COALESCE_S, eq->coalesce);
4781
4782         /* set eqc arm_state */
4783         roce_set_field(eqc->byte_4,
4784                        HNS_ROCE_EQC_ARM_ST_M,
4785                        HNS_ROCE_EQC_ARM_ST_S, eq->arm_st);
4786
4787         /* set eqn */
4788         roce_set_field(eqc->byte_4,
4789                        HNS_ROCE_EQC_EQN_M,
4790                        HNS_ROCE_EQC_EQN_S, eq->eqn);
4791
4792         /* set eqe_cnt */
4793         roce_set_field(eqc->byte_4,
4794                        HNS_ROCE_EQC_EQE_CNT_M,
4795                        HNS_ROCE_EQC_EQE_CNT_S,
4796                        HNS_ROCE_EQ_INIT_EQE_CNT);
4797
4798         /* set eqe_ba_pg_sz */
4799         roce_set_field(eqc->byte_8,
4800                        HNS_ROCE_EQC_BA_PG_SZ_M,
4801                        HNS_ROCE_EQC_BA_PG_SZ_S,
4802                        eq->eqe_ba_pg_sz + PG_SHIFT_OFFSET);
4803
4804         /* set eqe_buf_pg_sz */
4805         roce_set_field(eqc->byte_8,
4806                        HNS_ROCE_EQC_BUF_PG_SZ_M,
4807                        HNS_ROCE_EQC_BUF_PG_SZ_S,
4808                        eq->eqe_buf_pg_sz + PG_SHIFT_OFFSET);
4809
4810         /* set eq_producer_idx */
4811         roce_set_field(eqc->byte_8,
4812                        HNS_ROCE_EQC_PROD_INDX_M,
4813                        HNS_ROCE_EQC_PROD_INDX_S,
4814                        HNS_ROCE_EQ_INIT_PROD_IDX);
4815
4816         /* set eq_max_cnt */
4817         roce_set_field(eqc->byte_12,
4818                        HNS_ROCE_EQC_MAX_CNT_M,
4819                        HNS_ROCE_EQC_MAX_CNT_S, eq->eq_max_cnt);
4820
4821         /* set eq_period */
4822         roce_set_field(eqc->byte_12,
4823                        HNS_ROCE_EQC_PERIOD_M,
4824                        HNS_ROCE_EQC_PERIOD_S, eq->eq_period);
4825
4826         /* set eqe_report_timer */
4827         roce_set_field(eqc->eqe_report_timer,
4828                        HNS_ROCE_EQC_REPORT_TIMER_M,
4829                        HNS_ROCE_EQC_REPORT_TIMER_S,
4830                        HNS_ROCE_EQ_INIT_REPORT_TIMER);
4831
4832         /* set eqe_ba [34:3] */
4833         roce_set_field(eqc->eqe_ba0,
4834                        HNS_ROCE_EQC_EQE_BA_L_M,
4835                        HNS_ROCE_EQC_EQE_BA_L_S, eq->eqe_ba >> 3);
4836
4837         /* set eqe_ba [64:35] */
4838         roce_set_field(eqc->eqe_ba1,
4839                        HNS_ROCE_EQC_EQE_BA_H_M,
4840                        HNS_ROCE_EQC_EQE_BA_H_S, eq->eqe_ba >> 35);
4841
4842         /* set eq shift */
4843         roce_set_field(eqc->byte_28,
4844                        HNS_ROCE_EQC_SHIFT_M,
4845                        HNS_ROCE_EQC_SHIFT_S, eq->shift);
4846
4847         /* set eq MSI_IDX */
4848         roce_set_field(eqc->byte_28,
4849                        HNS_ROCE_EQC_MSI_INDX_M,
4850                        HNS_ROCE_EQC_MSI_INDX_S,
4851                        HNS_ROCE_EQ_INIT_MSI_IDX);
4852
4853         /* set cur_eqe_ba [27:12] */
4854         roce_set_field(eqc->byte_28,
4855                        HNS_ROCE_EQC_CUR_EQE_BA_L_M,
4856                        HNS_ROCE_EQC_CUR_EQE_BA_L_S, eq->cur_eqe_ba >> 12);
4857
4858         /* set cur_eqe_ba [59:28] */
4859         roce_set_field(eqc->byte_32,
4860                        HNS_ROCE_EQC_CUR_EQE_BA_M_M,
4861                        HNS_ROCE_EQC_CUR_EQE_BA_M_S, eq->cur_eqe_ba >> 28);
4862
4863         /* set cur_eqe_ba [63:60] */
4864         roce_set_field(eqc->byte_36,
4865                        HNS_ROCE_EQC_CUR_EQE_BA_H_M,
4866                        HNS_ROCE_EQC_CUR_EQE_BA_H_S, eq->cur_eqe_ba >> 60);
4867
4868         /* set eq consumer idx */
4869         roce_set_field(eqc->byte_36,
4870                        HNS_ROCE_EQC_CONS_INDX_M,
4871                        HNS_ROCE_EQC_CONS_INDX_S,
4872                        HNS_ROCE_EQ_INIT_CONS_IDX);
4873
4874         /* set nex_eqe_ba[43:12] */
4875         roce_set_field(eqc->nxt_eqe_ba0,
4876                        HNS_ROCE_EQC_NXT_EQE_BA_L_M,
4877                        HNS_ROCE_EQC_NXT_EQE_BA_L_S, eq->nxt_eqe_ba >> 12);
4878
4879         /* set nex_eqe_ba[63:44] */
4880         roce_set_field(eqc->nxt_eqe_ba1,
4881                        HNS_ROCE_EQC_NXT_EQE_BA_H_M,
4882                        HNS_ROCE_EQC_NXT_EQE_BA_H_S, eq->nxt_eqe_ba >> 44);
4883 }
4884
4885 static int hns_roce_mhop_alloc_eq(struct hns_roce_dev *hr_dev,
4886                                   struct hns_roce_eq *eq)
4887 {
4888         struct device *dev = hr_dev->dev;
4889         int eq_alloc_done = 0;
4890         int eq_buf_cnt = 0;
4891         int eqe_alloc;
4892         u32 buf_chk_sz;
4893         u32 bt_chk_sz;
4894         u32 mhop_num;
4895         u64 size;
4896         u64 idx;
4897         int ba_num;
4898         int bt_num;
4899         int record_i;
4900         int record_j;
4901         int i = 0;
4902         int j = 0;
4903
4904         mhop_num = hr_dev->caps.eqe_hop_num;
4905         buf_chk_sz = 1 << (hr_dev->caps.eqe_buf_pg_sz + PAGE_SHIFT);
4906         bt_chk_sz = 1 << (hr_dev->caps.eqe_ba_pg_sz + PAGE_SHIFT);
4907
4908         ba_num = (PAGE_ALIGN(eq->entries * eq->eqe_size) + buf_chk_sz - 1)
4909                   / buf_chk_sz;
4910         bt_num = (ba_num + bt_chk_sz / 8 - 1) / (bt_chk_sz / 8);
4911
4912         /* hop_num = 0 */
4913         if (mhop_num == HNS_ROCE_HOP_NUM_0) {
4914                 if (eq->entries > buf_chk_sz / eq->eqe_size) {
4915                         dev_err(dev, "eq entries %d is larger than buf_pg_sz!",
4916                                 eq->entries);
4917                         return -EINVAL;
4918                 }
4919                 eq->bt_l0 = dma_alloc_coherent(dev, eq->entries * eq->eqe_size,
4920                                                &(eq->l0_dma), GFP_KERNEL);
4921                 if (!eq->bt_l0)
4922                         return -ENOMEM;
4923
4924                 eq->cur_eqe_ba = eq->l0_dma;
4925                 eq->nxt_eqe_ba = 0;
4926
4927                 memset(eq->bt_l0, 0, eq->entries * eq->eqe_size);
4928
4929                 return 0;
4930         }
4931
4932         eq->buf_dma = kcalloc(ba_num, sizeof(*eq->buf_dma), GFP_KERNEL);
4933         if (!eq->buf_dma)
4934                 return -ENOMEM;
4935         eq->buf = kcalloc(ba_num, sizeof(*eq->buf), GFP_KERNEL);
4936         if (!eq->buf)
4937                 goto err_kcalloc_buf;
4938
4939         if (mhop_num == 2) {
4940                 eq->l1_dma = kcalloc(bt_num, sizeof(*eq->l1_dma), GFP_KERNEL);
4941                 if (!eq->l1_dma)
4942                         goto err_kcalloc_l1_dma;
4943
4944                 eq->bt_l1 = kcalloc(bt_num, sizeof(*eq->bt_l1), GFP_KERNEL);
4945                 if (!eq->bt_l1)
4946                         goto err_kcalloc_bt_l1;
4947         }
4948
4949         /* alloc L0 BT */
4950         eq->bt_l0 = dma_alloc_coherent(dev, bt_chk_sz, &eq->l0_dma, GFP_KERNEL);
4951         if (!eq->bt_l0)
4952                 goto err_dma_alloc_l0;
4953
4954         if (mhop_num == 1) {
4955                 if (ba_num > (bt_chk_sz / 8))
4956                         dev_err(dev, "ba_num %d is too large for 1 hop\n",
4957                                 ba_num);
4958
4959                 /* alloc buf */
4960                 for (i = 0; i < bt_chk_sz / 8; i++) {
4961                         if (eq_buf_cnt + 1 < ba_num) {
4962                                 size = buf_chk_sz;
4963                         } else {
4964                                 eqe_alloc = i * (buf_chk_sz / eq->eqe_size);
4965                                 size = (eq->entries - eqe_alloc) * eq->eqe_size;
4966                         }
4967                         eq->buf[i] = dma_alloc_coherent(dev, size,
4968                                                         &(eq->buf_dma[i]),
4969                                                         GFP_KERNEL);
4970                         if (!eq->buf[i])
4971                                 goto err_dma_alloc_buf;
4972
4973                         memset(eq->buf[i], 0, size);
4974                         *(eq->bt_l0 + i) = eq->buf_dma[i];
4975
4976                         eq_buf_cnt++;
4977                         if (eq_buf_cnt >= ba_num)
4978                                 break;
4979                 }
4980                 eq->cur_eqe_ba = eq->buf_dma[0];
4981                 eq->nxt_eqe_ba = eq->buf_dma[1];
4982
4983         } else if (mhop_num == 2) {
4984                 /* alloc L1 BT and buf */
4985                 for (i = 0; i < bt_chk_sz / 8; i++) {
4986                         eq->bt_l1[i] = dma_alloc_coherent(dev, bt_chk_sz,
4987                                                           &(eq->l1_dma[i]),
4988                                                           GFP_KERNEL);
4989                         if (!eq->bt_l1[i])
4990                                 goto err_dma_alloc_l1;
4991                         *(eq->bt_l0 + i) = eq->l1_dma[i];
4992
4993                         for (j = 0; j < bt_chk_sz / 8; j++) {
4994                                 idx = i * bt_chk_sz / 8 + j;
4995                                 if (eq_buf_cnt + 1 < ba_num) {
4996                                         size = buf_chk_sz;
4997                                 } else {
4998                                         eqe_alloc = (buf_chk_sz / eq->eqe_size)
4999                                                     * idx;
5000                                         size = (eq->entries - eqe_alloc)
5001                                                 * eq->eqe_size;
5002                                 }
5003                                 eq->buf[idx] = dma_alloc_coherent(dev, size,
5004                                                             &(eq->buf_dma[idx]),
5005                                                             GFP_KERNEL);
5006                                 if (!eq->buf[idx])
5007                                         goto err_dma_alloc_buf;
5008
5009                                 memset(eq->buf[idx], 0, size);
5010                                 *(eq->bt_l1[i] + j) = eq->buf_dma[idx];
5011
5012                                 eq_buf_cnt++;
5013                                 if (eq_buf_cnt >= ba_num) {
5014                                         eq_alloc_done = 1;
5015                                         break;
5016                                 }
5017                         }
5018
5019                         if (eq_alloc_done)
5020                                 break;
5021                 }
5022                 eq->cur_eqe_ba = eq->buf_dma[0];
5023                 eq->nxt_eqe_ba = eq->buf_dma[1];
5024         }
5025
5026         eq->l0_last_num = i + 1;
5027         if (mhop_num == 2)
5028                 eq->l1_last_num = j + 1;
5029
5030         return 0;
5031
5032 err_dma_alloc_l1:
5033         dma_free_coherent(dev, bt_chk_sz, eq->bt_l0, eq->l0_dma);
5034         eq->bt_l0 = NULL;
5035         eq->l0_dma = 0;
5036         for (i -= 1; i >= 0; i--) {
5037                 dma_free_coherent(dev, bt_chk_sz, eq->bt_l1[i],
5038                                   eq->l1_dma[i]);
5039
5040                 for (j = 0; j < bt_chk_sz / 8; j++) {
5041                         idx = i * bt_chk_sz / 8 + j;
5042                         dma_free_coherent(dev, buf_chk_sz, eq->buf[idx],
5043                                           eq->buf_dma[idx]);
5044                 }
5045         }
5046         goto err_dma_alloc_l0;
5047
5048 err_dma_alloc_buf:
5049         dma_free_coherent(dev, bt_chk_sz, eq->bt_l0, eq->l0_dma);
5050         eq->bt_l0 = NULL;
5051         eq->l0_dma = 0;
5052
5053         if (mhop_num == 1)
5054                 for (i -= 1; i >= 0; i--)
5055                         dma_free_coherent(dev, buf_chk_sz, eq->buf[i],
5056                                           eq->buf_dma[i]);
5057         else if (mhop_num == 2) {
5058                 record_i = i;
5059                 record_j = j;
5060                 for (; i >= 0; i--) {
5061                         dma_free_coherent(dev, bt_chk_sz, eq->bt_l1[i],
5062                                           eq->l1_dma[i]);
5063
5064                         for (j = 0; j < bt_chk_sz / 8; j++) {
5065                                 if (i == record_i && j >= record_j)
5066                                         break;
5067
5068                                 idx = i * bt_chk_sz / 8 + j;
5069                                 dma_free_coherent(dev, buf_chk_sz,
5070                                                   eq->buf[idx],
5071                                                   eq->buf_dma[idx]);
5072                         }
5073                 }
5074         }
5075
5076 err_dma_alloc_l0:
5077         kfree(eq->bt_l1);
5078         eq->bt_l1 = NULL;
5079
5080 err_kcalloc_bt_l1:
5081         kfree(eq->l1_dma);
5082         eq->l1_dma = NULL;
5083
5084 err_kcalloc_l1_dma:
5085         kfree(eq->buf);
5086         eq->buf = NULL;
5087
5088 err_kcalloc_buf:
5089         kfree(eq->buf_dma);
5090         eq->buf_dma = NULL;
5091
5092         return -ENOMEM;
5093 }
5094
5095 static int hns_roce_v2_create_eq(struct hns_roce_dev *hr_dev,
5096                                  struct hns_roce_eq *eq,
5097                                  unsigned int eq_cmd)
5098 {
5099         struct device *dev = hr_dev->dev;
5100         struct hns_roce_cmd_mailbox *mailbox;
5101         u32 buf_chk_sz = 0;
5102         int ret;
5103
5104         /* Allocate mailbox memory */
5105         mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
5106         if (IS_ERR(mailbox))
5107                 return PTR_ERR(mailbox);
5108
5109         if (!hr_dev->caps.eqe_hop_num) {
5110                 buf_chk_sz = 1 << (hr_dev->caps.eqe_buf_pg_sz + PAGE_SHIFT);
5111
5112                 eq->buf_list = kzalloc(sizeof(struct hns_roce_buf_list),
5113                                        GFP_KERNEL);
5114                 if (!eq->buf_list) {
5115                         ret = -ENOMEM;
5116                         goto free_cmd_mbox;
5117                 }
5118
5119                 eq->buf_list->buf = dma_alloc_coherent(dev, buf_chk_sz,
5120                                                        &(eq->buf_list->map),
5121                                                        GFP_KERNEL);
5122                 if (!eq->buf_list->buf) {
5123                         ret = -ENOMEM;
5124                         goto err_alloc_buf;
5125                 }
5126
5127                 memset(eq->buf_list->buf, 0, buf_chk_sz);
5128         } else {
5129                 ret = hns_roce_mhop_alloc_eq(hr_dev, eq);
5130                 if (ret) {
5131                         ret = -ENOMEM;
5132                         goto free_cmd_mbox;
5133                 }
5134         }
5135
5136         hns_roce_config_eqc(hr_dev, eq, mailbox->buf);
5137
5138         ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, eq->eqn, 0,
5139                                 eq_cmd, HNS_ROCE_CMD_TIMEOUT_MSECS);
5140         if (ret) {
5141                 dev_err(dev, "[mailbox cmd] create eqc failed.\n");
5142                 goto err_cmd_mbox;
5143         }
5144
5145         hns_roce_free_cmd_mailbox(hr_dev, mailbox);
5146
5147         return 0;
5148
5149 err_cmd_mbox:
5150         if (!hr_dev->caps.eqe_hop_num)
5151                 dma_free_coherent(dev, buf_chk_sz, eq->buf_list->buf,
5152                                   eq->buf_list->map);
5153         else {
5154                 hns_roce_mhop_free_eq(hr_dev, eq);
5155                 goto free_cmd_mbox;
5156         }
5157
5158 err_alloc_buf:
5159         kfree(eq->buf_list);
5160
5161 free_cmd_mbox:
5162         hns_roce_free_cmd_mailbox(hr_dev, mailbox);
5163
5164         return ret;
5165 }
5166
5167 static int hns_roce_v2_init_eq_table(struct hns_roce_dev *hr_dev)
5168 {
5169         struct hns_roce_eq_table *eq_table = &hr_dev->eq_table;
5170         struct device *dev = hr_dev->dev;
5171         struct hns_roce_eq *eq;
5172         unsigned int eq_cmd;
5173         int irq_num;
5174         int eq_num;
5175         int other_num;
5176         int comp_num;
5177         int aeq_num;
5178         int i, j, k;
5179         int ret;
5180
5181         other_num = hr_dev->caps.num_other_vectors;
5182         comp_num = hr_dev->caps.num_comp_vectors;
5183         aeq_num = hr_dev->caps.num_aeq_vectors;
5184
5185         eq_num = comp_num + aeq_num;
5186         irq_num = eq_num + other_num;
5187
5188         eq_table->eq = kcalloc(eq_num, sizeof(*eq_table->eq), GFP_KERNEL);
5189         if (!eq_table->eq)
5190                 return -ENOMEM;
5191
5192         for (i = 0; i < irq_num; i++) {
5193                 hr_dev->irq_names[i] = kzalloc(HNS_ROCE_INT_NAME_LEN,
5194                                                GFP_KERNEL);
5195                 if (!hr_dev->irq_names[i]) {
5196                         ret = -ENOMEM;
5197                         goto err_failed_kzalloc;
5198                 }
5199         }
5200
5201         /* create eq */
5202         for (j = 0; j < eq_num; j++) {
5203                 eq = &eq_table->eq[j];
5204                 eq->hr_dev = hr_dev;
5205                 eq->eqn = j;
5206                 if (j < comp_num) {
5207                         /* CEQ */
5208                         eq_cmd = HNS_ROCE_CMD_CREATE_CEQC;
5209                         eq->type_flag = HNS_ROCE_CEQ;
5210                         eq->entries = hr_dev->caps.ceqe_depth;
5211                         eq->eqe_size = HNS_ROCE_CEQ_ENTRY_SIZE;
5212                         eq->irq = hr_dev->irq[j + other_num + aeq_num];
5213                         eq->eq_max_cnt = HNS_ROCE_CEQ_DEFAULT_BURST_NUM;
5214                         eq->eq_period = HNS_ROCE_CEQ_DEFAULT_INTERVAL;
5215                 } else {
5216                         /* AEQ */
5217                         eq_cmd = HNS_ROCE_CMD_CREATE_AEQC;
5218                         eq->type_flag = HNS_ROCE_AEQ;
5219                         eq->entries = hr_dev->caps.aeqe_depth;
5220                         eq->eqe_size = HNS_ROCE_AEQ_ENTRY_SIZE;
5221                         eq->irq = hr_dev->irq[j - comp_num + other_num];
5222                         eq->eq_max_cnt = HNS_ROCE_AEQ_DEFAULT_BURST_NUM;
5223                         eq->eq_period = HNS_ROCE_AEQ_DEFAULT_INTERVAL;
5224                 }
5225
5226                 ret = hns_roce_v2_create_eq(hr_dev, eq, eq_cmd);
5227                 if (ret) {
5228                         dev_err(dev, "eq create failed.\n");
5229                         goto err_create_eq_fail;
5230                 }
5231         }
5232
5233         /* enable irq */
5234         hns_roce_v2_int_mask_enable(hr_dev, eq_num, EQ_ENABLE);
5235
5236         /* irq contains: abnormal + AEQ + CEQ*/
5237         for (k = 0; k < irq_num; k++)
5238                 if (k < other_num)
5239                         snprintf((char *)hr_dev->irq_names[k],
5240                                  HNS_ROCE_INT_NAME_LEN, "hns-abn-%d", k);
5241                 else if (k < (other_num + aeq_num))
5242                         snprintf((char *)hr_dev->irq_names[k],
5243                                  HNS_ROCE_INT_NAME_LEN, "hns-aeq-%d",
5244                                  k - other_num);
5245                 else
5246                         snprintf((char *)hr_dev->irq_names[k],
5247                                  HNS_ROCE_INT_NAME_LEN, "hns-ceq-%d",
5248                                  k - other_num - aeq_num);
5249
5250         for (k = 0; k < irq_num; k++) {
5251                 if (k < other_num)
5252                         ret = request_irq(hr_dev->irq[k],
5253                                           hns_roce_v2_msix_interrupt_abn,
5254                                           0, hr_dev->irq_names[k], hr_dev);
5255
5256                 else if (k < (other_num + comp_num))
5257                         ret = request_irq(eq_table->eq[k - other_num].irq,
5258                                           hns_roce_v2_msix_interrupt_eq,
5259                                           0, hr_dev->irq_names[k + aeq_num],
5260                                           &eq_table->eq[k - other_num]);
5261                 else
5262                         ret = request_irq(eq_table->eq[k - other_num].irq,
5263                                           hns_roce_v2_msix_interrupt_eq,
5264                                           0, hr_dev->irq_names[k - comp_num],
5265                                           &eq_table->eq[k - other_num]);
5266                 if (ret) {
5267                         dev_err(dev, "Request irq error!\n");
5268                         goto err_request_irq_fail;
5269                 }
5270         }
5271
5272         hr_dev->irq_workq =
5273                 create_singlethread_workqueue("hns_roce_irq_workqueue");
5274         if (!hr_dev->irq_workq) {
5275                 dev_err(dev, "Create irq workqueue failed!\n");
5276                 ret = -ENOMEM;
5277                 goto err_request_irq_fail;
5278         }
5279
5280         return 0;
5281
5282 err_request_irq_fail:
5283         for (k -= 1; k >= 0; k--)
5284                 if (k < other_num)
5285                         free_irq(hr_dev->irq[k], hr_dev);
5286                 else
5287                         free_irq(eq_table->eq[k - other_num].irq,
5288                                  &eq_table->eq[k - other_num]);
5289
5290 err_create_eq_fail:
5291         for (j -= 1; j >= 0; j--)
5292                 hns_roce_v2_free_eq(hr_dev, &eq_table->eq[j]);
5293
5294 err_failed_kzalloc:
5295         for (i -= 1; i >= 0; i--)
5296                 kfree(hr_dev->irq_names[i]);
5297         kfree(eq_table->eq);
5298
5299         return ret;
5300 }
5301
5302 static void hns_roce_v2_cleanup_eq_table(struct hns_roce_dev *hr_dev)
5303 {
5304         struct hns_roce_eq_table *eq_table = &hr_dev->eq_table;
5305         int irq_num;
5306         int eq_num;
5307         int i;
5308
5309         eq_num = hr_dev->caps.num_comp_vectors + hr_dev->caps.num_aeq_vectors;
5310         irq_num = eq_num + hr_dev->caps.num_other_vectors;
5311
5312         /* Disable irq */
5313         hns_roce_v2_int_mask_enable(hr_dev, eq_num, EQ_DISABLE);
5314
5315         for (i = 0; i < hr_dev->caps.num_other_vectors; i++)
5316                 free_irq(hr_dev->irq[i], hr_dev);
5317
5318         for (i = 0; i < eq_num; i++) {
5319                 hns_roce_v2_destroy_eqc(hr_dev, i);
5320
5321                 free_irq(eq_table->eq[i].irq, &eq_table->eq[i]);
5322
5323                 hns_roce_v2_free_eq(hr_dev, &eq_table->eq[i]);
5324         }
5325
5326         for (i = 0; i < irq_num; i++)
5327                 kfree(hr_dev->irq_names[i]);
5328
5329         kfree(eq_table->eq);
5330
5331         flush_workqueue(hr_dev->irq_workq);
5332         destroy_workqueue(hr_dev->irq_workq);
5333 }
5334
5335 static const struct hns_roce_hw hns_roce_hw_v2 = {
5336         .cmq_init = hns_roce_v2_cmq_init,
5337         .cmq_exit = hns_roce_v2_cmq_exit,
5338         .hw_profile = hns_roce_v2_profile,
5339         .hw_init = hns_roce_v2_init,
5340         .hw_exit = hns_roce_v2_exit,
5341         .post_mbox = hns_roce_v2_post_mbox,
5342         .chk_mbox = hns_roce_v2_chk_mbox,
5343         .set_gid = hns_roce_v2_set_gid,
5344         .set_mac = hns_roce_v2_set_mac,
5345         .write_mtpt = hns_roce_v2_write_mtpt,
5346         .rereg_write_mtpt = hns_roce_v2_rereg_write_mtpt,
5347         .frmr_write_mtpt = hns_roce_v2_frmr_write_mtpt,
5348         .mw_write_mtpt = hns_roce_v2_mw_write_mtpt,
5349         .write_cqc = hns_roce_v2_write_cqc,
5350         .set_hem = hns_roce_v2_set_hem,
5351         .clear_hem = hns_roce_v2_clear_hem,
5352         .modify_qp = hns_roce_v2_modify_qp,
5353         .query_qp = hns_roce_v2_query_qp,
5354         .destroy_qp = hns_roce_v2_destroy_qp,
5355         .modify_cq = hns_roce_v2_modify_cq,
5356         .post_send = hns_roce_v2_post_send,
5357         .post_recv = hns_roce_v2_post_recv,
5358         .req_notify_cq = hns_roce_v2_req_notify_cq,
5359         .poll_cq = hns_roce_v2_poll_cq,
5360         .init_eq = hns_roce_v2_init_eq_table,
5361         .cleanup_eq = hns_roce_v2_cleanup_eq_table,
5362 };
5363
5364 static const struct pci_device_id hns_roce_hw_v2_pci_tbl[] = {
5365         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA), 0},
5366         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA_MACSEC), 0},
5367         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA), 0},
5368         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA_MACSEC), 0},
5369         {PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_MACSEC), 0},
5370         /* required last entry */
5371         {0, }
5372 };
5373
5374 MODULE_DEVICE_TABLE(pci, hns_roce_hw_v2_pci_tbl);
5375
5376 static int hns_roce_hw_v2_get_cfg(struct hns_roce_dev *hr_dev,
5377                                   struct hnae3_handle *handle)
5378 {
5379         const struct pci_device_id *id;
5380         int i;
5381
5382         id = pci_match_id(hns_roce_hw_v2_pci_tbl, hr_dev->pci_dev);
5383         if (!id) {
5384                 dev_err(hr_dev->dev, "device is not compatible!\n");
5385                 return -ENXIO;
5386         }
5387
5388         hr_dev->hw = &hns_roce_hw_v2;
5389         hr_dev->sdb_offset = ROCEE_DB_SQ_L_0_REG;
5390         hr_dev->odb_offset = hr_dev->sdb_offset;
5391
5392         /* Get info from NIC driver. */
5393         hr_dev->reg_base = handle->rinfo.roce_io_base;
5394         hr_dev->caps.num_ports = 1;
5395         hr_dev->iboe.netdevs[0] = handle->rinfo.netdev;
5396         hr_dev->iboe.phy_port[0] = 0;
5397
5398         addrconf_addr_eui48((u8 *)&hr_dev->ib_dev.node_guid,
5399                             hr_dev->iboe.netdevs[0]->dev_addr);
5400
5401         for (i = 0; i < HNS_ROCE_V2_MAX_IRQ_NUM; i++)
5402                 hr_dev->irq[i] = pci_irq_vector(handle->pdev,
5403                                                 i + handle->rinfo.base_vector);
5404
5405         /* cmd issue mode: 0 is poll, 1 is event */
5406         hr_dev->cmd_mod = 1;
5407         hr_dev->loop_idc = 0;
5408
5409         return 0;
5410 }
5411
5412 static int hns_roce_hw_v2_init_instance(struct hnae3_handle *handle)
5413 {
5414         struct hns_roce_dev *hr_dev;
5415         int ret;
5416
5417         hr_dev = (struct hns_roce_dev *)ib_alloc_device(sizeof(*hr_dev));
5418         if (!hr_dev)
5419                 return -ENOMEM;
5420
5421         hr_dev->priv = kzalloc(sizeof(struct hns_roce_v2_priv), GFP_KERNEL);
5422         if (!hr_dev->priv) {
5423                 ret = -ENOMEM;
5424                 goto error_failed_kzalloc;
5425         }
5426
5427         hr_dev->pci_dev = handle->pdev;
5428         hr_dev->dev = &handle->pdev->dev;
5429         handle->priv = hr_dev;
5430
5431         ret = hns_roce_hw_v2_get_cfg(hr_dev, handle);
5432         if (ret) {
5433                 dev_err(hr_dev->dev, "Get Configuration failed!\n");
5434                 goto error_failed_get_cfg;
5435         }
5436
5437         ret = hns_roce_init(hr_dev);
5438         if (ret) {
5439                 dev_err(hr_dev->dev, "RoCE Engine init failed!\n");
5440                 goto error_failed_get_cfg;
5441         }
5442
5443         return 0;
5444
5445 error_failed_get_cfg:
5446         kfree(hr_dev->priv);
5447
5448 error_failed_kzalloc:
5449         ib_dealloc_device(&hr_dev->ib_dev);
5450
5451         return ret;
5452 }
5453
5454 static void hns_roce_hw_v2_uninit_instance(struct hnae3_handle *handle,
5455                                            bool reset)
5456 {
5457         struct hns_roce_dev *hr_dev = (struct hns_roce_dev *)handle->priv;
5458
5459         if (!hr_dev)
5460                 return;
5461
5462         hns_roce_exit(hr_dev);
5463         kfree(hr_dev->priv);
5464         ib_dealloc_device(&hr_dev->ib_dev);
5465 }
5466
5467 static int hns_roce_hw_v2_reset_notify_down(struct hnae3_handle *handle)
5468 {
5469         struct hns_roce_dev *hr_dev = (struct hns_roce_dev *)handle->priv;
5470         struct ib_event event;
5471
5472         if (!hr_dev) {
5473                 dev_err(&handle->pdev->dev,
5474                         "Input parameter handle->priv is NULL!\n");
5475                 return -EINVAL;
5476         }
5477
5478         hr_dev->active = false;
5479         hr_dev->is_reset = true;
5480
5481         event.event = IB_EVENT_DEVICE_FATAL;
5482         event.device = &hr_dev->ib_dev;
5483         event.element.port_num = 1;
5484         ib_dispatch_event(&event);
5485
5486         return 0;
5487 }
5488
5489 static int hns_roce_hw_v2_reset_notify_init(struct hnae3_handle *handle)
5490 {
5491         int ret;
5492
5493         ret = hns_roce_hw_v2_init_instance(handle);
5494         if (ret) {
5495                 /* when reset notify type is HNAE3_INIT_CLIENT In reset notify
5496                  * callback function, RoCE Engine reinitialize. If RoCE reinit
5497                  * failed, we should inform NIC driver.
5498                  */
5499                 handle->priv = NULL;
5500                 dev_err(&handle->pdev->dev,
5501                         "In reset process RoCE reinit failed %d.\n", ret);
5502         }
5503
5504         return ret;
5505 }
5506
5507 static int hns_roce_hw_v2_reset_notify_uninit(struct hnae3_handle *handle)
5508 {
5509         msleep(100);
5510         hns_roce_hw_v2_uninit_instance(handle, false);
5511         return 0;
5512 }
5513
5514 static int hns_roce_hw_v2_reset_notify(struct hnae3_handle *handle,
5515                                        enum hnae3_reset_notify_type type)
5516 {
5517         int ret = 0;
5518
5519         switch (type) {
5520         case HNAE3_DOWN_CLIENT:
5521                 ret = hns_roce_hw_v2_reset_notify_down(handle);
5522                 break;
5523         case HNAE3_INIT_CLIENT:
5524                 ret = hns_roce_hw_v2_reset_notify_init(handle);
5525                 break;
5526         case HNAE3_UNINIT_CLIENT:
5527                 ret = hns_roce_hw_v2_reset_notify_uninit(handle);
5528                 break;
5529         default:
5530                 break;
5531         }
5532
5533         return ret;
5534 }
5535
5536 static const struct hnae3_client_ops hns_roce_hw_v2_ops = {
5537         .init_instance = hns_roce_hw_v2_init_instance,
5538         .uninit_instance = hns_roce_hw_v2_uninit_instance,
5539         .reset_notify = hns_roce_hw_v2_reset_notify,
5540 };
5541
5542 static struct hnae3_client hns_roce_hw_v2_client = {
5543         .name = "hns_roce_hw_v2",
5544         .type = HNAE3_CLIENT_ROCE,
5545         .ops = &hns_roce_hw_v2_ops,
5546 };
5547
5548 static int __init hns_roce_hw_v2_init(void)
5549 {
5550         return hnae3_register_client(&hns_roce_hw_v2_client);
5551 }
5552
5553 static void __exit hns_roce_hw_v2_exit(void)
5554 {
5555         hnae3_unregister_client(&hns_roce_hw_v2_client);
5556 }
5557
5558 module_init(hns_roce_hw_v2_init);
5559 module_exit(hns_roce_hw_v2_exit);
5560
5561 MODULE_LICENSE("Dual BSD/GPL");
5562 MODULE_AUTHOR("Wei Hu <xavier.huwei@huawei.com>");
5563 MODULE_AUTHOR("Lijun Ou <oulijun@huawei.com>");
5564 MODULE_AUTHOR("Shaobo Xu <xushaobo2@huawei.com>");
5565 MODULE_DESCRIPTION("Hisilicon Hip08 Family RoCE Driver");