drivers/crypto/hisilicon/qm.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright (c) 2019 HiSilicon Limited. */
   3 #include <asm/page.h>
   4 #include <linux/bitmap.h>
   5 #include <linux/debugfs.h>
   6 #include <linux/dma-mapping.h>
   7 #include <linux/io.h>
   8 #include <linux/irqreturn.h>
   9 #include <linux/log2.h>
  10 #include <linux/seq_file.h>
  11 #include <linux/slab.h>
  12 #include "qm.h"
  13
  14 /* eq/aeq irq enable */
  15 #define QM_VF_AEQ_INT_SOURCE            0x0
  16 #define QM_VF_AEQ_INT_MASK              0x4
  17 #define QM_VF_EQ_INT_SOURCE             0x8
  18 #define QM_VF_EQ_INT_MASK               0xc
  19 #define QM_IRQ_NUM_V1                   1
  20 #define QM_IRQ_NUM_PF_V2                4
  21 #define QM_IRQ_NUM_VF_V2                2
  22
  23 #define QM_EQ_EVENT_IRQ_VECTOR          0
  24 #define QM_AEQ_EVENT_IRQ_VECTOR         1
  25 #define QM_ABNORMAL_EVENT_IRQ_VECTOR    3
  26
  27 /* mailbox */
  28 #define QM_MB_CMD_SQC                   0x0
  29 #define QM_MB_CMD_CQC                   0x1
  30 #define QM_MB_CMD_EQC                   0x2
  31 #define QM_MB_CMD_AEQC                  0x3
  32 #define QM_MB_CMD_SQC_BT                0x4
  33 #define QM_MB_CMD_CQC_BT                0x5
  34 #define QM_MB_CMD_SQC_VFT_V2            0x6
  35
  36 #define QM_MB_CMD_SEND_BASE             0x300
  37 #define QM_MB_EVENT_SHIFT               8
  38 #define QM_MB_BUSY_SHIFT                13
  39 #define QM_MB_OP_SHIFT                  14
  40 #define QM_MB_CMD_DATA_ADDR_L           0x304
  41 #define QM_MB_CMD_DATA_ADDR_H           0x308
  42
  43 /* sqc shift */
  44 #define QM_SQ_HOP_NUM_SHIFT             0
  45 #define QM_SQ_PAGE_SIZE_SHIFT           4
  46 #define QM_SQ_BUF_SIZE_SHIFT            8
  47 #define QM_SQ_SQE_SIZE_SHIFT            12
  48 #define QM_SQ_PRIORITY_SHIFT            0
  49 #define QM_SQ_ORDERS_SHIFT              4
  50 #define QM_SQ_TYPE_SHIFT                8
  51
  52 #define QM_SQ_TYPE_MASK                 GENMASK(3, 0)
  53
  54 /* cqc shift */
  55 #define QM_CQ_HOP_NUM_SHIFT             0
  56 #define QM_CQ_PAGE_SIZE_SHIFT           4
  57 #define QM_CQ_BUF_SIZE_SHIFT            8
  58 #define QM_CQ_CQE_SIZE_SHIFT            12
  59 #define QM_CQ_PHASE_SHIFT               0
  60 #define QM_CQ_FLAG_SHIFT                1
  61
  62 #define QM_CQE_PHASE(cqe)               ((cqe)->w7 & 0x1)
  63 #define QM_QC_CQE_SIZE                  4
  64
  65 /* eqc shift */
  66 #define QM_EQE_AEQE_SIZE                (2UL << 12)
  67 #define QM_EQC_PHASE_SHIFT              16
  68
  69 #define QM_EQE_PHASE(eqe)               (((eqe)->dw0 >> 16) & 0x1)
  70 #define QM_EQE_CQN_MASK                 GENMASK(15, 0)
  71
  72 #define QM_AEQE_PHASE(aeqe)             (((aeqe)->dw0 >> 16) & 0x1)
  73 #define QM_AEQE_TYPE_SHIFT              17
  74
  75 #define QM_DOORBELL_CMD_SQ              0
  76 #define QM_DOORBELL_CMD_CQ              1
  77 #define QM_DOORBELL_CMD_EQ              2
  78 #define QM_DOORBELL_CMD_AEQ             3
  79
  80 #define QM_DOORBELL_BASE_V1             0x340
  81 #define QM_DB_CMD_SHIFT_V1              16
  82 #define QM_DB_INDEX_SHIFT_V1            32
  83 #define QM_DB_PRIORITY_SHIFT_V1         48
  84 #define QM_DOORBELL_SQ_CQ_BASE_V2       0x1000
  85 #define QM_DOORBELL_EQ_AEQ_BASE_V2      0x2000
  86 #define QM_DB_CMD_SHIFT_V2              12
  87 #define QM_DB_RAND_SHIFT_V2             16
  88 #define QM_DB_INDEX_SHIFT_V2            32
  89 #define QM_DB_PRIORITY_SHIFT_V2         48
  90
  91 #define QM_MEM_START_INIT               0x100040
  92 #define QM_MEM_INIT_DONE                0x100044
  93 #define QM_VFT_CFG_RDY                  0x10006c
  94 #define QM_VFT_CFG_OP_WR                0x100058
  95 #define QM_VFT_CFG_TYPE                 0x10005c
  96 #define QM_SQC_VFT                      0x0
  97 #define QM_CQC_VFT                      0x1
  98 #define QM_VFT_CFG                      0x100060
  99 #define QM_VFT_CFG_OP_ENABLE            0x100054
 100
 101 #define QM_VFT_CFG_DATA_L               0x100064
 102 #define QM_VFT_CFG_DATA_H               0x100068
 103 #define QM_SQC_VFT_BUF_SIZE             (7ULL << 8)
 104 #define QM_SQC_VFT_SQC_SIZE             (5ULL << 12)
 105 #define QM_SQC_VFT_INDEX_NUMBER         (1ULL << 16)
 106 #define QM_SQC_VFT_START_SQN_SHIFT      28
 107 #define QM_SQC_VFT_VALID                (1ULL << 44)
 108 #define QM_SQC_VFT_SQN_SHIFT            45
 109 #define QM_CQC_VFT_BUF_SIZE             (7ULL << 8)
 110 #define QM_CQC_VFT_SQC_SIZE             (5ULL << 12)
 111 #define QM_CQC_VFT_INDEX_NUMBER         (1ULL << 16)
 112 #define QM_CQC_VFT_VALID                (1ULL << 28)
 113
 114 #define QM_SQC_VFT_BASE_SHIFT_V2        28
 115 #define QM_SQC_VFT_BASE_MASK_V2         GENMASK(5, 0)
 116 #define QM_SQC_VFT_NUM_SHIFT_V2         45
 117 #define QM_SQC_VFT_NUM_MASK_v2          GENMASK(9, 0)
 118
 119 #define QM_DFX_CNT_CLR_CE               0x100118
 120
 121 #define QM_ABNORMAL_INT_SOURCE          0x100000
 122 #define QM_ABNORMAL_INT_MASK            0x100004
 123 #define QM_ABNORMAL_INT_MASK_VALUE      0x1fff
 124 #define QM_ABNORMAL_INT_STATUS          0x100008
 125 #define QM_ABNORMAL_INF00               0x100010
 126 #define QM_FIFO_OVERFLOW_TYPE           0xc0
 127 #define QM_FIFO_OVERFLOW_TYPE_SHIFT     6
 128 #define QM_FIFO_OVERFLOW_VF             0x3f
 129 #define QM_ABNORMAL_INF01               0x100014
 130 #define QM_DB_TIMEOUT_TYPE              0xc0
 131 #define QM_DB_TIMEOUT_TYPE_SHIFT        6
 132 #define QM_DB_TIMEOUT_VF                0x3f
 133 #define QM_RAS_CE_ENABLE                0x1000ec
 134 #define QM_RAS_FE_ENABLE                0x1000f0
 135 #define QM_RAS_NFE_ENABLE               0x1000f4
 136 #define QM_RAS_CE_THRESHOLD             0x1000f8
 137 #define QM_RAS_CE_TIMES_PER_IRQ         1
 138 #define QM_RAS_MSI_INT_SEL              0x1040f4
 139
 140 #define QM_CACHE_WB_START               0x204
 141 #define QM_CACHE_WB_DONE                0x208
 142
 143 #define PCI_BAR_2                       2
 144 #define QM_SQE_DATA_ALIGN_MASK          GENMASK(6, 0)
 145 #define QMC_ALIGN(sz)                   ALIGN(sz, 32)
 146
 147 #define QM_DBG_TMP_BUF_LEN              22
 148
 149 #define QM_MK_CQC_DW3_V1(hop_num, pg_sz, buf_sz, cqe_sz) \
 150         (((hop_num) << QM_CQ_HOP_NUM_SHIFT)     | \
 151         ((pg_sz) << QM_CQ_PAGE_SIZE_SHIFT)      | \
 152         ((buf_sz) << QM_CQ_BUF_SIZE_SHIFT)      | \
 153         ((cqe_sz) << QM_CQ_CQE_SIZE_SHIFT))
 154
 155 #define QM_MK_CQC_DW3_V2(cqe_sz) \
 156         ((QM_Q_DEPTH - 1) | ((cqe_sz) << QM_CQ_CQE_SIZE_SHIFT))
 157
 158 #define QM_MK_SQC_W13(priority, orders, alg_type) \
 159         (((priority) << QM_SQ_PRIORITY_SHIFT)   | \
 160         ((orders) << QM_SQ_ORDERS_SHIFT)        | \
 161         (((alg_type) & QM_SQ_TYPE_MASK) << QM_SQ_TYPE_SHIFT))
 162
 163 #define QM_MK_SQC_DW3_V1(hop_num, pg_sz, buf_sz, sqe_sz) \
 164         (((hop_num) << QM_SQ_HOP_NUM_SHIFT)     | \
 165         ((pg_sz) << QM_SQ_PAGE_SIZE_SHIFT)      | \
 166         ((buf_sz) << QM_SQ_BUF_SIZE_SHIFT)      | \
 167         ((u32)ilog2(sqe_sz) << QM_SQ_SQE_SIZE_SHIFT))
 168
 169 #define QM_MK_SQC_DW3_V2(sqe_sz) \
 170         ((QM_Q_DEPTH - 1) | ((u32)ilog2(sqe_sz) << QM_SQ_SQE_SIZE_SHIFT))
 171
 172 #define INIT_QC_COMMON(qc, base, pasid) do {    \
 173         (qc)->head = 0;                         \
 174         (qc)->tail = 0;                         \
 175         (qc)->base_l = lower_32_bits(base);     \
 176         (qc)->base_h = upper_32_bits(base);     \
 177         (qc)->dw3 = 0;                          \
 178         (qc)->w8 = 0;                           \
 179         (qc)->rsvd0 = 0;                        \
 180         (qc)->pasid = pasid;                    \
 181         (qc)->w11 = 0;                          \
 182         (qc)->rsvd1 = 0;                        \
 183 } while (0)
 184
 185 enum vft_type {
 186         SQC_VFT = 0,
 187         CQC_VFT,
 188 };
 189
 190 struct qm_cqe {
 191         __le32 rsvd0;
 192         __le16 cmd_id;
 193         __le16 rsvd1;
 194         __le16 sq_head;
 195         __le16 sq_num;
 196         __le16 rsvd2;
 197         __le16 w7;
 198 };
 199
 200 struct qm_eqe {
 201         __le32 dw0;
 202 };
 203
 204 struct qm_aeqe {
 205         __le32 dw0;
 206 };
 207
 208 struct qm_sqc {
 209         __le16 head;
 210         __le16 tail;
 211         __le32 base_l;
 212         __le32 base_h;
 213         __le32 dw3;
 214         __le16 w8;
 215         __le16 rsvd0;
 216         __le16 pasid;
 217         __le16 w11;
 218         __le16 cq_num;
 219         __le16 w13;
 220         __le32 rsvd1;
 221 };
 222
 223 struct qm_cqc {
 224         __le16 head;
 225         __le16 tail;
 226         __le32 base_l;
 227         __le32 base_h;
 228         __le32 dw3;
 229         __le16 w8;
 230         __le16 rsvd0;
 231         __le16 pasid;
 232         __le16 w11;
 233         __le32 dw6;
 234         __le32 rsvd1;
 235 };
 236
 237 struct qm_eqc {
 238         __le16 head;
 239         __le16 tail;
 240         __le32 base_l;
 241         __le32 base_h;
 242         __le32 dw3;
 243         __le32 rsvd[2];
 244         __le32 dw6;
 245 };
 246
 247 struct qm_aeqc {
 248         __le16 head;
 249         __le16 tail;
 250         __le32 base_l;
 251         __le32 base_h;
 252         __le32 dw3;
 253         __le32 rsvd[2];
 254         __le32 dw6;
 255 };
 256
 257 struct qm_mailbox {
 258         __le16 w0;
 259         __le16 queue_num;
 260         __le32 base_l;
 261         __le32 base_h;
 262         __le32 rsvd;
 263 };
 264
 265 struct qm_doorbell {
 266         __le16 queue_num;
 267         __le16 cmd;
 268         __le16 index;
 269         __le16 priority;
 270 };
 271
 272 struct hisi_qm_hw_ops {
 273         int (*get_vft)(struct hisi_qm *qm, u32 *base, u32 *number);
 274         void (*qm_db)(struct hisi_qm *qm, u16 qn,
 275                       u8 cmd, u16 index, u8 priority);
 276         u32 (*get_irq_num)(struct hisi_qm *qm);
 277         int (*debug_init)(struct hisi_qm *qm);
 278         void (*hw_error_init)(struct hisi_qm *qm, u32 ce, u32 nfe, u32 fe,
 279                               u32 msi);
 280         pci_ers_result_t (*hw_error_handle)(struct hisi_qm *qm);
 281 };
 282
 283 static const char * const qm_debug_file_name[] = {
 284         [CURRENT_Q]    = "current_q",
 285         [CLEAR_ENABLE] = "clear_enable",
 286 };
 287
 288 struct hisi_qm_hw_error {
 289         u32 int_msk;
 290         const char *msg;
 291 };
 292
 293 static const struct hisi_qm_hw_error qm_hw_error[] = {
 294         { .int_msk = BIT(0), .msg = "qm_axi_rresp" },
 295         { .int_msk = BIT(1), .msg = "qm_axi_bresp" },
 296         { .int_msk = BIT(2), .msg = "qm_ecc_mbit" },
 297         { .int_msk = BIT(3), .msg = "qm_ecc_1bit" },
 298         { .int_msk = BIT(4), .msg = "qm_acc_get_task_timeout" },
 299         { .int_msk = BIT(5), .msg = "qm_acc_do_task_timeout" },
 300         { .int_msk = BIT(6), .msg = "qm_acc_wb_not_ready_timeout" },
 301         { .int_msk = BIT(7), .msg = "qm_sq_cq_vf_invalid" },
 302         { .int_msk = BIT(8), .msg = "qm_cq_vf_invalid" },
 303         { .int_msk = BIT(9), .msg = "qm_sq_vf_invalid" },
 304         { .int_msk = BIT(10), .msg = "qm_db_timeout" },
 305         { .int_msk = BIT(11), .msg = "qm_of_fifo_of" },
 306         { .int_msk = BIT(12), .msg = "qm_db_random_invalid" },
 307         { /* sentinel */ }
 308 };
 309
 310 static const char * const qm_db_timeout[] = {
 311         "sq", "cq", "eq", "aeq",
 312 };
 313
 314 static const char * const qm_fifo_overflow[] = {
 315         "cq", "eq", "aeq",
 316 };
 317
 318 /* return 0 mailbox ready, -ETIMEDOUT hardware timeout */
 319 static int qm_wait_mb_ready(struct hisi_qm *qm)
 320 {
 321         u32 val;
 322
 323         return readl_relaxed_poll_timeout(qm->io_base + QM_MB_CMD_SEND_BASE,
 324                                           val, !((val >> QM_MB_BUSY_SHIFT) &
 325                                           0x1), 10, 1000);
 326 }
 327
 328 /* 128 bit should be written to hardware at one time to trigger a mailbox */
 329 static void qm_mb_write(struct hisi_qm *qm, const void *src)
 330 {
 331         void __iomem *fun_base = qm->io_base + QM_MB_CMD_SEND_BASE;
 332         unsigned long tmp0 = 0, tmp1 = 0;
 333
 334         asm volatile("ldp %0, %1, %3\n"
 335                      "stp %0, %1, %2\n"
 336                      "dsb sy\n"
 337                      : "=&r" (tmp0),
 338                        "=&r" (tmp1),
 339                        "+Q" (*((char *)fun_base))
 340                      : "Q" (*((char *)src))
 341                      : "memory");
 342 }
 343
 344 static int qm_mb(struct hisi_qm *qm, u8 cmd, dma_addr_t dma_addr, u16 queue,
 345                  bool op)
 346 {
 347         struct qm_mailbox mailbox;
 348         int ret = 0;
 349
 350         dev_dbg(&qm->pdev->dev, "QM mailbox request to q%u: %u-%llx\n",
 351                 queue, cmd, (unsigned long long)dma_addr);
 352
 353         mailbox.w0 = cmd |
 354                      (op ? 0x1 << QM_MB_OP_SHIFT : 0) |
 355                      (0x1 << QM_MB_BUSY_SHIFT);
 356         mailbox.queue_num = queue;
 357         mailbox.base_l = lower_32_bits(dma_addr);
 358         mailbox.base_h = upper_32_bits(dma_addr);
 359         mailbox.rsvd = 0;
 360
 361         mutex_lock(&qm->mailbox_lock);
 362
 363         if (unlikely(qm_wait_mb_ready(qm))) {
 364                 ret = -EBUSY;
 365                 dev_err(&qm->pdev->dev, "QM mailbox is busy to start!\n");
 366                 goto busy_unlock;
 367         }
 368
 369         qm_mb_write(qm, &mailbox);
 370
 371         if (unlikely(qm_wait_mb_ready(qm))) {
 372                 ret = -EBUSY;
 373                 dev_err(&qm->pdev->dev, "QM mailbox operation timeout!\n");
 374                 goto busy_unlock;
 375         }
 376
 377 busy_unlock:
 378         mutex_unlock(&qm->mailbox_lock);
 379
 380         return ret;
 381 }
 382
 383 static void qm_db_v1(struct hisi_qm *qm, u16 qn, u8 cmd, u16 index, u8 priority)
 384 {
 385         u64 doorbell;
 386
 387         doorbell = qn | ((u64)cmd << QM_DB_CMD_SHIFT_V1) |
 388                    ((u64)index << QM_DB_INDEX_SHIFT_V1)  |
 389                    ((u64)priority << QM_DB_PRIORITY_SHIFT_V1);
 390
 391         writeq(doorbell, qm->io_base + QM_DOORBELL_BASE_V1);
 392 }
 393
 394 static void qm_db_v2(struct hisi_qm *qm, u16 qn, u8 cmd, u16 index, u8 priority)
 395 {
 396         u64 doorbell;
 397         u64 dbase;
 398         u16 randata = 0;
 399
 400         if (cmd == QM_DOORBELL_CMD_SQ || cmd == QM_DOORBELL_CMD_CQ)
 401                 dbase = QM_DOORBELL_SQ_CQ_BASE_V2;
 402         else
 403                 dbase = QM_DOORBELL_EQ_AEQ_BASE_V2;
 404
 405         doorbell = qn | ((u64)cmd << QM_DB_CMD_SHIFT_V2) |
 406                    ((u64)randata << QM_DB_RAND_SHIFT_V2) |
 407                    ((u64)index << QM_DB_INDEX_SHIFT_V2)  |
 408                    ((u64)priority << QM_DB_PRIORITY_SHIFT_V2);
 409
 410         writeq(doorbell, qm->io_base + dbase);
 411 }
 412
 413 static void qm_db(struct hisi_qm *qm, u16 qn, u8 cmd, u16 index, u8 priority)
 414 {
 415         dev_dbg(&qm->pdev->dev, "QM doorbell request: qn=%u, cmd=%u, index=%u\n",
 416                 qn, cmd, index);
 417
 418         qm->ops->qm_db(qm, qn, cmd, index, priority);
 419 }
 420
 421 static int qm_dev_mem_reset(struct hisi_qm *qm)
 422 {
 423         u32 val;
 424
 425         writel(0x1, qm->io_base + QM_MEM_START_INIT);
 426         return readl_relaxed_poll_timeout(qm->io_base + QM_MEM_INIT_DONE, val,
 427                                           val & BIT(0), 10, 1000);
 428 }
 429
 430 static u32 qm_get_irq_num_v1(struct hisi_qm *qm)
 431 {
 432         return QM_IRQ_NUM_V1;
 433 }
 434
 435 static u32 qm_get_irq_num_v2(struct hisi_qm *qm)
 436 {
 437         if (qm->fun_type == QM_HW_PF)
 438                 return QM_IRQ_NUM_PF_V2;
 439         else
 440                 return QM_IRQ_NUM_VF_V2;
 441 }
 442
 443 static struct hisi_qp *qm_to_hisi_qp(struct hisi_qm *qm, struct qm_eqe *eqe)
 444 {
 445         u16 cqn = eqe->dw0 & QM_EQE_CQN_MASK;
 446
 447         return qm->qp_array[cqn];
 448 }
 449
 450 static void qm_cq_head_update(struct hisi_qp *qp)
 451 {
 452         if (qp->qp_status.cq_head == QM_Q_DEPTH - 1) {
 453                 qp->qp_status.cqc_phase = !qp->qp_status.cqc_phase;
 454                 qp->qp_status.cq_head = 0;
 455         } else {
 456                 qp->qp_status.cq_head++;
 457         }
 458 }
 459
 460 static void qm_poll_qp(struct hisi_qp *qp, struct hisi_qm *qm)
 461 {
 462         struct qm_cqe *cqe = qp->cqe + qp->qp_status.cq_head;
 463
 464         if (qp->req_cb) {
 465                 while (QM_CQE_PHASE(cqe) == qp->qp_status.cqc_phase) {
 466                         dma_rmb();
 467                         qp->req_cb(qp, qp->sqe + qm->sqe_size * cqe->sq_head);
 468                         qm_cq_head_update(qp);
 469                         cqe = qp->cqe + qp->qp_status.cq_head;
 470                         qm_db(qm, qp->qp_id, QM_DOORBELL_CMD_CQ,
 471                               qp->qp_status.cq_head, 0);
 472                         atomic_dec(&qp->qp_status.used);
 473                 }
 474
 475                 /* set c_flag */
 476                 qm_db(qm, qp->qp_id, QM_DOORBELL_CMD_CQ,
 477                       qp->qp_status.cq_head, 1);
 478         }
 479 }
 480
 481 static void qm_qp_work_func(struct work_struct *work)
 482 {
 483         struct hisi_qp *qp;
 484
 485         qp = container_of(work, struct hisi_qp, work);
 486         qm_poll_qp(qp, qp->qm);
 487 }
 488
 489 static irqreturn_t qm_irq_handler(int irq, void *data)
 490 {
 491         struct hisi_qm *qm = data;
 492         struct qm_eqe *eqe = qm->eqe + qm->status.eq_head;
 493         struct hisi_qp *qp;
 494         int eqe_num = 0;
 495
 496         while (QM_EQE_PHASE(eqe) == qm->status.eqc_phase) {
 497                 eqe_num++;
 498                 qp = qm_to_hisi_qp(qm, eqe);
 499                 if (qp)
 500                         queue_work(qp->wq, &qp->work);
 501
 502                 if (qm->status.eq_head == QM_Q_DEPTH - 1) {
 503                         qm->status.eqc_phase = !qm->status.eqc_phase;
 504                         eqe = qm->eqe;
 505                         qm->status.eq_head = 0;
 506                 } else {
 507                         eqe++;
 508                         qm->status.eq_head++;
 509                 }
 510
 511                 if (eqe_num == QM_Q_DEPTH / 2 - 1) {
 512                         eqe_num = 0;
 513                         qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0);
 514                 }
 515         }
 516
 517         qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0);
 518
 519         return IRQ_HANDLED;
 520 }
 521
 522 static irqreturn_t qm_irq(int irq, void *data)
 523 {
 524         struct hisi_qm *qm = data;
 525
 526         if (readl(qm->io_base + QM_VF_EQ_INT_SOURCE))
 527                 return qm_irq_handler(irq, data);
 528
 529         dev_err(&qm->pdev->dev, "invalid int source\n");
 530         qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0);
 531
 532         return IRQ_NONE;
 533 }
 534
 535 static irqreturn_t qm_aeq_irq(int irq, void *data)
 536 {
 537         struct hisi_qm *qm = data;
 538         struct qm_aeqe *aeqe = qm->aeqe + qm->status.aeq_head;
 539         u32 type;
 540
 541         if (!readl(qm->io_base + QM_VF_AEQ_INT_SOURCE))
 542                 return IRQ_NONE;
 543
 544         while (QM_AEQE_PHASE(aeqe) == qm->status.aeqc_phase) {
 545                 type = aeqe->dw0 >> QM_AEQE_TYPE_SHIFT;
 546                 if (type < ARRAY_SIZE(qm_fifo_overflow))
 547                         dev_err(&qm->pdev->dev, "%s overflow\n",
 548                                 qm_fifo_overflow[type]);
 549                 else
 550                         dev_err(&qm->pdev->dev, "unknown error type %d\n",
 551                                 type);
 552
 553                 if (qm->status.aeq_head == QM_Q_DEPTH - 1) {
 554                         qm->status.aeqc_phase = !qm->status.aeqc_phase;
 555                         aeqe = qm->aeqe;
 556                         qm->status.aeq_head = 0;
 557                 } else {
 558                         aeqe++;
 559                         qm->status.aeq_head++;
 560                 }
 561
 562                 qm_db(qm, 0, QM_DOORBELL_CMD_AEQ, qm->status.aeq_head, 0);
 563         }
 564
 565         return IRQ_HANDLED;
 566 }
 567
 568 static irqreturn_t qm_abnormal_irq(int irq, void *data)
 569 {
 570         const struct hisi_qm_hw_error *err = qm_hw_error;
 571         struct hisi_qm *qm = data;
 572         struct device *dev = &qm->pdev->dev;
 573         u32 error_status, tmp;
 574
 575         /* read err sts */
 576         tmp = readl(qm->io_base + QM_ABNORMAL_INT_STATUS);
 577         error_status = qm->msi_mask & tmp;
 578
 579         while (err->msg) {
 580                 if (err->int_msk & error_status)
 581                         dev_err(dev, "%s [error status=0x%x] found\n",
 582                                 err->msg, err->int_msk);
 583
 584                 err++;
 585         }
 586
 587         /* clear err sts */
 588         writel(error_status, qm->io_base + QM_ABNORMAL_INT_SOURCE);
 589
 590         return IRQ_HANDLED;
 591 }
 592
 593 static int qm_irq_register(struct hisi_qm *qm)
 594 {
 595         struct pci_dev *pdev = qm->pdev;
 596         int ret;
 597
 598         ret = request_irq(pci_irq_vector(pdev, QM_EQ_EVENT_IRQ_VECTOR),
 599                           qm_irq, IRQF_SHARED, qm->dev_name, qm);
 600         if (ret)
 601                 return ret;
 602
 603         if (qm->ver == QM_HW_V2) {
 604                 ret = request_irq(pci_irq_vector(pdev, QM_AEQ_EVENT_IRQ_VECTOR),
 605                                   qm_aeq_irq, IRQF_SHARED, qm->dev_name, qm);
 606                 if (ret)
 607                         goto err_aeq_irq;
 608
 609                 if (qm->fun_type == QM_HW_PF) {
 610                         ret = request_irq(pci_irq_vector(pdev,
 611                                           QM_ABNORMAL_EVENT_IRQ_VECTOR),
 612                                           qm_abnormal_irq, IRQF_SHARED,
 613                                           qm->dev_name, qm);
 614                         if (ret)
 615                                 goto err_abonormal_irq;
 616                 }
 617         }
 618
 619         return 0;
 620
 621 err_abonormal_irq:
 622         free_irq(pci_irq_vector(pdev, QM_AEQ_EVENT_IRQ_VECTOR), qm);
 623 err_aeq_irq:
 624         free_irq(pci_irq_vector(pdev, QM_EQ_EVENT_IRQ_VECTOR), qm);
 625         return ret;
 626 }
 627
 628 static void qm_irq_unregister(struct hisi_qm *qm)
 629 {
 630         struct pci_dev *pdev = qm->pdev;
 631
 632         free_irq(pci_irq_vector(pdev, QM_EQ_EVENT_IRQ_VECTOR), qm);
 633
 634         if (qm->ver == QM_HW_V2) {
 635                 free_irq(pci_irq_vector(pdev, QM_AEQ_EVENT_IRQ_VECTOR), qm);
 636
 637                 if (qm->fun_type == QM_HW_PF)
 638                         free_irq(pci_irq_vector(pdev,
 639                                  QM_ABNORMAL_EVENT_IRQ_VECTOR), qm);
 640         }
 641 }
 642
 643 static void qm_init_qp_status(struct hisi_qp *qp)
 644 {
 645         struct hisi_qp_status *qp_status = &qp->qp_status;
 646
 647         qp_status->sq_tail = 0;
 648         qp_status->cq_head = 0;
 649         qp_status->cqc_phase = 1;
 650         qp_status->flags = 0;
 651 }
 652
 653 static void qm_vft_data_cfg(struct hisi_qm *qm, enum vft_type type, u32 base,
 654                             u32 number)
 655 {
 656         u64 tmp = 0;
 657
 658         if (number > 0) {
 659                 switch (type) {
 660                 case SQC_VFT:
 661                         switch (qm->ver) {
 662                         case QM_HW_V1:
 663                                 tmp = QM_SQC_VFT_BUF_SIZE       |
 664                                       QM_SQC_VFT_SQC_SIZE       |
 665                                       QM_SQC_VFT_INDEX_NUMBER   |
 666                                       QM_SQC_VFT_VALID          |
 667                                       (u64)base << QM_SQC_VFT_START_SQN_SHIFT;
 668                                 break;
 669                         case QM_HW_V2:
 670                                 tmp = (u64)base << QM_SQC_VFT_START_SQN_SHIFT |
 671                                       QM_SQC_VFT_VALID |
 672                                       (u64)(number - 1) << QM_SQC_VFT_SQN_SHIFT;
 673                                 break;
 674                         case QM_HW_UNKNOWN:
 675                                 break;
 676                         }
 677                         break;
 678                 case CQC_VFT:
 679                         switch (qm->ver) {
 680                         case QM_HW_V1:
 681                                 tmp = QM_CQC_VFT_BUF_SIZE       |
 682                                       QM_CQC_VFT_SQC_SIZE       |
 683                                       QM_CQC_VFT_INDEX_NUMBER   |
 684                                       QM_CQC_VFT_VALID;
 685                                 break;
 686                         case QM_HW_V2:
 687                                 tmp = QM_CQC_VFT_VALID;
 688                                 break;
 689                         case QM_HW_UNKNOWN:
 690                                 break;
 691                         }
 692                         break;
 693                 }
 694         }
 695
 696         writel(lower_32_bits(tmp), qm->io_base + QM_VFT_CFG_DATA_L);
 697         writel(upper_32_bits(tmp), qm->io_base + QM_VFT_CFG_DATA_H);
 698 }
 699
 700 static int qm_set_vft_common(struct hisi_qm *qm, enum vft_type type,
 701                              u32 fun_num, u32 base, u32 number)
 702 {
 703         unsigned int val;
 704         int ret;
 705
 706         ret = readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val,
 707                                          val & BIT(0), 10, 1000);
 708         if (ret)
 709                 return ret;
 710
 711         writel(0x0, qm->io_base + QM_VFT_CFG_OP_WR);
 712         writel(type, qm->io_base + QM_VFT_CFG_TYPE);
 713         writel(fun_num, qm->io_base + QM_VFT_CFG);
 714
 715         qm_vft_data_cfg(qm, type, base, number);
 716
 717         writel(0x0, qm->io_base + QM_VFT_CFG_RDY);
 718         writel(0x1, qm->io_base + QM_VFT_CFG_OP_ENABLE);
 719
 720         return readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val,
 721                                           val & BIT(0), 10, 1000);
 722 }
 723
 724 /* The config should be conducted after qm_dev_mem_reset() */
 725 static int qm_set_sqc_cqc_vft(struct hisi_qm *qm, u32 fun_num, u32 base,
 726                               u32 number)
 727 {
 728         int ret, i;
 729
 730         for (i = SQC_VFT; i <= CQC_VFT; i++) {
 731                 ret = qm_set_vft_common(qm, i, fun_num, base, number);
 732                 if (ret)
 733                         return ret;
 734         }
 735
 736         return 0;
 737 }
 738
 739 static int qm_get_vft_v2(struct hisi_qm *qm, u32 *base, u32 *number)
 740 {
 741         u64 sqc_vft;
 742         int ret;
 743
 744         ret = qm_mb(qm, QM_MB_CMD_SQC_VFT_V2, 0, 0, 1);
 745         if (ret)
 746                 return ret;
 747
 748         sqc_vft = readl(qm->io_base + QM_MB_CMD_DATA_ADDR_L) |
 749                   ((u64)readl(qm->io_base + QM_MB_CMD_DATA_ADDR_H) << 32);
 750         *base = QM_SQC_VFT_BASE_MASK_V2 & (sqc_vft >> QM_SQC_VFT_BASE_SHIFT_V2);
 751         *number = (QM_SQC_VFT_NUM_MASK_v2 &
 752                    (sqc_vft >> QM_SQC_VFT_NUM_SHIFT_V2)) + 1;
 753
 754         return 0;
 755 }
 756
 757 static struct hisi_qm *file_to_qm(struct debugfs_file *file)
 758 {
 759         struct qm_debug *debug = file->debug;
 760
 761         return container_of(debug, struct hisi_qm, debug);
 762 }
 763
 764 static u32 current_q_read(struct debugfs_file *file)
 765 {
 766         struct hisi_qm *qm = file_to_qm(file);
 767
 768         return readl(qm->io_base + QM_DFX_SQE_CNT_VF_SQN) >> QM_DFX_QN_SHIFT;
 769 }
 770
 771 static int current_q_write(struct debugfs_file *file, u32 val)
 772 {
 773         struct hisi_qm *qm = file_to_qm(file);
 774         u32 tmp;
 775
 776         if (val >= qm->debug.curr_qm_qp_num)
 777                 return -EINVAL;
 778
 779         tmp = val << QM_DFX_QN_SHIFT |
 780               (readl(qm->io_base + QM_DFX_SQE_CNT_VF_SQN) & CURRENT_FUN_MASK);
 781         writel(tmp, qm->io_base + QM_DFX_SQE_CNT_VF_SQN);
 782
 783         tmp = val << QM_DFX_QN_SHIFT |
 784               (readl(qm->io_base + QM_DFX_CQE_CNT_VF_CQN) & CURRENT_FUN_MASK);
 785         writel(tmp, qm->io_base + QM_DFX_CQE_CNT_VF_CQN);
 786
 787         return 0;
 788 }
 789
 790 static u32 clear_enable_read(struct debugfs_file *file)
 791 {
 792         struct hisi_qm *qm = file_to_qm(file);
 793
 794         return readl(qm->io_base + QM_DFX_CNT_CLR_CE);
 795 }
 796
 797 /* rd_clr_ctrl 1 enable read clear, otherwise 0 disable it */
 798 static int clear_enable_write(struct debugfs_file *file, u32 rd_clr_ctrl)
 799 {
 800         struct hisi_qm *qm = file_to_qm(file);
 801
 802         if (rd_clr_ctrl > 1)
 803                 return -EINVAL;
 804
 805         writel(rd_clr_ctrl, qm->io_base + QM_DFX_CNT_CLR_CE);
 806
 807         return 0;
 808 }
 809
 810 static ssize_t qm_debug_read(struct file *filp, char __user *buf,
 811                              size_t count, loff_t *pos)
 812 {
 813         struct debugfs_file *file = filp->private_data;
 814         enum qm_debug_file index = file->index;
 815         char tbuf[QM_DBG_TMP_BUF_LEN];
 816         u32 val;
 817         int ret;
 818
 819         mutex_lock(&file->lock);
 820         switch (index) {
 821         case CURRENT_Q:
 822                 val = current_q_read(file);
 823                 break;
 824         case CLEAR_ENABLE:
 825                 val = clear_enable_read(file);
 826                 break;
 827         default:
 828                 mutex_unlock(&file->lock);
 829                 return -EINVAL;
 830         }
 831         mutex_unlock(&file->lock);
 832         ret = sprintf(tbuf, "%u\n", val);
 833         return simple_read_from_buffer(buf, count, pos, tbuf, ret);
 834 }
 835
 836 static ssize_t qm_debug_write(struct file *filp, const char __user *buf,
 837                               size_t count, loff_t *pos)
 838 {
 839         struct debugfs_file *file = filp->private_data;
 840         enum qm_debug_file index = file->index;
 841         unsigned long val;
 842         char tbuf[QM_DBG_TMP_BUF_LEN];
 843         int len, ret;
 844
 845         if (*pos != 0)
 846                 return 0;
 847
 848         if (count >= QM_DBG_TMP_BUF_LEN)
 849                 return -ENOSPC;
 850
 851         len = simple_write_to_buffer(tbuf, QM_DBG_TMP_BUF_LEN - 1, pos, buf,
 852                                      count);
 853         if (len < 0)
 854                 return len;
 855
 856         tbuf[len] = '\0';
 857         if (kstrtoul(tbuf, 0, &val))
 858                 return -EFAULT;
 859
 860         mutex_lock(&file->lock);
 861         switch (index) {
 862         case CURRENT_Q:
 863                 ret = current_q_write(file, val);
 864                 if (ret)
 865                         goto err_input;
 866                 break;
 867         case CLEAR_ENABLE:
 868                 ret = clear_enable_write(file, val);
 869                 if (ret)
 870                         goto err_input;
 871                 break;
 872         default:
 873                 ret = -EINVAL;
 874                 goto err_input;
 875         }
 876         mutex_unlock(&file->lock);
 877
 878         return count;
 879
 880 err_input:
 881         mutex_unlock(&file->lock);
 882         return ret;
 883 }
 884
 885 static const struct file_operations qm_debug_fops = {
 886         .owner = THIS_MODULE,
 887         .open = simple_open,
 888         .read = qm_debug_read,
 889         .write = qm_debug_write,
 890 };
 891
 892 struct qm_dfx_registers {
 893         char  *reg_name;
 894         u64   reg_offset;
 895 };
 896
 897 #define CNT_CYC_REGS_NUM                10
 898 static struct qm_dfx_registers qm_dfx_regs[] = {
 899         /* XXX_CNT are reading clear register */
 900         {"QM_ECC_1BIT_CNT               ",  0x104000ull},
 901         {"QM_ECC_MBIT_CNT               ",  0x104008ull},
 902         {"QM_DFX_MB_CNT                 ",  0x104018ull},
 903         {"QM_DFX_DB_CNT                 ",  0x104028ull},
 904         {"QM_DFX_SQE_CNT                ",  0x104038ull},
 905         {"QM_DFX_CQE_CNT                ",  0x104048ull},
 906         {"QM_DFX_SEND_SQE_TO_ACC_CNT    ",  0x104050ull},
 907         {"QM_DFX_WB_SQE_FROM_ACC_CNT    ",  0x104058ull},
 908         {"QM_DFX_ACC_FINISH_CNT         ",  0x104060ull},
 909         {"QM_DFX_CQE_ERR_CNT            ",  0x1040b4ull},
 910         {"QM_DFX_FUNS_ACTIVE_ST         ",  0x200ull},
 911         {"QM_ECC_1BIT_INF               ",  0x104004ull},
 912         {"QM_ECC_MBIT_INF               ",  0x10400cull},
 913         {"QM_DFX_ACC_RDY_VLD0           ",  0x1040a0ull},
 914         {"QM_DFX_ACC_RDY_VLD1           ",  0x1040a4ull},
 915         {"QM_DFX_AXI_RDY_VLD            ",  0x1040a8ull},
 916         {"QM_DFX_FF_ST0                 ",  0x1040c8ull},
 917         {"QM_DFX_FF_ST1                 ",  0x1040ccull},
 918         {"QM_DFX_FF_ST2                 ",  0x1040d0ull},
 919         {"QM_DFX_FF_ST3                 ",  0x1040d4ull},
 920         {"QM_DFX_FF_ST4                 ",  0x1040d8ull},
 921         {"QM_DFX_FF_ST5                 ",  0x1040dcull},
 922         {"QM_DFX_FF_ST6                 ",  0x1040e0ull},
 923         {"QM_IN_IDLE_ST                 ",  0x1040e4ull},
 924         { NULL, 0}
 925 };
 926
 927 static struct qm_dfx_registers qm_vf_dfx_regs[] = {
 928         {"QM_DFX_FUNS_ACTIVE_ST         ",  0x200ull},
 929         { NULL, 0}
 930 };
 931
 932 static int qm_regs_show(struct seq_file *s, void *unused)
 933 {
 934         struct hisi_qm *qm = s->private;
 935         struct qm_dfx_registers *regs;
 936         u32 val;
 937
 938         if (qm->fun_type == QM_HW_PF)
 939                 regs = qm_dfx_regs;
 940         else
 941                 regs = qm_vf_dfx_regs;
 942
 943         while (regs->reg_name) {
 944                 val = readl(qm->io_base + regs->reg_offset);
 945                 seq_printf(s, "%s= 0x%08x\n", regs->reg_name, val);
 946                 regs++;
 947         }
 948
 949         return 0;
 950 }
 951
 952 static int qm_regs_open(struct inode *inode, struct file *file)
 953 {
 954         return single_open(file, qm_regs_show, inode->i_private);
 955 }
 956
 957 static const struct file_operations qm_regs_fops = {
 958         .owner = THIS_MODULE,
 959         .open = qm_regs_open,
 960         .read = seq_read,
 961         .release = single_release,
 962 };
 963
 964 static int qm_create_debugfs_file(struct hisi_qm *qm, enum qm_debug_file index)
 965 {
 966         struct dentry *qm_d = qm->debug.qm_d, *tmp;
 967         struct debugfs_file *file = qm->debug.files + index;
 968
 969         tmp = debugfs_create_file(qm_debug_file_name[index], 0600, qm_d, file,
 970                                   &qm_debug_fops);
 971         if (IS_ERR(tmp))
 972                 return -ENOENT;
 973
 974         file->index = index;
 975         mutex_init(&file->lock);
 976         file->debug = &qm->debug;
 977
 978         return 0;
 979 }
 980
 981 static void qm_hw_error_init_v1(struct hisi_qm *qm, u32 ce, u32 nfe, u32 fe,
 982                                 u32 msi)
 983 {
 984         dev_info(&qm->pdev->dev,
 985                  "QM v%d does not support hw error handle\n", qm->ver);
 986
 987         writel(QM_ABNORMAL_INT_MASK_VALUE, qm->io_base + QM_ABNORMAL_INT_MASK);
 988 }
 989
 990 static void qm_hw_error_init_v2(struct hisi_qm *qm, u32 ce, u32 nfe, u32 fe,
 991                                 u32 msi)
 992 {
 993         u32 irq_enable = ce | nfe | fe | msi;
 994         u32 irq_unmask = ~irq_enable;
 995
 996         qm->error_mask = ce | nfe | fe;
 997         qm->msi_mask = msi;
 998
 999         /* configure error type */
1000         writel(ce, qm->io_base + QM_RAS_CE_ENABLE);
1001         writel(QM_RAS_CE_TIMES_PER_IRQ, qm->io_base + QM_RAS_CE_THRESHOLD);
1002         writel(nfe, qm->io_base + QM_RAS_NFE_ENABLE);
1003         writel(fe, qm->io_base + QM_RAS_FE_ENABLE);
1004
1005         /* use RAS irq default, so only set QM_RAS_MSI_INT_SEL for MSI */
1006         writel(msi, qm->io_base + QM_RAS_MSI_INT_SEL);
1007
1008         irq_unmask &= readl(qm->io_base + QM_ABNORMAL_INT_MASK);
1009         writel(irq_unmask, qm->io_base + QM_ABNORMAL_INT_MASK);
1010 }
1011
1012 static void qm_log_hw_error(struct hisi_qm *qm, u32 error_status)
1013 {
1014         const struct hisi_qm_hw_error *err = qm_hw_error;
1015         struct device *dev = &qm->pdev->dev;
1016         u32 reg_val, type, vf_num;
1017
1018         while (err->msg) {
1019                 if (err->int_msk & error_status) {
1020                         dev_err(dev, "%s [error status=0x%x] found\n",
1021                                 err->msg, err->int_msk);
1022
1023                         if (error_status & QM_DB_TIMEOUT) {
1024                                 reg_val = readl(qm->io_base +
1025                                                 QM_ABNORMAL_INF01);
1026                                 type = (reg_val & QM_DB_TIMEOUT_TYPE) >>
1027                                        QM_DB_TIMEOUT_TYPE_SHIFT;
1028                                 vf_num = reg_val & QM_DB_TIMEOUT_VF;
1029                                 dev_err(dev, "qm %s doorbell timeout in function %u\n",
1030                                         qm_db_timeout[type], vf_num);
1031                         }
1032
1033                         if (error_status & QM_OF_FIFO_OF) {
1034                                 reg_val = readl(qm->io_base +
1035                                                 QM_ABNORMAL_INF00);
1036                                 type = (reg_val & QM_FIFO_OVERFLOW_TYPE) >>
1037                                        QM_FIFO_OVERFLOW_TYPE_SHIFT;
1038                                 vf_num = reg_val & QM_FIFO_OVERFLOW_VF;
1039
1040                                 if (type < ARRAY_SIZE(qm_fifo_overflow))
1041                                         dev_err(dev, "qm %s fifo overflow in function %u\n",
1042                                                 qm_fifo_overflow[type],
1043                                                 vf_num);
1044                                 else
1045                                         dev_err(dev, "unknown error type\n");
1046                         }
1047                 }
1048                 err++;
1049         }
1050 }
1051
1052 static pci_ers_result_t qm_hw_error_handle_v2(struct hisi_qm *qm)
1053 {
1054         u32 error_status, tmp;
1055
1056         /* read err sts */
1057         tmp = readl(qm->io_base + QM_ABNORMAL_INT_STATUS);
1058         error_status = qm->error_mask & tmp;
1059
1060         if (error_status) {
1061                 qm_log_hw_error(qm, error_status);
1062
1063                 /* clear err sts */
1064                 writel(error_status, qm->io_base + QM_ABNORMAL_INT_SOURCE);
1065
1066                 return PCI_ERS_RESULT_NEED_RESET;
1067         }
1068
1069         return PCI_ERS_RESULT_RECOVERED;
1070 }
1071
1072 static const struct hisi_qm_hw_ops qm_hw_ops_v1 = {
1073         .qm_db = qm_db_v1,
1074         .get_irq_num = qm_get_irq_num_v1,
1075         .hw_error_init = qm_hw_error_init_v1,
1076 };
1077
1078 static const struct hisi_qm_hw_ops qm_hw_ops_v2 = {
1079         .get_vft = qm_get_vft_v2,
1080         .qm_db = qm_db_v2,
1081         .get_irq_num = qm_get_irq_num_v2,
1082         .hw_error_init = qm_hw_error_init_v2,
1083         .hw_error_handle = qm_hw_error_handle_v2,
1084 };
1085
1086 static void *qm_get_avail_sqe(struct hisi_qp *qp)
1087 {
1088         struct hisi_qp_status *qp_status = &qp->qp_status;
1089         u16 sq_tail = qp_status->sq_tail;
1090
1091         if (unlikely(atomic_read(&qp->qp_status.used) == QM_Q_DEPTH))
1092                 return NULL;
1093
1094         return qp->sqe + sq_tail * qp->qm->sqe_size;
1095 }
1096
1097 /**
1098  * hisi_qm_create_qp() - Create a queue pair from qm.
1099  * @qm: The qm we create a qp from.
1100  * @alg_type: Accelerator specific algorithm type in sqc.
1101  *
1102  * return created qp, -EBUSY if all qps in qm allocated, -ENOMEM if allocating
1103  * qp memory fails.
1104  */
1105 struct hisi_qp *hisi_qm_create_qp(struct hisi_qm *qm, u8 alg_type)
1106 {
1107         struct device *dev = &qm->pdev->dev;
1108         struct hisi_qp *qp;
1109         int qp_id, ret;
1110
1111         qp = kzalloc(sizeof(*qp), GFP_KERNEL);
1112         if (!qp)
1113                 return ERR_PTR(-ENOMEM);
1114
1115         write_lock(&qm->qps_lock);
1116
1117         qp_id = find_first_zero_bit(qm->qp_bitmap, qm->qp_num);
1118         if (qp_id >= qm->qp_num) {
1119                 write_unlock(&qm->qps_lock);
1120                 dev_info(&qm->pdev->dev, "QM all queues are busy!\n");
1121                 ret = -EBUSY;
1122                 goto err_free_qp;
1123         }
1124         set_bit(qp_id, qm->qp_bitmap);
1125         qm->qp_array[qp_id] = qp;
1126
1127         write_unlock(&qm->qps_lock);
1128
1129         qp->qm = qm;
1130
1131         if (qm->use_dma_api) {
1132                 qp->qdma.size = qm->sqe_size * QM_Q_DEPTH +
1133                                 sizeof(struct qm_cqe) * QM_Q_DEPTH;
1134                 qp->qdma.va = dma_alloc_coherent(dev, qp->qdma.size,
1135                                                  &qp->qdma.dma, GFP_KERNEL);
1136                 if (!qp->qdma.va) {
1137                         ret = -ENOMEM;
1138                         goto err_clear_bit;
1139                 }
1140
1141                 dev_dbg(dev, "allocate qp dma buf(va=%pK, dma=%pad, size=%zx)\n",
1142                         qp->qdma.va, &qp->qdma.dma, qp->qdma.size);
1143         }
1144
1145         qp->qp_id = qp_id;
1146         qp->alg_type = alg_type;
1147         INIT_WORK(&qp->work, qm_qp_work_func);
1148         qp->wq = alloc_workqueue("hisi_qm", WQ_UNBOUND | WQ_HIGHPRI |
1149                                  WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM, 0);
1150         if (!qp->wq) {
1151                 ret = -EFAULT;
1152                 goto err_free_qp_mem;
1153         }
1154
1155         return qp;
1156
1157 err_free_qp_mem:
1158         if (qm->use_dma_api)
1159                 dma_free_coherent(dev, qp->qdma.size, qp->qdma.va,
1160                                   qp->qdma.dma);
1161 err_clear_bit:
1162         write_lock(&qm->qps_lock);
1163         qm->qp_array[qp_id] = NULL;
1164         clear_bit(qp_id, qm->qp_bitmap);
1165         write_unlock(&qm->qps_lock);
1166 err_free_qp:
1167         kfree(qp);
1168         return ERR_PTR(ret);
1169 }
1170 EXPORT_SYMBOL_GPL(hisi_qm_create_qp);
1171
1172 /**
1173  * hisi_qm_release_qp() - Release a qp back to its qm.
1174  * @qp: The qp we want to release.
1175  *
1176  * This function releases the resource of a qp.
1177  */
1178 void hisi_qm_release_qp(struct hisi_qp *qp)
1179 {
1180         struct hisi_qm *qm = qp->qm;
1181         struct qm_dma *qdma = &qp->qdma;
1182         struct device *dev = &qm->pdev->dev;
1183
1184         if (qm->use_dma_api && qdma->va)
1185                 dma_free_coherent(dev, qdma->size, qdma->va, qdma->dma);
1186
1187         write_lock(&qm->qps_lock);
1188         qm->qp_array[qp->qp_id] = NULL;
1189         clear_bit(qp->qp_id, qm->qp_bitmap);
1190         write_unlock(&qm->qps_lock);
1191
1192         kfree(qp);
1193 }
1194 EXPORT_SYMBOL_GPL(hisi_qm_release_qp);
1195
1196 static int qm_qp_ctx_cfg(struct hisi_qp *qp, int qp_id, int pasid)
1197 {
1198         struct hisi_qm *qm = qp->qm;
1199         struct device *dev = &qm->pdev->dev;
1200         enum qm_hw_ver ver = qm->ver;
1201         struct qm_sqc *sqc;
1202         struct qm_cqc *cqc;
1203         dma_addr_t sqc_dma;
1204         dma_addr_t cqc_dma;
1205         int ret;
1206
1207         qm_init_qp_status(qp);
1208
1209         sqc = kzalloc(sizeof(struct qm_sqc), GFP_KERNEL);
1210         if (!sqc)
1211                 return -ENOMEM;
1212         sqc_dma = dma_map_single(dev, sqc, sizeof(struct qm_sqc),
1213                                  DMA_TO_DEVICE);
1214         if (dma_mapping_error(dev, sqc_dma)) {
1215                 kfree(sqc);
1216                 return -ENOMEM;
1217         }
1218
1219         INIT_QC_COMMON(sqc, qp->sqe_dma, pasid);
1220         if (ver == QM_HW_V1) {
1221                 sqc->dw3 = QM_MK_SQC_DW3_V1(0, 0, 0, qm->sqe_size);
1222                 sqc->w8 = QM_Q_DEPTH - 1;
1223         } else if (ver == QM_HW_V2) {
1224                 sqc->dw3 = QM_MK_SQC_DW3_V2(qm->sqe_size);
1225                 sqc->w8 = 0; /* rand_qc */
1226         }
1227         sqc->cq_num = qp_id;
1228         sqc->w13 = QM_MK_SQC_W13(0, 1, qp->alg_type);
1229
1230         ret = qm_mb(qm, QM_MB_CMD_SQC, sqc_dma, qp_id, 0);
1231         dma_unmap_single(dev, sqc_dma, sizeof(struct qm_sqc), DMA_TO_DEVICE);
1232         kfree(sqc);
1233         if (ret)
1234                 return ret;
1235
1236         cqc = kzalloc(sizeof(struct qm_cqc), GFP_KERNEL);
1237         if (!cqc)
1238                 return -ENOMEM;
1239         cqc_dma = dma_map_single(dev, cqc, sizeof(struct qm_cqc),
1240                                  DMA_TO_DEVICE);
1241         if (dma_mapping_error(dev, cqc_dma)) {
1242                 kfree(cqc);
1243                 return -ENOMEM;
1244         }
1245
1246         INIT_QC_COMMON(cqc, qp->cqe_dma, pasid);
1247         if (ver == QM_HW_V1) {
1248                 cqc->dw3 = QM_MK_CQC_DW3_V1(0, 0, 0, 4);
1249                 cqc->w8 = QM_Q_DEPTH - 1;
1250         } else if (ver == QM_HW_V2) {
1251                 cqc->dw3 = QM_MK_CQC_DW3_V2(4);
1252                 cqc->w8 = 0;
1253         }
1254         cqc->dw6 = 1 << QM_CQ_PHASE_SHIFT | 1 << QM_CQ_FLAG_SHIFT;
1255
1256         ret = qm_mb(qm, QM_MB_CMD_CQC, cqc_dma, qp_id, 0);
1257         dma_unmap_single(dev, cqc_dma, sizeof(struct qm_cqc), DMA_TO_DEVICE);
1258         kfree(cqc);
1259
1260         return ret;
1261 }
1262
1263 /**
1264  * hisi_qm_start_qp() - Start a qp into running.
1265  * @qp: The qp we want to start to run.
1266  * @arg: Accelerator specific argument.
1267  *
1268  * After this function, qp can receive request from user. Return qp_id if
1269  * successful, Return -EBUSY if failed.
1270  */
1271 int hisi_qm_start_qp(struct hisi_qp *qp, unsigned long arg)
1272 {
1273         struct hisi_qm *qm = qp->qm;
1274         struct device *dev = &qm->pdev->dev;
1275         enum qm_hw_ver ver = qm->ver;
1276         int qp_id = qp->qp_id;
1277         int pasid = arg;
1278         size_t off = 0;
1279         int ret;
1280
1281 #define QP_INIT_BUF(qp, type, size) do { \
1282         (qp)->type = ((qp)->qdma.va + (off)); \
1283         (qp)->type##_dma = (qp)->qdma.dma + (off); \
1284         off += (size); \
1285 } while (0)
1286
1287         if (!qp->qdma.dma) {
1288                 dev_err(dev, "cannot get qm dma buffer\n");
1289                 return -EINVAL;
1290         }
1291
1292         /* sq need 128 bytes alignment */
1293         if (qp->qdma.dma & QM_SQE_DATA_ALIGN_MASK) {
1294                 dev_err(dev, "qm sq is not aligned to 128 byte\n");
1295                 return -EINVAL;
1296         }
1297
1298         QP_INIT_BUF(qp, sqe, qm->sqe_size * QM_Q_DEPTH);
1299         QP_INIT_BUF(qp, cqe, sizeof(struct qm_cqe) * QM_Q_DEPTH);
1300
1301         dev_dbg(dev, "init qp buffer(v%d):\n"
1302                      " sqe      (%pK, %lx)\n"
1303                      " cqe      (%pK, %lx)\n",
1304                      ver, qp->sqe, (unsigned long)qp->sqe_dma,
1305                      qp->cqe, (unsigned long)qp->cqe_dma);
1306
1307         ret = qm_qp_ctx_cfg(qp, qp_id, pasid);
1308         if (ret)
1309                 return ret;
1310
1311         dev_dbg(dev, "queue %d started\n", qp_id);
1312
1313         return qp_id;
1314 }
1315 EXPORT_SYMBOL_GPL(hisi_qm_start_qp);
1316
1317 /**
1318  * hisi_qm_stop_qp() - Stop a qp in qm.
1319  * @qp: The qp we want to stop.
1320  *
1321  * This function is reverse of hisi_qm_start_qp. Return 0 if successful.
1322  */
1323 int hisi_qm_stop_qp(struct hisi_qp *qp)
1324 {
1325         struct device *dev = &qp->qm->pdev->dev;
1326         int i = 0;
1327
1328         /* it is stopped */
1329         if (test_bit(QP_STOP, &qp->qp_status.flags))
1330                 return 0;
1331
1332         while (atomic_read(&qp->qp_status.used)) {
1333                 i++;
1334                 msleep(20);
1335                 if (i == 10) {
1336                         dev_err(dev, "Cannot drain out data for stopping, Force to stop!\n");
1337                         return 0;
1338                 }
1339         }
1340
1341         set_bit(QP_STOP, &qp->qp_status.flags);
1342
1343         dev_dbg(dev, "stop queue %u!", qp->qp_id);
1344
1345         return 0;
1346 }
1347 EXPORT_SYMBOL_GPL(hisi_qm_stop_qp);
1348
1349 /**
1350  * hisi_qp_send() - Queue up a task in the hardware queue.
1351  * @qp: The qp in which to put the message.
1352  * @msg: The message.
1353  *
1354  * This function will return -EBUSY if qp is currently full, and -EAGAIN
1355  * if qp related qm is resetting.
1356  */
1357 int hisi_qp_send(struct hisi_qp *qp, const void *msg)
1358 {
1359         struct hisi_qp_status *qp_status = &qp->qp_status;
1360         u16 sq_tail = qp_status->sq_tail;
1361         u16 sq_tail_next = (sq_tail + 1) % QM_Q_DEPTH;
1362         void *sqe = qm_get_avail_sqe(qp);
1363
1364         if (unlikely(test_bit(QP_STOP, &qp->qp_status.flags))) {
1365                 dev_info(&qp->qm->pdev->dev, "QP is stopped or resetting\n");
1366                 return -EAGAIN;
1367         }
1368
1369         if (!sqe)
1370                 return -EBUSY;
1371
1372         memcpy(sqe, msg, qp->qm->sqe_size);
1373
1374         qm_db(qp->qm, qp->qp_id, QM_DOORBELL_CMD_SQ, sq_tail_next, 0);
1375         atomic_inc(&qp->qp_status.used);
1376         qp_status->sq_tail = sq_tail_next;
1377
1378         return 0;
1379 }
1380 EXPORT_SYMBOL_GPL(hisi_qp_send);
1381
1382 static void hisi_qm_cache_wb(struct hisi_qm *qm)
1383 {
1384         unsigned int val;
1385
1386         if (qm->ver == QM_HW_V2) {
1387                 writel(0x1, qm->io_base + QM_CACHE_WB_START);
1388                 if (readl_relaxed_poll_timeout(qm->io_base + QM_CACHE_WB_DONE,
1389                                                val, val & BIT(0), 10, 1000))
1390                         dev_err(&qm->pdev->dev, "QM writeback sqc cache fail!\n");
1391         }
1392 }
1393
1394 /**
1395  * hisi_qm_init() - Initialize configures about qm.
1396  * @qm: The qm needing init.
1397  *
1398  * This function init qm, then we can call hisi_qm_start to put qm into work.
1399  */
1400 int hisi_qm_init(struct hisi_qm *qm)
1401 {
1402         struct pci_dev *pdev = qm->pdev;
1403         struct device *dev = &pdev->dev;
1404         unsigned int num_vec;
1405         int ret;
1406
1407         switch (qm->ver) {
1408         case QM_HW_V1:
1409                 qm->ops = &qm_hw_ops_v1;
1410                 break;
1411         case QM_HW_V2:
1412                 qm->ops = &qm_hw_ops_v2;
1413                 break;
1414         default:
1415                 return -EINVAL;
1416         }
1417
1418         ret = pci_enable_device_mem(pdev);
1419         if (ret < 0) {
1420                 dev_err(&pdev->dev, "Failed to enable device mem!\n");
1421                 return ret;
1422         }
1423
1424         ret = pci_request_mem_regions(pdev, qm->dev_name);
1425         if (ret < 0) {
1426                 dev_err(&pdev->dev, "Failed to request mem regions!\n");
1427                 goto err_disable_pcidev;
1428         }
1429
1430         qm->io_base = ioremap(pci_resource_start(pdev, PCI_BAR_2),
1431                               pci_resource_len(qm->pdev, PCI_BAR_2));
1432         if (!qm->io_base) {
1433                 ret = -EIO;
1434                 goto err_release_mem_regions;
1435         }
1436
1437         ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
1438         if (ret < 0)
1439                 goto err_iounmap;
1440         pci_set_master(pdev);
1441
1442         if (!qm->ops->get_irq_num) {
1443                 ret = -EOPNOTSUPP;
1444                 goto err_iounmap;
1445         }
1446         num_vec = qm->ops->get_irq_num(qm);
1447         ret = pci_alloc_irq_vectors(pdev, num_vec, num_vec, PCI_IRQ_MSI);
1448         if (ret < 0) {
1449                 dev_err(dev, "Failed to enable MSI vectors!\n");
1450                 goto err_iounmap;
1451         }
1452
1453         ret = qm_irq_register(qm);
1454         if (ret)
1455                 goto err_free_irq_vectors;
1456
1457         mutex_init(&qm->mailbox_lock);
1458         rwlock_init(&qm->qps_lock);
1459
1460         dev_dbg(dev, "init qm %s with %s\n", pdev->is_physfn ? "pf" : "vf",
1461                 qm->use_dma_api ? "dma api" : "iommu api");
1462
1463         return 0;
1464
1465 err_free_irq_vectors:
1466         pci_free_irq_vectors(pdev);
1467 err_iounmap:
1468         iounmap(qm->io_base);
1469 err_release_mem_regions:
1470         pci_release_mem_regions(pdev);
1471 err_disable_pcidev:
1472         pci_disable_device(pdev);
1473
1474         return ret;
1475 }
1476 EXPORT_SYMBOL_GPL(hisi_qm_init);
1477
1478 /**
1479  * hisi_qm_uninit() - Uninitialize qm.
1480  * @qm: The qm needed uninit.
1481  *
1482  * This function uninits qm related device resources.
1483  */
1484 void hisi_qm_uninit(struct hisi_qm *qm)
1485 {
1486         struct pci_dev *pdev = qm->pdev;
1487         struct device *dev = &pdev->dev;
1488
1489         if (qm->use_dma_api && qm->qdma.va) {
1490                 hisi_qm_cache_wb(qm);
1491                 dma_free_coherent(dev, qm->qdma.size,
1492                                   qm->qdma.va, qm->qdma.dma);
1493                 memset(&qm->qdma, 0, sizeof(qm->qdma));
1494         }
1495
1496         qm_irq_unregister(qm);
1497         pci_free_irq_vectors(pdev);
1498         iounmap(qm->io_base);
1499         pci_release_mem_regions(pdev);
1500         pci_disable_device(pdev);
1501 }
1502 EXPORT_SYMBOL_GPL(hisi_qm_uninit);
1503
1504 /**
1505  * hisi_qm_get_vft() - Get vft from a qm.
1506  * @qm: The qm we want to get its vft.
1507  * @base: The base number of queue in vft.
1508  * @number: The number of queues in vft.
1509  *
1510  * We can allocate multiple queues to a qm by configuring virtual function
1511  * table. We get related configures by this function. Normally, we call this
1512  * function in VF driver to get the queue information.
1513  *
1514  * qm hw v1 does not support this interface.
1515  */
1516 int hisi_qm_get_vft(struct hisi_qm *qm, u32 *base, u32 *number)
1517 {
1518         if (!base || !number)
1519                 return -EINVAL;
1520
1521         if (!qm->ops->get_vft) {
1522                 dev_err(&qm->pdev->dev, "Don't support vft read!\n");
1523                 return -EINVAL;
1524         }
1525
1526         return qm->ops->get_vft(qm, base, number);
1527 }
1528 EXPORT_SYMBOL_GPL(hisi_qm_get_vft);
1529
1530 /**
1531  * hisi_qm_set_vft() - Set "virtual function table" for a qm.
1532  * @fun_num: Number of operated function.
1533  * @qm: The qm in which to set vft, alway in a PF.
1534  * @base: The base number of queue in vft.
1535  * @number: The number of queues in vft. 0 means invalid vft.
1536  *
1537  * This function is alway called in PF driver, it is used to assign queues
1538  * among PF and VFs.
1539  *
1540  * Assign queues A~B to PF: hisi_qm_set_vft(qm, 0, A, B - A + 1)
1541  * Assign queues A~B to VF: hisi_qm_set_vft(qm, 2, A, B - A + 1)
1542  * (VF function number 0x2)
1543  */
1544 int hisi_qm_set_vft(struct hisi_qm *qm, u32 fun_num, u32 base,
1545                     u32 number)
1546 {
1547         u32 max_q_num = qm->ctrl_qp_num;
1548
1549         if (base >= max_q_num || number > max_q_num ||
1550             (base + number) > max_q_num)
1551                 return -EINVAL;
1552
1553         return qm_set_sqc_cqc_vft(qm, fun_num, base, number);
1554 }
1555 EXPORT_SYMBOL_GPL(hisi_qm_set_vft);
1556
1557 static void qm_init_eq_aeq_status(struct hisi_qm *qm)
1558 {
1559         struct hisi_qm_status *status = &qm->status;
1560
1561         status->eq_head = 0;
1562         status->aeq_head = 0;
1563         status->eqc_phase = 1;
1564         status->aeqc_phase = 1;
1565 }
1566
1567 static int qm_eq_ctx_cfg(struct hisi_qm *qm)
1568 {
1569         struct device *dev = &qm->pdev->dev;
1570         struct qm_eqc *eqc;
1571         struct qm_aeqc *aeqc;
1572         dma_addr_t eqc_dma;
1573         dma_addr_t aeqc_dma;
1574         int ret;
1575
1576         qm_init_eq_aeq_status(qm);
1577
1578         eqc = kzalloc(sizeof(struct qm_eqc), GFP_KERNEL);
1579         if (!eqc)
1580                 return -ENOMEM;
1581         eqc_dma = dma_map_single(dev, eqc, sizeof(struct qm_eqc),
1582                                  DMA_TO_DEVICE);
1583         if (dma_mapping_error(dev, eqc_dma)) {
1584                 kfree(eqc);
1585                 return -ENOMEM;
1586         }
1587
1588         eqc->base_l = lower_32_bits(qm->eqe_dma);
1589         eqc->base_h = upper_32_bits(qm->eqe_dma);
1590         if (qm->ver == QM_HW_V1)
1591                 eqc->dw3 = QM_EQE_AEQE_SIZE;
1592         eqc->dw6 = (QM_Q_DEPTH - 1) | (1 << QM_EQC_PHASE_SHIFT);
1593         ret = qm_mb(qm, QM_MB_CMD_EQC, eqc_dma, 0, 0);
1594         dma_unmap_single(dev, eqc_dma, sizeof(struct qm_eqc), DMA_TO_DEVICE);
1595         kfree(eqc);
1596         if (ret)
1597                 return ret;
1598
1599         aeqc = kzalloc(sizeof(struct qm_aeqc), GFP_KERNEL);
1600         if (!aeqc)
1601                 return -ENOMEM;
1602         aeqc_dma = dma_map_single(dev, aeqc, sizeof(struct qm_aeqc),
1603                                   DMA_TO_DEVICE);
1604         if (dma_mapping_error(dev, aeqc_dma)) {
1605                 kfree(aeqc);
1606                 return -ENOMEM;
1607         }
1608
1609         aeqc->base_l = lower_32_bits(qm->aeqe_dma);
1610         aeqc->base_h = upper_32_bits(qm->aeqe_dma);
1611         aeqc->dw6 = (QM_Q_DEPTH - 1) | (1 << QM_EQC_PHASE_SHIFT);
1612
1613         ret = qm_mb(qm, QM_MB_CMD_AEQC, aeqc_dma, 0, 0);
1614         dma_unmap_single(dev, aeqc_dma, sizeof(struct qm_aeqc), DMA_TO_DEVICE);
1615         kfree(aeqc);
1616
1617         return ret;
1618 }
1619
1620 static int __hisi_qm_start(struct hisi_qm *qm)
1621 {
1622         struct pci_dev *pdev = qm->pdev;
1623         struct device *dev = &pdev->dev;
1624         size_t off = 0;
1625         int ret;
1626
1627 #define QM_INIT_BUF(qm, type, num) do { \
1628         (qm)->type = ((qm)->qdma.va + (off)); \
1629         (qm)->type##_dma = (qm)->qdma.dma + (off); \
1630         off += QMC_ALIGN(sizeof(struct qm_##type) * (num)); \
1631 } while (0)
1632
1633         WARN_ON(!qm->qdma.dma);
1634
1635         if (qm->qp_num == 0)
1636                 return -EINVAL;
1637
1638         if (qm->fun_type == QM_HW_PF) {
1639                 ret = qm_dev_mem_reset(qm);
1640                 if (ret)
1641                         return ret;
1642
1643                 ret = hisi_qm_set_vft(qm, 0, qm->qp_base, qm->qp_num);
1644                 if (ret)
1645                         return ret;
1646         }
1647
1648         QM_INIT_BUF(qm, eqe, QM_Q_DEPTH);
1649         QM_INIT_BUF(qm, aeqe, QM_Q_DEPTH);
1650         QM_INIT_BUF(qm, sqc, qm->qp_num);
1651         QM_INIT_BUF(qm, cqc, qm->qp_num);
1652
1653         dev_dbg(dev, "init qm buffer:\n"
1654                      " eqe      (%pK, %lx)\n"
1655                      " aeqe     (%pK, %lx)\n"
1656                      " sqc      (%pK, %lx)\n"
1657                      " cqc      (%pK, %lx)\n",
1658                      qm->eqe, (unsigned long)qm->eqe_dma,
1659                      qm->aeqe, (unsigned long)qm->aeqe_dma,
1660                      qm->sqc, (unsigned long)qm->sqc_dma,
1661                      qm->cqc, (unsigned long)qm->cqc_dma);
1662
1663         ret = qm_eq_ctx_cfg(qm);
1664         if (ret)
1665                 return ret;
1666
1667         ret = qm_mb(qm, QM_MB_CMD_SQC_BT, qm->sqc_dma, 0, 0);
1668         if (ret)
1669                 return ret;
1670
1671         ret = qm_mb(qm, QM_MB_CMD_CQC_BT, qm->cqc_dma, 0, 0);
1672         if (ret)
1673                 return ret;
1674
1675         writel(0x0, qm->io_base + QM_VF_EQ_INT_MASK);
1676         writel(0x0, qm->io_base + QM_VF_AEQ_INT_MASK);
1677
1678         return 0;
1679 }
1680
1681 /**
1682  * hisi_qm_start() - start qm
1683  * @qm: The qm to be started.
1684  *
1685  * This function starts a qm, then we can allocate qp from this qm.
1686  */
1687 int hisi_qm_start(struct hisi_qm *qm)
1688 {
1689         struct device *dev = &qm->pdev->dev;
1690
1691         dev_dbg(dev, "qm start with %d queue pairs\n", qm->qp_num);
1692
1693         if (!qm->qp_num) {
1694                 dev_err(dev, "qp_num should not be 0\n");
1695                 return -EINVAL;
1696         }
1697
1698         if (!qm->qp_bitmap) {
1699                 qm->qp_bitmap = devm_kcalloc(dev, BITS_TO_LONGS(qm->qp_num),
1700                                              sizeof(long), GFP_KERNEL);
1701                 qm->qp_array = devm_kcalloc(dev, qm->qp_num,
1702                                             sizeof(struct hisi_qp *),
1703                                             GFP_KERNEL);
1704                 if (!qm->qp_bitmap || !qm->qp_array)
1705                         return -ENOMEM;
1706         }
1707
1708         if (!qm->use_dma_api) {
1709                 dev_dbg(&qm->pdev->dev, "qm delay start\n");
1710                 return 0;
1711         } else if (!qm->qdma.va) {
1712                 qm->qdma.size = QMC_ALIGN(sizeof(struct qm_eqe) * QM_Q_DEPTH) +
1713                                 QMC_ALIGN(sizeof(struct qm_aeqe) * QM_Q_DEPTH) +
1714                                 QMC_ALIGN(sizeof(struct qm_sqc) * qm->qp_num) +
1715                                 QMC_ALIGN(sizeof(struct qm_cqc) * qm->qp_num);
1716                 qm->qdma.va = dma_alloc_coherent(dev, qm->qdma.size,
1717                                                  &qm->qdma.dma, GFP_KERNEL);
1718                 dev_dbg(dev, "allocate qm dma buf(va=%pK, dma=%pad, size=%zx)\n",
1719                         qm->qdma.va, &qm->qdma.dma, qm->qdma.size);
1720                 if (!qm->qdma.va)
1721                         return -ENOMEM;
1722         }
1723
1724         return __hisi_qm_start(qm);
1725 }
1726 EXPORT_SYMBOL_GPL(hisi_qm_start);
1727
1728 /**
1729  * hisi_qm_stop() - Stop a qm.
1730  * @qm: The qm which will be stopped.
1731  *
1732  * This function stops qm and its qps, then qm can not accept request.
1733  * Related resources are not released at this state, we can use hisi_qm_start
1734  * to let qm start again.
1735  */
1736 int hisi_qm_stop(struct hisi_qm *qm)
1737 {
1738         struct device *dev;
1739         struct hisi_qp *qp;
1740         int ret = 0, i;
1741
1742         if (!qm || !qm->pdev) {
1743                 WARN_ON(1);
1744                 return -EINVAL;
1745         }
1746
1747         dev = &qm->pdev->dev;
1748
1749         /* Mask eq and aeq irq */
1750         writel(0x1, qm->io_base + QM_VF_EQ_INT_MASK);
1751         writel(0x1, qm->io_base + QM_VF_AEQ_INT_MASK);
1752
1753         /* Stop all qps belong to this qm */
1754         for (i = 0; i < qm->qp_num; i++) {
1755                 qp = qm->qp_array[i];
1756                 if (qp) {
1757                         ret = hisi_qm_stop_qp(qp);
1758                         if (ret < 0) {
1759                                 dev_err(dev, "Failed to stop qp%d!\n", i);
1760                                 return -EBUSY;
1761                         }
1762                 }
1763         }
1764
1765         if (qm->fun_type == QM_HW_PF) {
1766                 ret = hisi_qm_set_vft(qm, 0, 0, 0);
1767                 if (ret < 0)
1768                         dev_err(dev, "Failed to set vft!\n");
1769         }
1770
1771         return ret;
1772 }
1773 EXPORT_SYMBOL_GPL(hisi_qm_stop);
1774
1775 /**
1776  * hisi_qm_debug_init() - Initialize qm related debugfs files.
1777  * @qm: The qm for which we want to add debugfs files.
1778  *
1779  * Create qm related debugfs files.
1780  */
1781 int hisi_qm_debug_init(struct hisi_qm *qm)
1782 {
1783         struct dentry *qm_d, *qm_regs;
1784         int i, ret;
1785
1786         qm_d = debugfs_create_dir("qm", qm->debug.debug_root);
1787         if (IS_ERR(qm_d))
1788                 return -ENOENT;
1789         qm->debug.qm_d = qm_d;
1790
1791         /* only show this in PF */
1792         if (qm->fun_type == QM_HW_PF)
1793                 for (i = CURRENT_Q; i < DEBUG_FILE_NUM; i++)
1794                         if (qm_create_debugfs_file(qm, i)) {
1795                                 ret = -ENOENT;
1796                                 goto failed_to_create;
1797                         }
1798
1799         qm_regs = debugfs_create_file("qm_regs", 0444, qm->debug.qm_d, qm,
1800                                       &qm_regs_fops);
1801         if (IS_ERR(qm_regs)) {
1802                 ret = -ENOENT;
1803                 goto failed_to_create;
1804         }
1805
1806         return 0;
1807
1808 failed_to_create:
1809         debugfs_remove_recursive(qm_d);
1810         return ret;
1811 }
1812 EXPORT_SYMBOL_GPL(hisi_qm_debug_init);
1813
1814 /**
1815  * hisi_qm_debug_regs_clear() - clear qm debug related registers.
1816  * @qm: The qm for which we want to clear its debug registers.
1817  */
1818 void hisi_qm_debug_regs_clear(struct hisi_qm *qm)
1819 {
1820         struct qm_dfx_registers *regs;
1821         int i;
1822
1823         /* clear current_q */
1824         writel(0x0, qm->io_base + QM_DFX_SQE_CNT_VF_SQN);
1825         writel(0x0, qm->io_base + QM_DFX_CQE_CNT_VF_CQN);
1826
1827         /*
1828          * these registers are reading and clearing, so clear them after
1829          * reading them.
1830          */
1831         writel(0x1, qm->io_base + QM_DFX_CNT_CLR_CE);
1832
1833         regs = qm_dfx_regs;
1834         for (i = 0; i < CNT_CYC_REGS_NUM; i++) {
1835                 readl(qm->io_base + regs->reg_offset);
1836                 regs++;
1837         }
1838
1839         writel(0x0, qm->io_base + QM_DFX_CNT_CLR_CE);
1840 }
1841 EXPORT_SYMBOL_GPL(hisi_qm_debug_regs_clear);
1842
1843 /**
1844  * hisi_qm_hw_error_init() - Configure qm hardware error report method.
1845  * @qm: The qm which we want to configure.
1846  * @ce: Bit mask of correctable error configure.
1847  * @nfe: Bit mask of non-fatal error configure.
1848  * @fe: Bit mask of fatal error configure.
1849  * @msi: Bit mask of error reported by message signal interrupt.
1850  *
1851  * Hardware errors of qm can be reported either by RAS interrupts which will
1852  * be handled by UEFI and then PCIe AER or by device MSI. User can configure
1853  * each error to use either of above two methods. For RAS interrupts, we can
1854  * configure an error as one of correctable error, non-fatal error or
1855  * fatal error.
1856  *
1857  * Bits indicating errors can be configured to ce, nfe, fe and msi to enable
1858  * related report methods. Error report will be masked if related error bit
1859  * does not configure.
1860  */
1861 void hisi_qm_hw_error_init(struct hisi_qm *qm, u32 ce, u32 nfe, u32 fe,
1862                            u32 msi)
1863 {
1864         if (!qm->ops->hw_error_init) {
1865                 dev_err(&qm->pdev->dev, "QM version %d doesn't support hw error handling!\n",
1866                         qm->ver);
1867                 return;
1868         }
1869
1870         qm->ops->hw_error_init(qm, ce, nfe, fe, msi);
1871 }
1872 EXPORT_SYMBOL_GPL(hisi_qm_hw_error_init);
1873
1874 /**
1875  * hisi_qm_hw_error_handle() - Handle qm non-fatal hardware errors.
1876  * @qm: The qm which has non-fatal hardware errors.
1877  *
1878  * Accelerators use this function to handle qm non-fatal hardware errors.
1879  */
1880 int hisi_qm_hw_error_handle(struct hisi_qm *qm)
1881 {
1882         if (!qm->ops->hw_error_handle) {
1883                 dev_err(&qm->pdev->dev, "QM version %d doesn't support hw error report!\n",
1884                         qm->ver);
1885                 return PCI_ERS_RESULT_NONE;
1886         }
1887
1888         return qm->ops->hw_error_handle(qm);
1889 }
1890 EXPORT_SYMBOL_GPL(hisi_qm_hw_error_handle);
1891
1892 /**
1893  * hisi_qm_get_hw_version() - Get hardware version of a qm.
1894  * @pdev: The device which hardware version we want to get.
1895  *
1896  * This function gets the hardware version of a qm. Return QM_HW_UNKNOWN
1897  * if the hardware version is not supported.
1898  */
1899 enum qm_hw_ver hisi_qm_get_hw_version(struct pci_dev *pdev)
1900 {
1901         switch (pdev->revision) {
1902         case QM_HW_V1:
1903         case QM_HW_V2:
1904                 return pdev->revision;
1905         default:
1906                 return QM_HW_UNKNOWN;
1907         }
1908 }
1909 EXPORT_SYMBOL_GPL(hisi_qm_get_hw_version);
1910
1911 MODULE_LICENSE("GPL v2");
1912 MODULE_AUTHOR("Zhou Wang <wangzhou1@hisilicon.com>");
1913 MODULE_DESCRIPTION("HiSilicon Accelerator queue manager driver");