1 /* QLogic qedr NIC Driver
2 * Copyright (c) 2015-2016 QLogic Corporation
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and /or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
35 #include <linux/pci.h>
36 #include <linux/xarray.h>
37 #include <rdma/ib_addr.h>
38 #include <linux/qed/qed_if.h>
39 #include <linux/qed/qed_chain.h>
40 #include <linux/qed/qed_rdma_if.h>
41 #include <linux/qed/qede_rdma.h>
42 #include <linux/qed/roce_common.h>
43 #include <linux/completion.h>
44 #include "qedr_hsi_rdma.h"
46 #define QEDR_NODE_DESC "QLogic 579xx RoCE HCA"
47 #define DP_NAME(_dev) dev_name(&(_dev)->ibdev.dev)
48 #define IS_IWARP(_dev) ((_dev)->rdma_type == QED_RDMA_TYPE_IWARP)
49 #define IS_ROCE(_dev) ((_dev)->rdma_type == QED_RDMA_TYPE_ROCE)
51 #define DP_DEBUG(dev, module, fmt, ...) \
52 pr_debug("(%s) " module ": " fmt, \
53 DP_NAME(dev) ? DP_NAME(dev) : "", ## __VA_ARGS__)
55 #define QEDR_MSG_INIT "INIT"
56 #define QEDR_MSG_MISC "MISC"
57 #define QEDR_MSG_CQ " CQ"
58 #define QEDR_MSG_MR " MR"
59 #define QEDR_MSG_RQ " RQ"
60 #define QEDR_MSG_SQ " SQ"
61 #define QEDR_MSG_QP " QP"
62 #define QEDR_MSG_SRQ " SRQ"
63 #define QEDR_MSG_GSI " GSI"
64 #define QEDR_MSG_IWARP " IW"
66 #define QEDR_CQ_MAGIC_NUMBER (0x11223344)
68 #define FW_PAGE_SIZE (RDMA_RING_PAGE_SIZE)
69 #define FW_PAGE_SHIFT (12)
76 struct qed_sb_info *sb;
83 #define QEDR_MAX_SGID 128
85 struct qedr_device_attr {
97 u8 max_qp_resp_rd_atomic_resc;
98 u8 max_qp_req_rd_atomic_resc;
99 u64 max_dev_resp_rd_atomic_resc;
107 u32 max_mr_mw_fmr_pbl;
108 u64 max_mr_mw_fmr_size;
121 u32 bad_pkey_counter;
122 struct qed_rdma_events events;
125 #define QEDR_ENET_STATE_BIT (0)
128 struct ib_device ibdev;
129 struct qed_dev *cdev;
130 struct pci_dev *pdev;
131 struct net_device *ndev;
133 enum ib_atomic_cap atomic_cap;
136 struct qedr_device_attr attr;
138 const struct qed_rdma_ops *ops;
139 struct qed_int_info int_info;
141 struct qed_sb_info *sb_array;
142 struct qedr_cnq *cnq_array;
146 void __iomem *db_addr;
151 union ib_gid *sgid_tbl;
153 /* Lock for sgid table */
154 spinlock_t sgid_lock;
161 #define QEDR_IS_CMT(dev) ((dev)->num_hwfns > 1)
166 u8 gsi_ll2_mac_address[ETH_ALEN];
168 struct qedr_cq *gsi_sqcq;
169 struct qedr_cq *gsi_rqcq;
170 struct qedr_qp *gsi_qp;
171 enum qed_rdma_type rdma_type;
174 struct workqueue_struct *iwarp_wq;
177 unsigned long enet_state;
182 #define QEDR_MAX_SQ_PBL (0x8000)
183 #define QEDR_MAX_SQ_PBL_ENTRIES (0x10000 / sizeof(void *))
184 #define QEDR_SQE_ELEMENT_SIZE (sizeof(struct rdma_sq_sge))
185 #define QEDR_MAX_SQE_ELEMENTS_PER_SQE (ROCE_REQ_MAX_SINGLE_SQ_WQE_SIZE / \
186 QEDR_SQE_ELEMENT_SIZE)
187 #define QEDR_MAX_SQE_ELEMENTS_PER_PAGE ((RDMA_RING_PAGE_SIZE) / \
188 QEDR_SQE_ELEMENT_SIZE)
189 #define QEDR_MAX_SQE ((QEDR_MAX_SQ_PBL_ENTRIES) *\
190 (RDMA_RING_PAGE_SIZE) / \
191 (QEDR_SQE_ELEMENT_SIZE) /\
192 (QEDR_MAX_SQE_ELEMENTS_PER_SQE))
194 #define QEDR_MAX_RQ_PBL (0x2000)
195 #define QEDR_MAX_RQ_PBL_ENTRIES (0x10000 / sizeof(void *))
196 #define QEDR_RQE_ELEMENT_SIZE (sizeof(struct rdma_rq_sge))
197 #define QEDR_MAX_RQE_ELEMENTS_PER_RQE (RDMA_MAX_SGE_PER_RQ_WQE)
198 #define QEDR_MAX_RQE_ELEMENTS_PER_PAGE ((RDMA_RING_PAGE_SIZE) / \
199 QEDR_RQE_ELEMENT_SIZE)
200 #define QEDR_MAX_RQE ((QEDR_MAX_RQ_PBL_ENTRIES) *\
201 (RDMA_RING_PAGE_SIZE) / \
202 (QEDR_RQE_ELEMENT_SIZE) /\
203 (QEDR_MAX_RQE_ELEMENTS_PER_RQE))
205 #define QEDR_CQE_SIZE (sizeof(union rdma_cqe))
206 #define QEDR_MAX_CQE_PBL_SIZE (512 * 1024)
207 #define QEDR_MAX_CQE_PBL_ENTRIES (((QEDR_MAX_CQE_PBL_SIZE) / \
209 #define QEDR_MAX_CQES ((u32)((QEDR_MAX_CQE_PBL_ENTRIES) * \
210 (QED_CHAIN_PAGE_SIZE) / QEDR_CQE_SIZE))
212 #define QEDR_ROCE_MAX_CNQ_SIZE (0x4000)
214 #define QEDR_MAX_PORT (1)
215 #define QEDR_PORT (1)
217 #define QEDR_UVERBS(CMD_NAME) (1ull << IB_USER_VERBS_CMD_##CMD_NAME)
219 #define QEDR_ROCE_PKEY_MAX 1
220 #define QEDR_ROCE_PKEY_TABLE_LEN 1
221 #define QEDR_ROCE_PKEY_DEFAULT 0xffff
224 struct list_head list_entry;
229 struct qedr_ucontext {
230 struct ib_ucontext ibucontext;
231 struct qedr_dev *dev;
233 void __iomem *dpi_addr;
234 struct rdma_user_mmap_entry *db_mmap_entry;
242 struct rdma_pwm_val16_data data;
247 struct rdma_pwm_val32_data data;
257 struct qedr_pbl_info {
266 struct ib_umem *umem;
267 struct qedr_pbl_info pbl_info;
268 struct qedr_pbl *pbl_tbl;
272 /* doorbell recovery */
273 void __iomem *db_addr;
274 struct qedr_user_db_rec *db_rec_data;
275 struct rdma_user_mmap_entry *db_mmap_entry;
276 void __iomem *db_rec_db2_addr;
277 union db_prod32 db_rec_db2_data;
283 enum qedr_cq_type cq_type;
288 /* Lock to protect multiplem CQ's */
291 struct qed_chain pbl;
293 void __iomem *db_addr;
297 union rdma_cqe *latest_cqe;
298 union rdma_cqe *toggle_cqe;
310 struct qedr_ucontext *uctx;
313 struct qedr_qp_hwq_info {
315 struct qed_chain pbl;
328 union db_prod32 db_data;
330 void __iomem *iwarp_db2;
331 union db_prod32 iwarp_db2_data;
334 #define QEDR_INC_SW_IDX(p_info, index) \
336 p_info->index = (p_info->index + 1) & \
337 qed_chain_get_capacity(p_info->pbl) \
340 struct qedr_srq_hwq_info {
343 struct qed_chain pbl;
351 u32 *virt_prod_pair_addr;
352 dma_addr_t phy_prod_pair_addr;
357 struct qedr_dev *dev;
359 struct qedr_userq usrq;
360 struct qedr_srq_hwq_info hw_srq;
361 struct ib_umem *prod_umem;
364 /* lock to protect srq recv post */
368 enum qedr_qp_err_bitmap {
369 QEDR_QP_ERR_SQ_FULL = 1,
370 QEDR_QP_ERR_RQ_FULL = 2,
371 QEDR_QP_ERR_BAD_SR = 4,
372 QEDR_QP_ERR_BAD_RR = 8,
373 QEDR_QP_ERR_SQ_PBL_FULL = 16,
374 QEDR_QP_ERR_RQ_PBL_FULL = 32,
377 enum qedr_qp_create_type {
380 QEDR_QP_CREATE_KERNEL,
383 enum qedr_iwarp_cm_flags {
384 QEDR_IWARP_CM_WAIT_FOR_CONNECT = BIT(0),
385 QEDR_IWARP_CM_WAIT_FOR_DISCONNECT = BIT(1),
389 struct ib_qp ibqp; /* must be first */
390 struct qedr_dev *dev;
391 struct qedr_qp_hwq_info sq;
392 struct qedr_qp_hwq_info rq;
398 struct qedr_cq *sq_cq;
399 struct qedr_cq *rq_cq;
400 struct qedr_srq *srq;
401 enum qed_roce_qp_state state;
404 enum ib_qp_type qp_type;
405 enum qedr_qp_create_type create_type;
406 struct qed_rdma_qp *qed_qp;
416 /* Relevant to qps created from kernel space only (ULPs) */
425 enum ib_wc_opcode opcode;
429 dma_addr_t icrc_mapping;
437 struct ib_sge sg_list[RDMA_MAX_SGE_PER_RQ_WQE];
445 /* Relevant to qps created from user space only (applications) */
446 struct qedr_userq usq;
447 struct qedr_userq urq;
449 /* synchronization objects used with iwarp ep */
451 struct completion iwarp_cm_comp;
452 unsigned long iwarp_cm_flags; /* enum iwarp_cm_flags */
457 struct rdma_ah_attr attr;
468 struct qedr_pbl *pbl_table;
469 struct qedr_pbl_info pbl_info;
470 struct list_head free_pbl_list;
471 struct list_head inuse_pbl_list;
473 u32 completed_handled;
478 struct ib_umem *umem;
480 struct qed_rdma_register_tid_in_params hw_mr;
481 enum qedr_mr_type type;
483 struct qedr_dev *dev;
490 struct qedr_user_mmap_entry {
491 struct rdma_user_mmap_entry rdma_entry;
492 struct qedr_dev *dev;
502 #define SET_FIELD2(value, name, flag) ((value) |= ((flag) << (name ## _SHIFT)))
504 #define QEDR_RESP_IMM (RDMA_CQE_RESPONDER_IMM_FLG_MASK << \
505 RDMA_CQE_RESPONDER_IMM_FLG_SHIFT)
506 #define QEDR_RESP_RDMA (RDMA_CQE_RESPONDER_RDMA_FLG_MASK << \
507 RDMA_CQE_RESPONDER_RDMA_FLG_SHIFT)
508 #define QEDR_RESP_INV (RDMA_CQE_RESPONDER_INV_FLG_MASK << \
509 RDMA_CQE_RESPONDER_INV_FLG_SHIFT)
511 static inline void qedr_inc_sw_cons(struct qedr_qp_hwq_info *info)
513 info->cons = (info->cons + 1) % info->max_wr;
517 static inline void qedr_inc_sw_prod(struct qedr_qp_hwq_info *info)
519 info->prod = (info->prod + 1) % info->max_wr;
522 static inline int qedr_get_dmac(struct qedr_dev *dev,
523 struct rdma_ah_attr *ah_attr, u8 *mac_addr)
525 union ib_gid zero_sgid = { { 0 } };
527 const struct ib_global_route *grh = rdma_ah_read_grh(ah_attr);
530 if (!memcmp(&grh->dgid, &zero_sgid, sizeof(union ib_gid))) {
531 DP_ERR(dev, "Local port GID not supported\n");
532 eth_zero_addr(mac_addr);
536 memcpy(&in6, grh->dgid.raw, sizeof(in6));
537 dmac = rdma_ah_retrieve_dmac(ah_attr);
540 ether_addr_copy(mac_addr, dmac);
545 struct qedr_iw_listener {
546 struct qedr_dev *dev;
547 struct iw_cm_id *cm_id;
553 struct qedr_dev *dev;
554 struct iw_cm_id *cm_id;
561 struct qedr_ucontext *get_qedr_ucontext(struct ib_ucontext *ibucontext)
563 return container_of(ibucontext, struct qedr_ucontext, ibucontext);
566 static inline struct qedr_dev *get_qedr_dev(struct ib_device *ibdev)
568 return container_of(ibdev, struct qedr_dev, ibdev);
571 static inline struct qedr_pd *get_qedr_pd(struct ib_pd *ibpd)
573 return container_of(ibpd, struct qedr_pd, ibpd);
576 static inline struct qedr_cq *get_qedr_cq(struct ib_cq *ibcq)
578 return container_of(ibcq, struct qedr_cq, ibcq);
581 static inline struct qedr_qp *get_qedr_qp(struct ib_qp *ibqp)
583 return container_of(ibqp, struct qedr_qp, ibqp);
586 static inline struct qedr_ah *get_qedr_ah(struct ib_ah *ibah)
588 return container_of(ibah, struct qedr_ah, ibah);
591 static inline struct qedr_mr *get_qedr_mr(struct ib_mr *ibmr)
593 return container_of(ibmr, struct qedr_mr, ibmr);
596 static inline struct qedr_srq *get_qedr_srq(struct ib_srq *ibsrq)
598 return container_of(ibsrq, struct qedr_srq, ibsrq);
601 static inline struct qedr_user_mmap_entry *
602 get_qedr_mmap_entry(struct rdma_user_mmap_entry *rdma_entry)
604 return container_of(rdma_entry, struct qedr_user_mmap_entry,