1 // SPDX-License-Identifier: GPL-2.0
3 * Shared Memory Communications over RDMA (SMC-R) and RoCE
5 * Work Requests exploiting Infiniband API
7 * Work requests (WR) of type ib_post_send or ib_post_recv respectively
8 * are submitted to either RC SQ or RC RQ respectively
9 * (reliably connected send/receive queue)
10 * and become work queue entries (WQEs).
11 * While an SQ WR/WQE is pending, we track it until transmission completion.
12 * Through a send or receive completion queue (CQ) respectively,
13 * we get completion queue entries (CQEs) [aka work completions (WCs)].
14 * Since the CQ callback is called from IRQ context, we split work by using
15 * bottom halves implemented by tasklets.
17 * SMC uses this to exchange LLC (link layer control)
18 * and CDC (connection data control) messages.
20 * Copyright IBM Corp. 2016
22 * Author(s): Steffen Maier <maier@linux.vnet.ibm.com>
25 #include <linux/atomic.h>
26 #include <linux/hashtable.h>
27 #include <linux/wait.h>
28 #include <rdma/ib_verbs.h>
29 #include <asm/div64.h>
34 #define SMC_WR_MAX_POLL_CQE 10 /* max. # of compl. queue elements in 1 poll */
36 #define SMC_WR_RX_HASH_BITS 4
37 static DEFINE_HASHTABLE(smc_wr_rx_hash, SMC_WR_RX_HASH_BITS);
38 static DEFINE_SPINLOCK(smc_wr_rx_hash_lock);
40 struct smc_wr_tx_pend { /* control data for a pending send request */
41 u64 wr_id; /* work request id sent */
42 smc_wr_tx_handler handler;
43 enum ib_wc_status wc_status; /* CQE status */
44 struct smc_link *link;
46 struct smc_wr_tx_pend_priv priv;
49 /******************************** send queue *********************************/
51 /*------------------------------- completion --------------------------------*/
53 static inline int smc_wr_tx_find_pending_index(struct smc_link *link, u64 wr_id)
57 for (i = 0; i < link->wr_tx_cnt; i++) {
58 if (link->wr_tx_pends[i].wr_id == wr_id)
61 return link->wr_tx_cnt;
64 static inline void smc_wr_tx_process_cqe(struct ib_wc *wc)
66 struct smc_wr_tx_pend pnd_snd;
67 struct smc_link *link;
71 link = wc->qp->qp_context;
73 if (wc->opcode == IB_WC_REG_MR) {
75 link->wr_reg_state = FAILED;
77 link->wr_reg_state = CONFIRMED;
78 wake_up(&link->wr_reg_wait);
82 pnd_snd_idx = smc_wr_tx_find_pending_index(link, wc->wr_id);
83 if (pnd_snd_idx == link->wr_tx_cnt)
85 link->wr_tx_pends[pnd_snd_idx].wc_status = wc->status;
86 memcpy(&pnd_snd, &link->wr_tx_pends[pnd_snd_idx], sizeof(pnd_snd));
87 /* clear the full struct smc_wr_tx_pend including .priv */
88 memset(&link->wr_tx_pends[pnd_snd_idx], 0,
89 sizeof(link->wr_tx_pends[pnd_snd_idx]));
90 memset(&link->wr_tx_bufs[pnd_snd_idx], 0,
91 sizeof(link->wr_tx_bufs[pnd_snd_idx]));
92 if (!test_and_clear_bit(pnd_snd_idx, link->wr_tx_mask))
95 for_each_set_bit(i, link->wr_tx_mask, link->wr_tx_cnt) {
96 /* clear full struct smc_wr_tx_pend including .priv */
97 memset(&link->wr_tx_pends[i], 0,
98 sizeof(link->wr_tx_pends[i]));
99 memset(&link->wr_tx_bufs[i], 0,
100 sizeof(link->wr_tx_bufs[i]));
101 clear_bit(i, link->wr_tx_mask);
103 /* terminate connections of this link group abnormally */
104 smc_lgr_terminate(smc_get_lgr(link));
107 pnd_snd.handler(&pnd_snd.priv, link, wc->status);
108 wake_up(&link->wr_tx_wait);
111 static void smc_wr_tx_tasklet_fn(unsigned long data)
113 struct smc_ib_device *dev = (struct smc_ib_device *)data;
114 struct ib_wc wc[SMC_WR_MAX_POLL_CQE];
121 memset(&wc, 0, sizeof(wc));
122 rc = ib_poll_cq(dev->roce_cq_send, SMC_WR_MAX_POLL_CQE, wc);
124 ib_req_notify_cq(dev->roce_cq_send,
126 IB_CQ_REPORT_MISSED_EVENTS);
130 for (i = 0; i < rc; i++)
131 smc_wr_tx_process_cqe(&wc[i]);
137 void smc_wr_tx_cq_handler(struct ib_cq *ib_cq, void *cq_context)
139 struct smc_ib_device *dev = (struct smc_ib_device *)cq_context;
141 tasklet_schedule(&dev->send_tasklet);
144 /*---------------------------- request submission ---------------------------*/
146 static inline int smc_wr_tx_get_free_slot_index(struct smc_link *link, u32 *idx)
148 *idx = link->wr_tx_cnt;
149 for_each_clear_bit(*idx, link->wr_tx_mask, link->wr_tx_cnt) {
150 if (!test_and_set_bit(*idx, link->wr_tx_mask))
153 *idx = link->wr_tx_cnt;
158 * smc_wr_tx_get_free_slot() - returns buffer for message assembly,
159 * and sets info for pending transmit tracking
160 * @link: Pointer to smc_link used to later send the message.
161 * @handler: Send completion handler function pointer.
162 * @wr_buf: Out value returns pointer to message buffer.
163 * @wr_pend_priv: Out value returns pointer serving as handler context.
165 * Return: 0 on success, or -errno on error.
167 int smc_wr_tx_get_free_slot(struct smc_link *link,
168 smc_wr_tx_handler handler,
169 struct smc_wr_buf **wr_buf,
170 struct smc_wr_tx_pend_priv **wr_pend_priv)
172 struct smc_wr_tx_pend *wr_pend;
173 u32 idx = link->wr_tx_cnt;
174 struct ib_send_wr *wr_ib;
179 *wr_pend_priv = NULL;
181 rc = smc_wr_tx_get_free_slot_index(link, &idx);
185 struct smc_link_group *lgr;
187 lgr = smc_get_lgr(link);
188 rc = wait_event_timeout(
190 list_empty(&lgr->list) || /* lgr terminated */
191 (smc_wr_tx_get_free_slot_index(link, &idx) != -EBUSY),
192 SMC_WR_TX_WAIT_FREE_SLOT_TIME);
194 /* timeout - terminate connections */
195 smc_lgr_terminate(lgr);
198 if (idx == link->wr_tx_cnt)
201 wr_id = smc_wr_tx_get_next_wr_id(link);
202 wr_pend = &link->wr_tx_pends[idx];
203 wr_pend->wr_id = wr_id;
204 wr_pend->handler = handler;
205 wr_pend->link = link;
207 wr_ib = &link->wr_tx_ibs[idx];
208 wr_ib->wr_id = wr_id;
209 *wr_buf = &link->wr_tx_bufs[idx];
210 *wr_pend_priv = &wr_pend->priv;
214 int smc_wr_tx_put_slot(struct smc_link *link,
215 struct smc_wr_tx_pend_priv *wr_pend_priv)
217 struct smc_wr_tx_pend *pend;
219 pend = container_of(wr_pend_priv, struct smc_wr_tx_pend, priv);
220 if (pend->idx < link->wr_tx_cnt) {
221 /* clear the full struct smc_wr_tx_pend including .priv */
222 memset(&link->wr_tx_pends[pend->idx], 0,
223 sizeof(link->wr_tx_pends[pend->idx]));
224 memset(&link->wr_tx_bufs[pend->idx], 0,
225 sizeof(link->wr_tx_bufs[pend->idx]));
226 test_and_clear_bit(pend->idx, link->wr_tx_mask);
233 /* Send prepared WR slot via ib_post_send.
234 * @priv: pointer to smc_wr_tx_pend_priv identifying prepared message buffer
236 int smc_wr_tx_send(struct smc_link *link, struct smc_wr_tx_pend_priv *priv)
238 struct ib_send_wr *failed_wr = NULL;
239 struct smc_wr_tx_pend *pend;
242 ib_req_notify_cq(link->smcibdev->roce_cq_send,
243 IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS);
244 pend = container_of(priv, struct smc_wr_tx_pend, priv);
245 rc = ib_post_send(link->roce_qp, &link->wr_tx_ibs[pend->idx],
248 smc_wr_tx_put_slot(link, priv);
249 smc_lgr_terminate(smc_get_lgr(link));
254 /* Register a memory region and wait for result. */
255 int smc_wr_reg_send(struct smc_link *link, struct ib_mr *mr)
257 struct ib_send_wr *failed_wr = NULL;
260 ib_req_notify_cq(link->smcibdev->roce_cq_send,
261 IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS);
262 link->wr_reg_state = POSTED;
263 link->wr_reg.wr.wr_id = (u64)(uintptr_t)mr;
264 link->wr_reg.mr = mr;
265 link->wr_reg.key = mr->rkey;
266 failed_wr = &link->wr_reg.wr;
267 rc = ib_post_send(link->roce_qp, &link->wr_reg.wr, &failed_wr);
268 WARN_ON(failed_wr != &link->wr_reg.wr);
272 rc = wait_event_interruptible_timeout(link->wr_reg_wait,
273 (link->wr_reg_state != POSTED),
274 SMC_WR_REG_MR_WAIT_TIME);
276 /* timeout - terminate connections */
277 smc_lgr_terminate(smc_get_lgr(link));
280 if (rc == -ERESTARTSYS)
282 switch (link->wr_reg_state) {
296 void smc_wr_tx_dismiss_slots(struct smc_link *link, u8 wr_tx_hdr_type,
297 smc_wr_tx_filter filter,
298 smc_wr_tx_dismisser dismisser,
301 struct smc_wr_tx_pend_priv *tx_pend;
302 struct smc_wr_rx_hdr *wr_tx;
305 for_each_set_bit(i, link->wr_tx_mask, link->wr_tx_cnt) {
306 wr_tx = (struct smc_wr_rx_hdr *)&link->wr_tx_bufs[i];
307 if (wr_tx->type != wr_tx_hdr_type)
309 tx_pend = &link->wr_tx_pends[i].priv;
310 if (filter(tx_pend, data))
315 /****************************** receive queue ********************************/
317 int smc_wr_rx_register_handler(struct smc_wr_rx_handler *handler)
319 struct smc_wr_rx_handler *h_iter;
322 spin_lock(&smc_wr_rx_hash_lock);
323 hash_for_each_possible(smc_wr_rx_hash, h_iter, list, handler->type) {
324 if (h_iter->type == handler->type) {
329 hash_add(smc_wr_rx_hash, &handler->list, handler->type);
331 spin_unlock(&smc_wr_rx_hash_lock);
335 /* Demultiplex a received work request based on the message type to its handler.
336 * Relies on smc_wr_rx_hash having been completely filled before any IB WRs,
337 * and not being modified any more afterwards so we don't need to lock it.
339 static inline void smc_wr_rx_demultiplex(struct ib_wc *wc)
341 struct smc_link *link = (struct smc_link *)wc->qp->qp_context;
342 struct smc_wr_rx_handler *handler;
343 struct smc_wr_rx_hdr *wr_rx;
347 if (wc->byte_len < sizeof(*wr_rx))
348 return; /* short message */
349 temp_wr_id = wc->wr_id;
350 index = do_div(temp_wr_id, link->wr_rx_cnt);
351 wr_rx = (struct smc_wr_rx_hdr *)&link->wr_rx_bufs[index];
352 hash_for_each_possible(smc_wr_rx_hash, handler, list, wr_rx->type) {
353 if (handler->type == wr_rx->type)
354 handler->handler(wc, wr_rx);
358 static inline void smc_wr_rx_process_cqes(struct ib_wc wc[], int num)
360 struct smc_link *link;
363 for (i = 0; i < num; i++) {
364 link = wc[i].qp->qp_context;
365 if (wc[i].status == IB_WC_SUCCESS) {
366 link->wr_rx_tstamp = jiffies;
367 smc_wr_rx_demultiplex(&wc[i]);
368 smc_wr_rx_post(link); /* refill WR RX */
370 /* handle status errors */
371 switch (wc[i].status) {
372 case IB_WC_RETRY_EXC_ERR:
373 case IB_WC_RNR_RETRY_EXC_ERR:
374 case IB_WC_WR_FLUSH_ERR:
375 /* terminate connections of this link group
378 smc_lgr_terminate(smc_get_lgr(link));
381 smc_wr_rx_post(link); /* refill WR RX */
388 static void smc_wr_rx_tasklet_fn(unsigned long data)
390 struct smc_ib_device *dev = (struct smc_ib_device *)data;
391 struct ib_wc wc[SMC_WR_MAX_POLL_CQE];
398 memset(&wc, 0, sizeof(wc));
399 rc = ib_poll_cq(dev->roce_cq_recv, SMC_WR_MAX_POLL_CQE, wc);
401 ib_req_notify_cq(dev->roce_cq_recv,
403 | IB_CQ_REPORT_MISSED_EVENTS);
407 smc_wr_rx_process_cqes(&wc[0], rc);
413 void smc_wr_rx_cq_handler(struct ib_cq *ib_cq, void *cq_context)
415 struct smc_ib_device *dev = (struct smc_ib_device *)cq_context;
417 tasklet_schedule(&dev->recv_tasklet);
420 int smc_wr_rx_post_init(struct smc_link *link)
425 for (i = 0; i < link->wr_rx_cnt; i++)
426 rc = smc_wr_rx_post(link);
430 /***************************** init, exit, misc ******************************/
432 void smc_wr_remember_qp_attr(struct smc_link *lnk)
434 struct ib_qp_attr *attr = &lnk->qp_attr;
435 struct ib_qp_init_attr init_attr;
437 memset(attr, 0, sizeof(*attr));
438 memset(&init_attr, 0, sizeof(init_attr));
439 ib_query_qp(lnk->roce_qp, attr,
452 IB_QP_MIN_RNR_TIMER |
454 IB_QP_PATH_MIG_STATE |
459 lnk->wr_tx_cnt = min_t(size_t, SMC_WR_BUF_CNT,
460 lnk->qp_attr.cap.max_send_wr);
461 lnk->wr_rx_cnt = min_t(size_t, SMC_WR_BUF_CNT * 3,
462 lnk->qp_attr.cap.max_recv_wr);
465 static void smc_wr_init_sge(struct smc_link *lnk)
469 for (i = 0; i < lnk->wr_tx_cnt; i++) {
470 lnk->wr_tx_sges[i].addr =
471 lnk->wr_tx_dma_addr + i * SMC_WR_BUF_SIZE;
472 lnk->wr_tx_sges[i].length = SMC_WR_TX_SIZE;
473 lnk->wr_tx_sges[i].lkey = lnk->roce_pd->local_dma_lkey;
474 lnk->wr_tx_ibs[i].next = NULL;
475 lnk->wr_tx_ibs[i].sg_list = &lnk->wr_tx_sges[i];
476 lnk->wr_tx_ibs[i].num_sge = 1;
477 lnk->wr_tx_ibs[i].opcode = IB_WR_SEND;
478 lnk->wr_tx_ibs[i].send_flags =
479 IB_SEND_SIGNALED | IB_SEND_SOLICITED;
481 for (i = 0; i < lnk->wr_rx_cnt; i++) {
482 lnk->wr_rx_sges[i].addr =
483 lnk->wr_rx_dma_addr + i * SMC_WR_BUF_SIZE;
484 lnk->wr_rx_sges[i].length = SMC_WR_BUF_SIZE;
485 lnk->wr_rx_sges[i].lkey = lnk->roce_pd->local_dma_lkey;
486 lnk->wr_rx_ibs[i].next = NULL;
487 lnk->wr_rx_ibs[i].sg_list = &lnk->wr_rx_sges[i];
488 lnk->wr_rx_ibs[i].num_sge = 1;
490 lnk->wr_reg.wr.next = NULL;
491 lnk->wr_reg.wr.num_sge = 0;
492 lnk->wr_reg.wr.send_flags = IB_SEND_SIGNALED;
493 lnk->wr_reg.wr.opcode = IB_WR_REG_MR;
494 lnk->wr_reg.access = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_WRITE;
497 void smc_wr_free_link(struct smc_link *lnk)
499 struct ib_device *ibdev;
501 memset(lnk->wr_tx_mask, 0,
502 BITS_TO_LONGS(SMC_WR_BUF_CNT) * sizeof(*lnk->wr_tx_mask));
506 ibdev = lnk->smcibdev->ibdev;
508 if (lnk->wr_rx_dma_addr) {
509 ib_dma_unmap_single(ibdev, lnk->wr_rx_dma_addr,
510 SMC_WR_BUF_SIZE * lnk->wr_rx_cnt,
512 lnk->wr_rx_dma_addr = 0;
514 if (lnk->wr_tx_dma_addr) {
515 ib_dma_unmap_single(ibdev, lnk->wr_tx_dma_addr,
516 SMC_WR_BUF_SIZE * lnk->wr_tx_cnt,
518 lnk->wr_tx_dma_addr = 0;
522 void smc_wr_free_link_mem(struct smc_link *lnk)
524 kfree(lnk->wr_tx_pends);
525 lnk->wr_tx_pends = NULL;
526 kfree(lnk->wr_tx_mask);
527 lnk->wr_tx_mask = NULL;
528 kfree(lnk->wr_tx_sges);
529 lnk->wr_tx_sges = NULL;
530 kfree(lnk->wr_rx_sges);
531 lnk->wr_rx_sges = NULL;
532 kfree(lnk->wr_rx_ibs);
533 lnk->wr_rx_ibs = NULL;
534 kfree(lnk->wr_tx_ibs);
535 lnk->wr_tx_ibs = NULL;
536 kfree(lnk->wr_tx_bufs);
537 lnk->wr_tx_bufs = NULL;
538 kfree(lnk->wr_rx_bufs);
539 lnk->wr_rx_bufs = NULL;
542 int smc_wr_alloc_link_mem(struct smc_link *link)
544 /* allocate link related memory */
545 link->wr_tx_bufs = kcalloc(SMC_WR_BUF_CNT, SMC_WR_BUF_SIZE, GFP_KERNEL);
546 if (!link->wr_tx_bufs)
548 link->wr_rx_bufs = kcalloc(SMC_WR_BUF_CNT * 3, SMC_WR_BUF_SIZE,
550 if (!link->wr_rx_bufs)
551 goto no_mem_wr_tx_bufs;
552 link->wr_tx_ibs = kcalloc(SMC_WR_BUF_CNT, sizeof(link->wr_tx_ibs[0]),
554 if (!link->wr_tx_ibs)
555 goto no_mem_wr_rx_bufs;
556 link->wr_rx_ibs = kcalloc(SMC_WR_BUF_CNT * 3,
557 sizeof(link->wr_rx_ibs[0]),
559 if (!link->wr_rx_ibs)
560 goto no_mem_wr_tx_ibs;
561 link->wr_tx_sges = kcalloc(SMC_WR_BUF_CNT, sizeof(link->wr_tx_sges[0]),
563 if (!link->wr_tx_sges)
564 goto no_mem_wr_rx_ibs;
565 link->wr_rx_sges = kcalloc(SMC_WR_BUF_CNT * 3,
566 sizeof(link->wr_rx_sges[0]),
568 if (!link->wr_rx_sges)
569 goto no_mem_wr_tx_sges;
570 link->wr_tx_mask = kcalloc(BITS_TO_LONGS(SMC_WR_BUF_CNT),
571 sizeof(*link->wr_tx_mask),
573 if (!link->wr_tx_mask)
574 goto no_mem_wr_rx_sges;
575 link->wr_tx_pends = kcalloc(SMC_WR_BUF_CNT,
576 sizeof(link->wr_tx_pends[0]),
578 if (!link->wr_tx_pends)
579 goto no_mem_wr_tx_mask;
583 kfree(link->wr_tx_mask);
585 kfree(link->wr_rx_sges);
587 kfree(link->wr_tx_sges);
589 kfree(link->wr_rx_ibs);
591 kfree(link->wr_tx_ibs);
593 kfree(link->wr_rx_bufs);
595 kfree(link->wr_tx_bufs);
600 void smc_wr_remove_dev(struct smc_ib_device *smcibdev)
602 tasklet_kill(&smcibdev->recv_tasklet);
603 tasklet_kill(&smcibdev->send_tasklet);
606 void smc_wr_add_dev(struct smc_ib_device *smcibdev)
608 tasklet_init(&smcibdev->recv_tasklet, smc_wr_rx_tasklet_fn,
609 (unsigned long)smcibdev);
610 tasklet_init(&smcibdev->send_tasklet, smc_wr_tx_tasklet_fn,
611 (unsigned long)smcibdev);
614 int smc_wr_create_link(struct smc_link *lnk)
616 struct ib_device *ibdev = lnk->smcibdev->ibdev;
619 smc_wr_tx_set_wr_id(&lnk->wr_tx_id, 0);
621 lnk->wr_rx_dma_addr = ib_dma_map_single(
622 ibdev, lnk->wr_rx_bufs, SMC_WR_BUF_SIZE * lnk->wr_rx_cnt,
624 if (ib_dma_mapping_error(ibdev, lnk->wr_rx_dma_addr)) {
625 lnk->wr_rx_dma_addr = 0;
629 lnk->wr_tx_dma_addr = ib_dma_map_single(
630 ibdev, lnk->wr_tx_bufs, SMC_WR_BUF_SIZE * lnk->wr_tx_cnt,
632 if (ib_dma_mapping_error(ibdev, lnk->wr_tx_dma_addr)) {
636 smc_wr_init_sge(lnk);
637 memset(lnk->wr_tx_mask, 0,
638 BITS_TO_LONGS(SMC_WR_BUF_CNT) * sizeof(*lnk->wr_tx_mask));
639 init_waitqueue_head(&lnk->wr_tx_wait);
640 init_waitqueue_head(&lnk->wr_reg_wait);
644 ib_dma_unmap_single(ibdev, lnk->wr_rx_dma_addr,
645 SMC_WR_BUF_SIZE * lnk->wr_rx_cnt,
647 lnk->wr_rx_dma_addr = 0;