2 * Copyright (c) 2012, 2013 Intel Corporation. All rights reserved.
3 * Copyright (c) 2006 - 2012 QLogic Corporation. * All rights reserved.
4 * Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved.
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
35 #include <linux/err.h>
36 #include <linux/vmalloc.h>
37 #include <rdma/rdma_vt.h>
38 #ifdef CONFIG_DEBUG_FS
39 #include <linux/seq_file.h>
44 static inline unsigned mk_qpn(struct rvt_qpn_table *qpt,
45 struct rvt_qpn_map *map, unsigned off)
47 return (map - qpt->map) * RVT_BITS_PER_PAGE + off;
50 static inline unsigned find_next_offset(struct rvt_qpn_table *qpt,
51 struct rvt_qpn_map *map, unsigned off,
52 unsigned n, u16 qpt_mask)
56 if (((off & qpt_mask) >> 1) >= n)
57 off = (off | qpt_mask) + 2;
59 off = find_next_zero_bit(map->page, RVT_BITS_PER_PAGE, off);
64 const struct rvt_operation_params qib_post_parms[RVT_OPERATION_MAX] = {
65 [IB_WR_RDMA_WRITE] = {
66 .length = sizeof(struct ib_rdma_wr),
67 .qpt_support = BIT(IB_QPT_UC) | BIT(IB_QPT_RC),
71 .length = sizeof(struct ib_rdma_wr),
72 .qpt_support = BIT(IB_QPT_RC),
73 .flags = RVT_OPERATION_ATOMIC,
76 [IB_WR_ATOMIC_CMP_AND_SWP] = {
77 .length = sizeof(struct ib_atomic_wr),
78 .qpt_support = BIT(IB_QPT_RC),
79 .flags = RVT_OPERATION_ATOMIC | RVT_OPERATION_ATOMIC_SGE,
82 [IB_WR_ATOMIC_FETCH_AND_ADD] = {
83 .length = sizeof(struct ib_atomic_wr),
84 .qpt_support = BIT(IB_QPT_RC),
85 .flags = RVT_OPERATION_ATOMIC | RVT_OPERATION_ATOMIC_SGE,
88 [IB_WR_RDMA_WRITE_WITH_IMM] = {
89 .length = sizeof(struct ib_rdma_wr),
90 .qpt_support = BIT(IB_QPT_UC) | BIT(IB_QPT_RC),
94 .length = sizeof(struct ib_send_wr),
95 .qpt_support = BIT(IB_QPT_UD) | BIT(IB_QPT_SMI) | BIT(IB_QPT_GSI) |
96 BIT(IB_QPT_UC) | BIT(IB_QPT_RC),
99 [IB_WR_SEND_WITH_IMM] = {
100 .length = sizeof(struct ib_send_wr),
101 .qpt_support = BIT(IB_QPT_UD) | BIT(IB_QPT_SMI) | BIT(IB_QPT_GSI) |
102 BIT(IB_QPT_UC) | BIT(IB_QPT_RC),
107 static void get_map_page(struct rvt_qpn_table *qpt, struct rvt_qpn_map *map,
110 unsigned long page = get_zeroed_page(gfp);
113 * Free the page if someone raced with us installing it.
116 spin_lock(&qpt->lock);
120 map->page = (void *)page;
121 spin_unlock(&qpt->lock);
125 * Allocate the next available QPN or
126 * zero/one for QP type IB_QPT_SMI/IB_QPT_GSI.
128 int qib_alloc_qpn(struct rvt_dev_info *rdi, struct rvt_qpn_table *qpt,
129 enum ib_qp_type type, u8 port, gfp_t gfp)
131 u32 i, offset, max_scan, qpn;
132 struct rvt_qpn_map *map;
134 struct qib_ibdev *verbs_dev = container_of(rdi, struct qib_ibdev, rdi);
135 struct qib_devdata *dd = container_of(verbs_dev, struct qib_devdata,
137 u16 qpt_mask = dd->qpn_mask;
139 if (type == IB_QPT_SMI || type == IB_QPT_GSI) {
142 ret = type == IB_QPT_GSI;
143 n = 1 << (ret + 2 * (port - 1));
144 spin_lock(&qpt->lock);
149 spin_unlock(&qpt->lock);
154 if (qpn >= RVT_QPN_MAX)
156 if (qpt_mask && ((qpn & qpt_mask) >> 1) >= dd->n_krcv_queues)
157 qpn = (qpn | qpt_mask) + 2;
158 offset = qpn & RVT_BITS_PER_PAGE_MASK;
159 map = &qpt->map[qpn / RVT_BITS_PER_PAGE];
160 max_scan = qpt->nmaps - !offset;
162 if (unlikely(!map->page)) {
163 get_map_page(qpt, map, gfp);
164 if (unlikely(!map->page))
168 if (!test_and_set_bit(offset, map->page)) {
173 offset = find_next_offset(qpt, map, offset,
174 dd->n_krcv_queues, qpt_mask);
175 qpn = mk_qpn(qpt, map, offset);
177 * This test differs from alloc_pidmap().
178 * If find_next_offset() does find a zero
179 * bit, we don't need to check for QPN
180 * wrapping around past our starting QPN.
181 * We just need to be sure we don't loop
184 } while (offset < RVT_BITS_PER_PAGE && qpn < RVT_QPN_MAX);
186 * In order to keep the number of pages allocated to a
187 * minimum, we scan the all existing pages before increasing
188 * the size of the bitmap table.
190 if (++i > max_scan) {
191 if (qpt->nmaps == RVT_QPNMAP_ENTRIES)
193 map = &qpt->map[qpt->nmaps++];
195 } else if (map < &qpt->map[qpt->nmaps]) {
202 qpn = mk_qpn(qpt, map, offset);
212 * qib_free_all_qps - check for QPs still in use
214 unsigned qib_free_all_qps(struct rvt_dev_info *rdi)
216 struct qib_ibdev *verbs_dev = container_of(rdi, struct qib_ibdev, rdi);
217 struct qib_devdata *dd = container_of(verbs_dev, struct qib_devdata,
219 unsigned n, qp_inuse = 0;
221 for (n = 0; n < dd->num_pports; n++) {
222 struct qib_ibport *ibp = &dd->pport[n].ibport_data;
225 if (rcu_dereference(ibp->rvp.qp[0]))
227 if (rcu_dereference(ibp->rvp.qp[1]))
234 void qib_notify_qp_reset(struct rvt_qp *qp)
236 struct qib_qp_priv *priv = qp->priv;
238 atomic_set(&priv->s_dma_busy, 0);
241 void qib_notify_error_qp(struct rvt_qp *qp)
243 struct qib_qp_priv *priv = qp->priv;
244 struct qib_ibdev *dev = to_idev(qp->ibqp.device);
246 spin_lock(&dev->rdi.pending_lock);
247 if (!list_empty(&priv->iowait) && !(qp->s_flags & RVT_S_BUSY)) {
248 qp->s_flags &= ~RVT_S_ANY_WAIT_IO;
249 list_del_init(&priv->iowait);
251 spin_unlock(&dev->rdi.pending_lock);
253 if (!(qp->s_flags & RVT_S_BUSY)) {
256 rvt_put_mr(qp->s_rdma_mr);
257 qp->s_rdma_mr = NULL;
260 qib_put_txreq(priv->s_tx);
266 static int mtu_to_enum(u32 mtu)
272 enum_mtu = IB_MTU_4096;
275 enum_mtu = IB_MTU_2048;
278 enum_mtu = IB_MTU_1024;
281 enum_mtu = IB_MTU_512;
284 enum_mtu = IB_MTU_256;
287 enum_mtu = IB_MTU_2048;
292 int qib_get_pmtu_from_attr(struct rvt_dev_info *rdi, struct rvt_qp *qp,
293 struct ib_qp_attr *attr)
295 int mtu, pmtu, pidx = qp->port_num - 1;
296 struct qib_ibdev *verbs_dev = container_of(rdi, struct qib_ibdev, rdi);
297 struct qib_devdata *dd = container_of(verbs_dev, struct qib_devdata,
299 mtu = ib_mtu_enum_to_int(attr->path_mtu);
303 if (mtu > dd->pport[pidx].ibmtu)
304 pmtu = mtu_to_enum(dd->pport[pidx].ibmtu);
306 pmtu = attr->path_mtu;
310 int qib_mtu_to_path_mtu(u32 mtu)
312 return mtu_to_enum(mtu);
315 u32 qib_mtu_from_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp, u32 pmtu)
317 return ib_mtu_enum_to_int(pmtu);
320 void *qib_qp_priv_alloc(struct rvt_dev_info *rdi, struct rvt_qp *qp, gfp_t gfp)
322 struct qib_qp_priv *priv;
324 priv = kzalloc(sizeof(*priv), gfp);
326 return ERR_PTR(-ENOMEM);
329 priv->s_hdr = kzalloc(sizeof(*priv->s_hdr), gfp);
332 return ERR_PTR(-ENOMEM);
334 init_waitqueue_head(&priv->wait_dma);
335 INIT_WORK(&priv->s_work, _qib_do_send);
336 INIT_LIST_HEAD(&priv->iowait);
341 void qib_qp_priv_free(struct rvt_dev_info *rdi, struct rvt_qp *qp)
343 struct qib_qp_priv *priv = qp->priv;
349 void qib_stop_send_queue(struct rvt_qp *qp)
351 struct qib_qp_priv *priv = qp->priv;
353 cancel_work_sync(&priv->s_work);
354 del_timer_sync(&qp->s_timer);
357 void qib_quiesce_qp(struct rvt_qp *qp)
359 struct qib_qp_priv *priv = qp->priv;
361 wait_event(priv->wait_dma, !atomic_read(&priv->s_dma_busy));
363 qib_put_txreq(priv->s_tx);
368 void qib_flush_qp_waiters(struct rvt_qp *qp)
370 struct qib_qp_priv *priv = qp->priv;
371 struct qib_ibdev *dev = to_idev(qp->ibqp.device);
373 spin_lock(&dev->rdi.pending_lock);
374 if (!list_empty(&priv->iowait))
375 list_del_init(&priv->iowait);
376 spin_unlock(&dev->rdi.pending_lock);
380 * qib_check_send_wqe - validate wr/wqe
382 * @wqe - The built wqe
384 * validate wr/wqe. This is called
385 * prior to inserting the wqe into
386 * the ring but after the wqe has been
389 * Returns 1 to force direct progress, 0 otherwise, -EINVAL on failure
391 int qib_check_send_wqe(struct rvt_qp *qp,
392 struct rvt_swqe *wqe)
397 switch (qp->ibqp.qp_type) {
400 if (wqe->length > 0x80000000U)
406 ah = ibah_to_rvtah(wqe->ud_wr.ah);
407 if (wqe->length > (1 << ah->log_pmtu))
418 #ifdef CONFIG_DEBUG_FS
421 struct qib_ibdev *dev;
426 struct qib_qp_iter *qib_qp_iter_init(struct qib_ibdev *dev)
428 struct qib_qp_iter *iter;
430 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
439 int qib_qp_iter_next(struct qib_qp_iter *iter)
441 struct qib_ibdev *dev = iter->dev;
444 struct rvt_qp *pqp = iter->qp;
447 for (; n < dev->rdi.qp_dev->qp_table_size; n++) {
449 qp = rcu_dereference(pqp->next);
451 qp = rcu_dereference(dev->rdi.qp_dev->qp_table[n]);
462 static const char * const qp_type_str[] = {
463 "SMI", "GSI", "RC", "UC", "UD",
466 void qib_qp_iter_print(struct seq_file *s, struct qib_qp_iter *iter)
468 struct rvt_swqe *wqe;
469 struct rvt_qp *qp = iter->qp;
470 struct qib_qp_priv *priv = qp->priv;
472 wqe = rvt_get_swqe_ptr(qp, qp->s_last);
474 "N %d QP%u %s %u %u %u f=%x %u %u %u %u %u PSN %x %x %x %x %x (%u %u %u %u %u %u) QP%u LID %x\n",
477 qp_type_str[qp->ibqp.qp_type],
482 atomic_read(&priv->s_dma_busy),
483 !list_empty(&priv->iowait),
488 qp->s_psn, qp->s_next_psn,
489 qp->s_sending_psn, qp->s_sending_hpsn,
490 qp->s_last, qp->s_acked, qp->s_cur,
491 qp->s_tail, qp->s_head, qp->s_size,
493 qp->remote_ah_attr.dlid);