IB/rdmavt: Use per-CPU reference count for MRs

Having per-CPU reference count for each MR prevents
cache-line bouncing across the system. Thus, it
prevents bottlenecks. Use per-CPU reference counts
per MR.

The per-CPU reference count for FMRs is used in
atomic mode to allow accurate testing of the busy
state. Other MR types run in per-CPU mode MR until
they're freed.

Reviewed-by: Mike Marciniszyn <mike.marciniszyn@intel.com>
Signed-off-by: Sebastian Sanchez <sebastian.sanchez@intel.com>
Signed-off-by: Mike Marciniszyn <mike.marciniszyn@intel.com>
Signed-off-by: Dennis Dalessandro <dennis.dalessandro@intel.com>
Signed-off-by: Doug Ledford <dledford@redhat.com>

diff --git a/drivers/infiniband/sw/rdmavt/mr.c b/drivers/infiniband/sw/rdmavt/mr.c
index 52fd15276ee6ddd7ffe30eed28b9707a099536fa..c80a69b1ffcb060ebf111149e182b8a6daec5ce7 100644 (file)
--- a/drivers/infiniband/sw/rdmavt/mr.c
+++ b/drivers/infiniband/sw/rdmavt/mr.c
@@ -120,10 +120,19 @@ static void rvt_deinit_mregion(struct rvt_mregion *mr)
        mr->mapsz = 0;
        while (i)
                kfree(mr->map[--i]);
+       percpu_ref_exit(&mr->refcount);
+}
+
+static void __rvt_mregion_complete(struct percpu_ref *ref)
+{
+       struct rvt_mregion *mr = container_of(ref, struct rvt_mregion,
+                                             refcount);
+
+       complete(&mr->comp);
 }
 
 static int rvt_init_mregion(struct rvt_mregion *mr, struct ib_pd *pd,
-                           int count)
+                           int count, unsigned int percpu_flags)
 {
        int m, i = 0;
        struct rvt_dev_info *dev = ib_to_rvt(pd->device);
@@ -133,19 +142,23 @@ static int rvt_init_mregion(struct rvt_mregion *mr, struct ib_pd *pd,
        for (; i < m; i++) {
                mr->map[i] = kzalloc_node(sizeof(*mr->map[0]), GFP_KERNEL,
                                          dev->dparms.node);
-               if (!mr->map[i]) {
-                       rvt_deinit_mregion(mr);
-                       return -ENOMEM;
-               }
+               if (!mr->map[i])
+                       goto bail;
                mr->mapsz++;
        }
        init_completion(&mr->comp);
        /* count returning the ptr to user */
-       atomic_set(&mr->refcount, 1);
+       if (percpu_ref_init(&mr->refcount, &__rvt_mregion_complete,
+                           percpu_flags, GFP_KERNEL))
+               goto bail;
+
        atomic_set(&mr->lkey_invalid, 0);
        mr->pd = pd;
        mr->max_segs = count;
        return 0;
+bail:
+       rvt_deinit_mregion(mr);
+       return -ENOMEM;
 }
 
 /**
@@ -180,8 +193,7 @@ static int rvt_alloc_lkey(struct rvt_mregion *mr, int dma_region)
                if (!tmr) {
                        rcu_assign_pointer(dev->dma_mr, mr);
                        mr->lkey_published = 1;
-               } else {
-                       rvt_put_mr(mr);
+                       rvt_get_mr(mr);
                }
                goto success;
        }
@@ -239,11 +251,14 @@ static void rvt_free_lkey(struct rvt_mregion *mr)
        int freed = 0;
 
        spin_lock_irqsave(&rkt->lock, flags);
-       if (!mr->lkey_published)
-               goto out;
-       if (lkey == 0) {
-               RCU_INIT_POINTER(dev->dma_mr, NULL);
+       if (!lkey) {
+               if (mr->lkey_published) {
+                       RCU_INIT_POINTER(dev->dma_mr, NULL);
+                       rvt_put_mr(mr);
+               }
        } else {
+               if (!mr->lkey_published)
+                       goto out;
                r = lkey >> (32 - dev->dparms.lkey_table_size);
                RCU_INIT_POINTER(rkt->table[r], NULL);
        }
@@ -253,7 +268,7 @@ static void rvt_free_lkey(struct rvt_mregion *mr)
        spin_unlock_irqrestore(&rkt->lock, flags);
        if (freed) {
                synchronize_rcu();
-               rvt_put_mr(mr);
+               percpu_ref_kill(&mr->refcount);
        }
 }
 
@@ -269,7 +284,7 @@ static struct rvt_mr *__rvt_alloc_mr(int count, struct ib_pd *pd)
        if (!mr)
                goto bail;
 
-       rval = rvt_init_mregion(&mr->mr, pd, count);
+       rval = rvt_init_mregion(&mr->mr, pd, count, 0);
        if (rval)
                goto bail;
        /*
@@ -294,8 +309,8 @@ static struct rvt_mr *__rvt_alloc_mr(int count, struct ib_pd *pd)
 
 static void __rvt_free_mr(struct rvt_mr *mr)
 {
-       rvt_deinit_mregion(&mr->mr);
        rvt_free_lkey(&mr->mr);
+       rvt_deinit_mregion(&mr->mr);
        kfree(mr);
 }
 
@@ -323,7 +338,7 @@ struct ib_mr *rvt_get_dma_mr(struct ib_pd *pd, int acc)
                goto bail;
        }
 
-       rval = rvt_init_mregion(&mr->mr, pd, 0);
+       rval = rvt_init_mregion(&mr->mr, pd, 0, 0);
        if (rval) {
                ret = ERR_PTR(rval);
                goto bail;
@@ -445,8 +460,8 @@ int rvt_dereg_mr(struct ib_mr *ibmr)
        timeout = wait_for_completion_timeout(&mr->mr.comp, 5 * HZ);
        if (!timeout) {
                rvt_pr_err(rdi,
-                          "rvt_dereg_mr timeout mr %p pd %p refcount %u\n",
-                          mr, mr->mr.pd, atomic_read(&mr->mr.refcount));
+                          "rvt_dereg_mr timeout mr %p pd %p\n",
+                          mr, mr->mr.pd);
                rvt_get_mr(&mr->mr);
                ret = -EBUSY;
                goto out;
@@ -623,7 +638,8 @@ struct ib_fmr *rvt_alloc_fmr(struct ib_pd *pd, int mr_access_flags,
        if (!fmr)
                goto bail;
 
-       rval = rvt_init_mregion(&fmr->mr, pd, fmr_attr->max_pages);
+       rval = rvt_init_mregion(&fmr->mr, pd, fmr_attr->max_pages,
+                               PERCPU_REF_INIT_ATOMIC);
        if (rval)
                goto bail;
 
@@ -674,11 +690,12 @@ int rvt_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list,
        struct rvt_fmr *fmr = to_ifmr(ibfmr);
        struct rvt_lkey_table *rkt;
        unsigned long flags;
-       int m, n, i;
+       int m, n;
+       unsigned long i;
        u32 ps;
        struct rvt_dev_info *rdi = ib_to_rvt(ibfmr->device);
 
-       i = atomic_read(&fmr->mr.refcount);
+       i = atomic_long_read(&fmr->mr.refcount.count);
        if (i > 2)
                return -EBUSY;
 

diff --git a/include/rdma/rdmavt_mr.h b/include/rdma/rdmavt_mr.h
index de59de28b6a2ae8f25927c1d92dc2d63e02a1c45..05698d8d9c6f943cfd9c53af45aa8cff5ce3f069 100644 (file)
--- a/include/rdma/rdmavt_mr.h
+++ b/include/rdma/rdmavt_mr.h
@@ -52,6 +52,7 @@
  * For Memory Regions. This stuff should probably be moved into rdmavt/mr.h once
  * drivers no longer need access to the MR directly.
  */
+#include <linux/percpu-refcount.h>
 
 /*
  * A segment is a linear region of low physical memory.
@@ -79,11 +80,11 @@ struct rvt_mregion {
        int access_flags;
        u32 max_segs;           /* number of rvt_segs in all the arrays */
        u32 mapsz;              /* size of the map array */
+       atomic_t lkey_invalid;  /* true if current lkey is invalid */
        u8  page_shift;         /* 0 - non unform/non powerof2 sizes */
        u8  lkey_published;     /* in global table */
-       atomic_t lkey_invalid;  /* true if current lkey is invalid */
+       struct percpu_ref refcount;
        struct completion comp; /* complete when refcount goes to zero */
-       atomic_t refcount;
        struct rvt_segarray *map[0];    /* the segments */
 };
 
@@ -123,13 +124,12 @@ struct rvt_sge_state {
 
 static inline void rvt_put_mr(struct rvt_mregion *mr)
 {
-       if (unlikely(atomic_dec_and_test(&mr->refcount)))
-               complete(&mr->comp);
+       percpu_ref_put(&mr->refcount);
 }
 
 static inline void rvt_get_mr(struct rvt_mregion *mr)
 {
-       atomic_inc(&mr->refcount);
+       percpu_ref_get(&mr->refcount);
 }
 
 static inline void rvt_put_ss(struct rvt_sge_state *ss)