IB/srp: Make HCA completion vector configurable

Several InfiniBand HCAs allow configuring the completion vector per
CQ.  This allows spreading the workload created by IB completion
interrupts over multiple MSI-X vectors and hence over multiple CPU
cores.  In other words, configuring the completion vector properly not
only allows reducing latency on an initiator connected to multiple
SRP targets but also allows improving throughput.

Signed-off-by: Bart Van Assche <bvanassche@acm.org>
Acked-by: David Dillow <dillowda@ornl.gov>
Signed-off-by: Roland Dreier <roland@purestorage.com>

diff --git a/Documentation/ABI/stable/sysfs-driver-ib_srp b/Documentation/ABI/stable/sysfs-driver-ib_srp
index 481aae95c7d19c21511e3db8baf5317031fa0ccb..5c53d28f775cde36f4e77f46e41dc9ead66aa9c1 100644 (file)
--- a/Documentation/ABI/stable/sysfs-driver-ib_srp
+++ b/Documentation/ABI/stable/sysfs-driver-ib_srp
@@ -54,6 +54,13 @@ Description: Interface for making ib_srp connect to a new target.
                  ib_srp. Specifying a value that exceeds cmd_sg_entries is
                  only safe with partial memory descriptor list support enabled
                  (allow_ext_sg=1).
+               * comp_vector, a number in the range 0..n-1 specifying the
+                 MSI-X completion vector. Some HCA's allocate multiple (n)
+                 MSI-X vectors per HCA port. If the IRQ affinity masks of
+                 these interrupts have been configured such that each MSI-X
+                 interrupt is handled by a different CPU then the comp_vector
+                 parameter can be used to spread the SRP completion workload
+                 over multiple CPU's.
 
 What:          /sys/class/infiniband_srp/srp-<hca>-<port_number>/ibdev
 Date:          January 2, 2006

diff --git a/drivers/infiniband/ulp/srp/ib_srp.c b/drivers/infiniband/ulp/srp/ib_srp.c
index 830ef0037087fd47c2e370c59cc02d80c9fa1bbb..9b30366cb0177803dc0072a7a1741f161ee57b74 100644 (file)
--- a/drivers/infiniband/ulp/srp/ib_srp.c
+++ b/drivers/infiniband/ulp/srp/ib_srp.c
@@ -231,14 +231,16 @@ static int srp_create_target_ib(struct srp_target_port *target)
                return -ENOMEM;
 
        recv_cq = ib_create_cq(target->srp_host->srp_dev->dev,
-                              srp_recv_completion, NULL, target, SRP_RQ_SIZE, 0);
+                              srp_recv_completion, NULL, target, SRP_RQ_SIZE,
+                              target->comp_vector);
        if (IS_ERR(recv_cq)) {
                ret = PTR_ERR(recv_cq);
                goto err;
        }
 
        send_cq = ib_create_cq(target->srp_host->srp_dev->dev,
-                              srp_send_completion, NULL, target, SRP_SQ_SIZE, 0);
+                              srp_send_completion, NULL, target, SRP_SQ_SIZE,
+                              target->comp_vector);
        if (IS_ERR(send_cq)) {
                ret = PTR_ERR(send_cq);
                goto err_recv_cq;
@@ -1898,6 +1900,14 @@ static ssize_t show_local_ib_device(struct device *dev,
        return sprintf(buf, "%s\n", target->srp_host->srp_dev->dev->name);
 }
 
+static ssize_t show_comp_vector(struct device *dev,
+                               struct device_attribute *attr, char *buf)
+{
+       struct srp_target_port *target = host_to_target(class_to_shost(dev));
+
+       return sprintf(buf, "%d\n", target->comp_vector);
+}
+
 static ssize_t show_cmd_sg_entries(struct device *dev,
                                   struct device_attribute *attr, char *buf)
 {
@@ -1924,6 +1934,7 @@ static DEVICE_ATTR(req_lim,         S_IRUGO, show_req_lim,         NULL);
 static DEVICE_ATTR(zero_req_lim,    S_IRUGO, show_zero_req_lim,           NULL);
 static DEVICE_ATTR(local_ib_port,   S_IRUGO, show_local_ib_port,   NULL);
 static DEVICE_ATTR(local_ib_device, S_IRUGO, show_local_ib_device, NULL);
+static DEVICE_ATTR(comp_vector,     S_IRUGO, show_comp_vector,     NULL);
 static DEVICE_ATTR(cmd_sg_entries,  S_IRUGO, show_cmd_sg_entries,  NULL);
 static DEVICE_ATTR(allow_ext_sg,    S_IRUGO, show_allow_ext_sg,    NULL);
 
@@ -1938,6 +1949,7 @@ static struct device_attribute *srp_host_attrs[] = {
        &dev_attr_zero_req_lim,
        &dev_attr_local_ib_port,
        &dev_attr_local_ib_device,
+       &dev_attr_comp_vector,
        &dev_attr_cmd_sg_entries,
        &dev_attr_allow_ext_sg,
        NULL
@@ -2061,6 +2073,7 @@ enum {
        SRP_OPT_CMD_SG_ENTRIES  = 1 << 9,
        SRP_OPT_ALLOW_EXT_SG    = 1 << 10,
        SRP_OPT_SG_TABLESIZE    = 1 << 11,
+       SRP_OPT_COMP_VECTOR     = 1 << 12,
        SRP_OPT_ALL             = (SRP_OPT_ID_EXT       |
                                   SRP_OPT_IOC_GUID     |
                                   SRP_OPT_DGID         |
@@ -2081,6 +2094,7 @@ static const match_table_t srp_opt_tokens = {
        { SRP_OPT_CMD_SG_ENTRIES,       "cmd_sg_entries=%u"     },
        { SRP_OPT_ALLOW_EXT_SG,         "allow_ext_sg=%u"       },
        { SRP_OPT_SG_TABLESIZE,         "sg_tablesize=%u"       },
+       { SRP_OPT_COMP_VECTOR,          "comp_vector=%u"        },
        { SRP_OPT_ERR,                  NULL                    }
 };
 
@@ -2236,6 +2250,14 @@ static int srp_parse_options(const char *buf, struct srp_target_port *target)
                        target->sg_tablesize = token;
                        break;
 
+               case SRP_OPT_COMP_VECTOR:
+                       if (match_int(args, &token) || token < 0) {
+                               pr_warn("bad comp_vector parameter '%s'\n", p);
+                               goto out;
+                       }
+                       target->comp_vector = token;
+                       break;
+
                default:
                        pr_warn("unknown parameter or missing value '%s' in target creation request\n",
                                p);

diff --git a/drivers/infiniband/ulp/srp/ib_srp.h b/drivers/infiniband/ulp/srp/ib_srp.h
index 66fbedda457128c8d2695575d8bf7a217d78abf0..e641088c14dc37bda98e95134b4235f151e3c0fc 100644 (file)
--- a/drivers/infiniband/ulp/srp/ib_srp.h
+++ b/drivers/infiniband/ulp/srp/ib_srp.h
@@ -156,6 +156,7 @@ struct srp_target_port {
        char                    target_name[32];
        unsigned int            scsi_id;
        unsigned int            sg_tablesize;
+       int                     comp_vector;
 
        struct ib_sa_path_rec   path;
        __be16                  orig_dgid[8];