IB/mlx4: Create slave AH's directly

Since slave GID's do not exist in the core gid table we can no longer use
the core code to help do this without creating inconsistencies. Directly
create the AH using mlx4 internal APIs.

Signed-off-by: Jason Gunthorpe <jgg@mellanox.com>
Reviewed-by: Jack Morgenstein <jackm@dev.mellanox.co.il>
Signed-off-by: Leon Romanovsky <leonro@mellanox.com>

diff --git a/drivers/infiniband/hw/mlx4/ah.c b/drivers/infiniband/hw/mlx4/ah.c
index 5e9b0837ef61c40905705fcf56b06cca809e4145..e9e3a6f390dba73156b3c9394888516070761ea4 100644 (file)
--- a/drivers/infiniband/hw/mlx4/ah.c
+++ b/drivers/infiniband/hw/mlx4/ah.c
@@ -96,19 +96,29 @@ static struct ib_ah *create_iboe_ah(struct ib_pd *pd,
 
        memcpy(ah->av.eth.mac, ah_attr->roce.dmac, ETH_ALEN);
        eth_zero_addr(ah->av.eth.s_mac);
+
+       /*
+        * If sgid_attr is NULL we are being called by mlx4_ib_create_ah_slave
+        * and we are directly creating an AV for a slave's gid_index.
+        */
        gid_attr = ah_attr->grh.sgid_attr;
-       if (is_vlan_dev(gid_attr->ndev))
-               vlan_tag = vlan_dev_vlan_id(gid_attr->ndev);
-       memcpy(ah->av.eth.s_mac, gid_attr->ndev->dev_addr, ETH_ALEN);
+       if (gid_attr) {
+               if (is_vlan_dev(gid_attr->ndev))
+                       vlan_tag = vlan_dev_vlan_id(gid_attr->ndev);
+               memcpy(ah->av.eth.s_mac, gid_attr->ndev->dev_addr, ETH_ALEN);
+               ret = mlx4_ib_gid_index_to_real_index(ibdev, gid_attr);
+               if (ret < 0)
+                       return ERR_PTR(ret);
+               ah->av.eth.gid_index = ret;
+       } else {
+               /* mlx4_ib_create_ah_slave fills in the s_mac and the vlan */
+               ah->av.eth.gid_index = ah_attr->grh.sgid_index;
+       }
 
        if (vlan_tag < 0x1000)
                vlan_tag |= (rdma_ah_get_sl(ah_attr) & 7) << 13;
        ah->av.eth.port_pd = cpu_to_be32(to_mpd(pd)->pdn |
                                         (rdma_ah_get_port_num(ah_attr) << 24));
-       ret = mlx4_ib_gid_index_to_real_index(ibdev, gid_attr);
-       if (ret < 0)
-               return ERR_PTR(ret);
-       ah->av.eth.gid_index = ret;
        ah->av.eth.vlan = cpu_to_be16(vlan_tag);
        ah->av.eth.hop_limit = grh->hop_limit;
        if (rdma_ah_get_static_rate(ah_attr)) {
@@ -167,6 +177,40 @@ struct ib_ah *mlx4_ib_create_ah(struct ib_pd *pd, struct rdma_ah_attr *ah_attr,
                return create_ib_ah(pd, ah_attr, ah); /* never fails */
 }
 
+/* AH's created via this call must be free'd by mlx4_ib_destroy_ah. */
+struct ib_ah *mlx4_ib_create_ah_slave(struct ib_pd *pd,
+                                     struct rdma_ah_attr *ah_attr,
+                                     int slave_sgid_index, u8 *s_mac,
+                                     u16 vlan_tag)
+{
+       struct rdma_ah_attr slave_attr = *ah_attr;
+       struct mlx4_ib_ah *mah;
+       struct ib_ah *ah;
+
+       slave_attr.grh.sgid_attr = NULL;
+       slave_attr.grh.sgid_index = slave_sgid_index;
+       ah = mlx4_ib_create_ah(pd, &slave_attr, NULL);
+       if (IS_ERR(ah))
+               return ah;
+
+       ah->device = pd->device;
+       ah->pd = pd;
+       ah->type = ah_attr->type;
+       mah = to_mah(ah);
+
+       /* get rid of force-loopback bit */
+       mah->av.ib.port_pd &= cpu_to_be32(0x7FFFFFFF);
+
+       if (ah_attr->type == RDMA_AH_ATTR_TYPE_ROCE)
+               memcpy(mah->av.eth.s_mac, s_mac, 6);
+
+       if (vlan_tag < 0x1000)
+               vlan_tag |= (rdma_ah_get_sl(ah_attr) & 7) << 13;
+       mah->av.eth.vlan = cpu_to_be16(vlan_tag);
+
+       return ah;
+}
+
 int mlx4_ib_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr)
 {
        struct mlx4_ib_ah *ah = to_mah(ibah);

diff --git a/drivers/infiniband/hw/mlx4/mad.c b/drivers/infiniband/hw/mlx4/mad.c
index 90a3e2642c2e11bd3f54e3d0e874c212aa3bb401..8d730a69793d35648eb902c9783213b600a14051 100644 (file)
--- a/drivers/infiniband/hw/mlx4/mad.c
+++ b/drivers/infiniband/hw/mlx4/mad.c
@@ -1367,13 +1367,10 @@ int mlx4_ib_send_to_wire(struct mlx4_ib_dev *dev, int slave, u8 port,
        struct mlx4_mad_snd_buf *sqp_mad;
        struct ib_ah *ah;
        struct ib_qp *send_qp = NULL;
-       struct ib_global_route *grh;
        unsigned wire_tx_ix = 0;
        int ret = 0;
        u16 wire_pkey_ix;
        int src_qpnum;
-       u8 sgid_index;
-
 
        sqp_ctx = dev->sriov.sqps[port-1];
 
@@ -1394,16 +1391,11 @@ int mlx4_ib_send_to_wire(struct mlx4_ib_dev *dev, int slave, u8 port,
        send_qp = sqp->qp;
 
        /* create ah */
-       grh = rdma_ah_retrieve_grh(attr);
-       sgid_index = grh->sgid_index;
-       grh->sgid_index = 0;
-       ah = rdma_create_ah(sqp_ctx->pd, attr);
+       ah = mlx4_ib_create_ah_slave(sqp_ctx->pd, attr,
+                                    rdma_ah_retrieve_grh(attr)->sgid_index,
+                                    s_mac, vlan_id);
        if (IS_ERR(ah))
                return -ENOMEM;
-       grh->sgid_index = sgid_index;
-       to_mah(ah)->av.ib.gid_index = sgid_index;
-       /* get rid of force-loopback bit */
-       to_mah(ah)->av.ib.port_pd &= cpu_to_be32(0x7FFFFFFF);
        spin_lock(&sqp->tx_lock);
        if (sqp->tx_ix_head - sqp->tx_ix_tail >=
            (MLX4_NUM_TUNNEL_BUFS - 1))
@@ -1445,12 +1437,6 @@ int mlx4_ib_send_to_wire(struct mlx4_ib_dev *dev, int slave, u8 port,
        wr.wr.num_sge = 1;
        wr.wr.opcode = IB_WR_SEND;
        wr.wr.send_flags = IB_SEND_SIGNALED;
-       if (s_mac)
-               memcpy(to_mah(ah)->av.eth.s_mac, s_mac, 6);
-       if (vlan_id < 0x1000)
-               vlan_id |= (rdma_ah_get_sl(attr) & 7) << 13;
-       to_mah(ah)->av.eth.vlan = cpu_to_be16(vlan_id);
-
 
        ret = ib_post_send(send_qp, &wr.wr, &bad_wr);
        if (!ret)
@@ -1461,7 +1447,7 @@ int mlx4_ib_send_to_wire(struct mlx4_ib_dev *dev, int slave, u8 port,
        spin_unlock(&sqp->tx_lock);
        sqp->tx_ring[wire_tx_ix].ah = NULL;
 out:
-       rdma_destroy_ah(ah);
+       mlx4_ib_destroy_ah(ah);
        return ret;
 }
 

diff --git a/drivers/infiniband/hw/mlx4/mlx4_ib.h b/drivers/infiniband/hw/mlx4/mlx4_ib.h
index e5a365bdbe458a99db8168e408c79e4a3bf01b65..1a0fad30633b356c31b9cecf0a872788669c1a10 100644 (file)
--- a/drivers/infiniband/hw/mlx4/mlx4_ib.h
+++ b/drivers/infiniband/hw/mlx4/mlx4_ib.h
@@ -760,6 +760,10 @@ void mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq);
 
 struct ib_ah *mlx4_ib_create_ah(struct ib_pd *pd, struct rdma_ah_attr *ah_attr,
                                struct ib_udata *udata);
+struct ib_ah *mlx4_ib_create_ah_slave(struct ib_pd *pd,
+                                     struct rdma_ah_attr *ah_attr,
+                                     int slave_sgid_index, u8 *s_mac,
+                                     u16 vlan_tag);
 int mlx4_ib_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr);
 int mlx4_ib_destroy_ah(struct ib_ah *ah);