IB/rxe: vary the source udp port for receive scaling

Select the source udp port number for a QP based on the
source QPN. This provides a better spread of traffic
across NIC RX queues for RC/UC QPs.

Signed-off-by: Vijay Immanuel <vijayi@attalasystems.com>
Signed-off-by: Doug Ledford <dledford@redhat.com>

diff --git a/drivers/infiniband/sw/rxe/rxe_net.c b/drivers/infiniband/sw/rxe/rxe_net.c
index 8094cbaa54a9e2684bd41ae6774a886919461c46..4ddb6b4c6a462f599d277539140b7d9b9c112889 100644 (file)
--- a/drivers/infiniband/sw/rxe/rxe_net.c
+++ b/drivers/infiniband/sw/rxe/rxe_net.c
@@ -396,8 +396,8 @@ static int prepare4(struct rxe_dev *rxe, struct rxe_pkt_info *pkt,
        if (!memcmp(saddr, daddr, sizeof(*daddr)))
                pkt->mask |= RXE_LOOPBACK_MASK;
 
-       prepare_udp_hdr(skb, htons(RXE_ROCE_V2_SPORT),
-                       htons(ROCE_V2_UDP_DPORT));
+       prepare_udp_hdr(skb, cpu_to_be16(qp->src_port),
+                       cpu_to_be16(ROCE_V2_UDP_DPORT));
 
        prepare_ipv4_hdr(dst, skb, saddr->s_addr, daddr->s_addr, IPPROTO_UDP,
                         av->grh.traffic_class, av->grh.hop_limit, df, xnet);
@@ -423,8 +423,8 @@ static int prepare6(struct rxe_dev *rxe, struct rxe_pkt_info *pkt,
        if (!memcmp(saddr, daddr, sizeof(*daddr)))
                pkt->mask |= RXE_LOOPBACK_MASK;
 
-       prepare_udp_hdr(skb, htons(RXE_ROCE_V2_SPORT),
-                       htons(ROCE_V2_UDP_DPORT));
+       prepare_udp_hdr(skb, cpu_to_be16(qp->src_port),
+                       cpu_to_be16(ROCE_V2_UDP_DPORT));
 
        prepare_ipv6_hdr(dst, skb, saddr, daddr, IPPROTO_UDP,
                         av->grh.traffic_class,

diff --git a/drivers/infiniband/sw/rxe/rxe_qp.c b/drivers/infiniband/sw/rxe/rxe_qp.c
index c58452daffc7449875f362269b7cd51bb9f8803e..6ff88c8250f6320eeb01671fe66406fe79f2be29 100644 (file)
--- a/drivers/infiniband/sw/rxe/rxe_qp.c
+++ b/drivers/infiniband/sw/rxe/rxe_qp.c
@@ -227,6 +227,16 @@ static int rxe_qp_init_req(struct rxe_dev *rxe, struct rxe_qp *qp,
                return err;
        qp->sk->sk->sk_user_data = qp;
 
+       /* pick a source UDP port number for this QP based on
+        * the source QPN. this spreads traffic for different QPs
+        * across different NIC RX queues (while using a single
+        * flow for a given QP to maintain packet order).
+        * the port number must be in the Dynamic Ports range
+        * (0xc000 - 0xffff).
+        */
+       qp->src_port = RXE_ROCE_V2_SPORT +
+               (hash_32_generic(qp_num(qp), 14) & 0x3fff);
+
        qp->sq.max_wr           = init->cap.max_send_wr;
        qp->sq.max_sge          = init->cap.max_send_sge;
        qp->sq.max_inline       = init->cap.max_inline_data;

diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.h b/drivers/infiniband/sw/rxe/rxe_verbs.h
index af1470d293912e99ddaff4113b51d634b8c5743f..0d920087811a0ffaa8e14a24b703fc9a2ccd0655 100644 (file)
--- a/drivers/infiniband/sw/rxe/rxe_verbs.h
+++ b/drivers/infiniband/sw/rxe/rxe_verbs.h
@@ -248,6 +248,7 @@ struct rxe_qp {
 
        struct socket           *sk;
        u32                     dst_cookie;
+       u16                     src_port;
 
        struct rxe_av           pri_av;
        struct rxe_av           alt_av;