net/mlx5e: IPSec, Add support for ESP trailer removal by hardware

Current hardware decrypts and authenticates incoming ESP packets.
Subsequently, the software extracts the nexthdr field, truncates the
trailer and adjusts csum accordingly.

With this patch and a capable device, the trailer is being removed
by the hardware and the nexthdr field is conveyed via PET. This way
we avoid both the need to access the trailer (cache miss) and to
compute its relative checksum, which significantly improve
the performance.

Experiment shows that trailer removal improves the performance by
2Gbps, (netperf). Both forwarding and host-to-host configurations.

Signed-off-by: Yossi Kuperman <yossiku@mellanox.com>
Signed-off-by: Aviad Yehezkel <aviadye@mellanox.com>
Signed-off-by: Saeed Mahameed <saeedm@mellanox.com>

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/accel/ipsec.h b/drivers/net/ethernet/mellanox/mlx5/core/accel/ipsec.h
index 67cda8871f5a575c58657ebe2f26ec2bc4de535a..4da9611a753de3e5d9cc1b6953c17edf9e735885 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/accel/ipsec.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/accel/ipsec.h
@@ -43,6 +43,7 @@ enum {
        MLX5_ACCEL_IPSEC_IPV6 = BIT(2),
        MLX5_ACCEL_IPSEC_ESP = BIT(3),
        MLX5_ACCEL_IPSEC_LSO = BIT(4),
+       MLX5_ACCEL_IPSEC_NO_TRAILER = BIT(5),
 };
 
 #define MLX5_IPSEC_SADB_IP_AH       BIT(7)
@@ -55,6 +56,7 @@ enum {
 enum {
        MLX5_IPSEC_CMD_ADD_SA = 0,
        MLX5_IPSEC_CMD_DEL_SA = 1,
+       MLX5_IPSEC_CMD_SET_CAP = 5,
 };
 
 enum mlx5_accel_ipsec_enc_mode {

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c
index 1b49afca65c0b36621681109461df45f1231348a..460a613059fe9824bf725b9def3989f9adae5747 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c
@@ -378,6 +378,8 @@ int mlx5e_ipsec_init(struct mlx5e_priv *priv)
        ida_init(&ipsec->halloc);
        ipsec->en_priv = priv;
        ipsec->en_priv->ipsec = ipsec;
+       ipsec->no_trailer = !!(mlx5_accel_ipsec_device_caps(priv->mdev) &
+                              MLX5_ACCEL_IPSEC_NO_TRAILER);
        netdev_dbg(priv->netdev, "IPSec attached to netdevice\n");
        return 0;
 }

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h
index 56e00baf16cc44eb3e1709dc5ee1fcbec367f6e5..bffc3ed0574ab447bf427492ddef025d5e0025d8 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h
@@ -77,6 +77,7 @@ struct mlx5e_ipsec_stats {
 struct mlx5e_ipsec {
        struct mlx5e_priv *en_priv;
        DECLARE_HASHTABLE(sadb_rx, MLX5E_IPSEC_SADB_RX_BITS);
+       bool no_trailer;
        spinlock_t sadb_rx_lock; /* Protects sadb_rx and halloc */
        struct ida halloc;
        struct mlx5e_ipsec_sw_stats sw_stats;

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c
index 6a7c8b04447ebb4eae923cd455e1190982417bee..64c549a066783a458b419ecebcc694cab8b6cfb9 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c
@@ -42,10 +42,11 @@
 enum {
        MLX5E_IPSEC_RX_SYNDROME_DECRYPTED = 0x11,
        MLX5E_IPSEC_RX_SYNDROME_AUTH_FAILED = 0x12,
+       MLX5E_IPSEC_RX_SYNDROME_BAD_PROTO = 0x17,
 };
 
 struct mlx5e_ipsec_rx_metadata {
-       unsigned char   reserved;
+       unsigned char   nexthdr;
        __be32          sa_handle;
 } __packed;
 
@@ -301,10 +302,17 @@ mlx5e_ipsec_build_sp(struct net_device *netdev, struct sk_buff *skb,
        switch (mdata->syndrome) {
        case MLX5E_IPSEC_RX_SYNDROME_DECRYPTED:
                xo->status = CRYPTO_SUCCESS;
+               if (likely(priv->ipsec->no_trailer)) {
+                       xo->flags |= XFRM_ESP_NO_TRAILER;
+                       xo->proto = mdata->content.rx.nexthdr;
+               }
                break;
        case MLX5E_IPSEC_RX_SYNDROME_AUTH_FAILED:
                xo->status = CRYPTO_TUNNEL_ESP_AUTH_FAILED;
                break;
+       case MLX5E_IPSEC_RX_SYNDROME_BAD_PROTO:
+               xo->status = CRYPTO_INVALID_PROTOCOL;
+               break;
        default:
                atomic64_inc(&priv->ipsec->sw_stats.ipsec_rx_drop_syndrome);
                return NULL;

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fpga/ipsec.c b/drivers/net/ethernet/mellanox/mlx5/core/fpga/ipsec.c
index e0f32b025e06bebf6a9ba4e59fe379cd8e98cef4..3b10d46dc821847a3b4e8aec5b4ac187494ac476 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/fpga/ipsec.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/fpga/ipsec.c
@@ -273,6 +273,9 @@ u32 mlx5_fpga_ipsec_device_caps(struct mlx5_core_dev *mdev)
        if (MLX5_GET(ipsec_extended_cap, fdev->ipsec->caps, lso))
                ret |= MLX5_ACCEL_IPSEC_LSO;
 
+       if (MLX5_GET(ipsec_extended_cap, fdev->ipsec->caps, rx_no_trailer))
+               ret |= MLX5_ACCEL_IPSEC_NO_TRAILER;
+
        return ret;
 }
 
@@ -335,6 +338,46 @@ int mlx5_fpga_ipsec_counters_read(struct mlx5_core_dev *mdev, u64 *counters,
        return ret;
 }
 
+static int mlx5_fpga_ipsec_set_caps(struct mlx5_core_dev *mdev, u32 flags)
+{
+       struct mlx5_ipsec_command_context *context;
+       struct mlx5_ifc_fpga_ipsec_cmd_cap cmd = {0};
+       int err;
+
+       cmd.cmd = htonl(MLX5_IPSEC_CMD_SET_CAP);
+       cmd.flags = htonl(flags);
+       context = mlx5_fpga_ipsec_cmd_exec(mdev, &cmd, sizeof(cmd));
+       if (IS_ERR(context)) {
+               err = PTR_ERR(context);
+               goto out;
+       }
+
+       err = mlx5_fpga_ipsec_cmd_wait(context);
+       if (err)
+               goto out;
+
+       if ((context->resp.flags & cmd.flags) != cmd.flags) {
+               mlx5_fpga_err(context->dev, "Failed to set capabilities. cmd 0x%08x vs resp 0x%08x\n",
+                             cmd.flags,
+                             context->resp.flags);
+               err = -EIO;
+       }
+
+out:
+       return err;
+}
+
+static int mlx5_fpga_ipsec_enable_supported_caps(struct mlx5_core_dev *mdev)
+{
+       u32 dev_caps = mlx5_fpga_ipsec_device_caps(mdev);
+       u32 flags = 0;
+
+       if (dev_caps & MLX5_ACCEL_IPSEC_NO_TRAILER)
+               flags |= MLX5_FPGA_IPSEC_CAP_NO_TRAILER;
+
+       return mlx5_fpga_ipsec_set_caps(mdev, flags);
+}
+
 int mlx5_fpga_ipsec_init(struct mlx5_core_dev *mdev)
 {
        struct mlx5_fpga_conn_attr init_attr = {0};
@@ -372,8 +415,19 @@ int mlx5_fpga_ipsec_init(struct mlx5_core_dev *mdev)
                goto error;
        }
        fdev->ipsec->conn = conn;
+
+       err = mlx5_fpga_ipsec_enable_supported_caps(mdev);
+       if (err) {
+               mlx5_fpga_err(fdev, "Failed to enable IPSec extended capabilities: %d\n",
+                             err);
+               goto err_destroy_conn;
+       }
+
        return 0;
 
+err_destroy_conn:
+       mlx5_fpga_sbu_conn_destroy(conn);
+
 error:
        kfree(fdev->ipsec);
        fdev->ipsec = NULL;

diff --git a/include/linux/mlx5/mlx5_ifc_fpga.h b/include/linux/mlx5/mlx5_ifc_fpga.h
index 7283fe780f93939ea3bc420c98476204684b5f33..643544db180b95e89467352cb2c4eb22667f8a73 100644 (file)
--- a/include/linux/mlx5/mlx5_ifc_fpga.h
+++ b/include/linux/mlx5/mlx5_ifc_fpga.h
@@ -373,7 +373,8 @@ struct mlx5_ifc_fpga_destroy_qp_out_bits {
 struct mlx5_ifc_ipsec_extended_cap_bits {
        u8         encapsulation[0x20];
 
-       u8         reserved_0[0x15];
+       u8         reserved_0[0x14];
+       u8         rx_no_trailer[0x1];
        u8         ipv4_fragment[0x1];
        u8         ipv6[0x1];
        u8         esn[0x1];
@@ -445,4 +446,14 @@ struct mlx5_ifc_fpga_ipsec_cmd_resp {
        u8 reserved[24];
 } __packed;
 
+enum mlx5_ifc_fpga_ipsec_cap {
+       MLX5_FPGA_IPSEC_CAP_NO_TRAILER = BIT(0),
+};
+
+struct mlx5_ifc_fpga_ipsec_cmd_cap {
+       __be32 cmd;
+       __be32 flags;
+       u8 reserved[24];
+} __packed;
+
 #endif /* MLX5_IFC_FPGA_H */