nfp: bpf: eliminate zero extension code-gen

This patch eliminate zero extension code-gen for instructions including
both alu and load/store. The only exception is for ctx load, because
offload target doesn't go through host ctx convert logic so we do
customized load and ignores zext flag set by verifier.

Cc: Jakub Kicinski <jakub.kicinski@netronome.com>
Reviewed-by: Jakub Kicinski <jakub.kicinski@netronome.com>
Signed-off-by: Jiong Wang <jiong.wang@netronome.com>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

diff --git a/drivers/net/ethernet/netronome/nfp/bpf/jit.c b/drivers/net/ethernet/netronome/nfp/bpf/jit.c
index d4bf0e694541feb6e1d8b8031328f3305a85a893..4054b70d77196d2fb21d2c337487aaeb06393a50 100644 (file)
--- a/drivers/net/ethernet/netronome/nfp/bpf/jit.c
+++ b/drivers/net/ethernet/netronome/nfp/bpf/jit.c
@@ -622,6 +622,13 @@ static void wrp_immed(struct nfp_prog *nfp_prog, swreg dst, u32 imm)
        }
 }
 
+static void
+wrp_zext(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, u8 dst)
+{
+       if (meta->flags & FLAG_INSN_DO_ZEXT)
+               wrp_immed(nfp_prog, reg_both(dst + 1), 0);
+}
+
 static void
 wrp_immed_relo(struct nfp_prog *nfp_prog, swreg dst, u32 imm,
               enum nfp_relo_type relo)
@@ -858,7 +865,8 @@ static int nfp_cpp_memcpy(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
 }
 
 static int
-data_ld(struct nfp_prog *nfp_prog, swreg offset, u8 dst_gpr, int size)
+data_ld(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, swreg offset,
+       u8 dst_gpr, int size)
 {
        unsigned int i;
        u16 shift, sz;
@@ -881,14 +889,15 @@ data_ld(struct nfp_prog *nfp_prog, swreg offset, u8 dst_gpr, int size)
                        wrp_mov(nfp_prog, reg_both(dst_gpr + i), reg_xfer(i));
 
        if (i < 2)
-               wrp_immed(nfp_prog, reg_both(dst_gpr + 1), 0);
+               wrp_zext(nfp_prog, meta, dst_gpr);
 
        return 0;
 }
 
 static int
-data_ld_host_order(struct nfp_prog *nfp_prog, u8 dst_gpr,
-                  swreg lreg, swreg rreg, int size, enum cmd_mode mode)
+data_ld_host_order(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
+                  u8 dst_gpr, swreg lreg, swreg rreg, int size,
+                  enum cmd_mode mode)
 {
        unsigned int i;
        u8 mask, sz;
@@ -911,33 +920,34 @@ data_ld_host_order(struct nfp_prog *nfp_prog, u8 dst_gpr,
                        wrp_mov(nfp_prog, reg_both(dst_gpr + i), reg_xfer(i));
 
        if (i < 2)
-               wrp_immed(nfp_prog, reg_both(dst_gpr + 1), 0);
+               wrp_zext(nfp_prog, meta, dst_gpr);
 
        return 0;
 }
 
 static int
-data_ld_host_order_addr32(struct nfp_prog *nfp_prog, u8 src_gpr, swreg offset,
-                         u8 dst_gpr, u8 size)
+data_ld_host_order_addr32(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
+                         u8 src_gpr, swreg offset, u8 dst_gpr, u8 size)
 {
-       return data_ld_host_order(nfp_prog, dst_gpr, reg_a(src_gpr), offset,
-                                 size, CMD_MODE_32b);
+       return data_ld_host_order(nfp_prog, meta, dst_gpr, reg_a(src_gpr),
+                                 offset, size, CMD_MODE_32b);
 }
 
 static int
-data_ld_host_order_addr40(struct nfp_prog *nfp_prog, u8 src_gpr, swreg offset,
-                         u8 dst_gpr, u8 size)
+data_ld_host_order_addr40(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
+                         u8 src_gpr, swreg offset, u8 dst_gpr, u8 size)
 {
        swreg rega, regb;
 
        addr40_offset(nfp_prog, src_gpr, offset, &rega, &regb);
 
-       return data_ld_host_order(nfp_prog, dst_gpr, rega, regb,
+       return data_ld_host_order(nfp_prog, meta, dst_gpr, rega, regb,
                                  size, CMD_MODE_40b_BA);
 }
 
 static int
-construct_data_ind_ld(struct nfp_prog *nfp_prog, u16 offset, u16 src, u8 size)
+construct_data_ind_ld(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
+                     u16 offset, u16 src, u8 size)
 {
        swreg tmp_reg;
 
@@ -953,10 +963,12 @@ construct_data_ind_ld(struct nfp_prog *nfp_prog, u16 offset, u16 src, u8 size)
        emit_br_relo(nfp_prog, BR_BLO, BR_OFF_RELO, 0, RELO_BR_GO_ABORT);
 
        /* Load data */
-       return data_ld(nfp_prog, imm_b(nfp_prog), 0, size);
+       return data_ld(nfp_prog, meta, imm_b(nfp_prog), 0, size);
 }
 
-static int construct_data_ld(struct nfp_prog *nfp_prog, u16 offset, u8 size)
+static int
+construct_data_ld(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
+                 u16 offset, u8 size)
 {
        swreg tmp_reg;
 
@@ -967,7 +979,7 @@ static int construct_data_ld(struct nfp_prog *nfp_prog, u16 offset, u8 size)
 
        /* Load data */
        tmp_reg = re_load_imm_any(nfp_prog, offset, imm_b(nfp_prog));
-       return data_ld(nfp_prog, tmp_reg, 0, size);
+       return data_ld(nfp_prog, meta, tmp_reg, 0, size);
 }
 
 static int
@@ -1204,7 +1216,7 @@ mem_op_stack(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
        }
 
        if (clr_gpr && size < 8)
-               wrp_immed(nfp_prog, reg_both(gpr + 1), 0);
+               wrp_zext(nfp_prog, meta, gpr);
 
        while (size) {
                u32 slice_end;
@@ -1305,9 +1317,10 @@ wrp_alu32_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
              enum alu_op alu_op)
 {
        const struct bpf_insn *insn = &meta->insn;
+       u8 dst = insn->dst_reg * 2;
 
-       wrp_alu_imm(nfp_prog, insn->dst_reg * 2, alu_op, insn->imm);
-       wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2 + 1), 0);
+       wrp_alu_imm(nfp_prog, dst, alu_op, insn->imm);
+       wrp_zext(nfp_prog, meta, dst);
 
        return 0;
 }
@@ -1319,7 +1332,7 @@ wrp_alu32_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
        u8 dst = meta->insn.dst_reg * 2, src = meta->insn.src_reg * 2;
 
        emit_alu(nfp_prog, reg_both(dst), reg_a(dst), alu_op, reg_b(src));
-       wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0);
+       wrp_zext(nfp_prog, meta, dst);
 
        return 0;
 }
@@ -2396,12 +2409,14 @@ static int neg_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
        u8 dst = meta->insn.dst_reg * 2;
 
        emit_alu(nfp_prog, reg_both(dst), reg_imm(0), ALU_OP_SUB, reg_b(dst));
-       wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0);
+       wrp_zext(nfp_prog, meta, dst);
 
        return 0;
 }
 
-static int __ashr_imm(struct nfp_prog *nfp_prog, u8 dst, u8 shift_amt)
+static int
+__ashr_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, u8 dst,
+          u8 shift_amt)
 {
        if (shift_amt) {
                /* Set signedness bit (MSB of result). */
@@ -2410,7 +2425,7 @@ static int __ashr_imm(struct nfp_prog *nfp_prog, u8 dst, u8 shift_amt)
                emit_shf(nfp_prog, reg_both(dst), reg_none(), SHF_OP_ASHR,
                         reg_b(dst), SHF_SC_R_SHF, shift_amt);
        }
-       wrp_immed(nfp_prog, reg_both(dst + 1), 0);
+       wrp_zext(nfp_prog, meta, dst);
 
        return 0;
 }
@@ -2425,7 +2440,7 @@ static int ashr_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
        umin = meta->umin_src;
        umax = meta->umax_src;
        if (umin == umax)
-               return __ashr_imm(nfp_prog, dst, umin);
+               return __ashr_imm(nfp_prog, meta, dst, umin);
 
        src = insn->src_reg * 2;
        /* NOTE: the first insn will set both indirect shift amount (source A)
@@ -2434,7 +2449,7 @@ static int ashr_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
        emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_b(dst));
        emit_shf_indir(nfp_prog, reg_both(dst), reg_none(), SHF_OP_ASHR,
                       reg_b(dst), SHF_SC_R_SHF);
-       wrp_immed(nfp_prog, reg_both(dst + 1), 0);
+       wrp_zext(nfp_prog, meta, dst);
 
        return 0;
 }
@@ -2444,15 +2459,17 @@ static int ashr_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
        const struct bpf_insn *insn = &meta->insn;
        u8 dst = insn->dst_reg * 2;
 
-       return __ashr_imm(nfp_prog, dst, insn->imm);
+       return __ashr_imm(nfp_prog, meta, dst, insn->imm);
 }
 
-static int __shr_imm(struct nfp_prog *nfp_prog, u8 dst, u8 shift_amt)
+static int
+__shr_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, u8 dst,
+         u8 shift_amt)
 {
        if (shift_amt)
                emit_shf(nfp_prog, reg_both(dst), reg_none(), SHF_OP_NONE,
                         reg_b(dst), SHF_SC_R_SHF, shift_amt);
-       wrp_immed(nfp_prog, reg_both(dst + 1), 0);
+       wrp_zext(nfp_prog, meta, dst);
        return 0;
 }
 
@@ -2461,7 +2478,7 @@ static int shr_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
        const struct bpf_insn *insn = &meta->insn;
        u8 dst = insn->dst_reg * 2;
 
-       return __shr_imm(nfp_prog, dst, insn->imm);
+       return __shr_imm(nfp_prog, meta, dst, insn->imm);
 }
 
 static int shr_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
@@ -2474,22 +2491,24 @@ static int shr_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
        umin = meta->umin_src;
        umax = meta->umax_src;
        if (umin == umax)
-               return __shr_imm(nfp_prog, dst, umin);
+               return __shr_imm(nfp_prog, meta, dst, umin);
 
        src = insn->src_reg * 2;
        emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_imm(0));
        emit_shf_indir(nfp_prog, reg_both(dst), reg_none(), SHF_OP_NONE,
                       reg_b(dst), SHF_SC_R_SHF);
-       wrp_immed(nfp_prog, reg_both(dst + 1), 0);
+       wrp_zext(nfp_prog, meta, dst);
        return 0;
 }
 
-static int __shl_imm(struct nfp_prog *nfp_prog, u8 dst, u8 shift_amt)
+static int
+__shl_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, u8 dst,
+         u8 shift_amt)
 {
        if (shift_amt)
                emit_shf(nfp_prog, reg_both(dst), reg_none(), SHF_OP_NONE,
                         reg_b(dst), SHF_SC_L_SHF, shift_amt);
-       wrp_immed(nfp_prog, reg_both(dst + 1), 0);
+       wrp_zext(nfp_prog, meta, dst);
        return 0;
 }
 
@@ -2498,7 +2517,7 @@ static int shl_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
        const struct bpf_insn *insn = &meta->insn;
        u8 dst = insn->dst_reg * 2;
 
-       return __shl_imm(nfp_prog, dst, insn->imm);
+       return __shl_imm(nfp_prog, meta, dst, insn->imm);
 }
 
 static int shl_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
@@ -2511,11 +2530,11 @@ static int shl_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
        umin = meta->umin_src;
        umax = meta->umax_src;
        if (umin == umax)
-               return __shl_imm(nfp_prog, dst, umin);
+               return __shl_imm(nfp_prog, meta, dst, umin);
 
        src = insn->src_reg * 2;
        shl_reg64_lt32_low(nfp_prog, dst, src);
-       wrp_immed(nfp_prog, reg_both(dst + 1), 0);
+       wrp_zext(nfp_prog, meta, dst);
        return 0;
 }
 
@@ -2577,34 +2596,34 @@ static int imm_ld8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
 
 static int data_ld1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
 {
-       return construct_data_ld(nfp_prog, meta->insn.imm, 1);
+       return construct_data_ld(nfp_prog, meta, meta->insn.imm, 1);
 }
 
 static int data_ld2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
 {
-       return construct_data_ld(nfp_prog, meta->insn.imm, 2);
+       return construct_data_ld(nfp_prog, meta, meta->insn.imm, 2);
 }
 
 static int data_ld4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
 {
-       return construct_data_ld(nfp_prog, meta->insn.imm, 4);
+       return construct_data_ld(nfp_prog, meta, meta->insn.imm, 4);
 }
 
 static int data_ind_ld1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
 {
-       return construct_data_ind_ld(nfp_prog, meta->insn.imm,
+       return construct_data_ind_ld(nfp_prog, meta, meta->insn.imm,
                                     meta->insn.src_reg * 2, 1);
 }
 
 static int data_ind_ld2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
 {
-       return construct_data_ind_ld(nfp_prog, meta->insn.imm,
+       return construct_data_ind_ld(nfp_prog, meta, meta->insn.imm,
                                     meta->insn.src_reg * 2, 2);
 }
 
 static int data_ind_ld4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
 {
-       return construct_data_ind_ld(nfp_prog, meta->insn.imm,
+       return construct_data_ind_ld(nfp_prog, meta, meta->insn.imm,
                                     meta->insn.src_reg * 2, 4);
 }
 
@@ -2682,7 +2701,7 @@ mem_ldx_data(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
 
        tmp_reg = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog));
 
-       return data_ld_host_order_addr32(nfp_prog, meta->insn.src_reg * 2,
+       return data_ld_host_order_addr32(nfp_prog, meta, meta->insn.src_reg * 2,
                                         tmp_reg, meta->insn.dst_reg * 2, size);
 }
 
@@ -2694,7 +2713,7 @@ mem_ldx_emem(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
 
        tmp_reg = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog));
 
-       return data_ld_host_order_addr40(nfp_prog, meta->insn.src_reg * 2,
+       return data_ld_host_order_addr40(nfp_prog, meta, meta->insn.src_reg * 2,
                                         tmp_reg, meta->insn.dst_reg * 2, size);
 }
 
@@ -2755,7 +2774,7 @@ mem_ldx_data_from_pktcache_unaligned(struct nfp_prog *nfp_prog,
        wrp_reg_subpart(nfp_prog, dst_lo, src_lo, len_lo, off);
 
        if (!len_mid) {
-               wrp_immed(nfp_prog, dst_hi, 0);
+               wrp_zext(nfp_prog, meta, dst_gpr);
                return 0;
        }
 
@@ -2763,7 +2782,7 @@ mem_ldx_data_from_pktcache_unaligned(struct nfp_prog *nfp_prog,
 
        if (size <= REG_WIDTH) {
                wrp_reg_or_subpart(nfp_prog, dst_lo, src_mid, len_mid, len_lo);
-               wrp_immed(nfp_prog, dst_hi, 0);
+               wrp_zext(nfp_prog, meta, dst_gpr);
        } else {
                swreg src_hi = reg_xfer(idx + 2);
 
@@ -2794,10 +2813,10 @@ mem_ldx_data_from_pktcache_aligned(struct nfp_prog *nfp_prog,
 
        if (size < REG_WIDTH) {
                wrp_reg_subpart(nfp_prog, dst_lo, src_lo, size, 0);
-               wrp_immed(nfp_prog, dst_hi, 0);
+               wrp_zext(nfp_prog, meta, dst_gpr);
        } else if (size == REG_WIDTH) {
                wrp_mov(nfp_prog, dst_lo, src_lo);
-               wrp_immed(nfp_prog, dst_hi, 0);
+               wrp_zext(nfp_prog, meta, dst_gpr);
        } else {
                swreg src_hi = reg_xfer(idx + 1);
 

diff --git a/drivers/net/ethernet/netronome/nfp/bpf/main.h b/drivers/net/ethernet/netronome/nfp/bpf/main.h
index e54d1ac84df278737b39a5ee40bded7e1b43111e..57d6ff51e980c00236886d457c65245c2006bfe3 100644 (file)
--- a/drivers/net/ethernet/netronome/nfp/bpf/main.h
+++ b/drivers/net/ethernet/netronome/nfp/bpf/main.h
@@ -238,6 +238,8 @@ struct nfp_bpf_reg_state {
 #define FLAG_INSN_SKIP_PREC_DEPENDENT          BIT(4)
 /* Instruction is optimized by the verifier */
 #define FLAG_INSN_SKIP_VERIFIER_OPT            BIT(5)
+/* Instruction needs to zero extend to high 32-bit */
+#define FLAG_INSN_DO_ZEXT                      BIT(6)
 
 #define FLAG_INSN_SKIP_MASK            (FLAG_INSN_SKIP_NOOP | \
                                         FLAG_INSN_SKIP_PREC_DEPENDENT | \

diff --git a/drivers/net/ethernet/netronome/nfp/bpf/verifier.c b/drivers/net/ethernet/netronome/nfp/bpf/verifier.c
index 36f56eb4cbe2b8dff15d737feb429bbc27fecb17..e92ee510fd52a867ce0c8280503b142371b9830f 100644 (file)
--- a/drivers/net/ethernet/netronome/nfp/bpf/verifier.c
+++ b/drivers/net/ethernet/netronome/nfp/bpf/verifier.c
@@ -744,6 +744,17 @@ static unsigned int nfp_bpf_get_stack_usage(struct nfp_prog *nfp_prog)
        goto continue_subprog;
 }
 
+static void nfp_bpf_insn_flag_zext(struct nfp_prog *nfp_prog,
+                                  struct bpf_insn_aux_data *aux)
+{
+       struct nfp_insn_meta *meta;
+
+       list_for_each_entry(meta, &nfp_prog->insns, l) {
+               if (aux[meta->n].zext_dst)
+                       meta->flags |= FLAG_INSN_DO_ZEXT;
+       }
+}
+
 int nfp_bpf_finalize(struct bpf_verifier_env *env)
 {
        struct bpf_subprog_info *info;
@@ -784,6 +795,7 @@ int nfp_bpf_finalize(struct bpf_verifier_env *env)
                return -EOPNOTSUPP;
        }
 
+       nfp_bpf_insn_flag_zext(nfp_prog, env->insn_aux_data);
        return 0;
 }