bpf: Fix and simplifications on inline map lookup

[linux.git] / kernel / bpf / verifier.c

diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 2990fda1c6a5d4aa8c8460fa628d89771e25a122..9bf82267f2f9795c992043ab2d2360e924fffa2f 100644 (file)
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -143,6 +143,8 @@ struct bpf_verifier_stack_elem {
 #define BPF_COMPLEXITY_LIMIT_INSNS     65536
 #define BPF_COMPLEXITY_LIMIT_STACK     1024
 
+#define BPF_MAP_PTR_POISON ((void *)0xeB9F + POISON_POINTER_DELTA)
+
 struct bpf_call_arg_meta {
        struct bpf_map *map_ptr;
        bool raw_mode;
@@ -1273,7 +1275,7 @@ static void clear_all_pkt_pointers(struct bpf_verifier_env *env)
        }
 }
 
-static int check_call(struct bpf_verifier_env *env, int func_id)
+static int check_call(struct bpf_verifier_env *env, int func_id, int insn_idx)
 {
        struct bpf_verifier_state *state = &env->cur_state;
        const struct bpf_func_proto *fn = NULL;
@@ -1357,6 +1359,8 @@ static int check_call(struct bpf_verifier_env *env, int func_id)
        } else if (fn->ret_type == RET_VOID) {
                regs[BPF_REG_0].type = NOT_INIT;
        } else if (fn->ret_type == RET_PTR_TO_MAP_VALUE_OR_NULL) {
+               struct bpf_insn_aux_data *insn_aux;
+
                regs[BPF_REG_0].type = PTR_TO_MAP_VALUE_OR_NULL;
                regs[BPF_REG_0].max_value = regs[BPF_REG_0].min_value = 0;
                /* remember map_ptr, so that check_map_access()
@@ -1369,6 +1373,11 @@ static int check_call(struct bpf_verifier_env *env, int func_id)
                }
                regs[BPF_REG_0].map_ptr = meta.map_ptr;
                regs[BPF_REG_0].id = ++env->id_gen;
+               insn_aux = &env->insn_aux_data[insn_idx];
+               if (!insn_aux->map_ptr)
+                       insn_aux->map_ptr = meta.map_ptr;
+               else if (insn_aux->map_ptr != meta.map_ptr)
+                       insn_aux->map_ptr = BPF_MAP_PTR_POISON;
        } else {
                verbose("unknown return type %d of func %s#%d\n",
                        fn->ret_type, func_id_name(func_id), func_id);
@@ -2940,7 +2949,7 @@ static int do_check(struct bpf_verifier_env *env)
                                        return -EINVAL;
                                }
 
-                               err = check_call(env, insn->imm);
+                               err = check_call(env, insn->imm, insn_idx);
                                if (err)
                                        return err;
 
@@ -3268,6 +3277,7 @@ static int convert_ctx_accesses(struct bpf_verifier_env *env)
 }
 
 /* fixup insn->imm field of bpf_call instructions
+ * and inline eligible helpers as explicit sequence of BPF instructions
  *
  * this function is called after eBPF program passed verification
  */
@@ -3277,7 +3287,10 @@ static int fixup_bpf_calls(struct bpf_verifier_env *env)
        struct bpf_insn *insn = prog->insnsi;
        const struct bpf_func_proto *fn;
        const int insn_cnt = prog->len;
-       int i;
+       struct bpf_insn insn_buf[16];
+       struct bpf_prog *new_prog;
+       struct bpf_map *map_ptr;
+       int i, cnt, delta = 0;
 
        for (i = 0; i < insn_cnt; i++, insn++) {
                if (insn->code != (BPF_JMP | BPF_CALL))
@@ -3300,6 +3313,32 @@ static int fixup_bpf_calls(struct bpf_verifier_env *env)
                        continue;
                }
 
+               if (ebpf_jit_enabled() && insn->imm == BPF_FUNC_map_lookup_elem) {
+                       map_ptr = env->insn_aux_data[i + delta].map_ptr;
+                       if (map_ptr == BPF_MAP_PTR_POISON ||
+                           !map_ptr->ops->map_gen_lookup)
+                               goto patch_call_imm;
+
+                       cnt = map_ptr->ops->map_gen_lookup(map_ptr, insn_buf);
+                       if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf)) {
+                               verbose("bpf verifier is misconfigured\n");
+                               return -EINVAL;
+                       }
+
+                       new_prog = bpf_patch_insn_data(env, i + delta, insn_buf,
+                                                      cnt);
+                       if (!new_prog)
+                               return -ENOMEM;
+
+                       delta += cnt - 1;
+
+                       /* keep walking new program and skip insns we just inserted */
+                       env->prog = prog = new_prog;
+                       insn      = new_prog->insnsi + i + delta;
+                       continue;
+               }
+
+patch_call_imm:
                fn = prog->aux->ops->get_func_proto(insn->imm);
                /* all functions that have prototype and verifier allowed
                 * programs to call them, must be real in-kernel functions