bpf: properly enforce index mask to prevent out-of-bounds speculation

[linux.git] / kernel / bpf / verifier.c

diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 5dd1dcb902bf445ba50df106ee4aeb129164a4ac..dcebf3f7365c706551123fa52c4e3ffe73847cd3 100644 (file)
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -156,7 +156,29 @@ struct bpf_verifier_stack_elem {
 #define BPF_COMPLEXITY_LIMIT_INSNS     131072
 #define BPF_COMPLEXITY_LIMIT_STACK     1024
 
-#define BPF_MAP_PTR_POISON ((void *)0xeB9F + POISON_POINTER_DELTA)
+#define BPF_MAP_PTR_UNPRIV     1UL
+#define BPF_MAP_PTR_POISON     ((void *)((0xeB9FUL << 1) +     \
+                                         POISON_POINTER_DELTA))
+#define BPF_MAP_PTR(X)         ((struct bpf_map *)((X) & ~BPF_MAP_PTR_UNPRIV))
+
+static bool bpf_map_ptr_poisoned(const struct bpf_insn_aux_data *aux)
+{
+       return BPF_MAP_PTR(aux->map_state) == BPF_MAP_PTR_POISON;
+}
+
+static bool bpf_map_ptr_unpriv(const struct bpf_insn_aux_data *aux)
+{
+       return aux->map_state & BPF_MAP_PTR_UNPRIV;
+}
+
+static void bpf_map_ptr_store(struct bpf_insn_aux_data *aux,
+                             const struct bpf_map *map, bool unpriv)
+{
+       BUILD_BUG_ON((unsigned long)BPF_MAP_PTR_POISON & BPF_MAP_PTR_UNPRIV);
+       unpriv |= bpf_map_ptr_unpriv(aux);
+       aux->map_state = (unsigned long)map |
+                        (unpriv ? BPF_MAP_PTR_UNPRIV : 0UL);
+}
 
 struct bpf_call_arg_meta {
        struct bpf_map *map_ptr;
@@ -2333,6 +2355,29 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx)
        return 0;
 }
 
+static int
+record_func_map(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,
+               int func_id, int insn_idx)
+{
+       struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx];
+
+       if (func_id != BPF_FUNC_tail_call &&
+           func_id != BPF_FUNC_map_lookup_elem)
+               return 0;
+       if (meta->map_ptr == NULL) {
+               verbose(env, "kernel subsystem misconfigured verifier\n");
+               return -EINVAL;
+       }
+
+       if (!BPF_MAP_PTR(aux->map_state))
+               bpf_map_ptr_store(aux, meta->map_ptr,
+                                 meta->map_ptr->unpriv_array);
+       else if (BPF_MAP_PTR(aux->map_state) != meta->map_ptr)
+               bpf_map_ptr_store(aux, BPF_MAP_PTR_POISON,
+                                 meta->map_ptr->unpriv_array);
+       return 0;
+}
+
 static int check_helper_call(struct bpf_verifier_env *env, int func_id, int insn_idx)
 {
        const struct bpf_func_proto *fn = NULL;
@@ -2387,13 +2432,6 @@ static int check_helper_call(struct bpf_verifier_env *env, int func_id, int insn
        err = check_func_arg(env, BPF_REG_2, fn->arg2_type, &meta);
        if (err)
                return err;
-       if (func_id == BPF_FUNC_tail_call) {
-               if (meta.map_ptr == NULL) {
-                       verbose(env, "verifier bug\n");
-                       return -EINVAL;
-               }
-               env->insn_aux_data[insn_idx].map_ptr = meta.map_ptr;
-       }
        err = check_func_arg(env, BPF_REG_3, fn->arg3_type, &meta);
        if (err)
                return err;
@@ -2404,6 +2442,10 @@ static int check_helper_call(struct bpf_verifier_env *env, int func_id, int insn
        if (err)
                return err;
 
+       err = record_func_map(env, &meta, func_id, insn_idx);
+       if (err)
+               return err;
+
        /* Mark slots with STACK_MISC in case of raw mode, stack offset
         * is inferred from register state.
         */
@@ -2428,8 +2470,6 @@ static int check_helper_call(struct bpf_verifier_env *env, int func_id, int insn
        } else if (fn->ret_type == RET_VOID) {
                regs[BPF_REG_0].type = NOT_INIT;
        } else if (fn->ret_type == RET_PTR_TO_MAP_VALUE_OR_NULL) {
-               struct bpf_insn_aux_data *insn_aux;
-
                regs[BPF_REG_0].type = PTR_TO_MAP_VALUE_OR_NULL;
                /* There is no offset yet applied, variable or fixed */
                mark_reg_known_zero(env, regs, BPF_REG_0);
@@ -2445,11 +2485,6 @@ static int check_helper_call(struct bpf_verifier_env *env, int func_id, int insn
                }
                regs[BPF_REG_0].map_ptr = meta.map_ptr;
                regs[BPF_REG_0].id = ++env->id_gen;
-               insn_aux = &env->insn_aux_data[insn_idx];
-               if (!insn_aux->map_ptr)
-                       insn_aux->map_ptr = meta.map_ptr;
-               else if (insn_aux->map_ptr != meta.map_ptr)
-                       insn_aux->map_ptr = BPF_MAP_PTR_POISON;
        } else {
                verbose(env, "unknown return type %d of func %s#%d\n",
                        fn->ret_type, func_id_name(func_id), func_id);
@@ -5417,6 +5452,7 @@ static int fixup_bpf_calls(struct bpf_verifier_env *env)
        struct bpf_insn *insn = prog->insnsi;
        const struct bpf_func_proto *fn;
        const int insn_cnt = prog->len;
+       struct bpf_insn_aux_data *aux;
        struct bpf_insn insn_buf[16];
        struct bpf_prog *new_prog;
        struct bpf_map *map_ptr;
@@ -5491,19 +5527,22 @@ static int fixup_bpf_calls(struct bpf_verifier_env *env)
                        insn->imm = 0;
                        insn->code = BPF_JMP | BPF_TAIL_CALL;
 
+                       aux = &env->insn_aux_data[i + delta];
+                       if (!bpf_map_ptr_unpriv(aux))
+                               continue;
+
                        /* instead of changing every JIT dealing with tail_call
                         * emit two extra insns:
                         * if (index >= max_entries) goto out;
                         * index &= array->index_mask;
                         * to avoid out-of-bounds cpu speculation
                         */
-                       map_ptr = env->insn_aux_data[i + delta].map_ptr;
-                       if (map_ptr == BPF_MAP_PTR_POISON) {
+                       if (bpf_map_ptr_poisoned(aux)) {
                                verbose(env, "tail_call abusing map_ptr\n");
                                return -EINVAL;
                        }
-                       if (!map_ptr->unpriv_array)
-                               continue;
+
+                       map_ptr = BPF_MAP_PTR(aux->map_state);
                        insn_buf[0] = BPF_JMP_IMM(BPF_JGE, BPF_REG_3,
                                                  map_ptr->max_entries, 2);
                        insn_buf[1] = BPF_ALU32_IMM(BPF_AND, BPF_REG_3,
@@ -5527,9 +5566,12 @@ static int fixup_bpf_calls(struct bpf_verifier_env *env)
                 */
                if (prog->jit_requested && BITS_PER_LONG == 64 &&
                    insn->imm == BPF_FUNC_map_lookup_elem) {
-                       map_ptr = env->insn_aux_data[i + delta].map_ptr;
-                       if (map_ptr == BPF_MAP_PTR_POISON ||
-                           !map_ptr->ops->map_gen_lookup)
+                       aux = &env->insn_aux_data[i + delta];
+                       if (bpf_map_ptr_poisoned(aux))
+                               goto patch_call_imm;
+
+                       map_ptr = BPF_MAP_PTR(aux->map_state);
+                       if (!map_ptr->ops->map_gen_lookup)
                                goto patch_call_imm;
 
                        cnt = map_ptr->ops->map_gen_lookup(map_ptr, insn_buf);