1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (c) 2019 Facebook */
5 #include <linux/bpf_verifier.h>
7 #include <linux/filter.h>
8 #include <linux/slab.h>
9 #include <linux/numa.h>
10 #include <linux/seq_file.h>
11 #include <linux/refcount.h>
12 #include <linux/mutex.h>
14 enum bpf_struct_ops_state {
15 BPF_STRUCT_OPS_STATE_INIT,
16 BPF_STRUCT_OPS_STATE_INUSE,
17 BPF_STRUCT_OPS_STATE_TOBEFREE,
20 #define BPF_STRUCT_OPS_COMMON_VALUE \
22 enum bpf_struct_ops_state state
24 struct bpf_struct_ops_value {
25 BPF_STRUCT_OPS_COMMON_VALUE;
26 char data[0] ____cacheline_aligned_in_smp;
29 struct bpf_struct_ops_map {
31 const struct bpf_struct_ops *st_ops;
32 /* protect map_update */
34 /* progs has all the bpf_prog that is populated
35 * to the func ptr of the kernel's struct
38 struct bpf_prog **progs;
39 /* image is a page that has all the trampolines
40 * that stores the func args before calling the bpf_prog.
41 * A PAGE_SIZE "image" is enough to store all trampoline for
45 /* uvalue->data stores the kernel struct
46 * (e.g. tcp_congestion_ops) that is more useful
47 * to userspace than the kvalue. For example,
48 * the bpf_prog's id is stored instead of the kernel
49 * address of a func ptr.
51 struct bpf_struct_ops_value *uvalue;
52 /* kvalue.data stores the actual kernel's struct
53 * (e.g. tcp_congestion_ops) that will be
54 * registered to the kernel subsystem.
56 struct bpf_struct_ops_value kvalue;
59 #define VALUE_PREFIX "bpf_struct_ops_"
60 #define VALUE_PREFIX_LEN (sizeof(VALUE_PREFIX) - 1)
62 /* bpf_struct_ops_##_name (e.g. bpf_struct_ops_tcp_congestion_ops) is
63 * the map's value exposed to the userspace and its btf-type-id is
64 * stored at the map->btf_vmlinux_value_type_id.
67 #define BPF_STRUCT_OPS_TYPE(_name) \
68 extern struct bpf_struct_ops bpf_##_name; \
70 struct bpf_struct_ops_##_name { \
71 BPF_STRUCT_OPS_COMMON_VALUE; \
72 struct _name data ____cacheline_aligned_in_smp; \
74 #include "bpf_struct_ops_types.h"
75 #undef BPF_STRUCT_OPS_TYPE
78 #define BPF_STRUCT_OPS_TYPE(_name) BPF_STRUCT_OPS_TYPE_##_name,
79 #include "bpf_struct_ops_types.h"
80 #undef BPF_STRUCT_OPS_TYPE
81 __NR_BPF_STRUCT_OPS_TYPE,
84 static struct bpf_struct_ops * const bpf_struct_ops[] = {
85 #define BPF_STRUCT_OPS_TYPE(_name) \
86 [BPF_STRUCT_OPS_TYPE_##_name] = &bpf_##_name,
87 #include "bpf_struct_ops_types.h"
88 #undef BPF_STRUCT_OPS_TYPE
91 const struct bpf_verifier_ops bpf_struct_ops_verifier_ops = {
94 const struct bpf_prog_ops bpf_struct_ops_prog_ops = {
97 static const struct btf_type *module_type;
99 void bpf_struct_ops_init(struct btf *btf)
101 s32 type_id, value_id, module_id;
102 const struct btf_member *member;
103 struct bpf_struct_ops *st_ops;
104 struct bpf_verifier_log log = {};
105 const struct btf_type *t;
106 char value_name[128];
110 /* Ensure BTF type is emitted for "struct bpf_struct_ops_##_name" */
111 #define BPF_STRUCT_OPS_TYPE(_name) BTF_TYPE_EMIT(struct bpf_struct_ops_##_name);
112 #include "bpf_struct_ops_types.h"
113 #undef BPF_STRUCT_OPS_TYPE
115 module_id = btf_find_by_name_kind(btf, "module", BTF_KIND_STRUCT);
117 pr_warn("Cannot find struct module in btf_vmlinux\n");
120 module_type = btf_type_by_id(btf, module_id);
122 for (i = 0; i < ARRAY_SIZE(bpf_struct_ops); i++) {
123 st_ops = bpf_struct_ops[i];
125 if (strlen(st_ops->name) + VALUE_PREFIX_LEN >=
126 sizeof(value_name)) {
127 pr_warn("struct_ops name %s is too long\n",
131 sprintf(value_name, "%s%s", VALUE_PREFIX, st_ops->name);
133 value_id = btf_find_by_name_kind(btf, value_name,
136 pr_warn("Cannot find struct %s in btf_vmlinux\n",
141 type_id = btf_find_by_name_kind(btf, st_ops->name,
144 pr_warn("Cannot find struct %s in btf_vmlinux\n",
148 t = btf_type_by_id(btf, type_id);
149 if (btf_type_vlen(t) > BPF_STRUCT_OPS_MAX_NR_MEMBERS) {
150 pr_warn("Cannot support #%u members in struct %s\n",
151 btf_type_vlen(t), st_ops->name);
155 for_each_member(j, t, member) {
156 const struct btf_type *func_proto;
158 mname = btf_name_by_offset(btf, member->name_off);
160 pr_warn("anon member in struct %s is not supported\n",
165 if (btf_member_bitfield_size(t, member)) {
166 pr_warn("bit field member %s in struct %s is not supported\n",
167 mname, st_ops->name);
171 func_proto = btf_type_resolve_func_ptr(btf,
175 btf_distill_func_proto(&log, btf,
177 &st_ops->func_models[j])) {
178 pr_warn("Error in parsing func ptr %s in struct %s\n",
179 mname, st_ops->name);
184 if (j == btf_type_vlen(t)) {
185 if (st_ops->init(btf)) {
186 pr_warn("Error in init bpf_struct_ops %s\n",
189 st_ops->type_id = type_id;
191 st_ops->value_id = value_id;
192 st_ops->value_type = btf_type_by_id(btf,
199 extern struct btf *btf_vmlinux;
201 static const struct bpf_struct_ops *
202 bpf_struct_ops_find_value(u32 value_id)
206 if (!value_id || !btf_vmlinux)
209 for (i = 0; i < ARRAY_SIZE(bpf_struct_ops); i++) {
210 if (bpf_struct_ops[i]->value_id == value_id)
211 return bpf_struct_ops[i];
217 const struct bpf_struct_ops *bpf_struct_ops_find(u32 type_id)
221 if (!type_id || !btf_vmlinux)
224 for (i = 0; i < ARRAY_SIZE(bpf_struct_ops); i++) {
225 if (bpf_struct_ops[i]->type_id == type_id)
226 return bpf_struct_ops[i];
232 static int bpf_struct_ops_map_get_next_key(struct bpf_map *map, void *key,
235 if (key && *(u32 *)key == 0)
238 *(u32 *)next_key = 0;
242 int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map, void *key,
245 struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;
246 struct bpf_struct_ops_value *uvalue, *kvalue;
247 enum bpf_struct_ops_state state;
249 if (unlikely(*(u32 *)key != 0))
252 kvalue = &st_map->kvalue;
253 /* Pair with smp_store_release() during map_update */
254 state = smp_load_acquire(&kvalue->state);
255 if (state == BPF_STRUCT_OPS_STATE_INIT) {
256 memset(value, 0, map->value_size);
260 /* No lock is needed. state and refcnt do not need
261 * to be updated together under atomic context.
263 uvalue = (struct bpf_struct_ops_value *)value;
264 memcpy(uvalue, st_map->uvalue, map->value_size);
265 uvalue->state = state;
266 refcount_set(&uvalue->refcnt, refcount_read(&kvalue->refcnt));
271 static void *bpf_struct_ops_map_lookup_elem(struct bpf_map *map, void *key)
273 return ERR_PTR(-EINVAL);
276 static void bpf_struct_ops_map_put_progs(struct bpf_struct_ops_map *st_map)
278 const struct btf_type *t = st_map->st_ops->type;
281 for (i = 0; i < btf_type_vlen(t); i++) {
282 if (st_map->progs[i]) {
283 bpf_prog_put(st_map->progs[i]);
284 st_map->progs[i] = NULL;
289 static int check_zero_holes(const struct btf_type *t, void *data)
291 const struct btf_member *member;
292 u32 i, moff, msize, prev_mend = 0;
293 const struct btf_type *mtype;
295 for_each_member(i, t, member) {
296 moff = btf_member_bit_offset(t, member) / 8;
297 if (moff > prev_mend &&
298 memchr_inv(data + prev_mend, 0, moff - prev_mend))
301 mtype = btf_type_by_id(btf_vmlinux, member->type);
302 mtype = btf_resolve_size(btf_vmlinux, mtype, &msize,
305 return PTR_ERR(mtype);
306 prev_mend = moff + msize;
309 if (t->size > prev_mend &&
310 memchr_inv(data + prev_mend, 0, t->size - prev_mend))
316 static int bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key,
317 void *value, u64 flags)
319 struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;
320 const struct bpf_struct_ops *st_ops = st_map->st_ops;
321 struct bpf_struct_ops_value *uvalue, *kvalue;
322 const struct btf_member *member;
323 const struct btf_type *t = st_ops->type;
325 int prog_fd, err = 0;
332 if (*(u32 *)key != 0)
335 err = check_zero_holes(st_ops->value_type, value);
339 uvalue = (struct bpf_struct_ops_value *)value;
340 err = check_zero_holes(t, uvalue->data);
344 if (uvalue->state || refcount_read(&uvalue->refcnt))
347 uvalue = (struct bpf_struct_ops_value *)st_map->uvalue;
348 kvalue = (struct bpf_struct_ops_value *)&st_map->kvalue;
350 mutex_lock(&st_map->lock);
352 if (kvalue->state != BPF_STRUCT_OPS_STATE_INIT) {
357 memcpy(uvalue, value, map->value_size);
359 udata = &uvalue->data;
360 kdata = &kvalue->data;
361 image = st_map->image;
363 for_each_member(i, t, member) {
364 const struct btf_type *mtype, *ptype;
365 struct bpf_prog *prog;
368 moff = btf_member_bit_offset(t, member) / 8;
369 ptype = btf_type_resolve_ptr(btf_vmlinux, member->type, NULL);
370 if (ptype == module_type) {
371 if (*(void **)(udata + moff))
373 *(void **)(kdata + moff) = BPF_MODULE_OWNER;
377 err = st_ops->init_member(t, member, kdata, udata);
381 /* The ->init_member() has handled this member */
385 /* If st_ops->init_member does not handle it,
386 * we will only handle func ptrs and zero-ed members
387 * here. Reject everything else.
390 /* All non func ptr member must be 0 */
391 if (!ptype || !btf_type_is_func_proto(ptype)) {
394 mtype = btf_type_by_id(btf_vmlinux, member->type);
395 mtype = btf_resolve_size(btf_vmlinux, mtype, &msize,
398 err = PTR_ERR(mtype);
402 if (memchr_inv(udata + moff, 0, msize)) {
410 prog_fd = (int)(*(unsigned long *)(udata + moff));
411 /* Similar check as the attr->attach_prog_fd */
415 prog = bpf_prog_get(prog_fd);
420 st_map->progs[i] = prog;
422 if (prog->type != BPF_PROG_TYPE_STRUCT_OPS ||
423 prog->aux->attach_btf_id != st_ops->type_id ||
424 prog->expected_attach_type != i) {
429 err = arch_prepare_bpf_trampoline(image,
430 st_map->image + PAGE_SIZE,
431 &st_ops->func_models[i], 0,
432 &prog, 1, NULL, 0, NULL);
436 *(void **)(kdata + moff) = image;
439 /* put prog_id to udata */
440 *(unsigned long *)(udata + moff) = prog->aux->id;
443 refcount_set(&kvalue->refcnt, 1);
446 set_memory_ro((long)st_map->image, 1);
447 set_memory_x((long)st_map->image, 1);
448 err = st_ops->reg(kdata);
450 /* Pair with smp_load_acquire() during lookup_elem().
451 * It ensures the above udata updates (e.g. prog->aux->id)
452 * can be seen once BPF_STRUCT_OPS_STATE_INUSE is set.
454 smp_store_release(&kvalue->state, BPF_STRUCT_OPS_STATE_INUSE);
458 /* Error during st_ops->reg(). It is very unlikely since
459 * the above init_member() should have caught it earlier
460 * before reg(). The only possibility is if there was a race
461 * in registering the struct_ops (under the same name) to
462 * a sub-system through different struct_ops's maps.
464 set_memory_nx((long)st_map->image, 1);
465 set_memory_rw((long)st_map->image, 1);
469 bpf_struct_ops_map_put_progs(st_map);
470 memset(uvalue, 0, map->value_size);
471 memset(kvalue, 0, map->value_size);
473 mutex_unlock(&st_map->lock);
477 static int bpf_struct_ops_map_delete_elem(struct bpf_map *map, void *key)
479 enum bpf_struct_ops_state prev_state;
480 struct bpf_struct_ops_map *st_map;
482 st_map = (struct bpf_struct_ops_map *)map;
483 prev_state = cmpxchg(&st_map->kvalue.state,
484 BPF_STRUCT_OPS_STATE_INUSE,
485 BPF_STRUCT_OPS_STATE_TOBEFREE);
486 if (prev_state == BPF_STRUCT_OPS_STATE_INUSE) {
487 st_map->st_ops->unreg(&st_map->kvalue.data);
488 if (refcount_dec_and_test(&st_map->kvalue.refcnt))
495 static void bpf_struct_ops_map_seq_show_elem(struct bpf_map *map, void *key,
501 value = kmalloc(map->value_size, GFP_USER | __GFP_NOWARN);
505 err = bpf_struct_ops_map_sys_lookup_elem(map, key, value);
507 btf_type_seq_show(btf_vmlinux, map->btf_vmlinux_value_type_id,
515 static void bpf_struct_ops_map_free(struct bpf_map *map)
517 struct bpf_struct_ops_map *st_map = (struct bpf_struct_ops_map *)map;
520 bpf_struct_ops_map_put_progs(st_map);
521 bpf_map_area_free(st_map->progs);
522 bpf_jit_free_exec(st_map->image);
523 bpf_map_area_free(st_map->uvalue);
524 bpf_map_area_free(st_map);
527 static int bpf_struct_ops_map_alloc_check(union bpf_attr *attr)
529 if (attr->key_size != sizeof(unsigned int) || attr->max_entries != 1 ||
530 attr->map_flags || !attr->btf_vmlinux_value_type_id)
535 static struct bpf_map *bpf_struct_ops_map_alloc(union bpf_attr *attr)
537 const struct bpf_struct_ops *st_ops;
538 size_t map_total_size, st_map_size;
539 struct bpf_struct_ops_map *st_map;
540 const struct btf_type *t, *vt;
541 struct bpf_map_memory mem;
545 if (!capable(CAP_SYS_ADMIN))
546 return ERR_PTR(-EPERM);
548 st_ops = bpf_struct_ops_find_value(attr->btf_vmlinux_value_type_id);
550 return ERR_PTR(-ENOTSUPP);
552 vt = st_ops->value_type;
553 if (attr->value_size != vt->size)
554 return ERR_PTR(-EINVAL);
558 st_map_size = sizeof(*st_map) +
560 * struct bpf_struct_ops_tcp_congestions_ops
562 (vt->size - sizeof(struct bpf_struct_ops_value));
563 map_total_size = st_map_size +
566 /* struct bpf_progs **progs */
567 btf_type_vlen(t) * sizeof(struct bpf_prog *);
568 err = bpf_map_charge_init(&mem, map_total_size);
572 st_map = bpf_map_area_alloc(st_map_size, NUMA_NO_NODE);
574 bpf_map_charge_finish(&mem);
575 return ERR_PTR(-ENOMEM);
577 st_map->st_ops = st_ops;
580 st_map->uvalue = bpf_map_area_alloc(vt->size, NUMA_NO_NODE);
582 bpf_map_area_alloc(btf_type_vlen(t) * sizeof(struct bpf_prog *),
584 st_map->image = bpf_jit_alloc_exec(PAGE_SIZE);
585 if (!st_map->uvalue || !st_map->progs || !st_map->image) {
586 bpf_struct_ops_map_free(map);
587 bpf_map_charge_finish(&mem);
588 return ERR_PTR(-ENOMEM);
591 mutex_init(&st_map->lock);
592 set_vm_flush_reset_perms(st_map->image);
593 bpf_map_init_from_attr(map, attr);
594 bpf_map_charge_move(&map->memory, &mem);
599 const struct bpf_map_ops bpf_struct_ops_map_ops = {
600 .map_alloc_check = bpf_struct_ops_map_alloc_check,
601 .map_alloc = bpf_struct_ops_map_alloc,
602 .map_free = bpf_struct_ops_map_free,
603 .map_get_next_key = bpf_struct_ops_map_get_next_key,
604 .map_lookup_elem = bpf_struct_ops_map_lookup_elem,
605 .map_delete_elem = bpf_struct_ops_map_delete_elem,
606 .map_update_elem = bpf_struct_ops_map_update_elem,
607 .map_seq_show_elem = bpf_struct_ops_map_seq_show_elem,
610 /* "const void *" because some subsystem is
611 * passing a const (e.g. const struct tcp_congestion_ops *)
613 bool bpf_struct_ops_get(const void *kdata)
615 struct bpf_struct_ops_value *kvalue;
617 kvalue = container_of(kdata, struct bpf_struct_ops_value, data);
619 return refcount_inc_not_zero(&kvalue->refcnt);
622 void bpf_struct_ops_put(const void *kdata)
624 struct bpf_struct_ops_value *kvalue;
626 kvalue = container_of(kdata, struct bpf_struct_ops_value, data);
627 if (refcount_dec_and_test(&kvalue->refcnt)) {
628 struct bpf_struct_ops_map *st_map;
630 st_map = container_of(kvalue, struct bpf_struct_ops_map,
632 bpf_map_put(&st_map->map);