1 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
3 * This program is free software; you can redistribute it and/or
4 * modify it under the terms of version 2 of the GNU General Public
5 * License as published by the Free Software Foundation.
7 * This program is distributed in the hope that it will be useful, but
8 * WITHOUT ANY WARRANTY; without even the implied warranty of
9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
10 * General Public License for more details.
12 #include <linux/bpf.h>
13 #include <linux/rcupdate.h>
14 #include <linux/random.h>
15 #include <linux/smp.h>
16 #include <linux/topology.h>
17 #include <linux/ktime.h>
18 #include <linux/sched.h>
19 #include <linux/uidgid.h>
20 #include <linux/filter.h>
21 #include <linux/ctype.h>
23 #include "../../lib/kstrtox.h"
25 /* If kernel subsystem is allowing eBPF programs to call this function,
26 * inside its own verifier_ops->get_func_proto() callback it should return
27 * bpf_map_lookup_elem_proto, so that verifier can properly check the arguments
29 * Different map implementations will rely on rcu in map methods
30 * lookup/update/delete, therefore eBPF programs must run under rcu lock
31 * if program is allowed to access maps, so check rcu_read_lock_held in
32 * all three functions.
34 BPF_CALL_2(bpf_map_lookup_elem, struct bpf_map *, map, void *, key)
36 WARN_ON_ONCE(!rcu_read_lock_held());
37 return (unsigned long) map->ops->map_lookup_elem(map, key);
40 const struct bpf_func_proto bpf_map_lookup_elem_proto = {
41 .func = bpf_map_lookup_elem,
44 .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL,
45 .arg1_type = ARG_CONST_MAP_PTR,
46 .arg2_type = ARG_PTR_TO_MAP_KEY,
49 BPF_CALL_4(bpf_map_update_elem, struct bpf_map *, map, void *, key,
50 void *, value, u64, flags)
52 WARN_ON_ONCE(!rcu_read_lock_held());
53 return map->ops->map_update_elem(map, key, value, flags);
56 const struct bpf_func_proto bpf_map_update_elem_proto = {
57 .func = bpf_map_update_elem,
60 .ret_type = RET_INTEGER,
61 .arg1_type = ARG_CONST_MAP_PTR,
62 .arg2_type = ARG_PTR_TO_MAP_KEY,
63 .arg3_type = ARG_PTR_TO_MAP_VALUE,
64 .arg4_type = ARG_ANYTHING,
67 BPF_CALL_2(bpf_map_delete_elem, struct bpf_map *, map, void *, key)
69 WARN_ON_ONCE(!rcu_read_lock_held());
70 return map->ops->map_delete_elem(map, key);
73 const struct bpf_func_proto bpf_map_delete_elem_proto = {
74 .func = bpf_map_delete_elem,
77 .ret_type = RET_INTEGER,
78 .arg1_type = ARG_CONST_MAP_PTR,
79 .arg2_type = ARG_PTR_TO_MAP_KEY,
82 BPF_CALL_3(bpf_map_push_elem, struct bpf_map *, map, void *, value, u64, flags)
84 return map->ops->map_push_elem(map, value, flags);
87 const struct bpf_func_proto bpf_map_push_elem_proto = {
88 .func = bpf_map_push_elem,
91 .ret_type = RET_INTEGER,
92 .arg1_type = ARG_CONST_MAP_PTR,
93 .arg2_type = ARG_PTR_TO_MAP_VALUE,
94 .arg3_type = ARG_ANYTHING,
97 BPF_CALL_2(bpf_map_pop_elem, struct bpf_map *, map, void *, value)
99 return map->ops->map_pop_elem(map, value);
102 const struct bpf_func_proto bpf_map_pop_elem_proto = {
103 .func = bpf_map_pop_elem,
105 .ret_type = RET_INTEGER,
106 .arg1_type = ARG_CONST_MAP_PTR,
107 .arg2_type = ARG_PTR_TO_UNINIT_MAP_VALUE,
110 BPF_CALL_2(bpf_map_peek_elem, struct bpf_map *, map, void *, value)
112 return map->ops->map_peek_elem(map, value);
115 const struct bpf_func_proto bpf_map_peek_elem_proto = {
116 .func = bpf_map_pop_elem,
118 .ret_type = RET_INTEGER,
119 .arg1_type = ARG_CONST_MAP_PTR,
120 .arg2_type = ARG_PTR_TO_UNINIT_MAP_VALUE,
123 const struct bpf_func_proto bpf_get_prandom_u32_proto = {
124 .func = bpf_user_rnd_u32,
126 .ret_type = RET_INTEGER,
129 BPF_CALL_0(bpf_get_smp_processor_id)
131 return smp_processor_id();
134 const struct bpf_func_proto bpf_get_smp_processor_id_proto = {
135 .func = bpf_get_smp_processor_id,
137 .ret_type = RET_INTEGER,
140 BPF_CALL_0(bpf_get_numa_node_id)
142 return numa_node_id();
145 const struct bpf_func_proto bpf_get_numa_node_id_proto = {
146 .func = bpf_get_numa_node_id,
148 .ret_type = RET_INTEGER,
151 BPF_CALL_0(bpf_ktime_get_ns)
153 /* NMI safe access to clock monotonic */
154 return ktime_get_mono_fast_ns();
157 const struct bpf_func_proto bpf_ktime_get_ns_proto = {
158 .func = bpf_ktime_get_ns,
160 .ret_type = RET_INTEGER,
163 BPF_CALL_0(bpf_get_current_pid_tgid)
165 struct task_struct *task = current;
170 return (u64) task->tgid << 32 | task->pid;
173 const struct bpf_func_proto bpf_get_current_pid_tgid_proto = {
174 .func = bpf_get_current_pid_tgid,
176 .ret_type = RET_INTEGER,
179 BPF_CALL_0(bpf_get_current_uid_gid)
181 struct task_struct *task = current;
188 current_uid_gid(&uid, &gid);
189 return (u64) from_kgid(&init_user_ns, gid) << 32 |
190 from_kuid(&init_user_ns, uid);
193 const struct bpf_func_proto bpf_get_current_uid_gid_proto = {
194 .func = bpf_get_current_uid_gid,
196 .ret_type = RET_INTEGER,
199 BPF_CALL_2(bpf_get_current_comm, char *, buf, u32, size)
201 struct task_struct *task = current;
206 strncpy(buf, task->comm, size);
208 /* Verifier guarantees that size > 0. For task->comm exceeding
209 * size, guarantee that buf is %NUL-terminated. Unconditionally
210 * done here to save the size test.
215 memset(buf, 0, size);
219 const struct bpf_func_proto bpf_get_current_comm_proto = {
220 .func = bpf_get_current_comm,
222 .ret_type = RET_INTEGER,
223 .arg1_type = ARG_PTR_TO_UNINIT_MEM,
224 .arg2_type = ARG_CONST_SIZE,
227 #if defined(CONFIG_QUEUED_SPINLOCKS) || defined(CONFIG_BPF_ARCH_SPINLOCK)
229 static inline void __bpf_spin_lock(struct bpf_spin_lock *lock)
231 arch_spinlock_t *l = (void *)lock;
234 arch_spinlock_t lock;
235 } u = { .lock = __ARCH_SPIN_LOCK_UNLOCKED };
237 compiletime_assert(u.val == 0, "__ARCH_SPIN_LOCK_UNLOCKED not 0");
238 BUILD_BUG_ON(sizeof(*l) != sizeof(__u32));
239 BUILD_BUG_ON(sizeof(*lock) != sizeof(__u32));
243 static inline void __bpf_spin_unlock(struct bpf_spin_lock *lock)
245 arch_spinlock_t *l = (void *)lock;
252 static inline void __bpf_spin_lock(struct bpf_spin_lock *lock)
254 atomic_t *l = (void *)lock;
256 BUILD_BUG_ON(sizeof(*l) != sizeof(*lock));
258 atomic_cond_read_relaxed(l, !VAL);
259 } while (atomic_xchg(l, 1));
262 static inline void __bpf_spin_unlock(struct bpf_spin_lock *lock)
264 atomic_t *l = (void *)lock;
266 atomic_set_release(l, 0);
271 static DEFINE_PER_CPU(unsigned long, irqsave_flags);
273 notrace BPF_CALL_1(bpf_spin_lock, struct bpf_spin_lock *, lock)
277 local_irq_save(flags);
278 __bpf_spin_lock(lock);
279 __this_cpu_write(irqsave_flags, flags);
283 const struct bpf_func_proto bpf_spin_lock_proto = {
284 .func = bpf_spin_lock,
286 .ret_type = RET_VOID,
287 .arg1_type = ARG_PTR_TO_SPIN_LOCK,
290 notrace BPF_CALL_1(bpf_spin_unlock, struct bpf_spin_lock *, lock)
294 flags = __this_cpu_read(irqsave_flags);
295 __bpf_spin_unlock(lock);
296 local_irq_restore(flags);
300 const struct bpf_func_proto bpf_spin_unlock_proto = {
301 .func = bpf_spin_unlock,
303 .ret_type = RET_VOID,
304 .arg1_type = ARG_PTR_TO_SPIN_LOCK,
307 void copy_map_value_locked(struct bpf_map *map, void *dst, void *src,
310 struct bpf_spin_lock *lock;
313 lock = src + map->spin_lock_off;
315 lock = dst + map->spin_lock_off;
317 ____bpf_spin_lock(lock);
318 copy_map_value(map, dst, src);
319 ____bpf_spin_unlock(lock);
323 #ifdef CONFIG_CGROUPS
324 BPF_CALL_0(bpf_get_current_cgroup_id)
326 struct cgroup *cgrp = task_dfl_cgroup(current);
328 return cgrp->kn->id.id;
331 const struct bpf_func_proto bpf_get_current_cgroup_id_proto = {
332 .func = bpf_get_current_cgroup_id,
334 .ret_type = RET_INTEGER,
337 #ifdef CONFIG_CGROUP_BPF
338 DECLARE_PER_CPU(struct bpf_cgroup_storage*,
339 bpf_cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]);
341 BPF_CALL_2(bpf_get_local_storage, struct bpf_map *, map, u64, flags)
343 /* flags argument is not used now,
344 * but provides an ability to extend the API.
345 * verifier checks that its value is correct.
347 enum bpf_cgroup_storage_type stype = cgroup_storage_type(map);
348 struct bpf_cgroup_storage *storage;
351 storage = this_cpu_read(bpf_cgroup_storage[stype]);
353 if (stype == BPF_CGROUP_STORAGE_SHARED)
354 ptr = &READ_ONCE(storage->buf)->data[0];
356 ptr = this_cpu_ptr(storage->percpu_buf);
358 return (unsigned long)ptr;
361 const struct bpf_func_proto bpf_get_local_storage_proto = {
362 .func = bpf_get_local_storage,
364 .ret_type = RET_PTR_TO_MAP_VALUE,
365 .arg1_type = ARG_CONST_MAP_PTR,
366 .arg2_type = ARG_ANYTHING,
370 #define BPF_STRTOX_BASE_MASK 0x1F
372 static int __bpf_strtoull(const char *buf, size_t buf_len, u64 flags,
373 unsigned long long *res, bool *is_negative)
375 unsigned int base = flags & BPF_STRTOX_BASE_MASK;
376 const char *cur_buf = buf;
377 size_t cur_len = buf_len;
378 unsigned int consumed;
382 if (!buf || !buf_len || !res || !is_negative)
385 if (base != 0 && base != 8 && base != 10 && base != 16)
388 if (flags & ~BPF_STRTOX_BASE_MASK)
391 while (cur_buf < buf + buf_len && isspace(*cur_buf))
394 *is_negative = (cur_buf < buf + buf_len && *cur_buf == '-');
398 consumed = cur_buf - buf;
403 cur_len = min(cur_len, sizeof(str) - 1);
404 memcpy(str, cur_buf, cur_len);
408 cur_buf = _parse_integer_fixup_radix(cur_buf, &base);
409 val_len = _parse_integer(cur_buf, base, res);
411 if (val_len & KSTRTOX_OVERFLOW)
418 consumed += cur_buf - str;
423 static int __bpf_strtoll(const char *buf, size_t buf_len, u64 flags,
426 unsigned long long _res;
430 err = __bpf_strtoull(buf, buf_len, flags, &_res, &is_negative);
434 if ((long long)-_res > 0)
438 if ((long long)_res < 0)
445 BPF_CALL_4(bpf_strtol, const char *, buf, size_t, buf_len, u64, flags,
451 err = __bpf_strtoll(buf, buf_len, flags, &_res);
454 if (_res != (long)_res)
460 const struct bpf_func_proto bpf_strtol_proto = {
463 .ret_type = RET_INTEGER,
464 .arg1_type = ARG_PTR_TO_MEM,
465 .arg2_type = ARG_CONST_SIZE,
466 .arg3_type = ARG_ANYTHING,
467 .arg4_type = ARG_PTR_TO_LONG,
470 BPF_CALL_4(bpf_strtoul, const char *, buf, size_t, buf_len, u64, flags,
471 unsigned long *, res)
473 unsigned long long _res;
477 err = __bpf_strtoull(buf, buf_len, flags, &_res, &is_negative);
482 if (_res != (unsigned long)_res)
488 const struct bpf_func_proto bpf_strtoul_proto = {
491 .ret_type = RET_INTEGER,
492 .arg1_type = ARG_PTR_TO_MEM,
493 .arg2_type = ARG_CONST_SIZE,
494 .arg3_type = ARG_ANYTHING,
495 .arg4_type = ARG_PTR_TO_LONG,