1 /* SPDX-License-Identifier: GPL-2.0 */
6 #include <linux/errno.h>
7 #include <linux/jump_label.h>
8 #include <linux/percpu.h>
9 #include <linux/rbtree.h>
10 #include <uapi/linux/bpf.h>
18 struct bpf_sock_ops_kern;
19 struct bpf_cgroup_storage;
21 #ifdef CONFIG_CGROUP_BPF
23 extern struct static_key_false cgroup_bpf_enabled_key;
24 #define cgroup_bpf_enabled static_branch_unlikely(&cgroup_bpf_enabled_key)
26 DECLARE_PER_CPU(struct bpf_cgroup_storage*,
27 bpf_cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]);
29 #define for_each_cgroup_storage_type(stype) \
30 for (stype = 0; stype < MAX_BPF_CGROUP_STORAGE_TYPE; stype++)
32 struct bpf_cgroup_storage_map;
34 struct bpf_storage_buffer {
39 struct bpf_cgroup_storage {
41 struct bpf_storage_buffer *buf;
42 void __percpu *percpu_buf;
44 struct bpf_cgroup_storage_map *map;
45 struct bpf_cgroup_storage_key key;
46 struct list_head list;
51 struct bpf_prog_list {
52 struct list_head node;
53 struct bpf_prog *prog;
54 struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE];
57 struct bpf_prog_array;
60 /* array of effective progs in this cgroup */
61 struct bpf_prog_array __rcu *effective[MAX_BPF_ATTACH_TYPE];
63 /* attached progs to this cgroup and attach flags
64 * when flags == 0 or BPF_F_ALLOW_OVERRIDE the progs list will
65 * have either zero or one element
66 * when BPF_F_ALLOW_MULTI the list can have up to BPF_CGROUP_MAX_PROGS
68 struct list_head progs[MAX_BPF_ATTACH_TYPE];
69 u32 flags[MAX_BPF_ATTACH_TYPE];
71 /* temp storage for effective prog array used by prog_attach/detach */
72 struct bpf_prog_array __rcu *inactive;
75 void cgroup_bpf_put(struct cgroup *cgrp);
76 int cgroup_bpf_inherit(struct cgroup *cgrp);
78 int __cgroup_bpf_attach(struct cgroup *cgrp, struct bpf_prog *prog,
79 enum bpf_attach_type type, u32 flags);
80 int __cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog,
81 enum bpf_attach_type type, u32 flags);
82 int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr,
83 union bpf_attr __user *uattr);
85 /* Wrapper for __cgroup_bpf_*() protected by cgroup_mutex */
86 int cgroup_bpf_attach(struct cgroup *cgrp, struct bpf_prog *prog,
87 enum bpf_attach_type type, u32 flags);
88 int cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog,
89 enum bpf_attach_type type, u32 flags);
90 int cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr,
91 union bpf_attr __user *uattr);
93 int __cgroup_bpf_run_filter_skb(struct sock *sk,
95 enum bpf_attach_type type);
97 int __cgroup_bpf_run_filter_sk(struct sock *sk,
98 enum bpf_attach_type type);
100 int __cgroup_bpf_run_filter_sock_addr(struct sock *sk,
101 struct sockaddr *uaddr,
102 enum bpf_attach_type type,
105 int __cgroup_bpf_run_filter_sock_ops(struct sock *sk,
106 struct bpf_sock_ops_kern *sock_ops,
107 enum bpf_attach_type type);
109 int __cgroup_bpf_check_dev_permission(short dev_type, u32 major, u32 minor,
110 short access, enum bpf_attach_type type);
112 static inline enum bpf_cgroup_storage_type cgroup_storage_type(
115 if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE)
116 return BPF_CGROUP_STORAGE_PERCPU;
118 return BPF_CGROUP_STORAGE_SHARED;
121 static inline void bpf_cgroup_storage_set(struct bpf_cgroup_storage
122 *storage[MAX_BPF_CGROUP_STORAGE_TYPE])
124 enum bpf_cgroup_storage_type stype;
126 for_each_cgroup_storage_type(stype)
127 this_cpu_write(bpf_cgroup_storage[stype], storage[stype]);
130 struct bpf_cgroup_storage *bpf_cgroup_storage_alloc(struct bpf_prog *prog,
131 enum bpf_cgroup_storage_type stype);
132 void bpf_cgroup_storage_free(struct bpf_cgroup_storage *storage);
133 void bpf_cgroup_storage_link(struct bpf_cgroup_storage *storage,
134 struct cgroup *cgroup,
135 enum bpf_attach_type type);
136 void bpf_cgroup_storage_unlink(struct bpf_cgroup_storage *storage);
137 int bpf_cgroup_storage_assign(struct bpf_prog *prog, struct bpf_map *map);
138 void bpf_cgroup_storage_release(struct bpf_prog *prog, struct bpf_map *map);
140 int bpf_percpu_cgroup_storage_copy(struct bpf_map *map, void *key, void *value);
141 int bpf_percpu_cgroup_storage_update(struct bpf_map *map, void *key,
142 void *value, u64 flags);
144 /* Wrappers for __cgroup_bpf_run_filter_skb() guarded by cgroup_bpf_enabled. */
145 #define BPF_CGROUP_RUN_PROG_INET_INGRESS(sk, skb) \
148 if (cgroup_bpf_enabled) \
149 __ret = __cgroup_bpf_run_filter_skb(sk, skb, \
150 BPF_CGROUP_INET_INGRESS); \
155 #define BPF_CGROUP_RUN_PROG_INET_EGRESS(sk, skb) \
158 if (cgroup_bpf_enabled && sk && sk == skb->sk) { \
159 typeof(sk) __sk = sk_to_full_sk(sk); \
160 if (sk_fullsock(__sk)) \
161 __ret = __cgroup_bpf_run_filter_skb(__sk, skb, \
162 BPF_CGROUP_INET_EGRESS); \
167 #define BPF_CGROUP_RUN_SK_PROG(sk, type) \
170 if (cgroup_bpf_enabled) { \
171 __ret = __cgroup_bpf_run_filter_sk(sk, type); \
176 #define BPF_CGROUP_RUN_PROG_INET_SOCK(sk) \
177 BPF_CGROUP_RUN_SK_PROG(sk, BPF_CGROUP_INET_SOCK_CREATE)
179 #define BPF_CGROUP_RUN_PROG_INET4_POST_BIND(sk) \
180 BPF_CGROUP_RUN_SK_PROG(sk, BPF_CGROUP_INET4_POST_BIND)
182 #define BPF_CGROUP_RUN_PROG_INET6_POST_BIND(sk) \
183 BPF_CGROUP_RUN_SK_PROG(sk, BPF_CGROUP_INET6_POST_BIND)
185 #define BPF_CGROUP_RUN_SA_PROG(sk, uaddr, type) \
188 if (cgroup_bpf_enabled) \
189 __ret = __cgroup_bpf_run_filter_sock_addr(sk, uaddr, type, \
194 #define BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, type, t_ctx) \
197 if (cgroup_bpf_enabled) { \
199 __ret = __cgroup_bpf_run_filter_sock_addr(sk, uaddr, type, \
206 #define BPF_CGROUP_RUN_PROG_INET4_BIND(sk, uaddr) \
207 BPF_CGROUP_RUN_SA_PROG(sk, uaddr, BPF_CGROUP_INET4_BIND)
209 #define BPF_CGROUP_RUN_PROG_INET6_BIND(sk, uaddr) \
210 BPF_CGROUP_RUN_SA_PROG(sk, uaddr, BPF_CGROUP_INET6_BIND)
212 #define BPF_CGROUP_PRE_CONNECT_ENABLED(sk) (cgroup_bpf_enabled && \
213 sk->sk_prot->pre_connect)
215 #define BPF_CGROUP_RUN_PROG_INET4_CONNECT(sk, uaddr) \
216 BPF_CGROUP_RUN_SA_PROG(sk, uaddr, BPF_CGROUP_INET4_CONNECT)
218 #define BPF_CGROUP_RUN_PROG_INET6_CONNECT(sk, uaddr) \
219 BPF_CGROUP_RUN_SA_PROG(sk, uaddr, BPF_CGROUP_INET6_CONNECT)
221 #define BPF_CGROUP_RUN_PROG_INET4_CONNECT_LOCK(sk, uaddr) \
222 BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, BPF_CGROUP_INET4_CONNECT, NULL)
224 #define BPF_CGROUP_RUN_PROG_INET6_CONNECT_LOCK(sk, uaddr) \
225 BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, BPF_CGROUP_INET6_CONNECT, NULL)
227 #define BPF_CGROUP_RUN_PROG_UDP4_SENDMSG_LOCK(sk, uaddr, t_ctx) \
228 BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, BPF_CGROUP_UDP4_SENDMSG, t_ctx)
230 #define BPF_CGROUP_RUN_PROG_UDP6_SENDMSG_LOCK(sk, uaddr, t_ctx) \
231 BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, BPF_CGROUP_UDP6_SENDMSG, t_ctx)
233 #define BPF_CGROUP_RUN_PROG_SOCK_OPS(sock_ops) \
236 if (cgroup_bpf_enabled && (sock_ops)->sk) { \
237 typeof(sk) __sk = sk_to_full_sk((sock_ops)->sk); \
238 if (__sk && sk_fullsock(__sk)) \
239 __ret = __cgroup_bpf_run_filter_sock_ops(__sk, \
241 BPF_CGROUP_SOCK_OPS); \
246 #define BPF_CGROUP_RUN_PROG_DEVICE_CGROUP(type, major, minor, access) \
249 if (cgroup_bpf_enabled) \
250 __ret = __cgroup_bpf_check_dev_permission(type, major, minor, \
252 BPF_CGROUP_DEVICE); \
256 int cgroup_bpf_prog_attach(const union bpf_attr *attr,
257 enum bpf_prog_type ptype, struct bpf_prog *prog);
258 int cgroup_bpf_prog_detach(const union bpf_attr *attr,
259 enum bpf_prog_type ptype);
260 int cgroup_bpf_prog_query(const union bpf_attr *attr,
261 union bpf_attr __user *uattr);
265 struct cgroup_bpf {};
266 static inline void cgroup_bpf_put(struct cgroup *cgrp) {}
267 static inline int cgroup_bpf_inherit(struct cgroup *cgrp) { return 0; }
269 static inline int cgroup_bpf_prog_attach(const union bpf_attr *attr,
270 enum bpf_prog_type ptype,
271 struct bpf_prog *prog)
276 static inline int cgroup_bpf_prog_detach(const union bpf_attr *attr,
277 enum bpf_prog_type ptype)
282 static inline int cgroup_bpf_prog_query(const union bpf_attr *attr,
283 union bpf_attr __user *uattr)
288 static inline void bpf_cgroup_storage_set(
289 struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE]) {}
290 static inline int bpf_cgroup_storage_assign(struct bpf_prog *prog,
291 struct bpf_map *map) { return 0; }
292 static inline void bpf_cgroup_storage_release(struct bpf_prog *prog,
293 struct bpf_map *map) {}
294 static inline struct bpf_cgroup_storage *bpf_cgroup_storage_alloc(
295 struct bpf_prog *prog, enum bpf_cgroup_storage_type stype) { return 0; }
296 static inline void bpf_cgroup_storage_free(
297 struct bpf_cgroup_storage *storage) {}
298 static inline int bpf_percpu_cgroup_storage_copy(struct bpf_map *map, void *key,
302 static inline int bpf_percpu_cgroup_storage_update(struct bpf_map *map,
303 void *key, void *value, u64 flags) {
307 #define cgroup_bpf_enabled (0)
308 #define BPF_CGROUP_PRE_CONNECT_ENABLED(sk) (0)
309 #define BPF_CGROUP_RUN_PROG_INET_INGRESS(sk,skb) ({ 0; })
310 #define BPF_CGROUP_RUN_PROG_INET_EGRESS(sk,skb) ({ 0; })
311 #define BPF_CGROUP_RUN_PROG_INET_SOCK(sk) ({ 0; })
312 #define BPF_CGROUP_RUN_PROG_INET4_BIND(sk, uaddr) ({ 0; })
313 #define BPF_CGROUP_RUN_PROG_INET6_BIND(sk, uaddr) ({ 0; })
314 #define BPF_CGROUP_RUN_PROG_INET4_POST_BIND(sk) ({ 0; })
315 #define BPF_CGROUP_RUN_PROG_INET6_POST_BIND(sk) ({ 0; })
316 #define BPF_CGROUP_RUN_PROG_INET4_CONNECT(sk, uaddr) ({ 0; })
317 #define BPF_CGROUP_RUN_PROG_INET4_CONNECT_LOCK(sk, uaddr) ({ 0; })
318 #define BPF_CGROUP_RUN_PROG_INET6_CONNECT(sk, uaddr) ({ 0; })
319 #define BPF_CGROUP_RUN_PROG_INET6_CONNECT_LOCK(sk, uaddr) ({ 0; })
320 #define BPF_CGROUP_RUN_PROG_UDP4_SENDMSG_LOCK(sk, uaddr, t_ctx) ({ 0; })
321 #define BPF_CGROUP_RUN_PROG_UDP6_SENDMSG_LOCK(sk, uaddr, t_ctx) ({ 0; })
322 #define BPF_CGROUP_RUN_PROG_SOCK_OPS(sock_ops) ({ 0; })
323 #define BPF_CGROUP_RUN_PROG_DEVICE_CGROUP(type,major,minor,access) ({ 0; })
325 #define for_each_cgroup_storage_type(stype) for (; false; )
327 #endif /* CONFIG_CGROUP_BPF */
329 #endif /* _BPF_CGROUP_H */