kernel/bpf/local_storage.c

   1 //SPDX-License-Identifier: GPL-2.0
   2 #include <linux/bpf-cgroup.h>
   3 #include <linux/bpf.h>
   4 #include <linux/bug.h>
   5 #include <linux/filter.h>
   6 #include <linux/mm.h>
   7 #include <linux/rbtree.h>
   8 #include <linux/slab.h>
   9
  10 #ifdef CONFIG_CGROUP_BPF
  11
  12 #define LOCAL_STORAGE_CREATE_FLAG_MASK                                  \
  13         (BPF_F_NUMA_NODE | BPF_F_RDONLY | BPF_F_WRONLY)
  14
  15 struct bpf_cgroup_storage_map {
  16         struct bpf_map map;
  17
  18         spinlock_t lock;
  19         struct bpf_prog *prog;
  20         struct rb_root root;
  21         struct list_head list;
  22 };
  23
  24 static struct bpf_cgroup_storage_map *map_to_storage(struct bpf_map *map)
  25 {
  26         return container_of(map, struct bpf_cgroup_storage_map, map);
  27 }
  28
  29 static int bpf_cgroup_storage_key_cmp(
  30         const struct bpf_cgroup_storage_key *key1,
  31         const struct bpf_cgroup_storage_key *key2)
  32 {
  33         if (key1->cgroup_inode_id < key2->cgroup_inode_id)
  34                 return -1;
  35         else if (key1->cgroup_inode_id > key2->cgroup_inode_id)
  36                 return 1;
  37         else if (key1->attach_type < key2->attach_type)
  38                 return -1;
  39         else if (key1->attach_type > key2->attach_type)
  40                 return 1;
  41         return 0;
  42 }
  43
  44 static struct bpf_cgroup_storage *cgroup_storage_lookup(
  45         struct bpf_cgroup_storage_map *map, struct bpf_cgroup_storage_key *key,
  46         bool locked)
  47 {
  48         struct rb_root *root = &map->root;
  49         struct rb_node *node;
  50
  51         if (!locked)
  52                 spin_lock_bh(&map->lock);
  53
  54         node = root->rb_node;
  55         while (node) {
  56                 struct bpf_cgroup_storage *storage;
  57
  58                 storage = container_of(node, struct bpf_cgroup_storage, node);
  59
  60                 switch (bpf_cgroup_storage_key_cmp(key, &storage->key)) {
  61                 case -1:
  62                         node = node->rb_left;
  63                         break;
  64                 case 1:
  65                         node = node->rb_right;
  66                         break;
  67                 default:
  68                         if (!locked)
  69                                 spin_unlock_bh(&map->lock);
  70                         return storage;
  71                 }
  72         }
  73
  74         if (!locked)
  75                 spin_unlock_bh(&map->lock);
  76
  77         return NULL;
  78 }
  79
  80 static int cgroup_storage_insert(struct bpf_cgroup_storage_map *map,
  81                                  struct bpf_cgroup_storage *storage)
  82 {
  83         struct rb_root *root = &map->root;
  84         struct rb_node **new = &(root->rb_node), *parent = NULL;
  85
  86         while (*new) {
  87                 struct bpf_cgroup_storage *this;
  88
  89                 this = container_of(*new, struct bpf_cgroup_storage, node);
  90
  91                 parent = *new;
  92                 switch (bpf_cgroup_storage_key_cmp(&storage->key, &this->key)) {
  93                 case -1:
  94                         new = &((*new)->rb_left);
  95                         break;
  96                 case 1:
  97                         new = &((*new)->rb_right);
  98                         break;
  99                 default:
 100                         return -EEXIST;
 101                 }
 102         }
 103
 104         rb_link_node(&storage->node, parent, new);
 105         rb_insert_color(&storage->node, root);
 106
 107         return 0;
 108 }
 109
 110 static void *cgroup_storage_lookup_elem(struct bpf_map *_map, void *_key)
 111 {
 112         struct bpf_cgroup_storage_map *map = map_to_storage(_map);
 113         struct bpf_cgroup_storage_key *key = _key;
 114         struct bpf_cgroup_storage *storage;
 115
 116         storage = cgroup_storage_lookup(map, key, false);
 117         if (!storage)
 118                 return NULL;
 119
 120         return &READ_ONCE(storage->buf)->data[0];
 121 }
 122
 123 static int cgroup_storage_update_elem(struct bpf_map *map, void *_key,
 124                                       void *value, u64 flags)
 125 {
 126         struct bpf_cgroup_storage_key *key = _key;
 127         struct bpf_cgroup_storage *storage;
 128         struct bpf_storage_buffer *new;
 129
 130         if (flags & BPF_NOEXIST)
 131                 return -EINVAL;
 132
 133         storage = cgroup_storage_lookup((struct bpf_cgroup_storage_map *)map,
 134                                         key, false);
 135         if (!storage)
 136                 return -ENOENT;
 137
 138         new = kmalloc_node(sizeof(struct bpf_storage_buffer) +
 139                            map->value_size, __GFP_ZERO | GFP_USER,
 140                            map->numa_node);
 141         if (!new)
 142                 return -ENOMEM;
 143
 144         memcpy(&new->data[0], value, map->value_size);
 145
 146         new = xchg(&storage->buf, new);
 147         kfree_rcu(new, rcu);
 148
 149         return 0;
 150 }
 151
 152 static int cgroup_storage_get_next_key(struct bpf_map *_map, void *_key,
 153                                        void *_next_key)
 154 {
 155         struct bpf_cgroup_storage_map *map = map_to_storage(_map);
 156         struct bpf_cgroup_storage_key *key = _key;
 157         struct bpf_cgroup_storage_key *next = _next_key;
 158         struct bpf_cgroup_storage *storage;
 159
 160         spin_lock_bh(&map->lock);
 161
 162         if (list_empty(&map->list))
 163                 goto enoent;
 164
 165         if (key) {
 166                 storage = cgroup_storage_lookup(map, key, true);
 167                 if (!storage)
 168                         goto enoent;
 169
 170                 storage = list_next_entry(storage, list);
 171                 if (!storage)
 172                         goto enoent;
 173         } else {
 174                 storage = list_first_entry(&map->list,
 175                                          struct bpf_cgroup_storage, list);
 176         }
 177
 178         spin_unlock_bh(&map->lock);
 179         next->attach_type = storage->key.attach_type;
 180         next->cgroup_inode_id = storage->key.cgroup_inode_id;
 181         return 0;
 182
 183 enoent:
 184         spin_unlock_bh(&map->lock);
 185         return -ENOENT;
 186 }
 187
 188 static struct bpf_map *cgroup_storage_map_alloc(union bpf_attr *attr)
 189 {
 190         int numa_node = bpf_map_attr_numa_node(attr);
 191         struct bpf_cgroup_storage_map *map;
 192
 193         if (attr->key_size != sizeof(struct bpf_cgroup_storage_key))
 194                 return ERR_PTR(-EINVAL);
 195
 196         if (attr->value_size > PAGE_SIZE)
 197                 return ERR_PTR(-E2BIG);
 198
 199         if (attr->map_flags & ~LOCAL_STORAGE_CREATE_FLAG_MASK)
 200                 /* reserved bits should not be used */
 201                 return ERR_PTR(-EINVAL);
 202
 203         if (attr->max_entries)
 204                 /* max_entries is not used and enforced to be 0 */
 205                 return ERR_PTR(-EINVAL);
 206
 207         map = kmalloc_node(sizeof(struct bpf_cgroup_storage_map),
 208                            __GFP_ZERO | GFP_USER, numa_node);
 209         if (!map)
 210                 return ERR_PTR(-ENOMEM);
 211
 212         map->map.pages = round_up(sizeof(struct bpf_cgroup_storage_map),
 213                                   PAGE_SIZE) >> PAGE_SHIFT;
 214
 215         /* copy mandatory map attributes */
 216         bpf_map_init_from_attr(&map->map, attr);
 217
 218         spin_lock_init(&map->lock);
 219         map->root = RB_ROOT;
 220         INIT_LIST_HEAD(&map->list);
 221
 222         return &map->map;
 223 }
 224
 225 static void cgroup_storage_map_free(struct bpf_map *_map)
 226 {
 227         struct bpf_cgroup_storage_map *map = map_to_storage(_map);
 228
 229         WARN_ON(!RB_EMPTY_ROOT(&map->root));
 230         WARN_ON(!list_empty(&map->list));
 231
 232         kfree(map);
 233 }
 234
 235 static int cgroup_storage_delete_elem(struct bpf_map *map, void *key)
 236 {
 237         return -EINVAL;
 238 }
 239
 240 const struct bpf_map_ops cgroup_storage_map_ops = {
 241         .map_alloc = cgroup_storage_map_alloc,
 242         .map_free = cgroup_storage_map_free,
 243         .map_get_next_key = cgroup_storage_get_next_key,
 244         .map_lookup_elem = cgroup_storage_lookup_elem,
 245         .map_update_elem = cgroup_storage_update_elem,
 246         .map_delete_elem = cgroup_storage_delete_elem,
 247 };
 248
 249 int bpf_cgroup_storage_assign(struct bpf_prog *prog, struct bpf_map *_map)
 250 {
 251         struct bpf_cgroup_storage_map *map = map_to_storage(_map);
 252         int ret = -EBUSY;
 253
 254         spin_lock_bh(&map->lock);
 255
 256         if (map->prog && map->prog != prog)
 257                 goto unlock;
 258         if (prog->aux->cgroup_storage && prog->aux->cgroup_storage != _map)
 259                 goto unlock;
 260
 261         map->prog = prog;
 262         prog->aux->cgroup_storage = _map;
 263         ret = 0;
 264 unlock:
 265         spin_unlock_bh(&map->lock);
 266
 267         return ret;
 268 }
 269
 270 void bpf_cgroup_storage_release(struct bpf_prog *prog, struct bpf_map *_map)
 271 {
 272         struct bpf_cgroup_storage_map *map = map_to_storage(_map);
 273
 274         spin_lock_bh(&map->lock);
 275         if (map->prog == prog) {
 276                 WARN_ON(prog->aux->cgroup_storage != _map);
 277                 map->prog = NULL;
 278                 prog->aux->cgroup_storage = NULL;
 279         }
 280         spin_unlock_bh(&map->lock);
 281 }
 282
 283 struct bpf_cgroup_storage *bpf_cgroup_storage_alloc(struct bpf_prog *prog)
 284 {
 285         struct bpf_cgroup_storage *storage;
 286         struct bpf_map *map;
 287         u32 pages;
 288
 289         map = prog->aux->cgroup_storage;
 290         if (!map)
 291                 return NULL;
 292
 293         pages = round_up(sizeof(struct bpf_cgroup_storage) +
 294                          sizeof(struct bpf_storage_buffer) +
 295                          map->value_size, PAGE_SIZE) >> PAGE_SHIFT;
 296         if (bpf_map_charge_memlock(map, pages))
 297                 return ERR_PTR(-EPERM);
 298
 299         storage = kmalloc_node(sizeof(struct bpf_cgroup_storage),
 300                                __GFP_ZERO | GFP_USER, map->numa_node);
 301         if (!storage) {
 302                 bpf_map_uncharge_memlock(map, pages);
 303                 return ERR_PTR(-ENOMEM);
 304         }
 305
 306         storage->buf = kmalloc_node(sizeof(struct bpf_storage_buffer) +
 307                                     map->value_size, __GFP_ZERO | GFP_USER,
 308                                     map->numa_node);
 309         if (!storage->buf) {
 310                 bpf_map_uncharge_memlock(map, pages);
 311                 kfree(storage);
 312                 return ERR_PTR(-ENOMEM);
 313         }
 314
 315         storage->map = (struct bpf_cgroup_storage_map *)map;
 316
 317         return storage;
 318 }
 319
 320 void bpf_cgroup_storage_free(struct bpf_cgroup_storage *storage)
 321 {
 322         u32 pages;
 323         struct bpf_map *map;
 324
 325         if (!storage)
 326                 return;
 327
 328         map = &storage->map->map;
 329         pages = round_up(sizeof(struct bpf_cgroup_storage) +
 330                          sizeof(struct bpf_storage_buffer) +
 331                          map->value_size, PAGE_SIZE) >> PAGE_SHIFT;
 332         bpf_map_uncharge_memlock(map, pages);
 333
 334         kfree_rcu(storage->buf, rcu);
 335         kfree_rcu(storage, rcu);
 336 }
 337
 338 void bpf_cgroup_storage_link(struct bpf_cgroup_storage *storage,
 339                              struct cgroup *cgroup,
 340                              enum bpf_attach_type type)
 341 {
 342         struct bpf_cgroup_storage_map *map;
 343
 344         if (!storage)
 345                 return;
 346
 347         storage->key.attach_type = type;
 348         storage->key.cgroup_inode_id = cgroup->kn->id.id;
 349
 350         map = storage->map;
 351
 352         spin_lock_bh(&map->lock);
 353         WARN_ON(cgroup_storage_insert(map, storage));
 354         list_add(&storage->list, &map->list);
 355         spin_unlock_bh(&map->lock);
 356 }
 357
 358 void bpf_cgroup_storage_unlink(struct bpf_cgroup_storage *storage)
 359 {
 360         struct bpf_cgroup_storage_map *map;
 361         struct rb_root *root;
 362
 363         if (!storage)
 364                 return;
 365
 366         map = storage->map;
 367
 368         spin_lock_bh(&map->lock);
 369         root = &map->root;
 370         rb_erase(&storage->node, root);
 371
 372         list_del(&storage->list);
 373         spin_unlock_bh(&map->lock);
 374 }
 375
 376 #endif