kernel/bpf/syscall.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
   3  */
   4 #include <linux/bpf.h>
   5 #include <linux/bpf_trace.h>
   6 #include <linux/bpf_lirc.h>
   7 #include <linux/btf.h>
   8 #include <linux/syscalls.h>
   9 #include <linux/slab.h>
  10 #include <linux/sched/signal.h>
  11 #include <linux/vmalloc.h>
  12 #include <linux/mmzone.h>
  13 #include <linux/anon_inodes.h>
  14 #include <linux/fdtable.h>
  15 #include <linux/file.h>
  16 #include <linux/fs.h>
  17 #include <linux/license.h>
  18 #include <linux/filter.h>
  19 #include <linux/version.h>
  20 #include <linux/kernel.h>
  21 #include <linux/idr.h>
  22 #include <linux/cred.h>
  23 #include <linux/timekeeping.h>
  24 #include <linux/ctype.h>
  25 #include <linux/nospec.h>
  26 #include <linux/audit.h>
  27 #include <uapi/linux/btf.h>
  28
  29 #define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || \
  30                           (map)->map_type == BPF_MAP_TYPE_CGROUP_ARRAY || \
  31                           (map)->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS)
  32 #define IS_FD_PROG_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PROG_ARRAY)
  33 #define IS_FD_HASH(map) ((map)->map_type == BPF_MAP_TYPE_HASH_OF_MAPS)
  34 #define IS_FD_MAP(map) (IS_FD_ARRAY(map) || IS_FD_PROG_ARRAY(map) || \
  35                         IS_FD_HASH(map))
  36
  37 #define BPF_OBJ_FLAG_MASK   (BPF_F_RDONLY | BPF_F_WRONLY)
  38
  39 DEFINE_PER_CPU(int, bpf_prog_active);
  40 static DEFINE_IDR(prog_idr);
  41 static DEFINE_SPINLOCK(prog_idr_lock);
  42 static DEFINE_IDR(map_idr);
  43 static DEFINE_SPINLOCK(map_idr_lock);
  44
  45 int sysctl_unprivileged_bpf_disabled __read_mostly;
  46
  47 static const struct bpf_map_ops * const bpf_map_types[] = {
  48 #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type)
  49 #define BPF_MAP_TYPE(_id, _ops) \
  50         [_id] = &_ops,
  51 #include <linux/bpf_types.h>
  52 #undef BPF_PROG_TYPE
  53 #undef BPF_MAP_TYPE
  54 };
  55
  56 /*
  57  * If we're handed a bigger struct than we know of, ensure all the unknown bits
  58  * are 0 - i.e. new user-space does not rely on any kernel feature extensions
  59  * we don't know about yet.
  60  *
  61  * There is a ToCToU between this function call and the following
  62  * copy_from_user() call. However, this is not a concern since this function is
  63  * meant to be a future-proofing of bits.
  64  */
  65 int bpf_check_uarg_tail_zero(void __user *uaddr,
  66                              size_t expected_size,
  67                              size_t actual_size)
  68 {
  69         unsigned char __user *addr;
  70         unsigned char __user *end;
  71         unsigned char val;
  72         int err;
  73
  74         if (unlikely(actual_size > PAGE_SIZE))  /* silly large */
  75                 return -E2BIG;
  76
  77         if (unlikely(!access_ok(uaddr, actual_size)))
  78                 return -EFAULT;
  79
  80         if (actual_size <= expected_size)
  81                 return 0;
  82
  83         addr = uaddr + expected_size;
  84         end  = uaddr + actual_size;
  85
  86         for (; addr < end; addr++) {
  87                 err = get_user(val, addr);
  88                 if (err)
  89                         return err;
  90                 if (val)
  91                         return -E2BIG;
  92         }
  93
  94         return 0;
  95 }
  96
  97 const struct bpf_map_ops bpf_map_offload_ops = {
  98         .map_alloc = bpf_map_offload_map_alloc,
  99         .map_free = bpf_map_offload_map_free,
 100         .map_check_btf = map_check_no_btf,
 101 };
 102
 103 static struct bpf_map *find_and_alloc_map(union bpf_attr *attr)
 104 {
 105         const struct bpf_map_ops *ops;
 106         u32 type = attr->map_type;
 107         struct bpf_map *map;
 108         int err;
 109
 110         if (type >= ARRAY_SIZE(bpf_map_types))
 111                 return ERR_PTR(-EINVAL);
 112         type = array_index_nospec(type, ARRAY_SIZE(bpf_map_types));
 113         ops = bpf_map_types[type];
 114         if (!ops)
 115                 return ERR_PTR(-EINVAL);
 116
 117         if (ops->map_alloc_check) {
 118                 err = ops->map_alloc_check(attr);
 119                 if (err)
 120                         return ERR_PTR(err);
 121         }
 122         if (attr->map_ifindex)
 123                 ops = &bpf_map_offload_ops;
 124         map = ops->map_alloc(attr);
 125         if (IS_ERR(map))
 126                 return map;
 127         map->ops = ops;
 128         map->map_type = type;
 129         return map;
 130 }
 131
 132 static u32 bpf_map_value_size(struct bpf_map *map)
 133 {
 134         if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
 135             map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH ||
 136             map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY ||
 137             map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE)
 138                 return round_up(map->value_size, 8) * num_possible_cpus();
 139         else if (IS_FD_MAP(map))
 140                 return sizeof(u32);
 141         else
 142                 return  map->value_size;
 143 }
 144
 145 static void maybe_wait_bpf_programs(struct bpf_map *map)
 146 {
 147         /* Wait for any running BPF programs to complete so that
 148          * userspace, when we return to it, knows that all programs
 149          * that could be running use the new map value.
 150          */
 151         if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS ||
 152             map->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS)
 153                 synchronize_rcu();
 154 }
 155
 156 static int bpf_map_update_value(struct bpf_map *map, struct fd f, void *key,
 157                                 void *value, __u64 flags)
 158 {
 159         int err;
 160
 161         /* Need to create a kthread, thus must support schedule */
 162         if (bpf_map_is_dev_bound(map)) {
 163                 return bpf_map_offload_update_elem(map, key, value, flags);
 164         } else if (map->map_type == BPF_MAP_TYPE_CPUMAP ||
 165                    map->map_type == BPF_MAP_TYPE_SOCKHASH ||
 166                    map->map_type == BPF_MAP_TYPE_SOCKMAP ||
 167                    map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {
 168                 return map->ops->map_update_elem(map, key, value, flags);
 169         } else if (IS_FD_PROG_ARRAY(map)) {
 170                 return bpf_fd_array_map_update_elem(map, f.file, key, value,
 171                                                     flags);
 172         }
 173
 174         /* must increment bpf_prog_active to avoid kprobe+bpf triggering from
 175          * inside bpf map update or delete otherwise deadlocks are possible
 176          */
 177         preempt_disable();
 178         __this_cpu_inc(bpf_prog_active);
 179         if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
 180             map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
 181                 err = bpf_percpu_hash_update(map, key, value, flags);
 182         } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
 183                 err = bpf_percpu_array_update(map, key, value, flags);
 184         } else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) {
 185                 err = bpf_percpu_cgroup_storage_update(map, key, value,
 186                                                        flags);
 187         } else if (IS_FD_ARRAY(map)) {
 188                 rcu_read_lock();
 189                 err = bpf_fd_array_map_update_elem(map, f.file, key, value,
 190                                                    flags);
 191                 rcu_read_unlock();
 192         } else if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) {
 193                 rcu_read_lock();
 194                 err = bpf_fd_htab_map_update_elem(map, f.file, key, value,
 195                                                   flags);
 196                 rcu_read_unlock();
 197         } else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) {
 198                 /* rcu_read_lock() is not needed */
 199                 err = bpf_fd_reuseport_array_update_elem(map, key, value,
 200                                                          flags);
 201         } else if (map->map_type == BPF_MAP_TYPE_QUEUE ||
 202                    map->map_type == BPF_MAP_TYPE_STACK) {
 203                 err = map->ops->map_push_elem(map, value, flags);
 204         } else {
 205                 rcu_read_lock();
 206                 err = map->ops->map_update_elem(map, key, value, flags);
 207                 rcu_read_unlock();
 208         }
 209         __this_cpu_dec(bpf_prog_active);
 210         preempt_enable();
 211         maybe_wait_bpf_programs(map);
 212
 213         return err;
 214 }
 215
 216 static int bpf_map_copy_value(struct bpf_map *map, void *key, void *value,
 217                               __u64 flags)
 218 {
 219         void *ptr;
 220         int err;
 221
 222         if (bpf_map_is_dev_bound(map))
 223                 return bpf_map_offload_lookup_elem(map, key, value);
 224
 225         preempt_disable();
 226         this_cpu_inc(bpf_prog_active);
 227         if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
 228             map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
 229                 err = bpf_percpu_hash_copy(map, key, value);
 230         } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
 231                 err = bpf_percpu_array_copy(map, key, value);
 232         } else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) {
 233                 err = bpf_percpu_cgroup_storage_copy(map, key, value);
 234         } else if (map->map_type == BPF_MAP_TYPE_STACK_TRACE) {
 235                 err = bpf_stackmap_copy(map, key, value);
 236         } else if (IS_FD_ARRAY(map) || IS_FD_PROG_ARRAY(map)) {
 237                 err = bpf_fd_array_map_lookup_elem(map, key, value);
 238         } else if (IS_FD_HASH(map)) {
 239                 err = bpf_fd_htab_map_lookup_elem(map, key, value);
 240         } else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) {
 241                 err = bpf_fd_reuseport_array_lookup_elem(map, key, value);
 242         } else if (map->map_type == BPF_MAP_TYPE_QUEUE ||
 243                    map->map_type == BPF_MAP_TYPE_STACK) {
 244                 err = map->ops->map_peek_elem(map, value);
 245         } else if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {
 246                 /* struct_ops map requires directly updating "value" */
 247                 err = bpf_struct_ops_map_sys_lookup_elem(map, key, value);
 248         } else {
 249                 rcu_read_lock();
 250                 if (map->ops->map_lookup_elem_sys_only)
 251                         ptr = map->ops->map_lookup_elem_sys_only(map, key);
 252                 else
 253                         ptr = map->ops->map_lookup_elem(map, key);
 254                 if (IS_ERR(ptr)) {
 255                         err = PTR_ERR(ptr);
 256                 } else if (!ptr) {
 257                         err = -ENOENT;
 258                 } else {
 259                         err = 0;
 260                         if (flags & BPF_F_LOCK)
 261                                 /* lock 'ptr' and copy everything but lock */
 262                                 copy_map_value_locked(map, value, ptr, true);
 263                         else
 264                                 copy_map_value(map, value, ptr);
 265                         /* mask lock, since value wasn't zero inited */
 266                         check_and_init_map_lock(map, value);
 267                 }
 268                 rcu_read_unlock();
 269         }
 270
 271         this_cpu_dec(bpf_prog_active);
 272         preempt_enable();
 273         maybe_wait_bpf_programs(map);
 274
 275         return err;
 276 }
 277
 278 static void *__bpf_map_area_alloc(u64 size, int numa_node, bool mmapable)
 279 {
 280         /* We really just want to fail instead of triggering OOM killer
 281          * under memory pressure, therefore we set __GFP_NORETRY to kmalloc,
 282          * which is used for lower order allocation requests.
 283          *
 284          * It has been observed that higher order allocation requests done by
 285          * vmalloc with __GFP_NORETRY being set might fail due to not trying
 286          * to reclaim memory from the page cache, thus we set
 287          * __GFP_RETRY_MAYFAIL to avoid such situations.
 288          */
 289
 290         const gfp_t flags = __GFP_NOWARN | __GFP_ZERO;
 291         void *area;
 292
 293         if (size >= SIZE_MAX)
 294                 return NULL;
 295
 296         /* kmalloc()'ed memory can't be mmap()'ed */
 297         if (!mmapable && size <= (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) {
 298                 area = kmalloc_node(size, GFP_USER | __GFP_NORETRY | flags,
 299                                     numa_node);
 300                 if (area != NULL)
 301                         return area;
 302         }
 303         if (mmapable) {
 304                 BUG_ON(!PAGE_ALIGNED(size));
 305                 return vmalloc_user_node_flags(size, numa_node, GFP_KERNEL |
 306                                                __GFP_RETRY_MAYFAIL | flags);
 307         }
 308         return __vmalloc_node_flags_caller(size, numa_node,
 309                                            GFP_KERNEL | __GFP_RETRY_MAYFAIL |
 310                                            flags, __builtin_return_address(0));
 311 }
 312
 313 void *bpf_map_area_alloc(u64 size, int numa_node)
 314 {
 315         return __bpf_map_area_alloc(size, numa_node, false);
 316 }
 317
 318 void *bpf_map_area_mmapable_alloc(u64 size, int numa_node)
 319 {
 320         return __bpf_map_area_alloc(size, numa_node, true);
 321 }
 322
 323 void bpf_map_area_free(void *area)
 324 {
 325         kvfree(area);
 326 }
 327
 328 static u32 bpf_map_flags_retain_permanent(u32 flags)
 329 {
 330         /* Some map creation flags are not tied to the map object but
 331          * rather to the map fd instead, so they have no meaning upon
 332          * map object inspection since multiple file descriptors with
 333          * different (access) properties can exist here. Thus, given
 334          * this has zero meaning for the map itself, lets clear these
 335          * from here.
 336          */
 337         return flags & ~(BPF_F_RDONLY | BPF_F_WRONLY);
 338 }
 339
 340 void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr)
 341 {
 342         map->map_type = attr->map_type;
 343         map->key_size = attr->key_size;
 344         map->value_size = attr->value_size;
 345         map->max_entries = attr->max_entries;
 346         map->map_flags = bpf_map_flags_retain_permanent(attr->map_flags);
 347         map->numa_node = bpf_map_attr_numa_node(attr);
 348 }
 349
 350 static int bpf_charge_memlock(struct user_struct *user, u32 pages)
 351 {
 352         unsigned long memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
 353
 354         if (atomic_long_add_return(pages, &user->locked_vm) > memlock_limit) {
 355                 atomic_long_sub(pages, &user->locked_vm);
 356                 return -EPERM;
 357         }
 358         return 0;
 359 }
 360
 361 static void bpf_uncharge_memlock(struct user_struct *user, u32 pages)
 362 {
 363         if (user)
 364                 atomic_long_sub(pages, &user->locked_vm);
 365 }
 366
 367 int bpf_map_charge_init(struct bpf_map_memory *mem, u64 size)
 368 {
 369         u32 pages = round_up(size, PAGE_SIZE) >> PAGE_SHIFT;
 370         struct user_struct *user;
 371         int ret;
 372
 373         if (size >= U32_MAX - PAGE_SIZE)
 374                 return -E2BIG;
 375
 376         user = get_current_user();
 377         ret = bpf_charge_memlock(user, pages);
 378         if (ret) {
 379                 free_uid(user);
 380                 return ret;
 381         }
 382
 383         mem->pages = pages;
 384         mem->user = user;
 385
 386         return 0;
 387 }
 388
 389 void bpf_map_charge_finish(struct bpf_map_memory *mem)
 390 {
 391         bpf_uncharge_memlock(mem->user, mem->pages);
 392         free_uid(mem->user);
 393 }
 394
 395 void bpf_map_charge_move(struct bpf_map_memory *dst,
 396                          struct bpf_map_memory *src)
 397 {
 398         *dst = *src;
 399
 400         /* Make sure src will not be used for the redundant uncharging. */
 401         memset(src, 0, sizeof(struct bpf_map_memory));
 402 }
 403
 404 int bpf_map_charge_memlock(struct bpf_map *map, u32 pages)
 405 {
 406         int ret;
 407
 408         ret = bpf_charge_memlock(map->memory.user, pages);
 409         if (ret)
 410                 return ret;
 411         map->memory.pages += pages;
 412         return ret;
 413 }
 414
 415 void bpf_map_uncharge_memlock(struct bpf_map *map, u32 pages)
 416 {
 417         bpf_uncharge_memlock(map->memory.user, pages);
 418         map->memory.pages -= pages;
 419 }
 420
 421 static int bpf_map_alloc_id(struct bpf_map *map)
 422 {
 423         int id;
 424
 425         idr_preload(GFP_KERNEL);
 426         spin_lock_bh(&map_idr_lock);
 427         id = idr_alloc_cyclic(&map_idr, map, 1, INT_MAX, GFP_ATOMIC);
 428         if (id > 0)
 429                 map->id = id;
 430         spin_unlock_bh(&map_idr_lock);
 431         idr_preload_end();
 432
 433         if (WARN_ON_ONCE(!id))
 434                 return -ENOSPC;
 435
 436         return id > 0 ? 0 : id;
 437 }
 438
 439 void bpf_map_free_id(struct bpf_map *map, bool do_idr_lock)
 440 {
 441         unsigned long flags;
 442
 443         /* Offloaded maps are removed from the IDR store when their device
 444          * disappears - even if someone holds an fd to them they are unusable,
 445          * the memory is gone, all ops will fail; they are simply waiting for
 446          * refcnt to drop to be freed.
 447          */
 448         if (!map->id)
 449                 return;
 450
 451         if (do_idr_lock)
 452                 spin_lock_irqsave(&map_idr_lock, flags);
 453         else
 454                 __acquire(&map_idr_lock);
 455
 456         idr_remove(&map_idr, map->id);
 457         map->id = 0;
 458
 459         if (do_idr_lock)
 460                 spin_unlock_irqrestore(&map_idr_lock, flags);
 461         else
 462                 __release(&map_idr_lock);
 463 }
 464
 465 /* called from workqueue */
 466 static void bpf_map_free_deferred(struct work_struct *work)
 467 {
 468         struct bpf_map *map = container_of(work, struct bpf_map, work);
 469         struct bpf_map_memory mem;
 470
 471         bpf_map_charge_move(&mem, &map->memory);
 472         security_bpf_map_free(map);
 473         /* implementation dependent freeing */
 474         map->ops->map_free(map);
 475         bpf_map_charge_finish(&mem);
 476 }
 477
 478 static void bpf_map_put_uref(struct bpf_map *map)
 479 {
 480         if (atomic64_dec_and_test(&map->usercnt)) {
 481                 if (map->ops->map_release_uref)
 482                         map->ops->map_release_uref(map);
 483         }
 484 }
 485
 486 /* decrement map refcnt and schedule it for freeing via workqueue
 487  * (unrelying map implementation ops->map_free() might sleep)
 488  */
 489 static void __bpf_map_put(struct bpf_map *map, bool do_idr_lock)
 490 {
 491         if (atomic64_dec_and_test(&map->refcnt)) {
 492                 /* bpf_map_free_id() must be called first */
 493                 bpf_map_free_id(map, do_idr_lock);
 494                 btf_put(map->btf);
 495                 INIT_WORK(&map->work, bpf_map_free_deferred);
 496                 schedule_work(&map->work);
 497         }
 498 }
 499
 500 void bpf_map_put(struct bpf_map *map)
 501 {
 502         __bpf_map_put(map, true);
 503 }
 504 EXPORT_SYMBOL_GPL(bpf_map_put);
 505
 506 void bpf_map_put_with_uref(struct bpf_map *map)
 507 {
 508         bpf_map_put_uref(map);
 509         bpf_map_put(map);
 510 }
 511
 512 static int bpf_map_release(struct inode *inode, struct file *filp)
 513 {
 514         struct bpf_map *map = filp->private_data;
 515
 516         if (map->ops->map_release)
 517                 map->ops->map_release(map, filp);
 518
 519         bpf_map_put_with_uref(map);
 520         return 0;
 521 }
 522
 523 static fmode_t map_get_sys_perms(struct bpf_map *map, struct fd f)
 524 {
 525         fmode_t mode = f.file->f_mode;
 526
 527         /* Our file permissions may have been overridden by global
 528          * map permissions facing syscall side.
 529          */
 530         if (READ_ONCE(map->frozen))
 531                 mode &= ~FMODE_CAN_WRITE;
 532         return mode;
 533 }
 534
 535 #ifdef CONFIG_PROC_FS
 536 static void bpf_map_show_fdinfo(struct seq_file *m, struct file *filp)
 537 {
 538         const struct bpf_map *map = filp->private_data;
 539         const struct bpf_array *array;
 540         u32 type = 0, jited = 0;
 541
 542         if (map->map_type == BPF_MAP_TYPE_PROG_ARRAY) {
 543                 array = container_of(map, struct bpf_array, map);
 544                 type  = array->aux->type;
 545                 jited = array->aux->jited;
 546         }
 547
 548         seq_printf(m,
 549                    "map_type:\t%u\n"
 550                    "key_size:\t%u\n"
 551                    "value_size:\t%u\n"
 552                    "max_entries:\t%u\n"
 553                    "map_flags:\t%#x\n"
 554                    "memlock:\t%llu\n"
 555                    "map_id:\t%u\n"
 556                    "frozen:\t%u\n",
 557                    map->map_type,
 558                    map->key_size,
 559                    map->value_size,
 560                    map->max_entries,
 561                    map->map_flags,
 562                    map->memory.pages * 1ULL << PAGE_SHIFT,
 563                    map->id,
 564                    READ_ONCE(map->frozen));
 565         if (type) {
 566                 seq_printf(m, "owner_prog_type:\t%u\n", type);
 567                 seq_printf(m, "owner_jited:\t%u\n", jited);
 568         }
 569 }
 570 #endif
 571
 572 static ssize_t bpf_dummy_read(struct file *filp, char __user *buf, size_t siz,
 573                               loff_t *ppos)
 574 {
 575         /* We need this handler such that alloc_file() enables
 576          * f_mode with FMODE_CAN_READ.
 577          */
 578         return -EINVAL;
 579 }
 580
 581 static ssize_t bpf_dummy_write(struct file *filp, const char __user *buf,
 582                                size_t siz, loff_t *ppos)
 583 {
 584         /* We need this handler such that alloc_file() enables
 585          * f_mode with FMODE_CAN_WRITE.
 586          */
 587         return -EINVAL;
 588 }
 589
 590 /* called for any extra memory-mapped regions (except initial) */
 591 static void bpf_map_mmap_open(struct vm_area_struct *vma)
 592 {
 593         struct bpf_map *map = vma->vm_file->private_data;
 594
 595         bpf_map_inc_with_uref(map);
 596
 597         if (vma->vm_flags & VM_WRITE) {
 598                 mutex_lock(&map->freeze_mutex);
 599                 map->writecnt++;
 600                 mutex_unlock(&map->freeze_mutex);
 601         }
 602 }
 603
 604 /* called for all unmapped memory region (including initial) */
 605 static void bpf_map_mmap_close(struct vm_area_struct *vma)
 606 {
 607         struct bpf_map *map = vma->vm_file->private_data;
 608
 609         if (vma->vm_flags & VM_WRITE) {
 610                 mutex_lock(&map->freeze_mutex);
 611                 map->writecnt--;
 612                 mutex_unlock(&map->freeze_mutex);
 613         }
 614
 615         bpf_map_put_with_uref(map);
 616 }
 617
 618 static const struct vm_operations_struct bpf_map_default_vmops = {
 619         .open           = bpf_map_mmap_open,
 620         .close          = bpf_map_mmap_close,
 621 };
 622
 623 static int bpf_map_mmap(struct file *filp, struct vm_area_struct *vma)
 624 {
 625         struct bpf_map *map = filp->private_data;
 626         int err;
 627
 628         if (!map->ops->map_mmap || map_value_has_spin_lock(map))
 629                 return -ENOTSUPP;
 630
 631         if (!(vma->vm_flags & VM_SHARED))
 632                 return -EINVAL;
 633
 634         mutex_lock(&map->freeze_mutex);
 635
 636         if ((vma->vm_flags & VM_WRITE) && map->frozen) {
 637                 err = -EPERM;
 638                 goto out;
 639         }
 640
 641         /* set default open/close callbacks */
 642         vma->vm_ops = &bpf_map_default_vmops;
 643         vma->vm_private_data = map;
 644
 645         err = map->ops->map_mmap(map, vma);
 646         if (err)
 647                 goto out;
 648
 649         bpf_map_inc_with_uref(map);
 650
 651         if (vma->vm_flags & VM_WRITE)
 652                 map->writecnt++;
 653 out:
 654         mutex_unlock(&map->freeze_mutex);
 655         return err;
 656 }
 657
 658 const struct file_operations bpf_map_fops = {
 659 #ifdef CONFIG_PROC_FS
 660         .show_fdinfo    = bpf_map_show_fdinfo,
 661 #endif
 662         .release        = bpf_map_release,
 663         .read           = bpf_dummy_read,
 664         .write          = bpf_dummy_write,
 665         .mmap           = bpf_map_mmap,
 666 };
 667
 668 int bpf_map_new_fd(struct bpf_map *map, int flags)
 669 {
 670         int ret;
 671
 672         ret = security_bpf_map(map, OPEN_FMODE(flags));
 673         if (ret < 0)
 674                 return ret;
 675
 676         return anon_inode_getfd("bpf-map", &bpf_map_fops, map,
 677                                 flags | O_CLOEXEC);
 678 }
 679
 680 int bpf_get_file_flag(int flags)
 681 {
 682         if ((flags & BPF_F_RDONLY) && (flags & BPF_F_WRONLY))
 683                 return -EINVAL;
 684         if (flags & BPF_F_RDONLY)
 685                 return O_RDONLY;
 686         if (flags & BPF_F_WRONLY)
 687                 return O_WRONLY;
 688         return O_RDWR;
 689 }
 690
 691 /* helper macro to check that unused fields 'union bpf_attr' are zero */
 692 #define CHECK_ATTR(CMD) \
 693         memchr_inv((void *) &attr->CMD##_LAST_FIELD + \
 694                    sizeof(attr->CMD##_LAST_FIELD), 0, \
 695                    sizeof(*attr) - \
 696                    offsetof(union bpf_attr, CMD##_LAST_FIELD) - \
 697                    sizeof(attr->CMD##_LAST_FIELD)) != NULL
 698
 699 /* dst and src must have at least BPF_OBJ_NAME_LEN number of bytes.
 700  * Return 0 on success and < 0 on error.
 701  */
 702 static int bpf_obj_name_cpy(char *dst, const char *src)
 703 {
 704         const char *end = src + BPF_OBJ_NAME_LEN;
 705
 706         memset(dst, 0, BPF_OBJ_NAME_LEN);
 707         /* Copy all isalnum(), '_' and '.' chars. */
 708         while (src < end && *src) {
 709                 if (!isalnum(*src) &&
 710                     *src != '_' && *src != '.')
 711                         return -EINVAL;
 712                 *dst++ = *src++;
 713         }
 714
 715         /* No '\0' found in BPF_OBJ_NAME_LEN number of bytes */
 716         if (src == end)
 717                 return -EINVAL;
 718
 719         return 0;
 720 }
 721
 722 int map_check_no_btf(const struct bpf_map *map,
 723                      const struct btf *btf,
 724                      const struct btf_type *key_type,
 725                      const struct btf_type *value_type)
 726 {
 727         return -ENOTSUPP;
 728 }
 729
 730 static int map_check_btf(struct bpf_map *map, const struct btf *btf,
 731                          u32 btf_key_id, u32 btf_value_id)
 732 {
 733         const struct btf_type *key_type, *value_type;
 734         u32 key_size, value_size;
 735         int ret = 0;
 736
 737         /* Some maps allow key to be unspecified. */
 738         if (btf_key_id) {
 739                 key_type = btf_type_id_size(btf, &btf_key_id, &key_size);
 740                 if (!key_type || key_size != map->key_size)
 741                         return -EINVAL;
 742         } else {
 743                 key_type = btf_type_by_id(btf, 0);
 744                 if (!map->ops->map_check_btf)
 745                         return -EINVAL;
 746         }
 747
 748         value_type = btf_type_id_size(btf, &btf_value_id, &value_size);
 749         if (!value_type || value_size != map->value_size)
 750                 return -EINVAL;
 751
 752         map->spin_lock_off = btf_find_spin_lock(btf, value_type);
 753
 754         if (map_value_has_spin_lock(map)) {
 755                 if (map->map_flags & BPF_F_RDONLY_PROG)
 756                         return -EACCES;
 757                 if (map->map_type != BPF_MAP_TYPE_HASH &&
 758                     map->map_type != BPF_MAP_TYPE_ARRAY &&
 759                     map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE &&
 760                     map->map_type != BPF_MAP_TYPE_SK_STORAGE)
 761                         return -ENOTSUPP;
 762                 if (map->spin_lock_off + sizeof(struct bpf_spin_lock) >
 763                     map->value_size) {
 764                         WARN_ONCE(1,
 765                                   "verifier bug spin_lock_off %d value_size %d\n",
 766                                   map->spin_lock_off, map->value_size);
 767                         return -EFAULT;
 768                 }
 769         }
 770
 771         if (map->ops->map_check_btf)
 772                 ret = map->ops->map_check_btf(map, btf, key_type, value_type);
 773
 774         return ret;
 775 }
 776
 777 #define BPF_MAP_CREATE_LAST_FIELD btf_vmlinux_value_type_id
 778 /* called via syscall */
 779 static int map_create(union bpf_attr *attr)
 780 {
 781         int numa_node = bpf_map_attr_numa_node(attr);
 782         struct bpf_map_memory mem;
 783         struct bpf_map *map;
 784         int f_flags;
 785         int err;
 786
 787         err = CHECK_ATTR(BPF_MAP_CREATE);
 788         if (err)
 789                 return -EINVAL;
 790
 791         if (attr->btf_vmlinux_value_type_id) {
 792                 if (attr->map_type != BPF_MAP_TYPE_STRUCT_OPS ||
 793                     attr->btf_key_type_id || attr->btf_value_type_id)
 794                         return -EINVAL;
 795         } else if (attr->btf_key_type_id && !attr->btf_value_type_id) {
 796                 return -EINVAL;
 797         }
 798
 799         f_flags = bpf_get_file_flag(attr->map_flags);
 800         if (f_flags < 0)
 801                 return f_flags;
 802
 803         if (numa_node != NUMA_NO_NODE &&
 804             ((unsigned int)numa_node >= nr_node_ids ||
 805              !node_online(numa_node)))
 806                 return -EINVAL;
 807
 808         /* find map type and init map: hashtable vs rbtree vs bloom vs ... */
 809         map = find_and_alloc_map(attr);
 810         if (IS_ERR(map))
 811                 return PTR_ERR(map);
 812
 813         err = bpf_obj_name_cpy(map->name, attr->map_name);
 814         if (err)
 815                 goto free_map;
 816
 817         atomic64_set(&map->refcnt, 1);
 818         atomic64_set(&map->usercnt, 1);
 819         mutex_init(&map->freeze_mutex);
 820
 821         map->spin_lock_off = -EINVAL;
 822         if (attr->btf_key_type_id || attr->btf_value_type_id ||
 823             /* Even the map's value is a kernel's struct,
 824              * the bpf_prog.o must have BTF to begin with
 825              * to figure out the corresponding kernel's
 826              * counter part.  Thus, attr->btf_fd has
 827              * to be valid also.
 828              */
 829             attr->btf_vmlinux_value_type_id) {
 830                 struct btf *btf;
 831
 832                 btf = btf_get_by_fd(attr->btf_fd);
 833                 if (IS_ERR(btf)) {
 834                         err = PTR_ERR(btf);
 835                         goto free_map;
 836                 }
 837                 map->btf = btf;
 838
 839                 if (attr->btf_value_type_id) {
 840                         err = map_check_btf(map, btf, attr->btf_key_type_id,
 841                                             attr->btf_value_type_id);
 842                         if (err)
 843                                 goto free_map;
 844                 }
 845
 846                 map->btf_key_type_id = attr->btf_key_type_id;
 847                 map->btf_value_type_id = attr->btf_value_type_id;
 848                 map->btf_vmlinux_value_type_id =
 849                         attr->btf_vmlinux_value_type_id;
 850         }
 851
 852         err = security_bpf_map_alloc(map);
 853         if (err)
 854                 goto free_map;
 855
 856         err = bpf_map_alloc_id(map);
 857         if (err)
 858                 goto free_map_sec;
 859
 860         err = bpf_map_new_fd(map, f_flags);
 861         if (err < 0) {
 862                 /* failed to allocate fd.
 863                  * bpf_map_put_with_uref() is needed because the above
 864                  * bpf_map_alloc_id() has published the map
 865                  * to the userspace and the userspace may
 866                  * have refcnt-ed it through BPF_MAP_GET_FD_BY_ID.
 867                  */
 868                 bpf_map_put_with_uref(map);
 869                 return err;
 870         }
 871
 872         return err;
 873
 874 free_map_sec:
 875         security_bpf_map_free(map);
 876 free_map:
 877         btf_put(map->btf);
 878         bpf_map_charge_move(&mem, &map->memory);
 879         map->ops->map_free(map);
 880         bpf_map_charge_finish(&mem);
 881         return err;
 882 }
 883
 884 /* if error is returned, fd is released.
 885  * On success caller should complete fd access with matching fdput()
 886  */
 887 struct bpf_map *__bpf_map_get(struct fd f)
 888 {
 889         if (!f.file)
 890                 return ERR_PTR(-EBADF);
 891         if (f.file->f_op != &bpf_map_fops) {
 892                 fdput(f);
 893                 return ERR_PTR(-EINVAL);
 894         }
 895
 896         return f.file->private_data;
 897 }
 898
 899 void bpf_map_inc(struct bpf_map *map)
 900 {
 901         atomic64_inc(&map->refcnt);
 902 }
 903 EXPORT_SYMBOL_GPL(bpf_map_inc);
 904
 905 void bpf_map_inc_with_uref(struct bpf_map *map)
 906 {
 907         atomic64_inc(&map->refcnt);
 908         atomic64_inc(&map->usercnt);
 909 }
 910 EXPORT_SYMBOL_GPL(bpf_map_inc_with_uref);
 911
 912 struct bpf_map *bpf_map_get_with_uref(u32 ufd)
 913 {
 914         struct fd f = fdget(ufd);
 915         struct bpf_map *map;
 916
 917         map = __bpf_map_get(f);
 918         if (IS_ERR(map))
 919                 return map;
 920
 921         bpf_map_inc_with_uref(map);
 922         fdput(f);
 923
 924         return map;
 925 }
 926
 927 /* map_idr_lock should have been held */
 928 static struct bpf_map *__bpf_map_inc_not_zero(struct bpf_map *map, bool uref)
 929 {
 930         int refold;
 931
 932         refold = atomic64_fetch_add_unless(&map->refcnt, 1, 0);
 933         if (!refold)
 934                 return ERR_PTR(-ENOENT);
 935         if (uref)
 936                 atomic64_inc(&map->usercnt);
 937
 938         return map;
 939 }
 940
 941 struct bpf_map *bpf_map_inc_not_zero(struct bpf_map *map)
 942 {
 943         spin_lock_bh(&map_idr_lock);
 944         map = __bpf_map_inc_not_zero(map, false);
 945         spin_unlock_bh(&map_idr_lock);
 946
 947         return map;
 948 }
 949 EXPORT_SYMBOL_GPL(bpf_map_inc_not_zero);
 950
 951 int __weak bpf_stackmap_copy(struct bpf_map *map, void *key, void *value)
 952 {
 953         return -ENOTSUPP;
 954 }
 955
 956 static void *__bpf_copy_key(void __user *ukey, u64 key_size)
 957 {
 958         if (key_size)
 959                 return memdup_user(ukey, key_size);
 960
 961         if (ukey)
 962                 return ERR_PTR(-EINVAL);
 963
 964         return NULL;
 965 }
 966
 967 /* last field in 'union bpf_attr' used by this command */
 968 #define BPF_MAP_LOOKUP_ELEM_LAST_FIELD flags
 969
 970 static int map_lookup_elem(union bpf_attr *attr)
 971 {
 972         void __user *ukey = u64_to_user_ptr(attr->key);
 973         void __user *uvalue = u64_to_user_ptr(attr->value);
 974         int ufd = attr->map_fd;
 975         struct bpf_map *map;
 976         void *key, *value;
 977         u32 value_size;
 978         struct fd f;
 979         int err;
 980
 981         if (CHECK_ATTR(BPF_MAP_LOOKUP_ELEM))
 982                 return -EINVAL;
 983
 984         if (attr->flags & ~BPF_F_LOCK)
 985                 return -EINVAL;
 986
 987         f = fdget(ufd);
 988         map = __bpf_map_get(f);
 989         if (IS_ERR(map))
 990                 return PTR_ERR(map);
 991         if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ)) {
 992                 err = -EPERM;
 993                 goto err_put;
 994         }
 995
 996         if ((attr->flags & BPF_F_LOCK) &&
 997             !map_value_has_spin_lock(map)) {
 998                 err = -EINVAL;
 999                 goto err_put;
1000         }
1001
1002         key = __bpf_copy_key(ukey, map->key_size);
1003         if (IS_ERR(key)) {
1004                 err = PTR_ERR(key);
1005                 goto err_put;
1006         }
1007
1008         value_size = bpf_map_value_size(map);
1009
1010         err = -ENOMEM;
1011         value = kmalloc(value_size, GFP_USER | __GFP_NOWARN);
1012         if (!value)
1013                 goto free_key;
1014
1015         err = bpf_map_copy_value(map, key, value, attr->flags);
1016         if (err)
1017                 goto free_value;
1018
1019         err = -EFAULT;
1020         if (copy_to_user(uvalue, value, value_size) != 0)
1021                 goto free_value;
1022
1023         err = 0;
1024
1025 free_value:
1026         kfree(value);
1027 free_key:
1028         kfree(key);
1029 err_put:
1030         fdput(f);
1031         return err;
1032 }
1033
1034
1035 #define BPF_MAP_UPDATE_ELEM_LAST_FIELD flags
1036
1037 static int map_update_elem(union bpf_attr *attr)
1038 {
1039         void __user *ukey = u64_to_user_ptr(attr->key);
1040         void __user *uvalue = u64_to_user_ptr(attr->value);
1041         int ufd = attr->map_fd;
1042         struct bpf_map *map;
1043         void *key, *value;
1044         u32 value_size;
1045         struct fd f;
1046         int err;
1047
1048         if (CHECK_ATTR(BPF_MAP_UPDATE_ELEM))
1049                 return -EINVAL;
1050
1051         f = fdget(ufd);
1052         map = __bpf_map_get(f);
1053         if (IS_ERR(map))
1054                 return PTR_ERR(map);
1055         if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) {
1056                 err = -EPERM;
1057                 goto err_put;
1058         }
1059
1060         if ((attr->flags & BPF_F_LOCK) &&
1061             !map_value_has_spin_lock(map)) {
1062                 err = -EINVAL;
1063                 goto err_put;
1064         }
1065
1066         key = __bpf_copy_key(ukey, map->key_size);
1067         if (IS_ERR(key)) {
1068                 err = PTR_ERR(key);
1069                 goto err_put;
1070         }
1071
1072         if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
1073             map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH ||
1074             map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY ||
1075             map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE)
1076                 value_size = round_up(map->value_size, 8) * num_possible_cpus();
1077         else
1078                 value_size = map->value_size;
1079
1080         err = -ENOMEM;
1081         value = kmalloc(value_size, GFP_USER | __GFP_NOWARN);
1082         if (!value)
1083                 goto free_key;
1084
1085         err = -EFAULT;
1086         if (copy_from_user(value, uvalue, value_size) != 0)
1087                 goto free_value;
1088
1089         err = bpf_map_update_value(map, f, key, value, attr->flags);
1090
1091 free_value:
1092         kfree(value);
1093 free_key:
1094         kfree(key);
1095 err_put:
1096         fdput(f);
1097         return err;
1098 }
1099
1100 #define BPF_MAP_DELETE_ELEM_LAST_FIELD key
1101
1102 static int map_delete_elem(union bpf_attr *attr)
1103 {
1104         void __user *ukey = u64_to_user_ptr(attr->key);
1105         int ufd = attr->map_fd;
1106         struct bpf_map *map;
1107         struct fd f;
1108         void *key;
1109         int err;
1110
1111         if (CHECK_ATTR(BPF_MAP_DELETE_ELEM))
1112                 return -EINVAL;
1113
1114         f = fdget(ufd);
1115         map = __bpf_map_get(f);
1116         if (IS_ERR(map))
1117                 return PTR_ERR(map);
1118         if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) {
1119                 err = -EPERM;
1120                 goto err_put;
1121         }
1122
1123         key = __bpf_copy_key(ukey, map->key_size);
1124         if (IS_ERR(key)) {
1125                 err = PTR_ERR(key);
1126                 goto err_put;
1127         }
1128
1129         if (bpf_map_is_dev_bound(map)) {
1130                 err = bpf_map_offload_delete_elem(map, key);
1131                 goto out;
1132         } else if (IS_FD_PROG_ARRAY(map) ||
1133                    map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {
1134                 /* These maps require sleepable context */
1135                 err = map->ops->map_delete_elem(map, key);
1136                 goto out;
1137         }
1138
1139         preempt_disable();
1140         __this_cpu_inc(bpf_prog_active);
1141         rcu_read_lock();
1142         err = map->ops->map_delete_elem(map, key);
1143         rcu_read_unlock();
1144         __this_cpu_dec(bpf_prog_active);
1145         preempt_enable();
1146         maybe_wait_bpf_programs(map);
1147 out:
1148         kfree(key);
1149 err_put:
1150         fdput(f);
1151         return err;
1152 }
1153
1154 /* last field in 'union bpf_attr' used by this command */
1155 #define BPF_MAP_GET_NEXT_KEY_LAST_FIELD next_key
1156
1157 static int map_get_next_key(union bpf_attr *attr)
1158 {
1159         void __user *ukey = u64_to_user_ptr(attr->key);
1160         void __user *unext_key = u64_to_user_ptr(attr->next_key);
1161         int ufd = attr->map_fd;
1162         struct bpf_map *map;
1163         void *key, *next_key;
1164         struct fd f;
1165         int err;
1166
1167         if (CHECK_ATTR(BPF_MAP_GET_NEXT_KEY))
1168                 return -EINVAL;
1169
1170         f = fdget(ufd);
1171         map = __bpf_map_get(f);
1172         if (IS_ERR(map))
1173                 return PTR_ERR(map);
1174         if (!(map_get_sys_perms(map, f) & FMODE_CAN_READ)) {
1175                 err = -EPERM;
1176                 goto err_put;
1177         }
1178
1179         if (ukey) {
1180                 key = __bpf_copy_key(ukey, map->key_size);
1181                 if (IS_ERR(key)) {
1182                         err = PTR_ERR(key);
1183                         goto err_put;
1184                 }
1185         } else {
1186                 key = NULL;
1187         }
1188
1189         err = -ENOMEM;
1190         next_key = kmalloc(map->key_size, GFP_USER);
1191         if (!next_key)
1192                 goto free_key;
1193
1194         if (bpf_map_is_dev_bound(map)) {
1195                 err = bpf_map_offload_get_next_key(map, key, next_key);
1196                 goto out;
1197         }
1198
1199         rcu_read_lock();
1200         err = map->ops->map_get_next_key(map, key, next_key);
1201         rcu_read_unlock();
1202 out:
1203         if (err)
1204                 goto free_next_key;
1205
1206         err = -EFAULT;
1207         if (copy_to_user(unext_key, next_key, map->key_size) != 0)
1208                 goto free_next_key;
1209
1210         err = 0;
1211
1212 free_next_key:
1213         kfree(next_key);
1214 free_key:
1215         kfree(key);
1216 err_put:
1217         fdput(f);
1218         return err;
1219 }
1220
1221 int generic_map_delete_batch(struct bpf_map *map,
1222                              const union bpf_attr *attr,
1223                              union bpf_attr __user *uattr)
1224 {
1225         void __user *keys = u64_to_user_ptr(attr->batch.keys);
1226         u32 cp, max_count;
1227         int err = 0;
1228         void *key;
1229
1230         if (attr->batch.elem_flags & ~BPF_F_LOCK)
1231                 return -EINVAL;
1232
1233         if ((attr->batch.elem_flags & BPF_F_LOCK) &&
1234             !map_value_has_spin_lock(map)) {
1235                 return -EINVAL;
1236         }
1237
1238         max_count = attr->batch.count;
1239         if (!max_count)
1240                 return 0;
1241
1242         key = kmalloc(map->key_size, GFP_USER | __GFP_NOWARN);
1243         if (!key)
1244                 return -ENOMEM;
1245
1246         for (cp = 0; cp < max_count; cp++) {
1247                 err = -EFAULT;
1248                 if (copy_from_user(key, keys + cp * map->key_size,
1249                                    map->key_size))
1250                         break;
1251
1252                 if (bpf_map_is_dev_bound(map)) {
1253                         err = bpf_map_offload_delete_elem(map, key);
1254                         break;
1255                 }
1256
1257                 preempt_disable();
1258                 __this_cpu_inc(bpf_prog_active);
1259                 rcu_read_lock();
1260                 err = map->ops->map_delete_elem(map, key);
1261                 rcu_read_unlock();
1262                 __this_cpu_dec(bpf_prog_active);
1263                 preempt_enable();
1264                 maybe_wait_bpf_programs(map);
1265                 if (err)
1266                         break;
1267         }
1268         if (copy_to_user(&uattr->batch.count, &cp, sizeof(cp)))
1269                 err = -EFAULT;
1270
1271         kfree(key);
1272         return err;
1273 }
1274
1275 int generic_map_update_batch(struct bpf_map *map,
1276                              const union bpf_attr *attr,
1277                              union bpf_attr __user *uattr)
1278 {
1279         void __user *values = u64_to_user_ptr(attr->batch.values);
1280         void __user *keys = u64_to_user_ptr(attr->batch.keys);
1281         u32 value_size, cp, max_count;
1282         int ufd = attr->map_fd;
1283         void *key, *value;
1284         struct fd f;
1285         int err = 0;
1286
1287         f = fdget(ufd);
1288         if (attr->batch.elem_flags & ~BPF_F_LOCK)
1289                 return -EINVAL;
1290
1291         if ((attr->batch.elem_flags & BPF_F_LOCK) &&
1292             !map_value_has_spin_lock(map)) {
1293                 return -EINVAL;
1294         }
1295
1296         value_size = bpf_map_value_size(map);
1297
1298         max_count = attr->batch.count;
1299         if (!max_count)
1300                 return 0;
1301
1302         key = kmalloc(map->key_size, GFP_USER | __GFP_NOWARN);
1303         if (!key)
1304                 return -ENOMEM;
1305
1306         value = kmalloc(value_size, GFP_USER | __GFP_NOWARN);
1307         if (!value) {
1308                 kfree(key);
1309                 return -ENOMEM;
1310         }
1311
1312         for (cp = 0; cp < max_count; cp++) {
1313                 err = -EFAULT;
1314                 if (copy_from_user(key, keys + cp * map->key_size,
1315                     map->key_size) ||
1316                     copy_from_user(value, values + cp * value_size, value_size))
1317                         break;
1318
1319                 err = bpf_map_update_value(map, f, key, value,
1320                                            attr->batch.elem_flags);
1321
1322                 if (err)
1323                         break;
1324         }
1325
1326         if (copy_to_user(&uattr->batch.count, &cp, sizeof(cp)))
1327                 err = -EFAULT;
1328
1329         kfree(value);
1330         kfree(key);
1331         return err;
1332 }
1333
1334 #define MAP_LOOKUP_RETRIES 3
1335
1336 int generic_map_lookup_batch(struct bpf_map *map,
1337                                     const union bpf_attr *attr,
1338                                     union bpf_attr __user *uattr)
1339 {
1340         void __user *uobatch = u64_to_user_ptr(attr->batch.out_batch);
1341         void __user *ubatch = u64_to_user_ptr(attr->batch.in_batch);
1342         void __user *values = u64_to_user_ptr(attr->batch.values);
1343         void __user *keys = u64_to_user_ptr(attr->batch.keys);
1344         void *buf, *buf_prevkey, *prev_key, *key, *value;
1345         int err, retry = MAP_LOOKUP_RETRIES;
1346         u32 value_size, cp, max_count;
1347
1348         if (attr->batch.elem_flags & ~BPF_F_LOCK)
1349                 return -EINVAL;
1350
1351         if ((attr->batch.elem_flags & BPF_F_LOCK) &&
1352             !map_value_has_spin_lock(map))
1353                 return -EINVAL;
1354
1355         value_size = bpf_map_value_size(map);
1356
1357         max_count = attr->batch.count;
1358         if (!max_count)
1359                 return 0;
1360
1361         if (put_user(0, &uattr->batch.count))
1362                 return -EFAULT;
1363
1364         buf_prevkey = kmalloc(map->key_size, GFP_USER | __GFP_NOWARN);
1365         if (!buf_prevkey)
1366                 return -ENOMEM;
1367
1368         buf = kmalloc(map->key_size + value_size, GFP_USER | __GFP_NOWARN);
1369         if (!buf) {
1370                 kvfree(buf_prevkey);
1371                 return -ENOMEM;
1372         }
1373
1374         err = -EFAULT;
1375         prev_key = NULL;
1376         if (ubatch && copy_from_user(buf_prevkey, ubatch, map->key_size))
1377                 goto free_buf;
1378         key = buf;
1379         value = key + map->key_size;
1380         if (ubatch)
1381                 prev_key = buf_prevkey;
1382
1383         for (cp = 0; cp < max_count;) {
1384                 rcu_read_lock();
1385                 err = map->ops->map_get_next_key(map, prev_key, key);
1386                 rcu_read_unlock();
1387                 if (err)
1388                         break;
1389                 err = bpf_map_copy_value(map, key, value,
1390                                          attr->batch.elem_flags);
1391
1392                 if (err == -ENOENT) {
1393                         if (retry) {
1394                                 retry--;
1395                                 continue;
1396                         }
1397                         err = -EINTR;
1398                         break;
1399                 }
1400
1401                 if (err)
1402                         goto free_buf;
1403
1404                 if (copy_to_user(keys + cp * map->key_size, key,
1405                                  map->key_size)) {
1406                         err = -EFAULT;
1407                         goto free_buf;
1408                 }
1409                 if (copy_to_user(values + cp * value_size, value, value_size)) {
1410                         err = -EFAULT;
1411                         goto free_buf;
1412                 }
1413
1414                 if (!prev_key)
1415                         prev_key = buf_prevkey;
1416
1417                 swap(prev_key, key);
1418                 retry = MAP_LOOKUP_RETRIES;
1419                 cp++;
1420         }
1421
1422         if (err == -EFAULT)
1423                 goto free_buf;
1424
1425         if ((copy_to_user(&uattr->batch.count, &cp, sizeof(cp)) ||
1426                     (cp && copy_to_user(uobatch, prev_key, map->key_size))))
1427                 err = -EFAULT;
1428
1429 free_buf:
1430         kfree(buf_prevkey);
1431         kfree(buf);
1432         return err;
1433 }
1434
1435 #define BPF_MAP_LOOKUP_AND_DELETE_ELEM_LAST_FIELD value
1436
1437 static int map_lookup_and_delete_elem(union bpf_attr *attr)
1438 {
1439         void __user *ukey = u64_to_user_ptr(attr->key);
1440         void __user *uvalue = u64_to_user_ptr(attr->value);
1441         int ufd = attr->map_fd;
1442         struct bpf_map *map;
1443         void *key, *value;
1444         u32 value_size;
1445         struct fd f;
1446         int err;
1447
1448         if (CHECK_ATTR(BPF_MAP_LOOKUP_AND_DELETE_ELEM))
1449                 return -EINVAL;
1450
1451         f = fdget(ufd);
1452         map = __bpf_map_get(f);
1453         if (IS_ERR(map))
1454                 return PTR_ERR(map);
1455         if (!(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) {
1456                 err = -EPERM;
1457                 goto err_put;
1458         }
1459
1460         key = __bpf_copy_key(ukey, map->key_size);
1461         if (IS_ERR(key)) {
1462                 err = PTR_ERR(key);
1463                 goto err_put;
1464         }
1465
1466         value_size = map->value_size;
1467
1468         err = -ENOMEM;
1469         value = kmalloc(value_size, GFP_USER | __GFP_NOWARN);
1470         if (!value)
1471                 goto free_key;
1472
1473         if (map->map_type == BPF_MAP_TYPE_QUEUE ||
1474             map->map_type == BPF_MAP_TYPE_STACK) {
1475                 err = map->ops->map_pop_elem(map, value);
1476         } else {
1477                 err = -ENOTSUPP;
1478         }
1479
1480         if (err)
1481                 goto free_value;
1482
1483         if (copy_to_user(uvalue, value, value_size) != 0)
1484                 goto free_value;
1485
1486         err = 0;
1487
1488 free_value:
1489         kfree(value);
1490 free_key:
1491         kfree(key);
1492 err_put:
1493         fdput(f);
1494         return err;
1495 }
1496
1497 #define BPF_MAP_FREEZE_LAST_FIELD map_fd
1498
1499 static int map_freeze(const union bpf_attr *attr)
1500 {
1501         int err = 0, ufd = attr->map_fd;
1502         struct bpf_map *map;
1503         struct fd f;
1504
1505         if (CHECK_ATTR(BPF_MAP_FREEZE))
1506                 return -EINVAL;
1507
1508         f = fdget(ufd);
1509         map = __bpf_map_get(f);
1510         if (IS_ERR(map))
1511                 return PTR_ERR(map);
1512
1513         mutex_lock(&map->freeze_mutex);
1514
1515         if (map->writecnt) {
1516                 err = -EBUSY;
1517                 goto err_put;
1518         }
1519         if (READ_ONCE(map->frozen)) {
1520                 err = -EBUSY;
1521                 goto err_put;
1522         }
1523         if (!capable(CAP_SYS_ADMIN)) {
1524                 err = -EPERM;
1525                 goto err_put;
1526         }
1527
1528         WRITE_ONCE(map->frozen, true);
1529 err_put:
1530         mutex_unlock(&map->freeze_mutex);
1531         fdput(f);
1532         return err;
1533 }
1534
1535 static const struct bpf_prog_ops * const bpf_prog_types[] = {
1536 #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \
1537         [_id] = & _name ## _prog_ops,
1538 #define BPF_MAP_TYPE(_id, _ops)
1539 #include <linux/bpf_types.h>
1540 #undef BPF_PROG_TYPE
1541 #undef BPF_MAP_TYPE
1542 };
1543
1544 static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog)
1545 {
1546         const struct bpf_prog_ops *ops;
1547
1548         if (type >= ARRAY_SIZE(bpf_prog_types))
1549                 return -EINVAL;
1550         type = array_index_nospec(type, ARRAY_SIZE(bpf_prog_types));
1551         ops = bpf_prog_types[type];
1552         if (!ops)
1553                 return -EINVAL;
1554
1555         if (!bpf_prog_is_dev_bound(prog->aux))
1556                 prog->aux->ops = ops;
1557         else
1558                 prog->aux->ops = &bpf_offload_prog_ops;
1559         prog->type = type;
1560         return 0;
1561 }
1562
1563 enum bpf_audit {
1564         BPF_AUDIT_LOAD,
1565         BPF_AUDIT_UNLOAD,
1566         BPF_AUDIT_MAX,
1567 };
1568
1569 static const char * const bpf_audit_str[BPF_AUDIT_MAX] = {
1570         [BPF_AUDIT_LOAD]   = "LOAD",
1571         [BPF_AUDIT_UNLOAD] = "UNLOAD",
1572 };
1573
1574 static void bpf_audit_prog(const struct bpf_prog *prog, unsigned int op)
1575 {
1576         struct audit_context *ctx = NULL;
1577         struct audit_buffer *ab;
1578
1579         if (WARN_ON_ONCE(op >= BPF_AUDIT_MAX))
1580                 return;
1581         if (audit_enabled == AUDIT_OFF)
1582                 return;
1583         if (op == BPF_AUDIT_LOAD)
1584                 ctx = audit_context();
1585         ab = audit_log_start(ctx, GFP_ATOMIC, AUDIT_BPF);
1586         if (unlikely(!ab))
1587                 return;
1588         audit_log_format(ab, "prog-id=%u op=%s",
1589                          prog->aux->id, bpf_audit_str[op]);
1590         audit_log_end(ab);
1591 }
1592
1593 int __bpf_prog_charge(struct user_struct *user, u32 pages)
1594 {
1595         unsigned long memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
1596         unsigned long user_bufs;
1597
1598         if (user) {
1599                 user_bufs = atomic_long_add_return(pages, &user->locked_vm);
1600                 if (user_bufs > memlock_limit) {
1601                         atomic_long_sub(pages, &user->locked_vm);
1602                         return -EPERM;
1603                 }
1604         }
1605
1606         return 0;
1607 }
1608
1609 void __bpf_prog_uncharge(struct user_struct *user, u32 pages)
1610 {
1611         if (user)
1612                 atomic_long_sub(pages, &user->locked_vm);
1613 }
1614
1615 static int bpf_prog_charge_memlock(struct bpf_prog *prog)
1616 {
1617         struct user_struct *user = get_current_user();
1618         int ret;
1619
1620         ret = __bpf_prog_charge(user, prog->pages);
1621         if (ret) {
1622                 free_uid(user);
1623                 return ret;
1624         }
1625
1626         prog->aux->user = user;
1627         return 0;
1628 }
1629
1630 static void bpf_prog_uncharge_memlock(struct bpf_prog *prog)
1631 {
1632         struct user_struct *user = prog->aux->user;
1633
1634         __bpf_prog_uncharge(user, prog->pages);
1635         free_uid(user);
1636 }
1637
1638 static int bpf_prog_alloc_id(struct bpf_prog *prog)
1639 {
1640         int id;
1641
1642         idr_preload(GFP_KERNEL);
1643         spin_lock_bh(&prog_idr_lock);
1644         id = idr_alloc_cyclic(&prog_idr, prog, 1, INT_MAX, GFP_ATOMIC);
1645         if (id > 0)
1646                 prog->aux->id = id;
1647         spin_unlock_bh(&prog_idr_lock);
1648         idr_preload_end();
1649
1650         /* id is in [1, INT_MAX) */
1651         if (WARN_ON_ONCE(!id))
1652                 return -ENOSPC;
1653
1654         return id > 0 ? 0 : id;
1655 }
1656
1657 void bpf_prog_free_id(struct bpf_prog *prog, bool do_idr_lock)
1658 {
1659         /* cBPF to eBPF migrations are currently not in the idr store.
1660          * Offloaded programs are removed from the store when their device
1661          * disappears - even if someone grabs an fd to them they are unusable,
1662          * simply waiting for refcnt to drop to be freed.
1663          */
1664         if (!prog->aux->id)
1665                 return;
1666
1667         if (do_idr_lock)
1668                 spin_lock_bh(&prog_idr_lock);
1669         else
1670                 __acquire(&prog_idr_lock);
1671
1672         idr_remove(&prog_idr, prog->aux->id);
1673         prog->aux->id = 0;
1674
1675         if (do_idr_lock)
1676                 spin_unlock_bh(&prog_idr_lock);
1677         else
1678                 __release(&prog_idr_lock);
1679 }
1680
1681 static void __bpf_prog_put_rcu(struct rcu_head *rcu)
1682 {
1683         struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu);
1684
1685         kvfree(aux->func_info);
1686         kfree(aux->func_info_aux);
1687         bpf_prog_uncharge_memlock(aux->prog);
1688         security_bpf_prog_free(aux);
1689         bpf_prog_free(aux->prog);
1690 }
1691
1692 static void __bpf_prog_put_noref(struct bpf_prog *prog, bool deferred)
1693 {
1694         bpf_prog_kallsyms_del_all(prog);
1695         btf_put(prog->aux->btf);
1696         bpf_prog_free_linfo(prog);
1697
1698         if (deferred)
1699                 call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu);
1700         else
1701                 __bpf_prog_put_rcu(&prog->aux->rcu);
1702 }
1703
1704 static void __bpf_prog_put(struct bpf_prog *prog, bool do_idr_lock)
1705 {
1706         if (atomic64_dec_and_test(&prog->aux->refcnt)) {
1707                 perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_UNLOAD, 0);
1708                 bpf_audit_prog(prog, BPF_AUDIT_UNLOAD);
1709                 /* bpf_prog_free_id() must be called first */
1710                 bpf_prog_free_id(prog, do_idr_lock);
1711                 __bpf_prog_put_noref(prog, true);
1712         }
1713 }
1714
1715 void bpf_prog_put(struct bpf_prog *prog)
1716 {
1717         __bpf_prog_put(prog, true);
1718 }
1719 EXPORT_SYMBOL_GPL(bpf_prog_put);
1720
1721 static int bpf_prog_release(struct inode *inode, struct file *filp)
1722 {
1723         struct bpf_prog *prog = filp->private_data;
1724
1725         bpf_prog_put(prog);
1726         return 0;
1727 }
1728
1729 static void bpf_prog_get_stats(const struct bpf_prog *prog,
1730                                struct bpf_prog_stats *stats)
1731 {
1732         u64 nsecs = 0, cnt = 0;
1733         int cpu;
1734
1735         for_each_possible_cpu(cpu) {
1736                 const struct bpf_prog_stats *st;
1737                 unsigned int start;
1738                 u64 tnsecs, tcnt;
1739
1740                 st = per_cpu_ptr(prog->aux->stats, cpu);
1741                 do {
1742                         start = u64_stats_fetch_begin_irq(&st->syncp);
1743                         tnsecs = st->nsecs;
1744                         tcnt = st->cnt;
1745                 } while (u64_stats_fetch_retry_irq(&st->syncp, start));
1746                 nsecs += tnsecs;
1747                 cnt += tcnt;
1748         }
1749         stats->nsecs = nsecs;
1750         stats->cnt = cnt;
1751 }
1752
1753 #ifdef CONFIG_PROC_FS
1754 static void bpf_prog_show_fdinfo(struct seq_file *m, struct file *filp)
1755 {
1756         const struct bpf_prog *prog = filp->private_data;
1757         char prog_tag[sizeof(prog->tag) * 2 + 1] = { };
1758         struct bpf_prog_stats stats;
1759
1760         bpf_prog_get_stats(prog, &stats);
1761         bin2hex(prog_tag, prog->tag, sizeof(prog->tag));
1762         seq_printf(m,
1763                    "prog_type:\t%u\n"
1764                    "prog_jited:\t%u\n"
1765                    "prog_tag:\t%s\n"
1766                    "memlock:\t%llu\n"
1767                    "prog_id:\t%u\n"
1768                    "run_time_ns:\t%llu\n"
1769                    "run_cnt:\t%llu\n",
1770                    prog->type,
1771                    prog->jited,
1772                    prog_tag,
1773                    prog->pages * 1ULL << PAGE_SHIFT,
1774                    prog->aux->id,
1775                    stats.nsecs,
1776                    stats.cnt);
1777 }
1778 #endif
1779
1780 const struct file_operations bpf_prog_fops = {
1781 #ifdef CONFIG_PROC_FS
1782         .show_fdinfo    = bpf_prog_show_fdinfo,
1783 #endif
1784         .release        = bpf_prog_release,
1785         .read           = bpf_dummy_read,
1786         .write          = bpf_dummy_write,
1787 };
1788
1789 int bpf_prog_new_fd(struct bpf_prog *prog)
1790 {
1791         int ret;
1792
1793         ret = security_bpf_prog(prog);
1794         if (ret < 0)
1795                 return ret;
1796
1797         return anon_inode_getfd("bpf-prog", &bpf_prog_fops, prog,
1798                                 O_RDWR | O_CLOEXEC);
1799 }
1800
1801 static struct bpf_prog *____bpf_prog_get(struct fd f)
1802 {
1803         if (!f.file)
1804                 return ERR_PTR(-EBADF);
1805         if (f.file->f_op != &bpf_prog_fops) {
1806                 fdput(f);
1807                 return ERR_PTR(-EINVAL);
1808         }
1809
1810         return f.file->private_data;
1811 }
1812
1813 void bpf_prog_add(struct bpf_prog *prog, int i)
1814 {
1815         atomic64_add(i, &prog->aux->refcnt);
1816 }
1817 EXPORT_SYMBOL_GPL(bpf_prog_add);
1818
1819 void bpf_prog_sub(struct bpf_prog *prog, int i)
1820 {
1821         /* Only to be used for undoing previous bpf_prog_add() in some
1822          * error path. We still know that another entity in our call
1823          * path holds a reference to the program, thus atomic_sub() can
1824          * be safely used in such cases!
1825          */
1826         WARN_ON(atomic64_sub_return(i, &prog->aux->refcnt) == 0);
1827 }
1828 EXPORT_SYMBOL_GPL(bpf_prog_sub);
1829
1830 void bpf_prog_inc(struct bpf_prog *prog)
1831 {
1832         atomic64_inc(&prog->aux->refcnt);
1833 }
1834 EXPORT_SYMBOL_GPL(bpf_prog_inc);
1835
1836 /* prog_idr_lock should have been held */
1837 struct bpf_prog *bpf_prog_inc_not_zero(struct bpf_prog *prog)
1838 {
1839         int refold;
1840
1841         refold = atomic64_fetch_add_unless(&prog->aux->refcnt, 1, 0);
1842
1843         if (!refold)
1844                 return ERR_PTR(-ENOENT);
1845
1846         return prog;
1847 }
1848 EXPORT_SYMBOL_GPL(bpf_prog_inc_not_zero);
1849
1850 bool bpf_prog_get_ok(struct bpf_prog *prog,
1851                             enum bpf_prog_type *attach_type, bool attach_drv)
1852 {
1853         /* not an attachment, just a refcount inc, always allow */
1854         if (!attach_type)
1855                 return true;
1856
1857         if (prog->type != *attach_type)
1858                 return false;
1859         if (bpf_prog_is_dev_bound(prog->aux) && !attach_drv)
1860                 return false;
1861
1862         return true;
1863 }
1864
1865 static struct bpf_prog *__bpf_prog_get(u32 ufd, enum bpf_prog_type *attach_type,
1866                                        bool attach_drv)
1867 {
1868         struct fd f = fdget(ufd);
1869         struct bpf_prog *prog;
1870
1871         prog = ____bpf_prog_get(f);
1872         if (IS_ERR(prog))
1873                 return prog;
1874         if (!bpf_prog_get_ok(prog, attach_type, attach_drv)) {
1875                 prog = ERR_PTR(-EINVAL);
1876                 goto out;
1877         }
1878
1879         bpf_prog_inc(prog);
1880 out:
1881         fdput(f);
1882         return prog;
1883 }
1884
1885 struct bpf_prog *bpf_prog_get(u32 ufd)
1886 {
1887         return __bpf_prog_get(ufd, NULL, false);
1888 }
1889
1890 struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, enum bpf_prog_type type,
1891                                        bool attach_drv)
1892 {
1893         return __bpf_prog_get(ufd, &type, attach_drv);
1894 }
1895 EXPORT_SYMBOL_GPL(bpf_prog_get_type_dev);
1896
1897 /* Initially all BPF programs could be loaded w/o specifying
1898  * expected_attach_type. Later for some of them specifying expected_attach_type
1899  * at load time became required so that program could be validated properly.
1900  * Programs of types that are allowed to be loaded both w/ and w/o (for
1901  * backward compatibility) expected_attach_type, should have the default attach
1902  * type assigned to expected_attach_type for the latter case, so that it can be
1903  * validated later at attach time.
1904  *
1905  * bpf_prog_load_fixup_attach_type() sets expected_attach_type in @attr if
1906  * prog type requires it but has some attach types that have to be backward
1907  * compatible.
1908  */
1909 static void bpf_prog_load_fixup_attach_type(union bpf_attr *attr)
1910 {
1911         switch (attr->prog_type) {
1912         case BPF_PROG_TYPE_CGROUP_SOCK:
1913                 /* Unfortunately BPF_ATTACH_TYPE_UNSPEC enumeration doesn't
1914                  * exist so checking for non-zero is the way to go here.
1915                  */
1916                 if (!attr->expected_attach_type)
1917                         attr->expected_attach_type =
1918                                 BPF_CGROUP_INET_SOCK_CREATE;
1919                 break;
1920         }
1921 }
1922
1923 static int
1924 bpf_prog_load_check_attach(enum bpf_prog_type prog_type,
1925                            enum bpf_attach_type expected_attach_type,
1926                            u32 btf_id, u32 prog_fd)
1927 {
1928         if (btf_id) {
1929                 if (btf_id > BTF_MAX_TYPE)
1930                         return -EINVAL;
1931
1932                 switch (prog_type) {
1933                 case BPF_PROG_TYPE_TRACING:
1934                 case BPF_PROG_TYPE_STRUCT_OPS:
1935                 case BPF_PROG_TYPE_EXT:
1936                         break;
1937                 default:
1938                         return -EINVAL;
1939                 }
1940         }
1941
1942         if (prog_fd && prog_type != BPF_PROG_TYPE_TRACING &&
1943             prog_type != BPF_PROG_TYPE_EXT)
1944                 return -EINVAL;
1945
1946         switch (prog_type) {
1947         case BPF_PROG_TYPE_CGROUP_SOCK:
1948                 switch (expected_attach_type) {
1949                 case BPF_CGROUP_INET_SOCK_CREATE:
1950                 case BPF_CGROUP_INET4_POST_BIND:
1951                 case BPF_CGROUP_INET6_POST_BIND:
1952                         return 0;
1953                 default:
1954                         return -EINVAL;
1955                 }
1956         case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
1957                 switch (expected_attach_type) {
1958                 case BPF_CGROUP_INET4_BIND:
1959                 case BPF_CGROUP_INET6_BIND:
1960                 case BPF_CGROUP_INET4_CONNECT:
1961                 case BPF_CGROUP_INET6_CONNECT:
1962                 case BPF_CGROUP_UDP4_SENDMSG:
1963                 case BPF_CGROUP_UDP6_SENDMSG:
1964                 case BPF_CGROUP_UDP4_RECVMSG:
1965                 case BPF_CGROUP_UDP6_RECVMSG:
1966                         return 0;
1967                 default:
1968                         return -EINVAL;
1969                 }
1970         case BPF_PROG_TYPE_CGROUP_SKB:
1971                 switch (expected_attach_type) {
1972                 case BPF_CGROUP_INET_INGRESS:
1973                 case BPF_CGROUP_INET_EGRESS:
1974                         return 0;
1975                 default:
1976                         return -EINVAL;
1977                 }
1978         case BPF_PROG_TYPE_CGROUP_SOCKOPT:
1979                 switch (expected_attach_type) {
1980                 case BPF_CGROUP_SETSOCKOPT:
1981                 case BPF_CGROUP_GETSOCKOPT:
1982                         return 0;
1983                 default:
1984                         return -EINVAL;
1985                 }
1986         case BPF_PROG_TYPE_EXT:
1987                 if (expected_attach_type)
1988                         return -EINVAL;
1989                 /* fallthrough */
1990         default:
1991                 return 0;
1992         }
1993 }
1994
1995 /* last field in 'union bpf_attr' used by this command */
1996 #define BPF_PROG_LOAD_LAST_FIELD attach_prog_fd
1997
1998 static int bpf_prog_load(union bpf_attr *attr, union bpf_attr __user *uattr)
1999 {
2000         enum bpf_prog_type type = attr->prog_type;
2001         struct bpf_prog *prog;
2002         int err;
2003         char license[128];
2004         bool is_gpl;
2005
2006         if (CHECK_ATTR(BPF_PROG_LOAD))
2007                 return -EINVAL;
2008
2009         if (attr->prog_flags & ~(BPF_F_STRICT_ALIGNMENT |
2010                                  BPF_F_ANY_ALIGNMENT |
2011                                  BPF_F_TEST_STATE_FREQ |
2012                                  BPF_F_TEST_RND_HI32))
2013                 return -EINVAL;
2014
2015         if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) &&
2016             (attr->prog_flags & BPF_F_ANY_ALIGNMENT) &&
2017             !capable(CAP_SYS_ADMIN))
2018                 return -EPERM;
2019
2020         /* copy eBPF program license from user space */
2021         if (strncpy_from_user(license, u64_to_user_ptr(attr->license),
2022                               sizeof(license) - 1) < 0)
2023                 return -EFAULT;
2024         license[sizeof(license) - 1] = 0;
2025
2026         /* eBPF programs must be GPL compatible to use GPL-ed functions */
2027         is_gpl = license_is_gpl_compatible(license);
2028
2029         if (attr->insn_cnt == 0 ||
2030             attr->insn_cnt > (capable(CAP_SYS_ADMIN) ? BPF_COMPLEXITY_LIMIT_INSNS : BPF_MAXINSNS))
2031                 return -E2BIG;
2032         if (type != BPF_PROG_TYPE_SOCKET_FILTER &&
2033             type != BPF_PROG_TYPE_CGROUP_SKB &&
2034             !capable(CAP_SYS_ADMIN))
2035                 return -EPERM;
2036
2037         bpf_prog_load_fixup_attach_type(attr);
2038         if (bpf_prog_load_check_attach(type, attr->expected_attach_type,
2039                                        attr->attach_btf_id,
2040                                        attr->attach_prog_fd))
2041                 return -EINVAL;
2042
2043         /* plain bpf_prog allocation */
2044         prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER);
2045         if (!prog)
2046                 return -ENOMEM;
2047
2048         prog->expected_attach_type = attr->expected_attach_type;
2049         prog->aux->attach_btf_id = attr->attach_btf_id;
2050         if (attr->attach_prog_fd) {
2051                 struct bpf_prog *tgt_prog;
2052
2053                 tgt_prog = bpf_prog_get(attr->attach_prog_fd);
2054                 if (IS_ERR(tgt_prog)) {
2055                         err = PTR_ERR(tgt_prog);
2056                         goto free_prog_nouncharge;
2057                 }
2058                 prog->aux->linked_prog = tgt_prog;
2059         }
2060
2061         prog->aux->offload_requested = !!attr->prog_ifindex;
2062
2063         err = security_bpf_prog_alloc(prog->aux);
2064         if (err)
2065                 goto free_prog_nouncharge;
2066
2067         err = bpf_prog_charge_memlock(prog);
2068         if (err)
2069                 goto free_prog_sec;
2070
2071         prog->len = attr->insn_cnt;
2072
2073         err = -EFAULT;
2074         if (copy_from_user(prog->insns, u64_to_user_ptr(attr->insns),
2075                            bpf_prog_insn_size(prog)) != 0)
2076                 goto free_prog;
2077
2078         prog->orig_prog = NULL;
2079         prog->jited = 0;
2080
2081         atomic64_set(&prog->aux->refcnt, 1);
2082         prog->gpl_compatible = is_gpl ? 1 : 0;
2083
2084         if (bpf_prog_is_dev_bound(prog->aux)) {
2085                 err = bpf_prog_offload_init(prog, attr);
2086                 if (err)
2087                         goto free_prog;
2088         }
2089
2090         /* find program type: socket_filter vs tracing_filter */
2091         err = find_prog_type(type, prog);
2092         if (err < 0)
2093                 goto free_prog;
2094
2095         prog->aux->load_time = ktime_get_boottime_ns();
2096         err = bpf_obj_name_cpy(prog->aux->name, attr->prog_name);
2097         if (err)
2098                 goto free_prog;
2099
2100         /* run eBPF verifier */
2101         err = bpf_check(&prog, attr, uattr);
2102         if (err < 0)
2103                 goto free_used_maps;
2104
2105         prog = bpf_prog_select_runtime(prog, &err);
2106         if (err < 0)
2107                 goto free_used_maps;
2108
2109         err = bpf_prog_alloc_id(prog);
2110         if (err)
2111                 goto free_used_maps;
2112
2113         /* Upon success of bpf_prog_alloc_id(), the BPF prog is
2114          * effectively publicly exposed. However, retrieving via
2115          * bpf_prog_get_fd_by_id() will take another reference,
2116          * therefore it cannot be gone underneath us.
2117          *
2118          * Only for the time /after/ successful bpf_prog_new_fd()
2119          * and before returning to userspace, we might just hold
2120          * one reference and any parallel close on that fd could
2121          * rip everything out. Hence, below notifications must
2122          * happen before bpf_prog_new_fd().
2123          *
2124          * Also, any failure handling from this point onwards must
2125          * be using bpf_prog_put() given the program is exposed.
2126          */
2127         bpf_prog_kallsyms_add(prog);
2128         perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_LOAD, 0);
2129         bpf_audit_prog(prog, BPF_AUDIT_LOAD);
2130
2131         err = bpf_prog_new_fd(prog);
2132         if (err < 0)
2133                 bpf_prog_put(prog);
2134         return err;
2135
2136 free_used_maps:
2137         /* In case we have subprogs, we need to wait for a grace
2138          * period before we can tear down JIT memory since symbols
2139          * are already exposed under kallsyms.
2140          */
2141         __bpf_prog_put_noref(prog, prog->aux->func_cnt);
2142         return err;
2143 free_prog:
2144         bpf_prog_uncharge_memlock(prog);
2145 free_prog_sec:
2146         security_bpf_prog_free(prog->aux);
2147 free_prog_nouncharge:
2148         bpf_prog_free(prog);
2149         return err;
2150 }
2151
2152 #define BPF_OBJ_LAST_FIELD file_flags
2153
2154 static int bpf_obj_pin(const union bpf_attr *attr)
2155 {
2156         if (CHECK_ATTR(BPF_OBJ) || attr->file_flags != 0)
2157                 return -EINVAL;
2158
2159         return bpf_obj_pin_user(attr->bpf_fd, u64_to_user_ptr(attr->pathname));
2160 }
2161
2162 static int bpf_obj_get(const union bpf_attr *attr)
2163 {
2164         if (CHECK_ATTR(BPF_OBJ) || attr->bpf_fd != 0 ||
2165             attr->file_flags & ~BPF_OBJ_FLAG_MASK)
2166                 return -EINVAL;
2167
2168         return bpf_obj_get_user(u64_to_user_ptr(attr->pathname),
2169                                 attr->file_flags);
2170 }
2171
2172 static int bpf_tracing_prog_release(struct inode *inode, struct file *filp)
2173 {
2174         struct bpf_prog *prog = filp->private_data;
2175
2176         WARN_ON_ONCE(bpf_trampoline_unlink_prog(prog));
2177         bpf_prog_put(prog);
2178         return 0;
2179 }
2180
2181 static const struct file_operations bpf_tracing_prog_fops = {
2182         .release        = bpf_tracing_prog_release,
2183         .read           = bpf_dummy_read,
2184         .write          = bpf_dummy_write,
2185 };
2186
2187 static int bpf_tracing_prog_attach(struct bpf_prog *prog)
2188 {
2189         int tr_fd, err;
2190
2191         if (prog->expected_attach_type != BPF_TRACE_FENTRY &&
2192             prog->expected_attach_type != BPF_TRACE_FEXIT &&
2193             prog->type != BPF_PROG_TYPE_EXT) {
2194                 err = -EINVAL;
2195                 goto out_put_prog;
2196         }
2197
2198         err = bpf_trampoline_link_prog(prog);
2199         if (err)
2200                 goto out_put_prog;
2201
2202         tr_fd = anon_inode_getfd("bpf-tracing-prog", &bpf_tracing_prog_fops,
2203                                  prog, O_CLOEXEC);
2204         if (tr_fd < 0) {
2205                 WARN_ON_ONCE(bpf_trampoline_unlink_prog(prog));
2206                 err = tr_fd;
2207                 goto out_put_prog;
2208         }
2209         return tr_fd;
2210
2211 out_put_prog:
2212         bpf_prog_put(prog);
2213         return err;
2214 }
2215
2216 struct bpf_raw_tracepoint {
2217         struct bpf_raw_event_map *btp;
2218         struct bpf_prog *prog;
2219 };
2220
2221 static int bpf_raw_tracepoint_release(struct inode *inode, struct file *filp)
2222 {
2223         struct bpf_raw_tracepoint *raw_tp = filp->private_data;
2224
2225         if (raw_tp->prog) {
2226                 bpf_probe_unregister(raw_tp->btp, raw_tp->prog);
2227                 bpf_prog_put(raw_tp->prog);
2228         }
2229         bpf_put_raw_tracepoint(raw_tp->btp);
2230         kfree(raw_tp);
2231         return 0;
2232 }
2233
2234 static const struct file_operations bpf_raw_tp_fops = {
2235         .release        = bpf_raw_tracepoint_release,
2236         .read           = bpf_dummy_read,
2237         .write          = bpf_dummy_write,
2238 };
2239
2240 #define BPF_RAW_TRACEPOINT_OPEN_LAST_FIELD raw_tracepoint.prog_fd
2241
2242 static int bpf_raw_tracepoint_open(const union bpf_attr *attr)
2243 {
2244         struct bpf_raw_tracepoint *raw_tp;
2245         struct bpf_raw_event_map *btp;
2246         struct bpf_prog *prog;
2247         const char *tp_name;
2248         char buf[128];
2249         int tp_fd, err;
2250
2251         if (CHECK_ATTR(BPF_RAW_TRACEPOINT_OPEN))
2252                 return -EINVAL;
2253
2254         prog = bpf_prog_get(attr->raw_tracepoint.prog_fd);
2255         if (IS_ERR(prog))
2256                 return PTR_ERR(prog);
2257
2258         if (prog->type != BPF_PROG_TYPE_RAW_TRACEPOINT &&
2259             prog->type != BPF_PROG_TYPE_TRACING &&
2260             prog->type != BPF_PROG_TYPE_EXT &&
2261             prog->type != BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE) {
2262                 err = -EINVAL;
2263                 goto out_put_prog;
2264         }
2265
2266         if (prog->type == BPF_PROG_TYPE_TRACING ||
2267             prog->type == BPF_PROG_TYPE_EXT) {
2268                 if (attr->raw_tracepoint.name) {
2269                         /* The attach point for this category of programs
2270                          * should be specified via btf_id during program load.
2271                          */
2272                         err = -EINVAL;
2273                         goto out_put_prog;
2274                 }
2275                 if (prog->expected_attach_type == BPF_TRACE_RAW_TP)
2276                         tp_name = prog->aux->attach_func_name;
2277                 else
2278                         return bpf_tracing_prog_attach(prog);
2279         } else {
2280                 if (strncpy_from_user(buf,
2281                                       u64_to_user_ptr(attr->raw_tracepoint.name),
2282                                       sizeof(buf) - 1) < 0) {
2283                         err = -EFAULT;
2284                         goto out_put_prog;
2285                 }
2286                 buf[sizeof(buf) - 1] = 0;
2287                 tp_name = buf;
2288         }
2289
2290         btp = bpf_get_raw_tracepoint(tp_name);
2291         if (!btp) {
2292                 err = -ENOENT;
2293                 goto out_put_prog;
2294         }
2295
2296         raw_tp = kzalloc(sizeof(*raw_tp), GFP_USER);
2297         if (!raw_tp) {
2298                 err = -ENOMEM;
2299                 goto out_put_btp;
2300         }
2301         raw_tp->btp = btp;
2302         raw_tp->prog = prog;
2303
2304         err = bpf_probe_register(raw_tp->btp, prog);
2305         if (err)
2306                 goto out_free_tp;
2307
2308         tp_fd = anon_inode_getfd("bpf-raw-tracepoint", &bpf_raw_tp_fops, raw_tp,
2309                                  O_CLOEXEC);
2310         if (tp_fd < 0) {
2311                 bpf_probe_unregister(raw_tp->btp, prog);
2312                 err = tp_fd;
2313                 goto out_free_tp;
2314         }
2315         return tp_fd;
2316
2317 out_free_tp:
2318         kfree(raw_tp);
2319 out_put_btp:
2320         bpf_put_raw_tracepoint(btp);
2321 out_put_prog:
2322         bpf_prog_put(prog);
2323         return err;
2324 }
2325
2326 static int bpf_prog_attach_check_attach_type(const struct bpf_prog *prog,
2327                                              enum bpf_attach_type attach_type)
2328 {
2329         switch (prog->type) {
2330         case BPF_PROG_TYPE_CGROUP_SOCK:
2331         case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
2332         case BPF_PROG_TYPE_CGROUP_SOCKOPT:
2333                 return attach_type == prog->expected_attach_type ? 0 : -EINVAL;
2334         case BPF_PROG_TYPE_CGROUP_SKB:
2335                 return prog->enforce_expected_attach_type &&
2336                         prog->expected_attach_type != attach_type ?
2337                         -EINVAL : 0;
2338         default:
2339                 return 0;
2340         }
2341 }
2342
2343 #define BPF_PROG_ATTACH_LAST_FIELD replace_bpf_fd
2344
2345 #define BPF_F_ATTACH_MASK \
2346         (BPF_F_ALLOW_OVERRIDE | BPF_F_ALLOW_MULTI | BPF_F_REPLACE)
2347
2348 static int bpf_prog_attach(const union bpf_attr *attr)
2349 {
2350         enum bpf_prog_type ptype;
2351         struct bpf_prog *prog;
2352         int ret;
2353
2354         if (!capable(CAP_NET_ADMIN))
2355                 return -EPERM;
2356
2357         if (CHECK_ATTR(BPF_PROG_ATTACH))
2358                 return -EINVAL;
2359
2360         if (attr->attach_flags & ~BPF_F_ATTACH_MASK)
2361                 return -EINVAL;
2362
2363         switch (attr->attach_type) {
2364         case BPF_CGROUP_INET_INGRESS:
2365         case BPF_CGROUP_INET_EGRESS:
2366                 ptype = BPF_PROG_TYPE_CGROUP_SKB;
2367                 break;
2368         case BPF_CGROUP_INET_SOCK_CREATE:
2369         case BPF_CGROUP_INET4_POST_BIND:
2370         case BPF_CGROUP_INET6_POST_BIND:
2371                 ptype = BPF_PROG_TYPE_CGROUP_SOCK;
2372                 break;
2373         case BPF_CGROUP_INET4_BIND:
2374         case BPF_CGROUP_INET6_BIND:
2375         case BPF_CGROUP_INET4_CONNECT:
2376         case BPF_CGROUP_INET6_CONNECT:
2377         case BPF_CGROUP_UDP4_SENDMSG:
2378         case BPF_CGROUP_UDP6_SENDMSG:
2379         case BPF_CGROUP_UDP4_RECVMSG:
2380         case BPF_CGROUP_UDP6_RECVMSG:
2381                 ptype = BPF_PROG_TYPE_CGROUP_SOCK_ADDR;
2382                 break;
2383         case BPF_CGROUP_SOCK_OPS:
2384                 ptype = BPF_PROG_TYPE_SOCK_OPS;
2385                 break;
2386         case BPF_CGROUP_DEVICE:
2387                 ptype = BPF_PROG_TYPE_CGROUP_DEVICE;
2388                 break;
2389         case BPF_SK_MSG_VERDICT:
2390                 ptype = BPF_PROG_TYPE_SK_MSG;
2391                 break;
2392         case BPF_SK_SKB_STREAM_PARSER:
2393         case BPF_SK_SKB_STREAM_VERDICT:
2394                 ptype = BPF_PROG_TYPE_SK_SKB;
2395                 break;
2396         case BPF_LIRC_MODE2:
2397                 ptype = BPF_PROG_TYPE_LIRC_MODE2;
2398                 break;
2399         case BPF_FLOW_DISSECTOR:
2400                 ptype = BPF_PROG_TYPE_FLOW_DISSECTOR;
2401                 break;
2402         case BPF_CGROUP_SYSCTL:
2403                 ptype = BPF_PROG_TYPE_CGROUP_SYSCTL;
2404                 break;
2405         case BPF_CGROUP_GETSOCKOPT:
2406         case BPF_CGROUP_SETSOCKOPT:
2407                 ptype = BPF_PROG_TYPE_CGROUP_SOCKOPT;
2408                 break;
2409         default:
2410                 return -EINVAL;
2411         }
2412
2413         prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype);
2414         if (IS_ERR(prog))
2415                 return PTR_ERR(prog);
2416
2417         if (bpf_prog_attach_check_attach_type(prog, attr->attach_type)) {
2418                 bpf_prog_put(prog);
2419                 return -EINVAL;
2420         }
2421
2422         switch (ptype) {
2423         case BPF_PROG_TYPE_SK_SKB:
2424         case BPF_PROG_TYPE_SK_MSG:
2425                 ret = sock_map_get_from_fd(attr, prog);
2426                 break;
2427         case BPF_PROG_TYPE_LIRC_MODE2:
2428                 ret = lirc_prog_attach(attr, prog);
2429                 break;
2430         case BPF_PROG_TYPE_FLOW_DISSECTOR:
2431                 ret = skb_flow_dissector_bpf_prog_attach(attr, prog);
2432                 break;
2433         default:
2434                 ret = cgroup_bpf_prog_attach(attr, ptype, prog);
2435         }
2436
2437         if (ret)
2438                 bpf_prog_put(prog);
2439         return ret;
2440 }
2441
2442 #define BPF_PROG_DETACH_LAST_FIELD attach_type
2443
2444 static int bpf_prog_detach(const union bpf_attr *attr)
2445 {
2446         enum bpf_prog_type ptype;
2447
2448         if (!capable(CAP_NET_ADMIN))
2449                 return -EPERM;
2450
2451         if (CHECK_ATTR(BPF_PROG_DETACH))
2452                 return -EINVAL;
2453
2454         switch (attr->attach_type) {
2455         case BPF_CGROUP_INET_INGRESS:
2456         case BPF_CGROUP_INET_EGRESS:
2457                 ptype = BPF_PROG_TYPE_CGROUP_SKB;
2458                 break;
2459         case BPF_CGROUP_INET_SOCK_CREATE:
2460         case BPF_CGROUP_INET4_POST_BIND:
2461         case BPF_CGROUP_INET6_POST_BIND:
2462                 ptype = BPF_PROG_TYPE_CGROUP_SOCK;
2463                 break;
2464         case BPF_CGROUP_INET4_BIND:
2465         case BPF_CGROUP_INET6_BIND:
2466         case BPF_CGROUP_INET4_CONNECT:
2467         case BPF_CGROUP_INET6_CONNECT:
2468         case BPF_CGROUP_UDP4_SENDMSG:
2469         case BPF_CGROUP_UDP6_SENDMSG:
2470         case BPF_CGROUP_UDP4_RECVMSG:
2471         case BPF_CGROUP_UDP6_RECVMSG:
2472                 ptype = BPF_PROG_TYPE_CGROUP_SOCK_ADDR;
2473                 break;
2474         case BPF_CGROUP_SOCK_OPS:
2475                 ptype = BPF_PROG_TYPE_SOCK_OPS;
2476                 break;
2477         case BPF_CGROUP_DEVICE:
2478                 ptype = BPF_PROG_TYPE_CGROUP_DEVICE;
2479                 break;
2480         case BPF_SK_MSG_VERDICT:
2481                 return sock_map_get_from_fd(attr, NULL);
2482         case BPF_SK_SKB_STREAM_PARSER:
2483         case BPF_SK_SKB_STREAM_VERDICT:
2484                 return sock_map_get_from_fd(attr, NULL);
2485         case BPF_LIRC_MODE2:
2486                 return lirc_prog_detach(attr);
2487         case BPF_FLOW_DISSECTOR:
2488                 return skb_flow_dissector_bpf_prog_detach(attr);
2489         case BPF_CGROUP_SYSCTL:
2490                 ptype = BPF_PROG_TYPE_CGROUP_SYSCTL;
2491                 break;
2492         case BPF_CGROUP_GETSOCKOPT:
2493         case BPF_CGROUP_SETSOCKOPT:
2494                 ptype = BPF_PROG_TYPE_CGROUP_SOCKOPT;
2495                 break;
2496         default:
2497                 return -EINVAL;
2498         }
2499
2500         return cgroup_bpf_prog_detach(attr, ptype);
2501 }
2502
2503 #define BPF_PROG_QUERY_LAST_FIELD query.prog_cnt
2504
2505 static int bpf_prog_query(const union bpf_attr *attr,
2506                           union bpf_attr __user *uattr)
2507 {
2508         if (!capable(CAP_NET_ADMIN))
2509                 return -EPERM;
2510         if (CHECK_ATTR(BPF_PROG_QUERY))
2511                 return -EINVAL;
2512         if (attr->query.query_flags & ~BPF_F_QUERY_EFFECTIVE)
2513                 return -EINVAL;
2514
2515         switch (attr->query.attach_type) {
2516         case BPF_CGROUP_INET_INGRESS:
2517         case BPF_CGROUP_INET_EGRESS:
2518         case BPF_CGROUP_INET_SOCK_CREATE:
2519         case BPF_CGROUP_INET4_BIND:
2520         case BPF_CGROUP_INET6_BIND:
2521         case BPF_CGROUP_INET4_POST_BIND:
2522         case BPF_CGROUP_INET6_POST_BIND:
2523         case BPF_CGROUP_INET4_CONNECT:
2524         case BPF_CGROUP_INET6_CONNECT:
2525         case BPF_CGROUP_UDP4_SENDMSG:
2526         case BPF_CGROUP_UDP6_SENDMSG:
2527         case BPF_CGROUP_UDP4_RECVMSG:
2528         case BPF_CGROUP_UDP6_RECVMSG:
2529         case BPF_CGROUP_SOCK_OPS:
2530         case BPF_CGROUP_DEVICE:
2531         case BPF_CGROUP_SYSCTL:
2532         case BPF_CGROUP_GETSOCKOPT:
2533         case BPF_CGROUP_SETSOCKOPT:
2534                 break;
2535         case BPF_LIRC_MODE2:
2536                 return lirc_prog_query(attr, uattr);
2537         case BPF_FLOW_DISSECTOR:
2538                 return skb_flow_dissector_prog_query(attr, uattr);
2539         default:
2540                 return -EINVAL;
2541         }
2542
2543         return cgroup_bpf_prog_query(attr, uattr);
2544 }
2545
2546 #define BPF_PROG_TEST_RUN_LAST_FIELD test.ctx_out
2547
2548 static int bpf_prog_test_run(const union bpf_attr *attr,
2549                              union bpf_attr __user *uattr)
2550 {
2551         struct bpf_prog *prog;
2552         int ret = -ENOTSUPP;
2553
2554         if (!capable(CAP_SYS_ADMIN))
2555                 return -EPERM;
2556         if (CHECK_ATTR(BPF_PROG_TEST_RUN))
2557                 return -EINVAL;
2558
2559         if ((attr->test.ctx_size_in && !attr->test.ctx_in) ||
2560             (!attr->test.ctx_size_in && attr->test.ctx_in))
2561                 return -EINVAL;
2562
2563         if ((attr->test.ctx_size_out && !attr->test.ctx_out) ||
2564             (!attr->test.ctx_size_out && attr->test.ctx_out))
2565                 return -EINVAL;
2566
2567         prog = bpf_prog_get(attr->test.prog_fd);
2568         if (IS_ERR(prog))
2569                 return PTR_ERR(prog);
2570
2571         if (prog->aux->ops->test_run)
2572                 ret = prog->aux->ops->test_run(prog, attr, uattr);
2573
2574         bpf_prog_put(prog);
2575         return ret;
2576 }
2577
2578 #define BPF_OBJ_GET_NEXT_ID_LAST_FIELD next_id
2579
2580 static int bpf_obj_get_next_id(const union bpf_attr *attr,
2581                                union bpf_attr __user *uattr,
2582                                struct idr *idr,
2583                                spinlock_t *lock)
2584 {
2585         u32 next_id = attr->start_id;
2586         int err = 0;
2587
2588         if (CHECK_ATTR(BPF_OBJ_GET_NEXT_ID) || next_id >= INT_MAX)
2589                 return -EINVAL;
2590
2591         if (!capable(CAP_SYS_ADMIN))
2592                 return -EPERM;
2593
2594         next_id++;
2595         spin_lock_bh(lock);
2596         if (!idr_get_next(idr, &next_id))
2597                 err = -ENOENT;
2598         spin_unlock_bh(lock);
2599
2600         if (!err)
2601                 err = put_user(next_id, &uattr->next_id);
2602
2603         return err;
2604 }
2605
2606 #define BPF_PROG_GET_FD_BY_ID_LAST_FIELD prog_id
2607
2608 struct bpf_prog *bpf_prog_by_id(u32 id)
2609 {
2610         struct bpf_prog *prog;
2611
2612         if (!id)
2613                 return ERR_PTR(-ENOENT);
2614
2615         spin_lock_bh(&prog_idr_lock);
2616         prog = idr_find(&prog_idr, id);
2617         if (prog)
2618                 prog = bpf_prog_inc_not_zero(prog);
2619         else
2620                 prog = ERR_PTR(-ENOENT);
2621         spin_unlock_bh(&prog_idr_lock);
2622         return prog;
2623 }
2624
2625 static int bpf_prog_get_fd_by_id(const union bpf_attr *attr)
2626 {
2627         struct bpf_prog *prog;
2628         u32 id = attr->prog_id;
2629         int fd;
2630
2631         if (CHECK_ATTR(BPF_PROG_GET_FD_BY_ID))
2632                 return -EINVAL;
2633
2634         if (!capable(CAP_SYS_ADMIN))
2635                 return -EPERM;
2636
2637         prog = bpf_prog_by_id(id);
2638         if (IS_ERR(prog))
2639                 return PTR_ERR(prog);
2640
2641         fd = bpf_prog_new_fd(prog);
2642         if (fd < 0)
2643                 bpf_prog_put(prog);
2644
2645         return fd;
2646 }
2647
2648 #define BPF_MAP_GET_FD_BY_ID_LAST_FIELD open_flags
2649
2650 static int bpf_map_get_fd_by_id(const union bpf_attr *attr)
2651 {
2652         struct bpf_map *map;
2653         u32 id = attr->map_id;
2654         int f_flags;
2655         int fd;
2656
2657         if (CHECK_ATTR(BPF_MAP_GET_FD_BY_ID) ||
2658             attr->open_flags & ~BPF_OBJ_FLAG_MASK)
2659                 return -EINVAL;
2660
2661         if (!capable(CAP_SYS_ADMIN))
2662                 return -EPERM;
2663
2664         f_flags = bpf_get_file_flag(attr->open_flags);
2665         if (f_flags < 0)
2666                 return f_flags;
2667
2668         spin_lock_bh(&map_idr_lock);
2669         map = idr_find(&map_idr, id);
2670         if (map)
2671                 map = __bpf_map_inc_not_zero(map, true);
2672         else
2673                 map = ERR_PTR(-ENOENT);
2674         spin_unlock_bh(&map_idr_lock);
2675
2676         if (IS_ERR(map))
2677                 return PTR_ERR(map);
2678
2679         fd = bpf_map_new_fd(map, f_flags);
2680         if (fd < 0)
2681                 bpf_map_put_with_uref(map);
2682
2683         return fd;
2684 }
2685
2686 static const struct bpf_map *bpf_map_from_imm(const struct bpf_prog *prog,
2687                                               unsigned long addr, u32 *off,
2688                                               u32 *type)
2689 {
2690         const struct bpf_map *map;
2691         int i;
2692
2693         for (i = 0, *off = 0; i < prog->aux->used_map_cnt; i++) {
2694                 map = prog->aux->used_maps[i];
2695                 if (map == (void *)addr) {
2696                         *type = BPF_PSEUDO_MAP_FD;
2697                         return map;
2698                 }
2699                 if (!map->ops->map_direct_value_meta)
2700                         continue;
2701                 if (!map->ops->map_direct_value_meta(map, addr, off)) {
2702                         *type = BPF_PSEUDO_MAP_VALUE;
2703                         return map;
2704                 }
2705         }
2706
2707         return NULL;
2708 }
2709
2710 static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog)
2711 {
2712         const struct bpf_map *map;
2713         struct bpf_insn *insns;
2714         u32 off, type;
2715         u64 imm;
2716         int i;
2717
2718         insns = kmemdup(prog->insnsi, bpf_prog_insn_size(prog),
2719                         GFP_USER);
2720         if (!insns)
2721                 return insns;
2722
2723         for (i = 0; i < prog->len; i++) {
2724                 if (insns[i].code == (BPF_JMP | BPF_TAIL_CALL)) {
2725                         insns[i].code = BPF_JMP | BPF_CALL;
2726                         insns[i].imm = BPF_FUNC_tail_call;
2727                         /* fall-through */
2728                 }
2729                 if (insns[i].code == (BPF_JMP | BPF_CALL) ||
2730                     insns[i].code == (BPF_JMP | BPF_CALL_ARGS)) {
2731                         if (insns[i].code == (BPF_JMP | BPF_CALL_ARGS))
2732                                 insns[i].code = BPF_JMP | BPF_CALL;
2733                         if (!bpf_dump_raw_ok())
2734                                 insns[i].imm = 0;
2735                         continue;
2736                 }
2737
2738                 if (insns[i].code != (BPF_LD | BPF_IMM | BPF_DW))
2739                         continue;
2740
2741                 imm = ((u64)insns[i + 1].imm << 32) | (u32)insns[i].imm;
2742                 map = bpf_map_from_imm(prog, imm, &off, &type);
2743                 if (map) {
2744                         insns[i].src_reg = type;
2745                         insns[i].imm = map->id;
2746                         insns[i + 1].imm = off;
2747                         continue;
2748                 }
2749         }
2750
2751         return insns;
2752 }
2753
2754 static int set_info_rec_size(struct bpf_prog_info *info)
2755 {
2756         /*
2757          * Ensure info.*_rec_size is the same as kernel expected size
2758          *
2759          * or
2760          *
2761          * Only allow zero *_rec_size if both _rec_size and _cnt are
2762          * zero.  In this case, the kernel will set the expected
2763          * _rec_size back to the info.
2764          */
2765
2766         if ((info->nr_func_info || info->func_info_rec_size) &&
2767             info->func_info_rec_size != sizeof(struct bpf_func_info))
2768                 return -EINVAL;
2769
2770         if ((info->nr_line_info || info->line_info_rec_size) &&
2771             info->line_info_rec_size != sizeof(struct bpf_line_info))
2772                 return -EINVAL;
2773
2774         if ((info->nr_jited_line_info || info->jited_line_info_rec_size) &&
2775             info->jited_line_info_rec_size != sizeof(__u64))
2776                 return -EINVAL;
2777
2778         info->func_info_rec_size = sizeof(struct bpf_func_info);
2779         info->line_info_rec_size = sizeof(struct bpf_line_info);
2780         info->jited_line_info_rec_size = sizeof(__u64);
2781
2782         return 0;
2783 }
2784
2785 static int bpf_prog_get_info_by_fd(struct bpf_prog *prog,
2786                                    const union bpf_attr *attr,
2787                                    union bpf_attr __user *uattr)
2788 {
2789         struct bpf_prog_info __user *uinfo = u64_to_user_ptr(attr->info.info);
2790         struct bpf_prog_info info = {};
2791         u32 info_len = attr->info.info_len;
2792         struct bpf_prog_stats stats;
2793         char __user *uinsns;
2794         u32 ulen;
2795         int err;
2796
2797         err = bpf_check_uarg_tail_zero(uinfo, sizeof(info), info_len);
2798         if (err)
2799                 return err;
2800         info_len = min_t(u32, sizeof(info), info_len);
2801
2802         if (copy_from_user(&info, uinfo, info_len))
2803                 return -EFAULT;
2804
2805         info.type = prog->type;
2806         info.id = prog->aux->id;
2807         info.load_time = prog->aux->load_time;
2808         info.created_by_uid = from_kuid_munged(current_user_ns(),
2809                                                prog->aux->user->uid);
2810         info.gpl_compatible = prog->gpl_compatible;
2811
2812         memcpy(info.tag, prog->tag, sizeof(prog->tag));
2813         memcpy(info.name, prog->aux->name, sizeof(prog->aux->name));
2814
2815         ulen = info.nr_map_ids;
2816         info.nr_map_ids = prog->aux->used_map_cnt;
2817         ulen = min_t(u32, info.nr_map_ids, ulen);
2818         if (ulen) {
2819                 u32 __user *user_map_ids = u64_to_user_ptr(info.map_ids);
2820                 u32 i;
2821
2822                 for (i = 0; i < ulen; i++)
2823                         if (put_user(prog->aux->used_maps[i]->id,
2824                                      &user_map_ids[i]))
2825                                 return -EFAULT;
2826         }
2827
2828         err = set_info_rec_size(&info);
2829         if (err)
2830                 return err;
2831
2832         bpf_prog_get_stats(prog, &stats);
2833         info.run_time_ns = stats.nsecs;
2834         info.run_cnt = stats.cnt;
2835
2836         if (!capable(CAP_SYS_ADMIN)) {
2837                 info.jited_prog_len = 0;
2838                 info.xlated_prog_len = 0;
2839                 info.nr_jited_ksyms = 0;
2840                 info.nr_jited_func_lens = 0;
2841                 info.nr_func_info = 0;
2842                 info.nr_line_info = 0;
2843                 info.nr_jited_line_info = 0;
2844                 goto done;
2845         }
2846
2847         ulen = info.xlated_prog_len;
2848         info.xlated_prog_len = bpf_prog_insn_size(prog);
2849         if (info.xlated_prog_len && ulen) {
2850                 struct bpf_insn *insns_sanitized;
2851                 bool fault;
2852
2853                 if (prog->blinded && !bpf_dump_raw_ok()) {
2854                         info.xlated_prog_insns = 0;
2855                         goto done;
2856                 }
2857                 insns_sanitized = bpf_insn_prepare_dump(prog);
2858                 if (!insns_sanitized)
2859                         return -ENOMEM;
2860                 uinsns = u64_to_user_ptr(info.xlated_prog_insns);
2861                 ulen = min_t(u32, info.xlated_prog_len, ulen);
2862                 fault = copy_to_user(uinsns, insns_sanitized, ulen);
2863                 kfree(insns_sanitized);
2864                 if (fault)
2865                         return -EFAULT;
2866         }
2867
2868         if (bpf_prog_is_dev_bound(prog->aux)) {
2869                 err = bpf_prog_offload_info_fill(&info, prog);
2870                 if (err)
2871                         return err;
2872                 goto done;
2873         }
2874
2875         /* NOTE: the following code is supposed to be skipped for offload.
2876          * bpf_prog_offload_info_fill() is the place to fill similar fields
2877          * for offload.
2878          */
2879         ulen = info.jited_prog_len;
2880         if (prog->aux->func_cnt) {
2881                 u32 i;
2882
2883                 info.jited_prog_len = 0;
2884                 for (i = 0; i < prog->aux->func_cnt; i++)
2885                         info.jited_prog_len += prog->aux->func[i]->jited_len;
2886         } else {
2887                 info.jited_prog_len = prog->jited_len;
2888         }
2889
2890         if (info.jited_prog_len && ulen) {
2891                 if (bpf_dump_raw_ok()) {
2892                         uinsns = u64_to_user_ptr(info.jited_prog_insns);
2893                         ulen = min_t(u32, info.jited_prog_len, ulen);
2894
2895                         /* for multi-function programs, copy the JITed
2896                          * instructions for all the functions
2897                          */
2898                         if (prog->aux->func_cnt) {
2899                                 u32 len, free, i;
2900                                 u8 *img;
2901
2902                                 free = ulen;
2903                                 for (i = 0; i < prog->aux->func_cnt; i++) {
2904                                         len = prog->aux->func[i]->jited_len;
2905                                         len = min_t(u32, len, free);
2906                                         img = (u8 *) prog->aux->func[i]->bpf_func;
2907                                         if (copy_to_user(uinsns, img, len))
2908                                                 return -EFAULT;
2909                                         uinsns += len;
2910                                         free -= len;
2911                                         if (!free)
2912                                                 break;
2913                                 }
2914                         } else {
2915                                 if (copy_to_user(uinsns, prog->bpf_func, ulen))
2916                                         return -EFAULT;
2917                         }
2918                 } else {
2919                         info.jited_prog_insns = 0;
2920                 }
2921         }
2922
2923         ulen = info.nr_jited_ksyms;
2924         info.nr_jited_ksyms = prog->aux->func_cnt ? : 1;
2925         if (ulen) {
2926                 if (bpf_dump_raw_ok()) {
2927                         unsigned long ksym_addr;
2928                         u64 __user *user_ksyms;
2929                         u32 i;
2930
2931                         /* copy the address of the kernel symbol
2932                          * corresponding to each function
2933                          */
2934                         ulen = min_t(u32, info.nr_jited_ksyms, ulen);
2935                         user_ksyms = u64_to_user_ptr(info.jited_ksyms);
2936                         if (prog->aux->func_cnt) {
2937                                 for (i = 0; i < ulen; i++) {
2938                                         ksym_addr = (unsigned long)
2939                                                 prog->aux->func[i]->bpf_func;
2940                                         if (put_user((u64) ksym_addr,
2941                                                      &user_ksyms[i]))
2942                                                 return -EFAULT;
2943                                 }
2944                         } else {
2945                                 ksym_addr = (unsigned long) prog->bpf_func;
2946                                 if (put_user((u64) ksym_addr, &user_ksyms[0]))
2947                                         return -EFAULT;
2948                         }
2949                 } else {
2950                         info.jited_ksyms = 0;
2951                 }
2952         }
2953
2954         ulen = info.nr_jited_func_lens;
2955         info.nr_jited_func_lens = prog->aux->func_cnt ? : 1;
2956         if (ulen) {
2957                 if (bpf_dump_raw_ok()) {
2958                         u32 __user *user_lens;
2959                         u32 func_len, i;
2960
2961                         /* copy the JITed image lengths for each function */
2962                         ulen = min_t(u32, info.nr_jited_func_lens, ulen);
2963                         user_lens = u64_to_user_ptr(info.jited_func_lens);
2964                         if (prog->aux->func_cnt) {
2965                                 for (i = 0; i < ulen; i++) {
2966                                         func_len =
2967                                                 prog->aux->func[i]->jited_len;
2968                                         if (put_user(func_len, &user_lens[i]))
2969                                                 return -EFAULT;
2970                                 }
2971                         } else {
2972                                 func_len = prog->jited_len;
2973                                 if (put_user(func_len, &user_lens[0]))
2974                                         return -EFAULT;
2975                         }
2976                 } else {
2977                         info.jited_func_lens = 0;
2978                 }
2979         }
2980
2981         if (prog->aux->btf)
2982                 info.btf_id = btf_id(prog->aux->btf);
2983
2984         ulen = info.nr_func_info;
2985         info.nr_func_info = prog->aux->func_info_cnt;
2986         if (info.nr_func_info && ulen) {
2987                 char __user *user_finfo;
2988
2989                 user_finfo = u64_to_user_ptr(info.func_info);
2990                 ulen = min_t(u32, info.nr_func_info, ulen);
2991                 if (copy_to_user(user_finfo, prog->aux->func_info,
2992                                  info.func_info_rec_size * ulen))
2993                         return -EFAULT;
2994         }
2995
2996         ulen = info.nr_line_info;
2997         info.nr_line_info = prog->aux->nr_linfo;
2998         if (info.nr_line_info && ulen) {
2999                 __u8 __user *user_linfo;
3000
3001                 user_linfo = u64_to_user_ptr(info.line_info);
3002                 ulen = min_t(u32, info.nr_line_info, ulen);
3003                 if (copy_to_user(user_linfo, prog->aux->linfo,
3004                                  info.line_info_rec_size * ulen))
3005                         return -EFAULT;
3006         }
3007
3008         ulen = info.nr_jited_line_info;
3009         if (prog->aux->jited_linfo)
3010                 info.nr_jited_line_info = prog->aux->nr_linfo;
3011         else
3012                 info.nr_jited_line_info = 0;
3013         if (info.nr_jited_line_info && ulen) {
3014                 if (bpf_dump_raw_ok()) {
3015                         __u64 __user *user_linfo;
3016                         u32 i;
3017
3018                         user_linfo = u64_to_user_ptr(info.jited_line_info);
3019                         ulen = min_t(u32, info.nr_jited_line_info, ulen);
3020                         for (i = 0; i < ulen; i++) {
3021                                 if (put_user((__u64)(long)prog->aux->jited_linfo[i],
3022                                              &user_linfo[i]))
3023                                         return -EFAULT;
3024                         }
3025                 } else {
3026                         info.jited_line_info = 0;
3027                 }
3028         }
3029
3030         ulen = info.nr_prog_tags;
3031         info.nr_prog_tags = prog->aux->func_cnt ? : 1;
3032         if (ulen) {
3033                 __u8 __user (*user_prog_tags)[BPF_TAG_SIZE];
3034                 u32 i;
3035
3036                 user_prog_tags = u64_to_user_ptr(info.prog_tags);
3037                 ulen = min_t(u32, info.nr_prog_tags, ulen);
3038                 if (prog->aux->func_cnt) {
3039                         for (i = 0; i < ulen; i++) {
3040                                 if (copy_to_user(user_prog_tags[i],
3041                                                  prog->aux->func[i]->tag,
3042                                                  BPF_TAG_SIZE))
3043                                         return -EFAULT;
3044                         }
3045                 } else {
3046                         if (copy_to_user(user_prog_tags[0],
3047                                          prog->tag, BPF_TAG_SIZE))
3048                                 return -EFAULT;
3049                 }
3050         }
3051
3052 done:
3053         if (copy_to_user(uinfo, &info, info_len) ||
3054             put_user(info_len, &uattr->info.info_len))
3055                 return -EFAULT;
3056
3057         return 0;
3058 }
3059
3060 static int bpf_map_get_info_by_fd(struct bpf_map *map,
3061                                   const union bpf_attr *attr,
3062                                   union bpf_attr __user *uattr)
3063 {
3064         struct bpf_map_info __user *uinfo = u64_to_user_ptr(attr->info.info);
3065         struct bpf_map_info info = {};
3066         u32 info_len = attr->info.info_len;
3067         int err;
3068
3069         err = bpf_check_uarg_tail_zero(uinfo, sizeof(info), info_len);
3070         if (err)
3071                 return err;
3072         info_len = min_t(u32, sizeof(info), info_len);
3073
3074         info.type = map->map_type;
3075         info.id = map->id;
3076         info.key_size = map->key_size;
3077         info.value_size = map->value_size;
3078         info.max_entries = map->max_entries;
3079         info.map_flags = map->map_flags;
3080         memcpy(info.name, map->name, sizeof(map->name));
3081
3082         if (map->btf) {
3083                 info.btf_id = btf_id(map->btf);
3084                 info.btf_key_type_id = map->btf_key_type_id;
3085                 info.btf_value_type_id = map->btf_value_type_id;
3086         }
3087         info.btf_vmlinux_value_type_id = map->btf_vmlinux_value_type_id;
3088
3089         if (bpf_map_is_dev_bound(map)) {
3090                 err = bpf_map_offload_info_fill(&info, map);
3091                 if (err)
3092                         return err;
3093         }
3094
3095         if (copy_to_user(uinfo, &info, info_len) ||
3096             put_user(info_len, &uattr->info.info_len))
3097                 return -EFAULT;
3098
3099         return 0;
3100 }
3101
3102 static int bpf_btf_get_info_by_fd(struct btf *btf,
3103                                   const union bpf_attr *attr,
3104                                   union bpf_attr __user *uattr)
3105 {
3106         struct bpf_btf_info __user *uinfo = u64_to_user_ptr(attr->info.info);
3107         u32 info_len = attr->info.info_len;
3108         int err;
3109
3110         err = bpf_check_uarg_tail_zero(uinfo, sizeof(*uinfo), info_len);
3111         if (err)
3112                 return err;
3113
3114         return btf_get_info_by_fd(btf, attr, uattr);
3115 }
3116
3117 #define BPF_OBJ_GET_INFO_BY_FD_LAST_FIELD info.info
3118
3119 static int bpf_obj_get_info_by_fd(const union bpf_attr *attr,
3120                                   union bpf_attr __user *uattr)
3121 {
3122         int ufd = attr->info.bpf_fd;
3123         struct fd f;
3124         int err;
3125
3126         if (CHECK_ATTR(BPF_OBJ_GET_INFO_BY_FD))
3127                 return -EINVAL;
3128
3129         f = fdget(ufd);
3130         if (!f.file)
3131                 return -EBADFD;
3132
3133         if (f.file->f_op == &bpf_prog_fops)
3134                 err = bpf_prog_get_info_by_fd(f.file->private_data, attr,
3135                                               uattr);
3136         else if (f.file->f_op == &bpf_map_fops)
3137                 err = bpf_map_get_info_by_fd(f.file->private_data, attr,
3138                                              uattr);
3139         else if (f.file->f_op == &btf_fops)
3140                 err = bpf_btf_get_info_by_fd(f.file->private_data, attr, uattr);
3141         else
3142                 err = -EINVAL;
3143
3144         fdput(f);
3145         return err;
3146 }
3147
3148 #define BPF_BTF_LOAD_LAST_FIELD btf_log_level
3149
3150 static int bpf_btf_load(const union bpf_attr *attr)
3151 {
3152         if (CHECK_ATTR(BPF_BTF_LOAD))
3153                 return -EINVAL;
3154
3155         if (!capable(CAP_SYS_ADMIN))
3156                 return -EPERM;
3157
3158         return btf_new_fd(attr);
3159 }
3160
3161 #define BPF_BTF_GET_FD_BY_ID_LAST_FIELD btf_id
3162
3163 static int bpf_btf_get_fd_by_id(const union bpf_attr *attr)
3164 {
3165         if (CHECK_ATTR(BPF_BTF_GET_FD_BY_ID))
3166                 return -EINVAL;
3167
3168         if (!capable(CAP_SYS_ADMIN))
3169                 return -EPERM;
3170
3171         return btf_get_fd_by_id(attr->btf_id);
3172 }
3173
3174 static int bpf_task_fd_query_copy(const union bpf_attr *attr,
3175                                     union bpf_attr __user *uattr,
3176                                     u32 prog_id, u32 fd_type,
3177                                     const char *buf, u64 probe_offset,
3178                                     u64 probe_addr)
3179 {
3180         char __user *ubuf = u64_to_user_ptr(attr->task_fd_query.buf);
3181         u32 len = buf ? strlen(buf) : 0, input_len;
3182         int err = 0;
3183
3184         if (put_user(len, &uattr->task_fd_query.buf_len))
3185                 return -EFAULT;
3186         input_len = attr->task_fd_query.buf_len;
3187         if (input_len && ubuf) {
3188                 if (!len) {
3189                         /* nothing to copy, just make ubuf NULL terminated */
3190                         char zero = '\0';
3191
3192                         if (put_user(zero, ubuf))
3193                                 return -EFAULT;
3194                 } else if (input_len >= len + 1) {
3195                         /* ubuf can hold the string with NULL terminator */
3196                         if (copy_to_user(ubuf, buf, len + 1))
3197                                 return -EFAULT;
3198                 } else {
3199                         /* ubuf cannot hold the string with NULL terminator,
3200                          * do a partial copy with NULL terminator.
3201                          */
3202                         char zero = '\0';
3203
3204                         err = -ENOSPC;
3205                         if (copy_to_user(ubuf, buf, input_len - 1))
3206                                 return -EFAULT;
3207                         if (put_user(zero, ubuf + input_len - 1))
3208                                 return -EFAULT;
3209                 }
3210         }
3211
3212         if (put_user(prog_id, &uattr->task_fd_query.prog_id) ||
3213             put_user(fd_type, &uattr->task_fd_query.fd_type) ||
3214             put_user(probe_offset, &uattr->task_fd_query.probe_offset) ||
3215             put_user(probe_addr, &uattr->task_fd_query.probe_addr))
3216                 return -EFAULT;
3217
3218         return err;
3219 }
3220
3221 #define BPF_TASK_FD_QUERY_LAST_FIELD task_fd_query.probe_addr
3222
3223 static int bpf_task_fd_query(const union bpf_attr *attr,
3224                              union bpf_attr __user *uattr)
3225 {
3226         pid_t pid = attr->task_fd_query.pid;
3227         u32 fd = attr->task_fd_query.fd;
3228         const struct perf_event *event;
3229         struct files_struct *files;
3230         struct task_struct *task;
3231         struct file *file;
3232         int err;
3233
3234         if (CHECK_ATTR(BPF_TASK_FD_QUERY))
3235                 return -EINVAL;
3236
3237         if (!capable(CAP_SYS_ADMIN))
3238                 return -EPERM;
3239
3240         if (attr->task_fd_query.flags != 0)
3241                 return -EINVAL;
3242
3243         task = get_pid_task(find_vpid(pid), PIDTYPE_PID);
3244         if (!task)
3245                 return -ENOENT;
3246
3247         files = get_files_struct(task);
3248         put_task_struct(task);
3249         if (!files)
3250                 return -ENOENT;
3251
3252         err = 0;
3253         spin_lock(&files->file_lock);
3254         file = fcheck_files(files, fd);
3255         if (!file)
3256                 err = -EBADF;
3257         else
3258                 get_file(file);
3259         spin_unlock(&files->file_lock);
3260         put_files_struct(files);
3261
3262         if (err)
3263                 goto out;
3264
3265         if (file->f_op == &bpf_raw_tp_fops) {
3266                 struct bpf_raw_tracepoint *raw_tp = file->private_data;
3267                 struct bpf_raw_event_map *btp = raw_tp->btp;
3268
3269                 err = bpf_task_fd_query_copy(attr, uattr,
3270                                              raw_tp->prog->aux->id,
3271                                              BPF_FD_TYPE_RAW_TRACEPOINT,
3272                                              btp->tp->name, 0, 0);
3273                 goto put_file;
3274         }
3275
3276         event = perf_get_event(file);
3277         if (!IS_ERR(event)) {
3278                 u64 probe_offset, probe_addr;
3279                 u32 prog_id, fd_type;
3280                 const char *buf;
3281
3282                 err = bpf_get_perf_event_info(event, &prog_id, &fd_type,
3283                                               &buf, &probe_offset,
3284                                               &probe_addr);
3285                 if (!err)
3286                         err = bpf_task_fd_query_copy(attr, uattr, prog_id,
3287                                                      fd_type, buf,
3288                                                      probe_offset,
3289                                                      probe_addr);
3290                 goto put_file;
3291         }
3292
3293         err = -ENOTSUPP;
3294 put_file:
3295         fput(file);
3296 out:
3297         return err;
3298 }
3299
3300 #define BPF_MAP_BATCH_LAST_FIELD batch.flags
3301
3302 #define BPF_DO_BATCH(fn)                        \
3303         do {                                    \
3304                 if (!fn) {                      \
3305                         err = -ENOTSUPP;        \
3306                         goto err_put;           \
3307                 }                               \
3308                 err = fn(map, attr, uattr);     \
3309         } while (0)
3310
3311 static int bpf_map_do_batch(const union bpf_attr *attr,
3312                             union bpf_attr __user *uattr,
3313                             int cmd)
3314 {
3315         struct bpf_map *map;
3316         int err, ufd;
3317         struct fd f;
3318
3319         if (CHECK_ATTR(BPF_MAP_BATCH))
3320                 return -EINVAL;
3321
3322         ufd = attr->batch.map_fd;
3323         f = fdget(ufd);
3324         map = __bpf_map_get(f);
3325         if (IS_ERR(map))
3326                 return PTR_ERR(map);
3327
3328         if ((cmd == BPF_MAP_LOOKUP_BATCH ||
3329              cmd == BPF_MAP_LOOKUP_AND_DELETE_BATCH) &&
3330             !(map_get_sys_perms(map, f) & FMODE_CAN_READ)) {
3331                 err = -EPERM;
3332                 goto err_put;
3333         }
3334
3335         if (cmd != BPF_MAP_LOOKUP_BATCH &&
3336             !(map_get_sys_perms(map, f) & FMODE_CAN_WRITE)) {
3337                 err = -EPERM;
3338                 goto err_put;
3339         }
3340
3341         if (cmd == BPF_MAP_LOOKUP_BATCH)
3342                 BPF_DO_BATCH(map->ops->map_lookup_batch);
3343         else if (cmd == BPF_MAP_LOOKUP_AND_DELETE_BATCH)
3344                 BPF_DO_BATCH(map->ops->map_lookup_and_delete_batch);
3345         else if (cmd == BPF_MAP_UPDATE_BATCH)
3346                 BPF_DO_BATCH(map->ops->map_update_batch);
3347         else
3348                 BPF_DO_BATCH(map->ops->map_delete_batch);
3349
3350 err_put:
3351         fdput(f);
3352         return err;
3353 }
3354
3355 SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size)
3356 {
3357         union bpf_attr attr = {};
3358         int err;
3359
3360         if (sysctl_unprivileged_bpf_disabled && !capable(CAP_SYS_ADMIN))
3361                 return -EPERM;
3362
3363         err = bpf_check_uarg_tail_zero(uattr, sizeof(attr), size);
3364         if (err)
3365                 return err;
3366         size = min_t(u32, size, sizeof(attr));
3367
3368         /* copy attributes from user space, may be less than sizeof(bpf_attr) */
3369         if (copy_from_user(&attr, uattr, size) != 0)
3370                 return -EFAULT;
3371
3372         err = security_bpf(cmd, &attr, size);
3373         if (err < 0)
3374                 return err;
3375
3376         switch (cmd) {
3377         case BPF_MAP_CREATE:
3378                 err = map_create(&attr);
3379                 break;
3380         case BPF_MAP_LOOKUP_ELEM:
3381                 err = map_lookup_elem(&attr);
3382                 break;
3383         case BPF_MAP_UPDATE_ELEM:
3384                 err = map_update_elem(&attr);
3385                 break;
3386         case BPF_MAP_DELETE_ELEM:
3387                 err = map_delete_elem(&attr);
3388                 break;
3389         case BPF_MAP_GET_NEXT_KEY:
3390                 err = map_get_next_key(&attr);
3391                 break;
3392         case BPF_MAP_FREEZE:
3393                 err = map_freeze(&attr);
3394                 break;
3395         case BPF_PROG_LOAD:
3396                 err = bpf_prog_load(&attr, uattr);
3397                 break;
3398         case BPF_OBJ_PIN:
3399                 err = bpf_obj_pin(&attr);
3400                 break;
3401         case BPF_OBJ_GET:
3402                 err = bpf_obj_get(&attr);
3403                 break;
3404         case BPF_PROG_ATTACH:
3405                 err = bpf_prog_attach(&attr);
3406                 break;
3407         case BPF_PROG_DETACH:
3408                 err = bpf_prog_detach(&attr);
3409                 break;
3410         case BPF_PROG_QUERY:
3411                 err = bpf_prog_query(&attr, uattr);
3412                 break;
3413         case BPF_PROG_TEST_RUN:
3414                 err = bpf_prog_test_run(&attr, uattr);
3415                 break;
3416         case BPF_PROG_GET_NEXT_ID:
3417                 err = bpf_obj_get_next_id(&attr, uattr,
3418                                           &prog_idr, &prog_idr_lock);
3419                 break;
3420         case BPF_MAP_GET_NEXT_ID:
3421                 err = bpf_obj_get_next_id(&attr, uattr,
3422                                           &map_idr, &map_idr_lock);
3423                 break;
3424         case BPF_BTF_GET_NEXT_ID:
3425                 err = bpf_obj_get_next_id(&attr, uattr,
3426                                           &btf_idr, &btf_idr_lock);
3427                 break;
3428         case BPF_PROG_GET_FD_BY_ID:
3429                 err = bpf_prog_get_fd_by_id(&attr);
3430                 break;
3431         case BPF_MAP_GET_FD_BY_ID:
3432                 err = bpf_map_get_fd_by_id(&attr);
3433                 break;
3434         case BPF_OBJ_GET_INFO_BY_FD:
3435                 err = bpf_obj_get_info_by_fd(&attr, uattr);
3436                 break;
3437         case BPF_RAW_TRACEPOINT_OPEN:
3438                 err = bpf_raw_tracepoint_open(&attr);
3439                 break;
3440         case BPF_BTF_LOAD:
3441                 err = bpf_btf_load(&attr);
3442                 break;
3443         case BPF_BTF_GET_FD_BY_ID:
3444                 err = bpf_btf_get_fd_by_id(&attr);
3445                 break;
3446         case BPF_TASK_FD_QUERY:
3447                 err = bpf_task_fd_query(&attr, uattr);
3448                 break;
3449         case BPF_MAP_LOOKUP_AND_DELETE_ELEM:
3450                 err = map_lookup_and_delete_elem(&attr);
3451                 break;
3452         case BPF_MAP_LOOKUP_BATCH:
3453                 err = bpf_map_do_batch(&attr, uattr, BPF_MAP_LOOKUP_BATCH);
3454                 break;
3455         case BPF_MAP_LOOKUP_AND_DELETE_BATCH:
3456                 err = bpf_map_do_batch(&attr, uattr,
3457                                        BPF_MAP_LOOKUP_AND_DELETE_BATCH);
3458                 break;
3459         case BPF_MAP_UPDATE_BATCH:
3460                 err = bpf_map_do_batch(&attr, uattr, BPF_MAP_UPDATE_BATCH);
3461                 break;
3462         case BPF_MAP_DELETE_BATCH:
3463                 err = bpf_map_do_batch(&attr, uattr, BPF_MAP_DELETE_BATCH);
3464                 break;
3465         default:
3466                 err = -EINVAL;
3467                 break;
3468         }
3469
3470         return err;
3471 }