kernel/bpf/syscall.c

   1 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
   2  *
   3  * This program is free software; you can redistribute it and/or
   4  * modify it under the terms of version 2 of the GNU General Public
   5  * License as published by the Free Software Foundation.
   6  *
   7  * This program is distributed in the hope that it will be useful, but
   8  * WITHOUT ANY WARRANTY; without even the implied warranty of
   9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  10  * General Public License for more details.
  11  */
  12 #include <linux/bpf.h>
  13 #include <linux/bpf_trace.h>
  14 #include <linux/bpf_lirc.h>
  15 #include <linux/btf.h>
  16 #include <linux/syscalls.h>
  17 #include <linux/slab.h>
  18 #include <linux/sched/signal.h>
  19 #include <linux/vmalloc.h>
  20 #include <linux/mmzone.h>
  21 #include <linux/anon_inodes.h>
  22 #include <linux/fdtable.h>
  23 #include <linux/file.h>
  24 #include <linux/fs.h>
  25 #include <linux/license.h>
  26 #include <linux/filter.h>
  27 #include <linux/version.h>
  28 #include <linux/kernel.h>
  29 #include <linux/idr.h>
  30 #include <linux/cred.h>
  31 #include <linux/timekeeping.h>
  32 #include <linux/ctype.h>
  33 #include <linux/nospec.h>
  34
  35 #define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PROG_ARRAY || \
  36                            (map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || \
  37                            (map)->map_type == BPF_MAP_TYPE_CGROUP_ARRAY || \
  38                            (map)->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS)
  39 #define IS_FD_HASH(map) ((map)->map_type == BPF_MAP_TYPE_HASH_OF_MAPS)
  40 #define IS_FD_MAP(map) (IS_FD_ARRAY(map) || IS_FD_HASH(map))
  41
  42 #define BPF_OBJ_FLAG_MASK   (BPF_F_RDONLY | BPF_F_WRONLY)
  43
  44 DEFINE_PER_CPU(int, bpf_prog_active);
  45 static DEFINE_IDR(prog_idr);
  46 static DEFINE_SPINLOCK(prog_idr_lock);
  47 static DEFINE_IDR(map_idr);
  48 static DEFINE_SPINLOCK(map_idr_lock);
  49
  50 int sysctl_unprivileged_bpf_disabled __read_mostly;
  51
  52 static const struct bpf_map_ops * const bpf_map_types[] = {
  53 #define BPF_PROG_TYPE(_id, _ops)
  54 #define BPF_MAP_TYPE(_id, _ops) \
  55         [_id] = &_ops,
  56 #include <linux/bpf_types.h>
  57 #undef BPF_PROG_TYPE
  58 #undef BPF_MAP_TYPE
  59 };
  60
  61 /*
  62  * If we're handed a bigger struct than we know of, ensure all the unknown bits
  63  * are 0 - i.e. new user-space does not rely on any kernel feature extensions
  64  * we don't know about yet.
  65  *
  66  * There is a ToCToU between this function call and the following
  67  * copy_from_user() call. However, this is not a concern since this function is
  68  * meant to be a future-proofing of bits.
  69  */
  70 int bpf_check_uarg_tail_zero(void __user *uaddr,
  71                              size_t expected_size,
  72                              size_t actual_size)
  73 {
  74         unsigned char __user *addr;
  75         unsigned char __user *end;
  76         unsigned char val;
  77         int err;
  78
  79         if (unlikely(actual_size > PAGE_SIZE))  /* silly large */
  80                 return -E2BIG;
  81
  82         if (unlikely(!access_ok(uaddr, actual_size)))
  83                 return -EFAULT;
  84
  85         if (actual_size <= expected_size)
  86                 return 0;
  87
  88         addr = uaddr + expected_size;
  89         end  = uaddr + actual_size;
  90
  91         for (; addr < end; addr++) {
  92                 err = get_user(val, addr);
  93                 if (err)
  94                         return err;
  95                 if (val)
  96                         return -E2BIG;
  97         }
  98
  99         return 0;
 100 }
 101
 102 const struct bpf_map_ops bpf_map_offload_ops = {
 103         .map_alloc = bpf_map_offload_map_alloc,
 104         .map_free = bpf_map_offload_map_free,
 105         .map_check_btf = map_check_no_btf,
 106 };
 107
 108 static struct bpf_map *find_and_alloc_map(union bpf_attr *attr)
 109 {
 110         const struct bpf_map_ops *ops;
 111         u32 type = attr->map_type;
 112         struct bpf_map *map;
 113         int err;
 114
 115         if (type >= ARRAY_SIZE(bpf_map_types))
 116                 return ERR_PTR(-EINVAL);
 117         type = array_index_nospec(type, ARRAY_SIZE(bpf_map_types));
 118         ops = bpf_map_types[type];
 119         if (!ops)
 120                 return ERR_PTR(-EINVAL);
 121
 122         if (ops->map_alloc_check) {
 123                 err = ops->map_alloc_check(attr);
 124                 if (err)
 125                         return ERR_PTR(err);
 126         }
 127         if (attr->map_ifindex)
 128                 ops = &bpf_map_offload_ops;
 129         map = ops->map_alloc(attr);
 130         if (IS_ERR(map))
 131                 return map;
 132         map->ops = ops;
 133         map->map_type = type;
 134         return map;
 135 }
 136
 137 void *bpf_map_area_alloc(size_t size, int numa_node)
 138 {
 139         /* We really just want to fail instead of triggering OOM killer
 140          * under memory pressure, therefore we set __GFP_NORETRY to kmalloc,
 141          * which is used for lower order allocation requests.
 142          *
 143          * It has been observed that higher order allocation requests done by
 144          * vmalloc with __GFP_NORETRY being set might fail due to not trying
 145          * to reclaim memory from the page cache, thus we set
 146          * __GFP_RETRY_MAYFAIL to avoid such situations.
 147          */
 148
 149         const gfp_t flags = __GFP_NOWARN | __GFP_ZERO;
 150         void *area;
 151
 152         if (size <= (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) {
 153                 area = kmalloc_node(size, GFP_USER | __GFP_NORETRY | flags,
 154                                     numa_node);
 155                 if (area != NULL)
 156                         return area;
 157         }
 158
 159         return __vmalloc_node_flags_caller(size, numa_node,
 160                                            GFP_KERNEL | __GFP_RETRY_MAYFAIL |
 161                                            flags, __builtin_return_address(0));
 162 }
 163
 164 void bpf_map_area_free(void *area)
 165 {
 166         kvfree(area);
 167 }
 168
 169 static u32 bpf_map_flags_retain_permanent(u32 flags)
 170 {
 171         /* Some map creation flags are not tied to the map object but
 172          * rather to the map fd instead, so they have no meaning upon
 173          * map object inspection since multiple file descriptors with
 174          * different (access) properties can exist here. Thus, given
 175          * this has zero meaning for the map itself, lets clear these
 176          * from here.
 177          */
 178         return flags & ~(BPF_F_RDONLY | BPF_F_WRONLY);
 179 }
 180
 181 void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr)
 182 {
 183         map->map_type = attr->map_type;
 184         map->key_size = attr->key_size;
 185         map->value_size = attr->value_size;
 186         map->max_entries = attr->max_entries;
 187         map->map_flags = bpf_map_flags_retain_permanent(attr->map_flags);
 188         map->numa_node = bpf_map_attr_numa_node(attr);
 189 }
 190
 191 int bpf_map_precharge_memlock(u32 pages)
 192 {
 193         struct user_struct *user = get_current_user();
 194         unsigned long memlock_limit, cur;
 195
 196         memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
 197         cur = atomic_long_read(&user->locked_vm);
 198         free_uid(user);
 199         if (cur + pages > memlock_limit)
 200                 return -EPERM;
 201         return 0;
 202 }
 203
 204 static int bpf_charge_memlock(struct user_struct *user, u32 pages)
 205 {
 206         unsigned long memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
 207
 208         if (atomic_long_add_return(pages, &user->locked_vm) > memlock_limit) {
 209                 atomic_long_sub(pages, &user->locked_vm);
 210                 return -EPERM;
 211         }
 212         return 0;
 213 }
 214
 215 static void bpf_uncharge_memlock(struct user_struct *user, u32 pages)
 216 {
 217         atomic_long_sub(pages, &user->locked_vm);
 218 }
 219
 220 static int bpf_map_init_memlock(struct bpf_map *map)
 221 {
 222         struct user_struct *user = get_current_user();
 223         int ret;
 224
 225         ret = bpf_charge_memlock(user, map->pages);
 226         if (ret) {
 227                 free_uid(user);
 228                 return ret;
 229         }
 230         map->user = user;
 231         return ret;
 232 }
 233
 234 static void bpf_map_release_memlock(struct bpf_map *map)
 235 {
 236         struct user_struct *user = map->user;
 237         bpf_uncharge_memlock(user, map->pages);
 238         free_uid(user);
 239 }
 240
 241 int bpf_map_charge_memlock(struct bpf_map *map, u32 pages)
 242 {
 243         int ret;
 244
 245         ret = bpf_charge_memlock(map->user, pages);
 246         if (ret)
 247                 return ret;
 248         map->pages += pages;
 249         return ret;
 250 }
 251
 252 void bpf_map_uncharge_memlock(struct bpf_map *map, u32 pages)
 253 {
 254         bpf_uncharge_memlock(map->user, pages);
 255         map->pages -= pages;
 256 }
 257
 258 static int bpf_map_alloc_id(struct bpf_map *map)
 259 {
 260         int id;
 261
 262         idr_preload(GFP_KERNEL);
 263         spin_lock_bh(&map_idr_lock);
 264         id = idr_alloc_cyclic(&map_idr, map, 1, INT_MAX, GFP_ATOMIC);
 265         if (id > 0)
 266                 map->id = id;
 267         spin_unlock_bh(&map_idr_lock);
 268         idr_preload_end();
 269
 270         if (WARN_ON_ONCE(!id))
 271                 return -ENOSPC;
 272
 273         return id > 0 ? 0 : id;
 274 }
 275
 276 void bpf_map_free_id(struct bpf_map *map, bool do_idr_lock)
 277 {
 278         unsigned long flags;
 279
 280         /* Offloaded maps are removed from the IDR store when their device
 281          * disappears - even if someone holds an fd to them they are unusable,
 282          * the memory is gone, all ops will fail; they are simply waiting for
 283          * refcnt to drop to be freed.
 284          */
 285         if (!map->id)
 286                 return;
 287
 288         if (do_idr_lock)
 289                 spin_lock_irqsave(&map_idr_lock, flags);
 290         else
 291                 __acquire(&map_idr_lock);
 292
 293         idr_remove(&map_idr, map->id);
 294         map->id = 0;
 295
 296         if (do_idr_lock)
 297                 spin_unlock_irqrestore(&map_idr_lock, flags);
 298         else
 299                 __release(&map_idr_lock);
 300 }
 301
 302 /* called from workqueue */
 303 static void bpf_map_free_deferred(struct work_struct *work)
 304 {
 305         struct bpf_map *map = container_of(work, struct bpf_map, work);
 306
 307         bpf_map_release_memlock(map);
 308         security_bpf_map_free(map);
 309         /* implementation dependent freeing */
 310         map->ops->map_free(map);
 311 }
 312
 313 static void bpf_map_put_uref(struct bpf_map *map)
 314 {
 315         if (atomic_dec_and_test(&map->usercnt)) {
 316                 if (map->ops->map_release_uref)
 317                         map->ops->map_release_uref(map);
 318         }
 319 }
 320
 321 /* decrement map refcnt and schedule it for freeing via workqueue
 322  * (unrelying map implementation ops->map_free() might sleep)
 323  */
 324 static void __bpf_map_put(struct bpf_map *map, bool do_idr_lock)
 325 {
 326         if (atomic_dec_and_test(&map->refcnt)) {
 327                 /* bpf_map_free_id() must be called first */
 328                 bpf_map_free_id(map, do_idr_lock);
 329                 btf_put(map->btf);
 330                 INIT_WORK(&map->work, bpf_map_free_deferred);
 331                 schedule_work(&map->work);
 332         }
 333 }
 334
 335 void bpf_map_put(struct bpf_map *map)
 336 {
 337         __bpf_map_put(map, true);
 338 }
 339 EXPORT_SYMBOL_GPL(bpf_map_put);
 340
 341 void bpf_map_put_with_uref(struct bpf_map *map)
 342 {
 343         bpf_map_put_uref(map);
 344         bpf_map_put(map);
 345 }
 346
 347 static int bpf_map_release(struct inode *inode, struct file *filp)
 348 {
 349         struct bpf_map *map = filp->private_data;
 350
 351         if (map->ops->map_release)
 352                 map->ops->map_release(map, filp);
 353
 354         bpf_map_put_with_uref(map);
 355         return 0;
 356 }
 357
 358 #ifdef CONFIG_PROC_FS
 359 static void bpf_map_show_fdinfo(struct seq_file *m, struct file *filp)
 360 {
 361         const struct bpf_map *map = filp->private_data;
 362         const struct bpf_array *array;
 363         u32 owner_prog_type = 0;
 364         u32 owner_jited = 0;
 365
 366         if (map->map_type == BPF_MAP_TYPE_PROG_ARRAY) {
 367                 array = container_of(map, struct bpf_array, map);
 368                 owner_prog_type = array->owner_prog_type;
 369                 owner_jited = array->owner_jited;
 370         }
 371
 372         seq_printf(m,
 373                    "map_type:\t%u\n"
 374                    "key_size:\t%u\n"
 375                    "value_size:\t%u\n"
 376                    "max_entries:\t%u\n"
 377                    "map_flags:\t%#x\n"
 378                    "memlock:\t%llu\n"
 379                    "map_id:\t%u\n",
 380                    map->map_type,
 381                    map->key_size,
 382                    map->value_size,
 383                    map->max_entries,
 384                    map->map_flags,
 385                    map->pages * 1ULL << PAGE_SHIFT,
 386                    map->id);
 387
 388         if (owner_prog_type) {
 389                 seq_printf(m, "owner_prog_type:\t%u\n",
 390                            owner_prog_type);
 391                 seq_printf(m, "owner_jited:\t%u\n",
 392                            owner_jited);
 393         }
 394 }
 395 #endif
 396
 397 static ssize_t bpf_dummy_read(struct file *filp, char __user *buf, size_t siz,
 398                               loff_t *ppos)
 399 {
 400         /* We need this handler such that alloc_file() enables
 401          * f_mode with FMODE_CAN_READ.
 402          */
 403         return -EINVAL;
 404 }
 405
 406 static ssize_t bpf_dummy_write(struct file *filp, const char __user *buf,
 407                                size_t siz, loff_t *ppos)
 408 {
 409         /* We need this handler such that alloc_file() enables
 410          * f_mode with FMODE_CAN_WRITE.
 411          */
 412         return -EINVAL;
 413 }
 414
 415 const struct file_operations bpf_map_fops = {
 416 #ifdef CONFIG_PROC_FS
 417         .show_fdinfo    = bpf_map_show_fdinfo,
 418 #endif
 419         .release        = bpf_map_release,
 420         .read           = bpf_dummy_read,
 421         .write          = bpf_dummy_write,
 422 };
 423
 424 int bpf_map_new_fd(struct bpf_map *map, int flags)
 425 {
 426         int ret;
 427
 428         ret = security_bpf_map(map, OPEN_FMODE(flags));
 429         if (ret < 0)
 430                 return ret;
 431
 432         return anon_inode_getfd("bpf-map", &bpf_map_fops, map,
 433                                 flags | O_CLOEXEC);
 434 }
 435
 436 int bpf_get_file_flag(int flags)
 437 {
 438         if ((flags & BPF_F_RDONLY) && (flags & BPF_F_WRONLY))
 439                 return -EINVAL;
 440         if (flags & BPF_F_RDONLY)
 441                 return O_RDONLY;
 442         if (flags & BPF_F_WRONLY)
 443                 return O_WRONLY;
 444         return O_RDWR;
 445 }
 446
 447 /* helper macro to check that unused fields 'union bpf_attr' are zero */
 448 #define CHECK_ATTR(CMD) \
 449         memchr_inv((void *) &attr->CMD##_LAST_FIELD + \
 450                    sizeof(attr->CMD##_LAST_FIELD), 0, \
 451                    sizeof(*attr) - \
 452                    offsetof(union bpf_attr, CMD##_LAST_FIELD) - \
 453                    sizeof(attr->CMD##_LAST_FIELD)) != NULL
 454
 455 /* dst and src must have at least BPF_OBJ_NAME_LEN number of bytes.
 456  * Return 0 on success and < 0 on error.
 457  */
 458 static int bpf_obj_name_cpy(char *dst, const char *src)
 459 {
 460         const char *end = src + BPF_OBJ_NAME_LEN;
 461
 462         memset(dst, 0, BPF_OBJ_NAME_LEN);
 463
 464         /* Copy all isalnum() and '_' char */
 465         while (src < end && *src) {
 466                 if (!isalnum(*src) && *src != '_')
 467                         return -EINVAL;
 468                 *dst++ = *src++;
 469         }
 470
 471         /* No '\0' found in BPF_OBJ_NAME_LEN number of bytes */
 472         if (src == end)
 473                 return -EINVAL;
 474
 475         return 0;
 476 }
 477
 478 int map_check_no_btf(const struct bpf_map *map,
 479                      const struct btf *btf,
 480                      const struct btf_type *key_type,
 481                      const struct btf_type *value_type)
 482 {
 483         return -ENOTSUPP;
 484 }
 485
 486 static int map_check_btf(struct bpf_map *map, const struct btf *btf,
 487                          u32 btf_key_id, u32 btf_value_id)
 488 {
 489         const struct btf_type *key_type, *value_type;
 490         u32 key_size, value_size;
 491         int ret = 0;
 492
 493         key_type = btf_type_id_size(btf, &btf_key_id, &key_size);
 494         if (!key_type || key_size != map->key_size)
 495                 return -EINVAL;
 496
 497         value_type = btf_type_id_size(btf, &btf_value_id, &value_size);
 498         if (!value_type || value_size != map->value_size)
 499                 return -EINVAL;
 500
 501         map->spin_lock_off = btf_find_spin_lock(btf, value_type);
 502
 503         if (map_value_has_spin_lock(map)) {
 504                 if (map->map_type != BPF_MAP_TYPE_HASH &&
 505                     map->map_type != BPF_MAP_TYPE_ARRAY &&
 506                     map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE)
 507                         return -ENOTSUPP;
 508                 if (map->spin_lock_off + sizeof(struct bpf_spin_lock) >
 509                     map->value_size) {
 510                         WARN_ONCE(1,
 511                                   "verifier bug spin_lock_off %d value_size %d\n",
 512                                   map->spin_lock_off, map->value_size);
 513                         return -EFAULT;
 514                 }
 515         }
 516
 517         if (map->ops->map_check_btf)
 518                 ret = map->ops->map_check_btf(map, btf, key_type, value_type);
 519
 520         return ret;
 521 }
 522
 523 #define BPF_MAP_CREATE_LAST_FIELD btf_value_type_id
 524 /* called via syscall */
 525 static int map_create(union bpf_attr *attr)
 526 {
 527         int numa_node = bpf_map_attr_numa_node(attr);
 528         struct bpf_map *map;
 529         int f_flags;
 530         int err;
 531
 532         err = CHECK_ATTR(BPF_MAP_CREATE);
 533         if (err)
 534                 return -EINVAL;
 535
 536         f_flags = bpf_get_file_flag(attr->map_flags);
 537         if (f_flags < 0)
 538                 return f_flags;
 539
 540         if (numa_node != NUMA_NO_NODE &&
 541             ((unsigned int)numa_node >= nr_node_ids ||
 542              !node_online(numa_node)))
 543                 return -EINVAL;
 544
 545         /* find map type and init map: hashtable vs rbtree vs bloom vs ... */
 546         map = find_and_alloc_map(attr);
 547         if (IS_ERR(map))
 548                 return PTR_ERR(map);
 549
 550         err = bpf_obj_name_cpy(map->name, attr->map_name);
 551         if (err)
 552                 goto free_map_nouncharge;
 553
 554         atomic_set(&map->refcnt, 1);
 555         atomic_set(&map->usercnt, 1);
 556
 557         if (attr->btf_key_type_id || attr->btf_value_type_id) {
 558                 struct btf *btf;
 559
 560                 if (!attr->btf_key_type_id || !attr->btf_value_type_id) {
 561                         err = -EINVAL;
 562                         goto free_map_nouncharge;
 563                 }
 564
 565                 btf = btf_get_by_fd(attr->btf_fd);
 566                 if (IS_ERR(btf)) {
 567                         err = PTR_ERR(btf);
 568                         goto free_map_nouncharge;
 569                 }
 570
 571                 err = map_check_btf(map, btf, attr->btf_key_type_id,
 572                                     attr->btf_value_type_id);
 573                 if (err) {
 574                         btf_put(btf);
 575                         goto free_map_nouncharge;
 576                 }
 577
 578                 map->btf = btf;
 579                 map->btf_key_type_id = attr->btf_key_type_id;
 580                 map->btf_value_type_id = attr->btf_value_type_id;
 581         } else {
 582                 map->spin_lock_off = -EINVAL;
 583         }
 584
 585         err = security_bpf_map_alloc(map);
 586         if (err)
 587                 goto free_map_nouncharge;
 588
 589         err = bpf_map_init_memlock(map);
 590         if (err)
 591                 goto free_map_sec;
 592
 593         err = bpf_map_alloc_id(map);
 594         if (err)
 595                 goto free_map;
 596
 597         err = bpf_map_new_fd(map, f_flags);
 598         if (err < 0) {
 599                 /* failed to allocate fd.
 600                  * bpf_map_put_with_uref() is needed because the above
 601                  * bpf_map_alloc_id() has published the map
 602                  * to the userspace and the userspace may
 603                  * have refcnt-ed it through BPF_MAP_GET_FD_BY_ID.
 604                  */
 605                 bpf_map_put_with_uref(map);
 606                 return err;
 607         }
 608
 609         return err;
 610
 611 free_map:
 612         bpf_map_release_memlock(map);
 613 free_map_sec:
 614         security_bpf_map_free(map);
 615 free_map_nouncharge:
 616         btf_put(map->btf);
 617         map->ops->map_free(map);
 618         return err;
 619 }
 620
 621 /* if error is returned, fd is released.
 622  * On success caller should complete fd access with matching fdput()
 623  */
 624 struct bpf_map *__bpf_map_get(struct fd f)
 625 {
 626         if (!f.file)
 627                 return ERR_PTR(-EBADF);
 628         if (f.file->f_op != &bpf_map_fops) {
 629                 fdput(f);
 630                 return ERR_PTR(-EINVAL);
 631         }
 632
 633         return f.file->private_data;
 634 }
 635
 636 /* prog's and map's refcnt limit */
 637 #define BPF_MAX_REFCNT 32768
 638
 639 struct bpf_map *bpf_map_inc(struct bpf_map *map, bool uref)
 640 {
 641         if (atomic_inc_return(&map->refcnt) > BPF_MAX_REFCNT) {
 642                 atomic_dec(&map->refcnt);
 643                 return ERR_PTR(-EBUSY);
 644         }
 645         if (uref)
 646                 atomic_inc(&map->usercnt);
 647         return map;
 648 }
 649 EXPORT_SYMBOL_GPL(bpf_map_inc);
 650
 651 struct bpf_map *bpf_map_get_with_uref(u32 ufd)
 652 {
 653         struct fd f = fdget(ufd);
 654         struct bpf_map *map;
 655
 656         map = __bpf_map_get(f);
 657         if (IS_ERR(map))
 658                 return map;
 659
 660         map = bpf_map_inc(map, true);
 661         fdput(f);
 662
 663         return map;
 664 }
 665
 666 /* map_idr_lock should have been held */
 667 static struct bpf_map *bpf_map_inc_not_zero(struct bpf_map *map,
 668                                             bool uref)
 669 {
 670         int refold;
 671
 672         refold = atomic_fetch_add_unless(&map->refcnt, 1, 0);
 673
 674         if (refold >= BPF_MAX_REFCNT) {
 675                 __bpf_map_put(map, false);
 676                 return ERR_PTR(-EBUSY);
 677         }
 678
 679         if (!refold)
 680                 return ERR_PTR(-ENOENT);
 681
 682         if (uref)
 683                 atomic_inc(&map->usercnt);
 684
 685         return map;
 686 }
 687
 688 int __weak bpf_stackmap_copy(struct bpf_map *map, void *key, void *value)
 689 {
 690         return -ENOTSUPP;
 691 }
 692
 693 static void *__bpf_copy_key(void __user *ukey, u64 key_size)
 694 {
 695         if (key_size)
 696                 return memdup_user(ukey, key_size);
 697
 698         if (ukey)
 699                 return ERR_PTR(-EINVAL);
 700
 701         return NULL;
 702 }
 703
 704 /* last field in 'union bpf_attr' used by this command */
 705 #define BPF_MAP_LOOKUP_ELEM_LAST_FIELD flags
 706
 707 static int map_lookup_elem(union bpf_attr *attr)
 708 {
 709         void __user *ukey = u64_to_user_ptr(attr->key);
 710         void __user *uvalue = u64_to_user_ptr(attr->value);
 711         int ufd = attr->map_fd;
 712         struct bpf_map *map;
 713         void *key, *value, *ptr;
 714         u32 value_size;
 715         struct fd f;
 716         int err;
 717
 718         if (CHECK_ATTR(BPF_MAP_LOOKUP_ELEM))
 719                 return -EINVAL;
 720
 721         if (attr->flags & ~BPF_F_LOCK)
 722                 return -EINVAL;
 723
 724         f = fdget(ufd);
 725         map = __bpf_map_get(f);
 726         if (IS_ERR(map))
 727                 return PTR_ERR(map);
 728
 729         if (!(f.file->f_mode & FMODE_CAN_READ)) {
 730                 err = -EPERM;
 731                 goto err_put;
 732         }
 733
 734         if ((attr->flags & BPF_F_LOCK) &&
 735             !map_value_has_spin_lock(map)) {
 736                 err = -EINVAL;
 737                 goto err_put;
 738         }
 739
 740         key = __bpf_copy_key(ukey, map->key_size);
 741         if (IS_ERR(key)) {
 742                 err = PTR_ERR(key);
 743                 goto err_put;
 744         }
 745
 746         if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
 747             map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH ||
 748             map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY ||
 749             map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE)
 750                 value_size = round_up(map->value_size, 8) * num_possible_cpus();
 751         else if (IS_FD_MAP(map))
 752                 value_size = sizeof(u32);
 753         else
 754                 value_size = map->value_size;
 755
 756         err = -ENOMEM;
 757         value = kmalloc(value_size, GFP_USER | __GFP_NOWARN);
 758         if (!value)
 759                 goto free_key;
 760
 761         if (bpf_map_is_dev_bound(map)) {
 762                 err = bpf_map_offload_lookup_elem(map, key, value);
 763                 goto done;
 764         }
 765
 766         preempt_disable();
 767         this_cpu_inc(bpf_prog_active);
 768         if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
 769             map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
 770                 err = bpf_percpu_hash_copy(map, key, value);
 771         } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
 772                 err = bpf_percpu_array_copy(map, key, value);
 773         } else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) {
 774                 err = bpf_percpu_cgroup_storage_copy(map, key, value);
 775         } else if (map->map_type == BPF_MAP_TYPE_STACK_TRACE) {
 776                 err = bpf_stackmap_copy(map, key, value);
 777         } else if (IS_FD_ARRAY(map)) {
 778                 err = bpf_fd_array_map_lookup_elem(map, key, value);
 779         } else if (IS_FD_HASH(map)) {
 780                 err = bpf_fd_htab_map_lookup_elem(map, key, value);
 781         } else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) {
 782                 err = bpf_fd_reuseport_array_lookup_elem(map, key, value);
 783         } else if (map->map_type == BPF_MAP_TYPE_QUEUE ||
 784                    map->map_type == BPF_MAP_TYPE_STACK) {
 785                 err = map->ops->map_peek_elem(map, value);
 786         } else {
 787                 rcu_read_lock();
 788                 ptr = map->ops->map_lookup_elem(map, key);
 789                 if (IS_ERR(ptr)) {
 790                         err = PTR_ERR(ptr);
 791                 } else if (!ptr) {
 792                         err = -ENOENT;
 793                 } else {
 794                         err = 0;
 795                         if (attr->flags & BPF_F_LOCK)
 796                                 /* lock 'ptr' and copy everything but lock */
 797                                 copy_map_value_locked(map, value, ptr, true);
 798                         else
 799                                 copy_map_value(map, value, ptr);
 800                         /* mask lock, since value wasn't zero inited */
 801                         check_and_init_map_lock(map, value);
 802                 }
 803                 rcu_read_unlock();
 804         }
 805         this_cpu_dec(bpf_prog_active);
 806         preempt_enable();
 807
 808 done:
 809         if (err)
 810                 goto free_value;
 811
 812         err = -EFAULT;
 813         if (copy_to_user(uvalue, value, value_size) != 0)
 814                 goto free_value;
 815
 816         err = 0;
 817
 818 free_value:
 819         kfree(value);
 820 free_key:
 821         kfree(key);
 822 err_put:
 823         fdput(f);
 824         return err;
 825 }
 826
 827 static void maybe_wait_bpf_programs(struct bpf_map *map)
 828 {
 829         /* Wait for any running BPF programs to complete so that
 830          * userspace, when we return to it, knows that all programs
 831          * that could be running use the new map value.
 832          */
 833         if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS ||
 834             map->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS)
 835                 synchronize_rcu();
 836 }
 837
 838 #define BPF_MAP_UPDATE_ELEM_LAST_FIELD flags
 839
 840 static int map_update_elem(union bpf_attr *attr)
 841 {
 842         void __user *ukey = u64_to_user_ptr(attr->key);
 843         void __user *uvalue = u64_to_user_ptr(attr->value);
 844         int ufd = attr->map_fd;
 845         struct bpf_map *map;
 846         void *key, *value;
 847         u32 value_size;
 848         struct fd f;
 849         int err;
 850
 851         if (CHECK_ATTR(BPF_MAP_UPDATE_ELEM))
 852                 return -EINVAL;
 853
 854         f = fdget(ufd);
 855         map = __bpf_map_get(f);
 856         if (IS_ERR(map))
 857                 return PTR_ERR(map);
 858
 859         if (!(f.file->f_mode & FMODE_CAN_WRITE)) {
 860                 err = -EPERM;
 861                 goto err_put;
 862         }
 863
 864         if ((attr->flags & BPF_F_LOCK) &&
 865             !map_value_has_spin_lock(map)) {
 866                 err = -EINVAL;
 867                 goto err_put;
 868         }
 869
 870         key = __bpf_copy_key(ukey, map->key_size);
 871         if (IS_ERR(key)) {
 872                 err = PTR_ERR(key);
 873                 goto err_put;
 874         }
 875
 876         if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
 877             map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH ||
 878             map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY ||
 879             map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE)
 880                 value_size = round_up(map->value_size, 8) * num_possible_cpus();
 881         else
 882                 value_size = map->value_size;
 883
 884         err = -ENOMEM;
 885         value = kmalloc(value_size, GFP_USER | __GFP_NOWARN);
 886         if (!value)
 887                 goto free_key;
 888
 889         err = -EFAULT;
 890         if (copy_from_user(value, uvalue, value_size) != 0)
 891                 goto free_value;
 892
 893         /* Need to create a kthread, thus must support schedule */
 894         if (bpf_map_is_dev_bound(map)) {
 895                 err = bpf_map_offload_update_elem(map, key, value, attr->flags);
 896                 goto out;
 897         } else if (map->map_type == BPF_MAP_TYPE_CPUMAP ||
 898                    map->map_type == BPF_MAP_TYPE_SOCKHASH ||
 899                    map->map_type == BPF_MAP_TYPE_SOCKMAP) {
 900                 err = map->ops->map_update_elem(map, key, value, attr->flags);
 901                 goto out;
 902         }
 903
 904         /* must increment bpf_prog_active to avoid kprobe+bpf triggering from
 905          * inside bpf map update or delete otherwise deadlocks are possible
 906          */
 907         preempt_disable();
 908         __this_cpu_inc(bpf_prog_active);
 909         if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH ||
 910             map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) {
 911                 err = bpf_percpu_hash_update(map, key, value, attr->flags);
 912         } else if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
 913                 err = bpf_percpu_array_update(map, key, value, attr->flags);
 914         } else if (map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) {
 915                 err = bpf_percpu_cgroup_storage_update(map, key, value,
 916                                                        attr->flags);
 917         } else if (IS_FD_ARRAY(map)) {
 918                 rcu_read_lock();
 919                 err = bpf_fd_array_map_update_elem(map, f.file, key, value,
 920                                                    attr->flags);
 921                 rcu_read_unlock();
 922         } else if (map->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) {
 923                 rcu_read_lock();
 924                 err = bpf_fd_htab_map_update_elem(map, f.file, key, value,
 925                                                   attr->flags);
 926                 rcu_read_unlock();
 927         } else if (map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) {
 928                 /* rcu_read_lock() is not needed */
 929                 err = bpf_fd_reuseport_array_update_elem(map, key, value,
 930                                                          attr->flags);
 931         } else if (map->map_type == BPF_MAP_TYPE_QUEUE ||
 932                    map->map_type == BPF_MAP_TYPE_STACK) {
 933                 err = map->ops->map_push_elem(map, value, attr->flags);
 934         } else {
 935                 rcu_read_lock();
 936                 err = map->ops->map_update_elem(map, key, value, attr->flags);
 937                 rcu_read_unlock();
 938         }
 939         __this_cpu_dec(bpf_prog_active);
 940         preempt_enable();
 941         maybe_wait_bpf_programs(map);
 942 out:
 943 free_value:
 944         kfree(value);
 945 free_key:
 946         kfree(key);
 947 err_put:
 948         fdput(f);
 949         return err;
 950 }
 951
 952 #define BPF_MAP_DELETE_ELEM_LAST_FIELD key
 953
 954 static int map_delete_elem(union bpf_attr *attr)
 955 {
 956         void __user *ukey = u64_to_user_ptr(attr->key);
 957         int ufd = attr->map_fd;
 958         struct bpf_map *map;
 959         struct fd f;
 960         void *key;
 961         int err;
 962
 963         if (CHECK_ATTR(BPF_MAP_DELETE_ELEM))
 964                 return -EINVAL;
 965
 966         f = fdget(ufd);
 967         map = __bpf_map_get(f);
 968         if (IS_ERR(map))
 969                 return PTR_ERR(map);
 970
 971         if (!(f.file->f_mode & FMODE_CAN_WRITE)) {
 972                 err = -EPERM;
 973                 goto err_put;
 974         }
 975
 976         key = __bpf_copy_key(ukey, map->key_size);
 977         if (IS_ERR(key)) {
 978                 err = PTR_ERR(key);
 979                 goto err_put;
 980         }
 981
 982         if (bpf_map_is_dev_bound(map)) {
 983                 err = bpf_map_offload_delete_elem(map, key);
 984                 goto out;
 985         }
 986
 987         preempt_disable();
 988         __this_cpu_inc(bpf_prog_active);
 989         rcu_read_lock();
 990         err = map->ops->map_delete_elem(map, key);
 991         rcu_read_unlock();
 992         __this_cpu_dec(bpf_prog_active);
 993         preempt_enable();
 994         maybe_wait_bpf_programs(map);
 995 out:
 996         kfree(key);
 997 err_put:
 998         fdput(f);
 999         return err;
1000 }
1001
1002 /* last field in 'union bpf_attr' used by this command */
1003 #define BPF_MAP_GET_NEXT_KEY_LAST_FIELD next_key
1004
1005 static int map_get_next_key(union bpf_attr *attr)
1006 {
1007         void __user *ukey = u64_to_user_ptr(attr->key);
1008         void __user *unext_key = u64_to_user_ptr(attr->next_key);
1009         int ufd = attr->map_fd;
1010         struct bpf_map *map;
1011         void *key, *next_key;
1012         struct fd f;
1013         int err;
1014
1015         if (CHECK_ATTR(BPF_MAP_GET_NEXT_KEY))
1016                 return -EINVAL;
1017
1018         f = fdget(ufd);
1019         map = __bpf_map_get(f);
1020         if (IS_ERR(map))
1021                 return PTR_ERR(map);
1022
1023         if (!(f.file->f_mode & FMODE_CAN_READ)) {
1024                 err = -EPERM;
1025                 goto err_put;
1026         }
1027
1028         if (ukey) {
1029                 key = __bpf_copy_key(ukey, map->key_size);
1030                 if (IS_ERR(key)) {
1031                         err = PTR_ERR(key);
1032                         goto err_put;
1033                 }
1034         } else {
1035                 key = NULL;
1036         }
1037
1038         err = -ENOMEM;
1039         next_key = kmalloc(map->key_size, GFP_USER);
1040         if (!next_key)
1041                 goto free_key;
1042
1043         if (bpf_map_is_dev_bound(map)) {
1044                 err = bpf_map_offload_get_next_key(map, key, next_key);
1045                 goto out;
1046         }
1047
1048         rcu_read_lock();
1049         err = map->ops->map_get_next_key(map, key, next_key);
1050         rcu_read_unlock();
1051 out:
1052         if (err)
1053                 goto free_next_key;
1054
1055         err = -EFAULT;
1056         if (copy_to_user(unext_key, next_key, map->key_size) != 0)
1057                 goto free_next_key;
1058
1059         err = 0;
1060
1061 free_next_key:
1062         kfree(next_key);
1063 free_key:
1064         kfree(key);
1065 err_put:
1066         fdput(f);
1067         return err;
1068 }
1069
1070 #define BPF_MAP_LOOKUP_AND_DELETE_ELEM_LAST_FIELD value
1071
1072 static int map_lookup_and_delete_elem(union bpf_attr *attr)
1073 {
1074         void __user *ukey = u64_to_user_ptr(attr->key);
1075         void __user *uvalue = u64_to_user_ptr(attr->value);
1076         int ufd = attr->map_fd;
1077         struct bpf_map *map;
1078         void *key, *value;
1079         u32 value_size;
1080         struct fd f;
1081         int err;
1082
1083         if (CHECK_ATTR(BPF_MAP_LOOKUP_AND_DELETE_ELEM))
1084                 return -EINVAL;
1085
1086         f = fdget(ufd);
1087         map = __bpf_map_get(f);
1088         if (IS_ERR(map))
1089                 return PTR_ERR(map);
1090
1091         if (!(f.file->f_mode & FMODE_CAN_WRITE)) {
1092                 err = -EPERM;
1093                 goto err_put;
1094         }
1095
1096         key = __bpf_copy_key(ukey, map->key_size);
1097         if (IS_ERR(key)) {
1098                 err = PTR_ERR(key);
1099                 goto err_put;
1100         }
1101
1102         value_size = map->value_size;
1103
1104         err = -ENOMEM;
1105         value = kmalloc(value_size, GFP_USER | __GFP_NOWARN);
1106         if (!value)
1107                 goto free_key;
1108
1109         if (map->map_type == BPF_MAP_TYPE_QUEUE ||
1110             map->map_type == BPF_MAP_TYPE_STACK) {
1111                 err = map->ops->map_pop_elem(map, value);
1112         } else {
1113                 err = -ENOTSUPP;
1114         }
1115
1116         if (err)
1117                 goto free_value;
1118
1119         if (copy_to_user(uvalue, value, value_size) != 0)
1120                 goto free_value;
1121
1122         err = 0;
1123
1124 free_value:
1125         kfree(value);
1126 free_key:
1127         kfree(key);
1128 err_put:
1129         fdput(f);
1130         return err;
1131 }
1132
1133 static const struct bpf_prog_ops * const bpf_prog_types[] = {
1134 #define BPF_PROG_TYPE(_id, _name) \
1135         [_id] = & _name ## _prog_ops,
1136 #define BPF_MAP_TYPE(_id, _ops)
1137 #include <linux/bpf_types.h>
1138 #undef BPF_PROG_TYPE
1139 #undef BPF_MAP_TYPE
1140 };
1141
1142 static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog)
1143 {
1144         const struct bpf_prog_ops *ops;
1145
1146         if (type >= ARRAY_SIZE(bpf_prog_types))
1147                 return -EINVAL;
1148         type = array_index_nospec(type, ARRAY_SIZE(bpf_prog_types));
1149         ops = bpf_prog_types[type];
1150         if (!ops)
1151                 return -EINVAL;
1152
1153         if (!bpf_prog_is_dev_bound(prog->aux))
1154                 prog->aux->ops = ops;
1155         else
1156                 prog->aux->ops = &bpf_offload_prog_ops;
1157         prog->type = type;
1158         return 0;
1159 }
1160
1161 /* drop refcnt on maps used by eBPF program and free auxilary data */
1162 static void free_used_maps(struct bpf_prog_aux *aux)
1163 {
1164         enum bpf_cgroup_storage_type stype;
1165         int i;
1166
1167         for_each_cgroup_storage_type(stype) {
1168                 if (!aux->cgroup_storage[stype])
1169                         continue;
1170                 bpf_cgroup_storage_release(aux->prog,
1171                                            aux->cgroup_storage[stype]);
1172         }
1173
1174         for (i = 0; i < aux->used_map_cnt; i++)
1175                 bpf_map_put(aux->used_maps[i]);
1176
1177         kfree(aux->used_maps);
1178 }
1179
1180 int __bpf_prog_charge(struct user_struct *user, u32 pages)
1181 {
1182         unsigned long memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
1183         unsigned long user_bufs;
1184
1185         if (user) {
1186                 user_bufs = atomic_long_add_return(pages, &user->locked_vm);
1187                 if (user_bufs > memlock_limit) {
1188                         atomic_long_sub(pages, &user->locked_vm);
1189                         return -EPERM;
1190                 }
1191         }
1192
1193         return 0;
1194 }
1195
1196 void __bpf_prog_uncharge(struct user_struct *user, u32 pages)
1197 {
1198         if (user)
1199                 atomic_long_sub(pages, &user->locked_vm);
1200 }
1201
1202 static int bpf_prog_charge_memlock(struct bpf_prog *prog)
1203 {
1204         struct user_struct *user = get_current_user();
1205         int ret;
1206
1207         ret = __bpf_prog_charge(user, prog->pages);
1208         if (ret) {
1209                 free_uid(user);
1210                 return ret;
1211         }
1212
1213         prog->aux->user = user;
1214         return 0;
1215 }
1216
1217 static void bpf_prog_uncharge_memlock(struct bpf_prog *prog)
1218 {
1219         struct user_struct *user = prog->aux->user;
1220
1221         __bpf_prog_uncharge(user, prog->pages);
1222         free_uid(user);
1223 }
1224
1225 static int bpf_prog_alloc_id(struct bpf_prog *prog)
1226 {
1227         int id;
1228
1229         idr_preload(GFP_KERNEL);
1230         spin_lock_bh(&prog_idr_lock);
1231         id = idr_alloc_cyclic(&prog_idr, prog, 1, INT_MAX, GFP_ATOMIC);
1232         if (id > 0)
1233                 prog->aux->id = id;
1234         spin_unlock_bh(&prog_idr_lock);
1235         idr_preload_end();
1236
1237         /* id is in [1, INT_MAX) */
1238         if (WARN_ON_ONCE(!id))
1239                 return -ENOSPC;
1240
1241         return id > 0 ? 0 : id;
1242 }
1243
1244 void bpf_prog_free_id(struct bpf_prog *prog, bool do_idr_lock)
1245 {
1246         /* cBPF to eBPF migrations are currently not in the idr store.
1247          * Offloaded programs are removed from the store when their device
1248          * disappears - even if someone grabs an fd to them they are unusable,
1249          * simply waiting for refcnt to drop to be freed.
1250          */
1251         if (!prog->aux->id)
1252                 return;
1253
1254         if (do_idr_lock)
1255                 spin_lock_bh(&prog_idr_lock);
1256         else
1257                 __acquire(&prog_idr_lock);
1258
1259         idr_remove(&prog_idr, prog->aux->id);
1260         prog->aux->id = 0;
1261
1262         if (do_idr_lock)
1263                 spin_unlock_bh(&prog_idr_lock);
1264         else
1265                 __release(&prog_idr_lock);
1266 }
1267
1268 static void __bpf_prog_put_rcu(struct rcu_head *rcu)
1269 {
1270         struct bpf_prog_aux *aux = container_of(rcu, struct bpf_prog_aux, rcu);
1271
1272         free_used_maps(aux);
1273         bpf_prog_uncharge_memlock(aux->prog);
1274         security_bpf_prog_free(aux);
1275         bpf_prog_free(aux->prog);
1276 }
1277
1278 static void __bpf_prog_put(struct bpf_prog *prog, bool do_idr_lock)
1279 {
1280         if (atomic_dec_and_test(&prog->aux->refcnt)) {
1281                 perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_UNLOAD, 0);
1282                 /* bpf_prog_free_id() must be called first */
1283                 bpf_prog_free_id(prog, do_idr_lock);
1284                 bpf_prog_kallsyms_del_all(prog);
1285                 btf_put(prog->aux->btf);
1286                 kvfree(prog->aux->func_info);
1287                 bpf_prog_free_linfo(prog);
1288
1289                 call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu);
1290         }
1291 }
1292
1293 void bpf_prog_put(struct bpf_prog *prog)
1294 {
1295         __bpf_prog_put(prog, true);
1296 }
1297 EXPORT_SYMBOL_GPL(bpf_prog_put);
1298
1299 static int bpf_prog_release(struct inode *inode, struct file *filp)
1300 {
1301         struct bpf_prog *prog = filp->private_data;
1302
1303         bpf_prog_put(prog);
1304         return 0;
1305 }
1306
1307 static void bpf_prog_get_stats(const struct bpf_prog *prog,
1308                                struct bpf_prog_stats *stats)
1309 {
1310         u64 nsecs = 0, cnt = 0;
1311         int cpu;
1312
1313         for_each_possible_cpu(cpu) {
1314                 const struct bpf_prog_stats *st;
1315                 unsigned int start;
1316                 u64 tnsecs, tcnt;
1317
1318                 st = per_cpu_ptr(prog->aux->stats, cpu);
1319                 do {
1320                         start = u64_stats_fetch_begin_irq(&st->syncp);
1321                         tnsecs = st->nsecs;
1322                         tcnt = st->cnt;
1323                 } while (u64_stats_fetch_retry_irq(&st->syncp, start));
1324                 nsecs += tnsecs;
1325                 cnt += tcnt;
1326         }
1327         stats->nsecs = nsecs;
1328         stats->cnt = cnt;
1329 }
1330
1331 #ifdef CONFIG_PROC_FS
1332 static void bpf_prog_show_fdinfo(struct seq_file *m, struct file *filp)
1333 {
1334         const struct bpf_prog *prog = filp->private_data;
1335         char prog_tag[sizeof(prog->tag) * 2 + 1] = { };
1336         struct bpf_prog_stats stats;
1337
1338         bpf_prog_get_stats(prog, &stats);
1339         bin2hex(prog_tag, prog->tag, sizeof(prog->tag));
1340         seq_printf(m,
1341                    "prog_type:\t%u\n"
1342                    "prog_jited:\t%u\n"
1343                    "prog_tag:\t%s\n"
1344                    "memlock:\t%llu\n"
1345                    "prog_id:\t%u\n"
1346                    "run_time_ns:\t%llu\n"
1347                    "run_cnt:\t%llu\n",
1348                    prog->type,
1349                    prog->jited,
1350                    prog_tag,
1351                    prog->pages * 1ULL << PAGE_SHIFT,
1352                    prog->aux->id,
1353                    stats.nsecs,
1354                    stats.cnt);
1355 }
1356 #endif
1357
1358 const struct file_operations bpf_prog_fops = {
1359 #ifdef CONFIG_PROC_FS
1360         .show_fdinfo    = bpf_prog_show_fdinfo,
1361 #endif
1362         .release        = bpf_prog_release,
1363         .read           = bpf_dummy_read,
1364         .write          = bpf_dummy_write,
1365 };
1366
1367 int bpf_prog_new_fd(struct bpf_prog *prog)
1368 {
1369         int ret;
1370
1371         ret = security_bpf_prog(prog);
1372         if (ret < 0)
1373                 return ret;
1374
1375         return anon_inode_getfd("bpf-prog", &bpf_prog_fops, prog,
1376                                 O_RDWR | O_CLOEXEC);
1377 }
1378
1379 static struct bpf_prog *____bpf_prog_get(struct fd f)
1380 {
1381         if (!f.file)
1382                 return ERR_PTR(-EBADF);
1383         if (f.file->f_op != &bpf_prog_fops) {
1384                 fdput(f);
1385                 return ERR_PTR(-EINVAL);
1386         }
1387
1388         return f.file->private_data;
1389 }
1390
1391 struct bpf_prog *bpf_prog_add(struct bpf_prog *prog, int i)
1392 {
1393         if (atomic_add_return(i, &prog->aux->refcnt) > BPF_MAX_REFCNT) {
1394                 atomic_sub(i, &prog->aux->refcnt);
1395                 return ERR_PTR(-EBUSY);
1396         }
1397         return prog;
1398 }
1399 EXPORT_SYMBOL_GPL(bpf_prog_add);
1400
1401 void bpf_prog_sub(struct bpf_prog *prog, int i)
1402 {
1403         /* Only to be used for undoing previous bpf_prog_add() in some
1404          * error path. We still know that another entity in our call
1405          * path holds a reference to the program, thus atomic_sub() can
1406          * be safely used in such cases!
1407          */
1408         WARN_ON(atomic_sub_return(i, &prog->aux->refcnt) == 0);
1409 }
1410 EXPORT_SYMBOL_GPL(bpf_prog_sub);
1411
1412 struct bpf_prog *bpf_prog_inc(struct bpf_prog *prog)
1413 {
1414         return bpf_prog_add(prog, 1);
1415 }
1416 EXPORT_SYMBOL_GPL(bpf_prog_inc);
1417
1418 /* prog_idr_lock should have been held */
1419 struct bpf_prog *bpf_prog_inc_not_zero(struct bpf_prog *prog)
1420 {
1421         int refold;
1422
1423         refold = atomic_fetch_add_unless(&prog->aux->refcnt, 1, 0);
1424
1425         if (refold >= BPF_MAX_REFCNT) {
1426                 __bpf_prog_put(prog, false);
1427                 return ERR_PTR(-EBUSY);
1428         }
1429
1430         if (!refold)
1431                 return ERR_PTR(-ENOENT);
1432
1433         return prog;
1434 }
1435 EXPORT_SYMBOL_GPL(bpf_prog_inc_not_zero);
1436
1437 bool bpf_prog_get_ok(struct bpf_prog *prog,
1438                             enum bpf_prog_type *attach_type, bool attach_drv)
1439 {
1440         /* not an attachment, just a refcount inc, always allow */
1441         if (!attach_type)
1442                 return true;
1443
1444         if (prog->type != *attach_type)
1445                 return false;
1446         if (bpf_prog_is_dev_bound(prog->aux) && !attach_drv)
1447                 return false;
1448
1449         return true;
1450 }
1451
1452 static struct bpf_prog *__bpf_prog_get(u32 ufd, enum bpf_prog_type *attach_type,
1453                                        bool attach_drv)
1454 {
1455         struct fd f = fdget(ufd);
1456         struct bpf_prog *prog;
1457
1458         prog = ____bpf_prog_get(f);
1459         if (IS_ERR(prog))
1460                 return prog;
1461         if (!bpf_prog_get_ok(prog, attach_type, attach_drv)) {
1462                 prog = ERR_PTR(-EINVAL);
1463                 goto out;
1464         }
1465
1466         prog = bpf_prog_inc(prog);
1467 out:
1468         fdput(f);
1469         return prog;
1470 }
1471
1472 struct bpf_prog *bpf_prog_get(u32 ufd)
1473 {
1474         return __bpf_prog_get(ufd, NULL, false);
1475 }
1476
1477 struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, enum bpf_prog_type type,
1478                                        bool attach_drv)
1479 {
1480         return __bpf_prog_get(ufd, &type, attach_drv);
1481 }
1482 EXPORT_SYMBOL_GPL(bpf_prog_get_type_dev);
1483
1484 /* Initially all BPF programs could be loaded w/o specifying
1485  * expected_attach_type. Later for some of them specifying expected_attach_type
1486  * at load time became required so that program could be validated properly.
1487  * Programs of types that are allowed to be loaded both w/ and w/o (for
1488  * backward compatibility) expected_attach_type, should have the default attach
1489  * type assigned to expected_attach_type for the latter case, so that it can be
1490  * validated later at attach time.
1491  *
1492  * bpf_prog_load_fixup_attach_type() sets expected_attach_type in @attr if
1493  * prog type requires it but has some attach types that have to be backward
1494  * compatible.
1495  */
1496 static void bpf_prog_load_fixup_attach_type(union bpf_attr *attr)
1497 {
1498         switch (attr->prog_type) {
1499         case BPF_PROG_TYPE_CGROUP_SOCK:
1500                 /* Unfortunately BPF_ATTACH_TYPE_UNSPEC enumeration doesn't
1501                  * exist so checking for non-zero is the way to go here.
1502                  */
1503                 if (!attr->expected_attach_type)
1504                         attr->expected_attach_type =
1505                                 BPF_CGROUP_INET_SOCK_CREATE;
1506                 break;
1507         }
1508 }
1509
1510 static int
1511 bpf_prog_load_check_attach_type(enum bpf_prog_type prog_type,
1512                                 enum bpf_attach_type expected_attach_type)
1513 {
1514         switch (prog_type) {
1515         case BPF_PROG_TYPE_CGROUP_SOCK:
1516                 switch (expected_attach_type) {
1517                 case BPF_CGROUP_INET_SOCK_CREATE:
1518                 case BPF_CGROUP_INET4_POST_BIND:
1519                 case BPF_CGROUP_INET6_POST_BIND:
1520                         return 0;
1521                 default:
1522                         return -EINVAL;
1523                 }
1524         case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
1525                 switch (expected_attach_type) {
1526                 case BPF_CGROUP_INET4_BIND:
1527                 case BPF_CGROUP_INET6_BIND:
1528                 case BPF_CGROUP_INET4_CONNECT:
1529                 case BPF_CGROUP_INET6_CONNECT:
1530                 case BPF_CGROUP_UDP4_SENDMSG:
1531                 case BPF_CGROUP_UDP6_SENDMSG:
1532                         return 0;
1533                 default:
1534                         return -EINVAL;
1535                 }
1536         default:
1537                 return 0;
1538         }
1539 }
1540
1541 /* last field in 'union bpf_attr' used by this command */
1542 #define BPF_PROG_LOAD_LAST_FIELD line_info_cnt
1543
1544 static int bpf_prog_load(union bpf_attr *attr, union bpf_attr __user *uattr)
1545 {
1546         enum bpf_prog_type type = attr->prog_type;
1547         struct bpf_prog *prog;
1548         int err;
1549         char license[128];
1550         bool is_gpl;
1551
1552         if (CHECK_ATTR(BPF_PROG_LOAD))
1553                 return -EINVAL;
1554
1555         if (attr->prog_flags & ~(BPF_F_STRICT_ALIGNMENT | BPF_F_ANY_ALIGNMENT))
1556                 return -EINVAL;
1557
1558         if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) &&
1559             (attr->prog_flags & BPF_F_ANY_ALIGNMENT) &&
1560             !capable(CAP_SYS_ADMIN))
1561                 return -EPERM;
1562
1563         /* copy eBPF program license from user space */
1564         if (strncpy_from_user(license, u64_to_user_ptr(attr->license),
1565                               sizeof(license) - 1) < 0)
1566                 return -EFAULT;
1567         license[sizeof(license) - 1] = 0;
1568
1569         /* eBPF programs must be GPL compatible to use GPL-ed functions */
1570         is_gpl = license_is_gpl_compatible(license);
1571
1572         if (attr->insn_cnt == 0 ||
1573             attr->insn_cnt > (capable(CAP_SYS_ADMIN) ? BPF_COMPLEXITY_LIMIT_INSNS : BPF_MAXINSNS))
1574                 return -E2BIG;
1575         if (type != BPF_PROG_TYPE_SOCKET_FILTER &&
1576             type != BPF_PROG_TYPE_CGROUP_SKB &&
1577             !capable(CAP_SYS_ADMIN))
1578                 return -EPERM;
1579
1580         bpf_prog_load_fixup_attach_type(attr);
1581         if (bpf_prog_load_check_attach_type(type, attr->expected_attach_type))
1582                 return -EINVAL;
1583
1584         /* plain bpf_prog allocation */
1585         prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER);
1586         if (!prog)
1587                 return -ENOMEM;
1588
1589         prog->expected_attach_type = attr->expected_attach_type;
1590
1591         prog->aux->offload_requested = !!attr->prog_ifindex;
1592
1593         err = security_bpf_prog_alloc(prog->aux);
1594         if (err)
1595                 goto free_prog_nouncharge;
1596
1597         err = bpf_prog_charge_memlock(prog);
1598         if (err)
1599                 goto free_prog_sec;
1600
1601         prog->len = attr->insn_cnt;
1602
1603         err = -EFAULT;
1604         if (copy_from_user(prog->insns, u64_to_user_ptr(attr->insns),
1605                            bpf_prog_insn_size(prog)) != 0)
1606                 goto free_prog;
1607
1608         prog->orig_prog = NULL;
1609         prog->jited = 0;
1610
1611         atomic_set(&prog->aux->refcnt, 1);
1612         prog->gpl_compatible = is_gpl ? 1 : 0;
1613
1614         if (bpf_prog_is_dev_bound(prog->aux)) {
1615                 err = bpf_prog_offload_init(prog, attr);
1616                 if (err)
1617                         goto free_prog;
1618         }
1619
1620         /* find program type: socket_filter vs tracing_filter */
1621         err = find_prog_type(type, prog);
1622         if (err < 0)
1623                 goto free_prog;
1624
1625         prog->aux->load_time = ktime_get_boot_ns();
1626         err = bpf_obj_name_cpy(prog->aux->name, attr->prog_name);
1627         if (err)
1628                 goto free_prog;
1629
1630         /* run eBPF verifier */
1631         err = bpf_check(&prog, attr, uattr);
1632         if (err < 0)
1633                 goto free_used_maps;
1634
1635         prog = bpf_prog_select_runtime(prog, &err);
1636         if (err < 0)
1637                 goto free_used_maps;
1638
1639         err = bpf_prog_alloc_id(prog);
1640         if (err)
1641                 goto free_used_maps;
1642
1643         err = bpf_prog_new_fd(prog);
1644         if (err < 0) {
1645                 /* failed to allocate fd.
1646                  * bpf_prog_put() is needed because the above
1647                  * bpf_prog_alloc_id() has published the prog
1648                  * to the userspace and the userspace may
1649                  * have refcnt-ed it through BPF_PROG_GET_FD_BY_ID.
1650                  */
1651                 bpf_prog_put(prog);
1652                 return err;
1653         }
1654
1655         bpf_prog_kallsyms_add(prog);
1656         perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_LOAD, 0);
1657         return err;
1658
1659 free_used_maps:
1660         bpf_prog_free_linfo(prog);
1661         kvfree(prog->aux->func_info);
1662         btf_put(prog->aux->btf);
1663         bpf_prog_kallsyms_del_subprogs(prog);
1664         free_used_maps(prog->aux);
1665 free_prog:
1666         bpf_prog_uncharge_memlock(prog);
1667 free_prog_sec:
1668         security_bpf_prog_free(prog->aux);
1669 free_prog_nouncharge:
1670         bpf_prog_free(prog);
1671         return err;
1672 }
1673
1674 #define BPF_OBJ_LAST_FIELD file_flags
1675
1676 static int bpf_obj_pin(const union bpf_attr *attr)
1677 {
1678         if (CHECK_ATTR(BPF_OBJ) || attr->file_flags != 0)
1679                 return -EINVAL;
1680
1681         return bpf_obj_pin_user(attr->bpf_fd, u64_to_user_ptr(attr->pathname));
1682 }
1683
1684 static int bpf_obj_get(const union bpf_attr *attr)
1685 {
1686         if (CHECK_ATTR(BPF_OBJ) || attr->bpf_fd != 0 ||
1687             attr->file_flags & ~BPF_OBJ_FLAG_MASK)
1688                 return -EINVAL;
1689
1690         return bpf_obj_get_user(u64_to_user_ptr(attr->pathname),
1691                                 attr->file_flags);
1692 }
1693
1694 struct bpf_raw_tracepoint {
1695         struct bpf_raw_event_map *btp;
1696         struct bpf_prog *prog;
1697 };
1698
1699 static int bpf_raw_tracepoint_release(struct inode *inode, struct file *filp)
1700 {
1701         struct bpf_raw_tracepoint *raw_tp = filp->private_data;
1702
1703         if (raw_tp->prog) {
1704                 bpf_probe_unregister(raw_tp->btp, raw_tp->prog);
1705                 bpf_prog_put(raw_tp->prog);
1706         }
1707         bpf_put_raw_tracepoint(raw_tp->btp);
1708         kfree(raw_tp);
1709         return 0;
1710 }
1711
1712 static const struct file_operations bpf_raw_tp_fops = {
1713         .release        = bpf_raw_tracepoint_release,
1714         .read           = bpf_dummy_read,
1715         .write          = bpf_dummy_write,
1716 };
1717
1718 #define BPF_RAW_TRACEPOINT_OPEN_LAST_FIELD raw_tracepoint.prog_fd
1719
1720 static int bpf_raw_tracepoint_open(const union bpf_attr *attr)
1721 {
1722         struct bpf_raw_tracepoint *raw_tp;
1723         struct bpf_raw_event_map *btp;
1724         struct bpf_prog *prog;
1725         char tp_name[128];
1726         int tp_fd, err;
1727
1728         if (strncpy_from_user(tp_name, u64_to_user_ptr(attr->raw_tracepoint.name),
1729                               sizeof(tp_name) - 1) < 0)
1730                 return -EFAULT;
1731         tp_name[sizeof(tp_name) - 1] = 0;
1732
1733         btp = bpf_get_raw_tracepoint(tp_name);
1734         if (!btp)
1735                 return -ENOENT;
1736
1737         raw_tp = kzalloc(sizeof(*raw_tp), GFP_USER);
1738         if (!raw_tp) {
1739                 err = -ENOMEM;
1740                 goto out_put_btp;
1741         }
1742         raw_tp->btp = btp;
1743
1744         prog = bpf_prog_get_type(attr->raw_tracepoint.prog_fd,
1745                                  BPF_PROG_TYPE_RAW_TRACEPOINT);
1746         if (IS_ERR(prog)) {
1747                 err = PTR_ERR(prog);
1748                 goto out_free_tp;
1749         }
1750
1751         err = bpf_probe_register(raw_tp->btp, prog);
1752         if (err)
1753                 goto out_put_prog;
1754
1755         raw_tp->prog = prog;
1756         tp_fd = anon_inode_getfd("bpf-raw-tracepoint", &bpf_raw_tp_fops, raw_tp,
1757                                  O_CLOEXEC);
1758         if (tp_fd < 0) {
1759                 bpf_probe_unregister(raw_tp->btp, prog);
1760                 err = tp_fd;
1761                 goto out_put_prog;
1762         }
1763         return tp_fd;
1764
1765 out_put_prog:
1766         bpf_prog_put(prog);
1767 out_free_tp:
1768         kfree(raw_tp);
1769 out_put_btp:
1770         bpf_put_raw_tracepoint(btp);
1771         return err;
1772 }
1773
1774 static int bpf_prog_attach_check_attach_type(const struct bpf_prog *prog,
1775                                              enum bpf_attach_type attach_type)
1776 {
1777         switch (prog->type) {
1778         case BPF_PROG_TYPE_CGROUP_SOCK:
1779         case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
1780                 return attach_type == prog->expected_attach_type ? 0 : -EINVAL;
1781         default:
1782                 return 0;
1783         }
1784 }
1785
1786 #define BPF_PROG_ATTACH_LAST_FIELD attach_flags
1787
1788 #define BPF_F_ATTACH_MASK \
1789         (BPF_F_ALLOW_OVERRIDE | BPF_F_ALLOW_MULTI)
1790
1791 static int bpf_prog_attach(const union bpf_attr *attr)
1792 {
1793         enum bpf_prog_type ptype;
1794         struct bpf_prog *prog;
1795         int ret;
1796
1797         if (!capable(CAP_NET_ADMIN))
1798                 return -EPERM;
1799
1800         if (CHECK_ATTR(BPF_PROG_ATTACH))
1801                 return -EINVAL;
1802
1803         if (attr->attach_flags & ~BPF_F_ATTACH_MASK)
1804                 return -EINVAL;
1805
1806         switch (attr->attach_type) {
1807         case BPF_CGROUP_INET_INGRESS:
1808         case BPF_CGROUP_INET_EGRESS:
1809                 ptype = BPF_PROG_TYPE_CGROUP_SKB;
1810                 break;
1811         case BPF_CGROUP_INET_SOCK_CREATE:
1812         case BPF_CGROUP_INET4_POST_BIND:
1813         case BPF_CGROUP_INET6_POST_BIND:
1814                 ptype = BPF_PROG_TYPE_CGROUP_SOCK;
1815                 break;
1816         case BPF_CGROUP_INET4_BIND:
1817         case BPF_CGROUP_INET6_BIND:
1818         case BPF_CGROUP_INET4_CONNECT:
1819         case BPF_CGROUP_INET6_CONNECT:
1820         case BPF_CGROUP_UDP4_SENDMSG:
1821         case BPF_CGROUP_UDP6_SENDMSG:
1822                 ptype = BPF_PROG_TYPE_CGROUP_SOCK_ADDR;
1823                 break;
1824         case BPF_CGROUP_SOCK_OPS:
1825                 ptype = BPF_PROG_TYPE_SOCK_OPS;
1826                 break;
1827         case BPF_CGROUP_DEVICE:
1828                 ptype = BPF_PROG_TYPE_CGROUP_DEVICE;
1829                 break;
1830         case BPF_SK_MSG_VERDICT:
1831                 ptype = BPF_PROG_TYPE_SK_MSG;
1832                 break;
1833         case BPF_SK_SKB_STREAM_PARSER:
1834         case BPF_SK_SKB_STREAM_VERDICT:
1835                 ptype = BPF_PROG_TYPE_SK_SKB;
1836                 break;
1837         case BPF_LIRC_MODE2:
1838                 ptype = BPF_PROG_TYPE_LIRC_MODE2;
1839                 break;
1840         case BPF_FLOW_DISSECTOR:
1841                 ptype = BPF_PROG_TYPE_FLOW_DISSECTOR;
1842                 break;
1843         default:
1844                 return -EINVAL;
1845         }
1846
1847         prog = bpf_prog_get_type(attr->attach_bpf_fd, ptype);
1848         if (IS_ERR(prog))
1849                 return PTR_ERR(prog);
1850
1851         if (bpf_prog_attach_check_attach_type(prog, attr->attach_type)) {
1852                 bpf_prog_put(prog);
1853                 return -EINVAL;
1854         }
1855
1856         switch (ptype) {
1857         case BPF_PROG_TYPE_SK_SKB:
1858         case BPF_PROG_TYPE_SK_MSG:
1859                 ret = sock_map_get_from_fd(attr, prog);
1860                 break;
1861         case BPF_PROG_TYPE_LIRC_MODE2:
1862                 ret = lirc_prog_attach(attr, prog);
1863                 break;
1864         case BPF_PROG_TYPE_FLOW_DISSECTOR:
1865                 ret = skb_flow_dissector_bpf_prog_attach(attr, prog);
1866                 break;
1867         default:
1868                 ret = cgroup_bpf_prog_attach(attr, ptype, prog);
1869         }
1870
1871         if (ret)
1872                 bpf_prog_put(prog);
1873         return ret;
1874 }
1875
1876 #define BPF_PROG_DETACH_LAST_FIELD attach_type
1877
1878 static int bpf_prog_detach(const union bpf_attr *attr)
1879 {
1880         enum bpf_prog_type ptype;
1881
1882         if (!capable(CAP_NET_ADMIN))
1883                 return -EPERM;
1884
1885         if (CHECK_ATTR(BPF_PROG_DETACH))
1886                 return -EINVAL;
1887
1888         switch (attr->attach_type) {
1889         case BPF_CGROUP_INET_INGRESS:
1890         case BPF_CGROUP_INET_EGRESS:
1891                 ptype = BPF_PROG_TYPE_CGROUP_SKB;
1892                 break;
1893         case BPF_CGROUP_INET_SOCK_CREATE:
1894         case BPF_CGROUP_INET4_POST_BIND:
1895         case BPF_CGROUP_INET6_POST_BIND:
1896                 ptype = BPF_PROG_TYPE_CGROUP_SOCK;
1897                 break;
1898         case BPF_CGROUP_INET4_BIND:
1899         case BPF_CGROUP_INET6_BIND:
1900         case BPF_CGROUP_INET4_CONNECT:
1901         case BPF_CGROUP_INET6_CONNECT:
1902         case BPF_CGROUP_UDP4_SENDMSG:
1903         case BPF_CGROUP_UDP6_SENDMSG:
1904                 ptype = BPF_PROG_TYPE_CGROUP_SOCK_ADDR;
1905                 break;
1906         case BPF_CGROUP_SOCK_OPS:
1907                 ptype = BPF_PROG_TYPE_SOCK_OPS;
1908                 break;
1909         case BPF_CGROUP_DEVICE:
1910                 ptype = BPF_PROG_TYPE_CGROUP_DEVICE;
1911                 break;
1912         case BPF_SK_MSG_VERDICT:
1913                 return sock_map_get_from_fd(attr, NULL);
1914         case BPF_SK_SKB_STREAM_PARSER:
1915         case BPF_SK_SKB_STREAM_VERDICT:
1916                 return sock_map_get_from_fd(attr, NULL);
1917         case BPF_LIRC_MODE2:
1918                 return lirc_prog_detach(attr);
1919         case BPF_FLOW_DISSECTOR:
1920                 return skb_flow_dissector_bpf_prog_detach(attr);
1921         default:
1922                 return -EINVAL;
1923         }
1924
1925         return cgroup_bpf_prog_detach(attr, ptype);
1926 }
1927
1928 #define BPF_PROG_QUERY_LAST_FIELD query.prog_cnt
1929
1930 static int bpf_prog_query(const union bpf_attr *attr,
1931                           union bpf_attr __user *uattr)
1932 {
1933         if (!capable(CAP_NET_ADMIN))
1934                 return -EPERM;
1935         if (CHECK_ATTR(BPF_PROG_QUERY))
1936                 return -EINVAL;
1937         if (attr->query.query_flags & ~BPF_F_QUERY_EFFECTIVE)
1938                 return -EINVAL;
1939
1940         switch (attr->query.attach_type) {
1941         case BPF_CGROUP_INET_INGRESS:
1942         case BPF_CGROUP_INET_EGRESS:
1943         case BPF_CGROUP_INET_SOCK_CREATE:
1944         case BPF_CGROUP_INET4_BIND:
1945         case BPF_CGROUP_INET6_BIND:
1946         case BPF_CGROUP_INET4_POST_BIND:
1947         case BPF_CGROUP_INET6_POST_BIND:
1948         case BPF_CGROUP_INET4_CONNECT:
1949         case BPF_CGROUP_INET6_CONNECT:
1950         case BPF_CGROUP_UDP4_SENDMSG:
1951         case BPF_CGROUP_UDP6_SENDMSG:
1952         case BPF_CGROUP_SOCK_OPS:
1953         case BPF_CGROUP_DEVICE:
1954                 break;
1955         case BPF_LIRC_MODE2:
1956                 return lirc_prog_query(attr, uattr);
1957         default:
1958                 return -EINVAL;
1959         }
1960
1961         return cgroup_bpf_prog_query(attr, uattr);
1962 }
1963
1964 #define BPF_PROG_TEST_RUN_LAST_FIELD test.duration
1965
1966 static int bpf_prog_test_run(const union bpf_attr *attr,
1967                              union bpf_attr __user *uattr)
1968 {
1969         struct bpf_prog *prog;
1970         int ret = -ENOTSUPP;
1971
1972         if (!capable(CAP_SYS_ADMIN))
1973                 return -EPERM;
1974         if (CHECK_ATTR(BPF_PROG_TEST_RUN))
1975                 return -EINVAL;
1976
1977         prog = bpf_prog_get(attr->test.prog_fd);
1978         if (IS_ERR(prog))
1979                 return PTR_ERR(prog);
1980
1981         if (prog->aux->ops->test_run)
1982                 ret = prog->aux->ops->test_run(prog, attr, uattr);
1983
1984         bpf_prog_put(prog);
1985         return ret;
1986 }
1987
1988 #define BPF_OBJ_GET_NEXT_ID_LAST_FIELD next_id
1989
1990 static int bpf_obj_get_next_id(const union bpf_attr *attr,
1991                                union bpf_attr __user *uattr,
1992                                struct idr *idr,
1993                                spinlock_t *lock)
1994 {
1995         u32 next_id = attr->start_id;
1996         int err = 0;
1997
1998         if (CHECK_ATTR(BPF_OBJ_GET_NEXT_ID) || next_id >= INT_MAX)
1999                 return -EINVAL;
2000
2001         if (!capable(CAP_SYS_ADMIN))
2002                 return -EPERM;
2003
2004         next_id++;
2005         spin_lock_bh(lock);
2006         if (!idr_get_next(idr, &next_id))
2007                 err = -ENOENT;
2008         spin_unlock_bh(lock);
2009
2010         if (!err)
2011                 err = put_user(next_id, &uattr->next_id);
2012
2013         return err;
2014 }
2015
2016 #define BPF_PROG_GET_FD_BY_ID_LAST_FIELD prog_id
2017
2018 static int bpf_prog_get_fd_by_id(const union bpf_attr *attr)
2019 {
2020         struct bpf_prog *prog;
2021         u32 id = attr->prog_id;
2022         int fd;
2023
2024         if (CHECK_ATTR(BPF_PROG_GET_FD_BY_ID))
2025                 return -EINVAL;
2026
2027         if (!capable(CAP_SYS_ADMIN))
2028                 return -EPERM;
2029
2030         spin_lock_bh(&prog_idr_lock);
2031         prog = idr_find(&prog_idr, id);
2032         if (prog)
2033                 prog = bpf_prog_inc_not_zero(prog);
2034         else
2035                 prog = ERR_PTR(-ENOENT);
2036         spin_unlock_bh(&prog_idr_lock);
2037
2038         if (IS_ERR(prog))
2039                 return PTR_ERR(prog);
2040
2041         fd = bpf_prog_new_fd(prog);
2042         if (fd < 0)
2043                 bpf_prog_put(prog);
2044
2045         return fd;
2046 }
2047
2048 #define BPF_MAP_GET_FD_BY_ID_LAST_FIELD open_flags
2049
2050 static int bpf_map_get_fd_by_id(const union bpf_attr *attr)
2051 {
2052         struct bpf_map *map;
2053         u32 id = attr->map_id;
2054         int f_flags;
2055         int fd;
2056
2057         if (CHECK_ATTR(BPF_MAP_GET_FD_BY_ID) ||
2058             attr->open_flags & ~BPF_OBJ_FLAG_MASK)
2059                 return -EINVAL;
2060
2061         if (!capable(CAP_SYS_ADMIN))
2062                 return -EPERM;
2063
2064         f_flags = bpf_get_file_flag(attr->open_flags);
2065         if (f_flags < 0)
2066                 return f_flags;
2067
2068         spin_lock_bh(&map_idr_lock);
2069         map = idr_find(&map_idr, id);
2070         if (map)
2071                 map = bpf_map_inc_not_zero(map, true);
2072         else
2073                 map = ERR_PTR(-ENOENT);
2074         spin_unlock_bh(&map_idr_lock);
2075
2076         if (IS_ERR(map))
2077                 return PTR_ERR(map);
2078
2079         fd = bpf_map_new_fd(map, f_flags);
2080         if (fd < 0)
2081                 bpf_map_put_with_uref(map);
2082
2083         return fd;
2084 }
2085
2086 static const struct bpf_map *bpf_map_from_imm(const struct bpf_prog *prog,
2087                                               unsigned long addr, u32 *off,
2088                                               u32 *type)
2089 {
2090         const struct bpf_map *map;
2091         int i;
2092
2093         for (i = 0, *off = 0; i < prog->aux->used_map_cnt; i++) {
2094                 map = prog->aux->used_maps[i];
2095                 if (map == (void *)addr) {
2096                         *type = BPF_PSEUDO_MAP_FD;
2097                         return map;
2098                 }
2099                 if (!map->ops->map_direct_value_meta)
2100                         continue;
2101                 if (!map->ops->map_direct_value_meta(map, addr, off)) {
2102                         *type = BPF_PSEUDO_MAP_VALUE;
2103                         return map;
2104                 }
2105         }
2106
2107         return NULL;
2108 }
2109
2110 static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog)
2111 {
2112         const struct bpf_map *map;
2113         struct bpf_insn *insns;
2114         u32 off, type;
2115         u64 imm;
2116         int i;
2117
2118         insns = kmemdup(prog->insnsi, bpf_prog_insn_size(prog),
2119                         GFP_USER);
2120         if (!insns)
2121                 return insns;
2122
2123         for (i = 0; i < prog->len; i++) {
2124                 if (insns[i].code == (BPF_JMP | BPF_TAIL_CALL)) {
2125                         insns[i].code = BPF_JMP | BPF_CALL;
2126                         insns[i].imm = BPF_FUNC_tail_call;
2127                         /* fall-through */
2128                 }
2129                 if (insns[i].code == (BPF_JMP | BPF_CALL) ||
2130                     insns[i].code == (BPF_JMP | BPF_CALL_ARGS)) {
2131                         if (insns[i].code == (BPF_JMP | BPF_CALL_ARGS))
2132                                 insns[i].code = BPF_JMP | BPF_CALL;
2133                         if (!bpf_dump_raw_ok())
2134                                 insns[i].imm = 0;
2135                         continue;
2136                 }
2137
2138                 if (insns[i].code != (BPF_LD | BPF_IMM | BPF_DW))
2139                         continue;
2140
2141                 imm = ((u64)insns[i + 1].imm << 32) | (u32)insns[i].imm;
2142                 map = bpf_map_from_imm(prog, imm, &off, &type);
2143                 if (map) {
2144                         insns[i].src_reg = type;
2145                         insns[i].imm = map->id;
2146                         insns[i + 1].imm = off;
2147                         continue;
2148                 }
2149         }
2150
2151         return insns;
2152 }
2153
2154 static int set_info_rec_size(struct bpf_prog_info *info)
2155 {
2156         /*
2157          * Ensure info.*_rec_size is the same as kernel expected size
2158          *
2159          * or
2160          *
2161          * Only allow zero *_rec_size if both _rec_size and _cnt are
2162          * zero.  In this case, the kernel will set the expected
2163          * _rec_size back to the info.
2164          */
2165
2166         if ((info->nr_func_info || info->func_info_rec_size) &&
2167             info->func_info_rec_size != sizeof(struct bpf_func_info))
2168                 return -EINVAL;
2169
2170         if ((info->nr_line_info || info->line_info_rec_size) &&
2171             info->line_info_rec_size != sizeof(struct bpf_line_info))
2172                 return -EINVAL;
2173
2174         if ((info->nr_jited_line_info || info->jited_line_info_rec_size) &&
2175             info->jited_line_info_rec_size != sizeof(__u64))
2176                 return -EINVAL;
2177
2178         info->func_info_rec_size = sizeof(struct bpf_func_info);
2179         info->line_info_rec_size = sizeof(struct bpf_line_info);
2180         info->jited_line_info_rec_size = sizeof(__u64);
2181
2182         return 0;
2183 }
2184
2185 static int bpf_prog_get_info_by_fd(struct bpf_prog *prog,
2186                                    const union bpf_attr *attr,
2187                                    union bpf_attr __user *uattr)
2188 {
2189         struct bpf_prog_info __user *uinfo = u64_to_user_ptr(attr->info.info);
2190         struct bpf_prog_info info = {};
2191         u32 info_len = attr->info.info_len;
2192         struct bpf_prog_stats stats;
2193         char __user *uinsns;
2194         u32 ulen;
2195         int err;
2196
2197         err = bpf_check_uarg_tail_zero(uinfo, sizeof(info), info_len);
2198         if (err)
2199                 return err;
2200         info_len = min_t(u32, sizeof(info), info_len);
2201
2202         if (copy_from_user(&info, uinfo, info_len))
2203                 return -EFAULT;
2204
2205         info.type = prog->type;
2206         info.id = prog->aux->id;
2207         info.load_time = prog->aux->load_time;
2208         info.created_by_uid = from_kuid_munged(current_user_ns(),
2209                                                prog->aux->user->uid);
2210         info.gpl_compatible = prog->gpl_compatible;
2211
2212         memcpy(info.tag, prog->tag, sizeof(prog->tag));
2213         memcpy(info.name, prog->aux->name, sizeof(prog->aux->name));
2214
2215         ulen = info.nr_map_ids;
2216         info.nr_map_ids = prog->aux->used_map_cnt;
2217         ulen = min_t(u32, info.nr_map_ids, ulen);
2218         if (ulen) {
2219                 u32 __user *user_map_ids = u64_to_user_ptr(info.map_ids);
2220                 u32 i;
2221
2222                 for (i = 0; i < ulen; i++)
2223                         if (put_user(prog->aux->used_maps[i]->id,
2224                                      &user_map_ids[i]))
2225                                 return -EFAULT;
2226         }
2227
2228         err = set_info_rec_size(&info);
2229         if (err)
2230                 return err;
2231
2232         bpf_prog_get_stats(prog, &stats);
2233         info.run_time_ns = stats.nsecs;
2234         info.run_cnt = stats.cnt;
2235
2236         if (!capable(CAP_SYS_ADMIN)) {
2237                 info.jited_prog_len = 0;
2238                 info.xlated_prog_len = 0;
2239                 info.nr_jited_ksyms = 0;
2240                 info.nr_jited_func_lens = 0;
2241                 info.nr_func_info = 0;
2242                 info.nr_line_info = 0;
2243                 info.nr_jited_line_info = 0;
2244                 goto done;
2245         }
2246
2247         ulen = info.xlated_prog_len;
2248         info.xlated_prog_len = bpf_prog_insn_size(prog);
2249         if (info.xlated_prog_len && ulen) {
2250                 struct bpf_insn *insns_sanitized;
2251                 bool fault;
2252
2253                 if (prog->blinded && !bpf_dump_raw_ok()) {
2254                         info.xlated_prog_insns = 0;
2255                         goto done;
2256                 }
2257                 insns_sanitized = bpf_insn_prepare_dump(prog);
2258                 if (!insns_sanitized)
2259                         return -ENOMEM;
2260                 uinsns = u64_to_user_ptr(info.xlated_prog_insns);
2261                 ulen = min_t(u32, info.xlated_prog_len, ulen);
2262                 fault = copy_to_user(uinsns, insns_sanitized, ulen);
2263                 kfree(insns_sanitized);
2264                 if (fault)
2265                         return -EFAULT;
2266         }
2267
2268         if (bpf_prog_is_dev_bound(prog->aux)) {
2269                 err = bpf_prog_offload_info_fill(&info, prog);
2270                 if (err)
2271                         return err;
2272                 goto done;
2273         }
2274
2275         /* NOTE: the following code is supposed to be skipped for offload.
2276          * bpf_prog_offload_info_fill() is the place to fill similar fields
2277          * for offload.
2278          */
2279         ulen = info.jited_prog_len;
2280         if (prog->aux->func_cnt) {
2281                 u32 i;
2282
2283                 info.jited_prog_len = 0;
2284                 for (i = 0; i < prog->aux->func_cnt; i++)
2285                         info.jited_prog_len += prog->aux->func[i]->jited_len;
2286         } else {
2287                 info.jited_prog_len = prog->jited_len;
2288         }
2289
2290         if (info.jited_prog_len && ulen) {
2291                 if (bpf_dump_raw_ok()) {
2292                         uinsns = u64_to_user_ptr(info.jited_prog_insns);
2293                         ulen = min_t(u32, info.jited_prog_len, ulen);
2294
2295                         /* for multi-function programs, copy the JITed
2296                          * instructions for all the functions
2297                          */
2298                         if (prog->aux->func_cnt) {
2299                                 u32 len, free, i;
2300                                 u8 *img;
2301
2302                                 free = ulen;
2303                                 for (i = 0; i < prog->aux->func_cnt; i++) {
2304                                         len = prog->aux->func[i]->jited_len;
2305                                         len = min_t(u32, len, free);
2306                                         img = (u8 *) prog->aux->func[i]->bpf_func;
2307                                         if (copy_to_user(uinsns, img, len))
2308                                                 return -EFAULT;
2309                                         uinsns += len;
2310                                         free -= len;
2311                                         if (!free)
2312                                                 break;
2313                                 }
2314                         } else {
2315                                 if (copy_to_user(uinsns, prog->bpf_func, ulen))
2316                                         return -EFAULT;
2317                         }
2318                 } else {
2319                         info.jited_prog_insns = 0;
2320                 }
2321         }
2322
2323         ulen = info.nr_jited_ksyms;
2324         info.nr_jited_ksyms = prog->aux->func_cnt ? : 1;
2325         if (ulen) {
2326                 if (bpf_dump_raw_ok()) {
2327                         unsigned long ksym_addr;
2328                         u64 __user *user_ksyms;
2329                         u32 i;
2330
2331                         /* copy the address of the kernel symbol
2332                          * corresponding to each function
2333                          */
2334                         ulen = min_t(u32, info.nr_jited_ksyms, ulen);
2335                         user_ksyms = u64_to_user_ptr(info.jited_ksyms);
2336                         if (prog->aux->func_cnt) {
2337                                 for (i = 0; i < ulen; i++) {
2338                                         ksym_addr = (unsigned long)
2339                                                 prog->aux->func[i]->bpf_func;
2340                                         if (put_user((u64) ksym_addr,
2341                                                      &user_ksyms[i]))
2342                                                 return -EFAULT;
2343                                 }
2344                         } else {
2345                                 ksym_addr = (unsigned long) prog->bpf_func;
2346                                 if (put_user((u64) ksym_addr, &user_ksyms[0]))
2347                                         return -EFAULT;
2348                         }
2349                 } else {
2350                         info.jited_ksyms = 0;
2351                 }
2352         }
2353
2354         ulen = info.nr_jited_func_lens;
2355         info.nr_jited_func_lens = prog->aux->func_cnt ? : 1;
2356         if (ulen) {
2357                 if (bpf_dump_raw_ok()) {
2358                         u32 __user *user_lens;
2359                         u32 func_len, i;
2360
2361                         /* copy the JITed image lengths for each function */
2362                         ulen = min_t(u32, info.nr_jited_func_lens, ulen);
2363                         user_lens = u64_to_user_ptr(info.jited_func_lens);
2364                         if (prog->aux->func_cnt) {
2365                                 for (i = 0; i < ulen; i++) {
2366                                         func_len =
2367                                                 prog->aux->func[i]->jited_len;
2368                                         if (put_user(func_len, &user_lens[i]))
2369                                                 return -EFAULT;
2370                                 }
2371                         } else {
2372                                 func_len = prog->jited_len;
2373                                 if (put_user(func_len, &user_lens[0]))
2374                                         return -EFAULT;
2375                         }
2376                 } else {
2377                         info.jited_func_lens = 0;
2378                 }
2379         }
2380
2381         if (prog->aux->btf)
2382                 info.btf_id = btf_id(prog->aux->btf);
2383
2384         ulen = info.nr_func_info;
2385         info.nr_func_info = prog->aux->func_info_cnt;
2386         if (info.nr_func_info && ulen) {
2387                 char __user *user_finfo;
2388
2389                 user_finfo = u64_to_user_ptr(info.func_info);
2390                 ulen = min_t(u32, info.nr_func_info, ulen);
2391                 if (copy_to_user(user_finfo, prog->aux->func_info,
2392                                  info.func_info_rec_size * ulen))
2393                         return -EFAULT;
2394         }
2395
2396         ulen = info.nr_line_info;
2397         info.nr_line_info = prog->aux->nr_linfo;
2398         if (info.nr_line_info && ulen) {
2399                 __u8 __user *user_linfo;
2400
2401                 user_linfo = u64_to_user_ptr(info.line_info);
2402                 ulen = min_t(u32, info.nr_line_info, ulen);
2403                 if (copy_to_user(user_linfo, prog->aux->linfo,
2404                                  info.line_info_rec_size * ulen))
2405                         return -EFAULT;
2406         }
2407
2408         ulen = info.nr_jited_line_info;
2409         if (prog->aux->jited_linfo)
2410                 info.nr_jited_line_info = prog->aux->nr_linfo;
2411         else
2412                 info.nr_jited_line_info = 0;
2413         if (info.nr_jited_line_info && ulen) {
2414                 if (bpf_dump_raw_ok()) {
2415                         __u64 __user *user_linfo;
2416                         u32 i;
2417
2418                         user_linfo = u64_to_user_ptr(info.jited_line_info);
2419                         ulen = min_t(u32, info.nr_jited_line_info, ulen);
2420                         for (i = 0; i < ulen; i++) {
2421                                 if (put_user((__u64)(long)prog->aux->jited_linfo[i],
2422                                              &user_linfo[i]))
2423                                         return -EFAULT;
2424                         }
2425                 } else {
2426                         info.jited_line_info = 0;
2427                 }
2428         }
2429
2430         ulen = info.nr_prog_tags;
2431         info.nr_prog_tags = prog->aux->func_cnt ? : 1;
2432         if (ulen) {
2433                 __u8 __user (*user_prog_tags)[BPF_TAG_SIZE];
2434                 u32 i;
2435
2436                 user_prog_tags = u64_to_user_ptr(info.prog_tags);
2437                 ulen = min_t(u32, info.nr_prog_tags, ulen);
2438                 if (prog->aux->func_cnt) {
2439                         for (i = 0; i < ulen; i++) {
2440                                 if (copy_to_user(user_prog_tags[i],
2441                                                  prog->aux->func[i]->tag,
2442                                                  BPF_TAG_SIZE))
2443                                         return -EFAULT;
2444                         }
2445                 } else {
2446                         if (copy_to_user(user_prog_tags[0],
2447                                          prog->tag, BPF_TAG_SIZE))
2448                                 return -EFAULT;
2449                 }
2450         }
2451
2452 done:
2453         if (copy_to_user(uinfo, &info, info_len) ||
2454             put_user(info_len, &uattr->info.info_len))
2455                 return -EFAULT;
2456
2457         return 0;
2458 }
2459
2460 static int bpf_map_get_info_by_fd(struct bpf_map *map,
2461                                   const union bpf_attr *attr,
2462                                   union bpf_attr __user *uattr)
2463 {
2464         struct bpf_map_info __user *uinfo = u64_to_user_ptr(attr->info.info);
2465         struct bpf_map_info info = {};
2466         u32 info_len = attr->info.info_len;
2467         int err;
2468
2469         err = bpf_check_uarg_tail_zero(uinfo, sizeof(info), info_len);
2470         if (err)
2471                 return err;
2472         info_len = min_t(u32, sizeof(info), info_len);
2473
2474         info.type = map->map_type;
2475         info.id = map->id;
2476         info.key_size = map->key_size;
2477         info.value_size = map->value_size;
2478         info.max_entries = map->max_entries;
2479         info.map_flags = map->map_flags;
2480         memcpy(info.name, map->name, sizeof(map->name));
2481
2482         if (map->btf) {
2483                 info.btf_id = btf_id(map->btf);
2484                 info.btf_key_type_id = map->btf_key_type_id;
2485                 info.btf_value_type_id = map->btf_value_type_id;
2486         }
2487
2488         if (bpf_map_is_dev_bound(map)) {
2489                 err = bpf_map_offload_info_fill(&info, map);
2490                 if (err)
2491                         return err;
2492         }
2493
2494         if (copy_to_user(uinfo, &info, info_len) ||
2495             put_user(info_len, &uattr->info.info_len))
2496                 return -EFAULT;
2497
2498         return 0;
2499 }
2500
2501 static int bpf_btf_get_info_by_fd(struct btf *btf,
2502                                   const union bpf_attr *attr,
2503                                   union bpf_attr __user *uattr)
2504 {
2505         struct bpf_btf_info __user *uinfo = u64_to_user_ptr(attr->info.info);
2506         u32 info_len = attr->info.info_len;
2507         int err;
2508
2509         err = bpf_check_uarg_tail_zero(uinfo, sizeof(*uinfo), info_len);
2510         if (err)
2511                 return err;
2512
2513         return btf_get_info_by_fd(btf, attr, uattr);
2514 }
2515
2516 #define BPF_OBJ_GET_INFO_BY_FD_LAST_FIELD info.info
2517
2518 static int bpf_obj_get_info_by_fd(const union bpf_attr *attr,
2519                                   union bpf_attr __user *uattr)
2520 {
2521         int ufd = attr->info.bpf_fd;
2522         struct fd f;
2523         int err;
2524
2525         if (CHECK_ATTR(BPF_OBJ_GET_INFO_BY_FD))
2526                 return -EINVAL;
2527
2528         f = fdget(ufd);
2529         if (!f.file)
2530                 return -EBADFD;
2531
2532         if (f.file->f_op == &bpf_prog_fops)
2533                 err = bpf_prog_get_info_by_fd(f.file->private_data, attr,
2534                                               uattr);
2535         else if (f.file->f_op == &bpf_map_fops)
2536                 err = bpf_map_get_info_by_fd(f.file->private_data, attr,
2537                                              uattr);
2538         else if (f.file->f_op == &btf_fops)
2539                 err = bpf_btf_get_info_by_fd(f.file->private_data, attr, uattr);
2540         else
2541                 err = -EINVAL;
2542
2543         fdput(f);
2544         return err;
2545 }
2546
2547 #define BPF_BTF_LOAD_LAST_FIELD btf_log_level
2548
2549 static int bpf_btf_load(const union bpf_attr *attr)
2550 {
2551         if (CHECK_ATTR(BPF_BTF_LOAD))
2552                 return -EINVAL;
2553
2554         if (!capable(CAP_SYS_ADMIN))
2555                 return -EPERM;
2556
2557         return btf_new_fd(attr);
2558 }
2559
2560 #define BPF_BTF_GET_FD_BY_ID_LAST_FIELD btf_id
2561
2562 static int bpf_btf_get_fd_by_id(const union bpf_attr *attr)
2563 {
2564         if (CHECK_ATTR(BPF_BTF_GET_FD_BY_ID))
2565                 return -EINVAL;
2566
2567         if (!capable(CAP_SYS_ADMIN))
2568                 return -EPERM;
2569
2570         return btf_get_fd_by_id(attr->btf_id);
2571 }
2572
2573 static int bpf_task_fd_query_copy(const union bpf_attr *attr,
2574                                     union bpf_attr __user *uattr,
2575                                     u32 prog_id, u32 fd_type,
2576                                     const char *buf, u64 probe_offset,
2577                                     u64 probe_addr)
2578 {
2579         char __user *ubuf = u64_to_user_ptr(attr->task_fd_query.buf);
2580         u32 len = buf ? strlen(buf) : 0, input_len;
2581         int err = 0;
2582
2583         if (put_user(len, &uattr->task_fd_query.buf_len))
2584                 return -EFAULT;
2585         input_len = attr->task_fd_query.buf_len;
2586         if (input_len && ubuf) {
2587                 if (!len) {
2588                         /* nothing to copy, just make ubuf NULL terminated */
2589                         char zero = '\0';
2590
2591                         if (put_user(zero, ubuf))
2592                                 return -EFAULT;
2593                 } else if (input_len >= len + 1) {
2594                         /* ubuf can hold the string with NULL terminator */
2595                         if (copy_to_user(ubuf, buf, len + 1))
2596                                 return -EFAULT;
2597                 } else {
2598                         /* ubuf cannot hold the string with NULL terminator,
2599                          * do a partial copy with NULL terminator.
2600                          */
2601                         char zero = '\0';
2602
2603                         err = -ENOSPC;
2604                         if (copy_to_user(ubuf, buf, input_len - 1))
2605                                 return -EFAULT;
2606                         if (put_user(zero, ubuf + input_len - 1))
2607                                 return -EFAULT;
2608                 }
2609         }
2610
2611         if (put_user(prog_id, &uattr->task_fd_query.prog_id) ||
2612             put_user(fd_type, &uattr->task_fd_query.fd_type) ||
2613             put_user(probe_offset, &uattr->task_fd_query.probe_offset) ||
2614             put_user(probe_addr, &uattr->task_fd_query.probe_addr))
2615                 return -EFAULT;
2616
2617         return err;
2618 }
2619
2620 #define BPF_TASK_FD_QUERY_LAST_FIELD task_fd_query.probe_addr
2621
2622 static int bpf_task_fd_query(const union bpf_attr *attr,
2623                              union bpf_attr __user *uattr)
2624 {
2625         pid_t pid = attr->task_fd_query.pid;
2626         u32 fd = attr->task_fd_query.fd;
2627         const struct perf_event *event;
2628         struct files_struct *files;
2629         struct task_struct *task;
2630         struct file *file;
2631         int err;
2632
2633         if (CHECK_ATTR(BPF_TASK_FD_QUERY))
2634                 return -EINVAL;
2635
2636         if (!capable(CAP_SYS_ADMIN))
2637                 return -EPERM;
2638
2639         if (attr->task_fd_query.flags != 0)
2640                 return -EINVAL;
2641
2642         task = get_pid_task(find_vpid(pid), PIDTYPE_PID);
2643         if (!task)
2644                 return -ENOENT;
2645
2646         files = get_files_struct(task);
2647         put_task_struct(task);
2648         if (!files)
2649                 return -ENOENT;
2650
2651         err = 0;
2652         spin_lock(&files->file_lock);
2653         file = fcheck_files(files, fd);
2654         if (!file)
2655                 err = -EBADF;
2656         else
2657                 get_file(file);
2658         spin_unlock(&files->file_lock);
2659         put_files_struct(files);
2660
2661         if (err)
2662                 goto out;
2663
2664         if (file->f_op == &bpf_raw_tp_fops) {
2665                 struct bpf_raw_tracepoint *raw_tp = file->private_data;
2666                 struct bpf_raw_event_map *btp = raw_tp->btp;
2667
2668                 err = bpf_task_fd_query_copy(attr, uattr,
2669                                              raw_tp->prog->aux->id,
2670                                              BPF_FD_TYPE_RAW_TRACEPOINT,
2671                                              btp->tp->name, 0, 0);
2672                 goto put_file;
2673         }
2674
2675         event = perf_get_event(file);
2676         if (!IS_ERR(event)) {
2677                 u64 probe_offset, probe_addr;
2678                 u32 prog_id, fd_type;
2679                 const char *buf;
2680
2681                 err = bpf_get_perf_event_info(event, &prog_id, &fd_type,
2682                                               &buf, &probe_offset,
2683                                               &probe_addr);
2684                 if (!err)
2685                         err = bpf_task_fd_query_copy(attr, uattr, prog_id,
2686                                                      fd_type, buf,
2687                                                      probe_offset,
2688                                                      probe_addr);
2689                 goto put_file;
2690         }
2691
2692         err = -ENOTSUPP;
2693 put_file:
2694         fput(file);
2695 out:
2696         return err;
2697 }
2698
2699 SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size)
2700 {
2701         union bpf_attr attr = {};
2702         int err;
2703
2704         if (sysctl_unprivileged_bpf_disabled && !capable(CAP_SYS_ADMIN))
2705                 return -EPERM;
2706
2707         err = bpf_check_uarg_tail_zero(uattr, sizeof(attr), size);
2708         if (err)
2709                 return err;
2710         size = min_t(u32, size, sizeof(attr));
2711
2712         /* copy attributes from user space, may be less than sizeof(bpf_attr) */
2713         if (copy_from_user(&attr, uattr, size) != 0)
2714                 return -EFAULT;
2715
2716         err = security_bpf(cmd, &attr, size);
2717         if (err < 0)
2718                 return err;
2719
2720         switch (cmd) {
2721         case BPF_MAP_CREATE:
2722                 err = map_create(&attr);
2723                 break;
2724         case BPF_MAP_LOOKUP_ELEM:
2725                 err = map_lookup_elem(&attr);
2726                 break;
2727         case BPF_MAP_UPDATE_ELEM:
2728                 err = map_update_elem(&attr);
2729                 break;
2730         case BPF_MAP_DELETE_ELEM:
2731                 err = map_delete_elem(&attr);
2732                 break;
2733         case BPF_MAP_GET_NEXT_KEY:
2734                 err = map_get_next_key(&attr);
2735                 break;
2736         case BPF_PROG_LOAD:
2737                 err = bpf_prog_load(&attr, uattr);
2738                 break;
2739         case BPF_OBJ_PIN:
2740                 err = bpf_obj_pin(&attr);
2741                 break;
2742         case BPF_OBJ_GET:
2743                 err = bpf_obj_get(&attr);
2744                 break;
2745         case BPF_PROG_ATTACH:
2746                 err = bpf_prog_attach(&attr);
2747                 break;
2748         case BPF_PROG_DETACH:
2749                 err = bpf_prog_detach(&attr);
2750                 break;
2751         case BPF_PROG_QUERY:
2752                 err = bpf_prog_query(&attr, uattr);
2753                 break;
2754         case BPF_PROG_TEST_RUN:
2755                 err = bpf_prog_test_run(&attr, uattr);
2756                 break;
2757         case BPF_PROG_GET_NEXT_ID:
2758                 err = bpf_obj_get_next_id(&attr, uattr,
2759                                           &prog_idr, &prog_idr_lock);
2760                 break;
2761         case BPF_MAP_GET_NEXT_ID:
2762                 err = bpf_obj_get_next_id(&attr, uattr,
2763                                           &map_idr, &map_idr_lock);
2764                 break;
2765         case BPF_PROG_GET_FD_BY_ID:
2766                 err = bpf_prog_get_fd_by_id(&attr);
2767                 break;
2768         case BPF_MAP_GET_FD_BY_ID:
2769                 err = bpf_map_get_fd_by_id(&attr);
2770                 break;
2771         case BPF_OBJ_GET_INFO_BY_FD:
2772                 err = bpf_obj_get_info_by_fd(&attr, uattr);
2773                 break;
2774         case BPF_RAW_TRACEPOINT_OPEN:
2775                 err = bpf_raw_tracepoint_open(&attr);
2776                 break;
2777         case BPF_BTF_LOAD:
2778                 err = bpf_btf_load(&attr);
2779                 break;
2780         case BPF_BTF_GET_FD_BY_ID:
2781                 err = bpf_btf_get_fd_by_id(&attr);
2782                 break;
2783         case BPF_TASK_FD_QUERY:
2784                 err = bpf_task_fd_query(&attr, uattr);
2785                 break;
2786         case BPF_MAP_LOOKUP_AND_DELETE_ELEM:
2787                 err = map_lookup_and_delete_elem(&attr);
2788                 break;
2789         default:
2790                 err = -EINVAL;
2791                 break;
2792         }
2793
2794         return err;
2795 }