tools/lib/bpf/libbpf.c

   1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
   2
   3 /*
   4  * Common eBPF ELF object loading operations.
   5  *
   6  * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org>
   7  * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com>
   8  * Copyright (C) 2015 Huawei Inc.
   9  * Copyright (C) 2017 Nicira, Inc.
  10  */
  11
  12 #ifndef _GNU_SOURCE
  13 #define _GNU_SOURCE
  14 #endif
  15 #include <stdlib.h>
  16 #include <stdio.h>
  17 #include <stdarg.h>
  18 #include <libgen.h>
  19 #include <inttypes.h>
  20 #include <string.h>
  21 #include <unistd.h>
  22 #include <fcntl.h>
  23 #include <errno.h>
  24 #include <asm/unistd.h>
  25 #include <linux/err.h>
  26 #include <linux/kernel.h>
  27 #include <linux/bpf.h>
  28 #include <linux/btf.h>
  29 #include <linux/filter.h>
  30 #include <linux/list.h>
  31 #include <linux/limits.h>
  32 #include <linux/perf_event.h>
  33 #include <linux/ring_buffer.h>
  34 #include <sys/stat.h>
  35 #include <sys/types.h>
  36 #include <sys/vfs.h>
  37 #include <tools/libc_compat.h>
  38 #include <libelf.h>
  39 #include <gelf.h>
  40
  41 #include "libbpf.h"
  42 #include "bpf.h"
  43 #include "btf.h"
  44 #include "str_error.h"
  45 #include "libbpf_util.h"
  46
  47 #ifndef EM_BPF
  48 #define EM_BPF 247
  49 #endif
  50
  51 #ifndef BPF_FS_MAGIC
  52 #define BPF_FS_MAGIC            0xcafe4a11
  53 #endif
  54
  55 #define __printf(a, b)  __attribute__((format(printf, a, b)))
  56
  57 static int __base_pr(enum libbpf_print_level level, const char *format,
  58                      va_list args)
  59 {
  60         if (level == LIBBPF_DEBUG)
  61                 return 0;
  62
  63         return vfprintf(stderr, format, args);
  64 }
  65
  66 static libbpf_print_fn_t __libbpf_pr = __base_pr;
  67
  68 void libbpf_set_print(libbpf_print_fn_t fn)
  69 {
  70         __libbpf_pr = fn;
  71 }
  72
  73 __printf(2, 3)
  74 void libbpf_print(enum libbpf_print_level level, const char *format, ...)
  75 {
  76         va_list args;
  77
  78         if (!__libbpf_pr)
  79                 return;
  80
  81         va_start(args, format);
  82         __libbpf_pr(level, format, args);
  83         va_end(args);
  84 }
  85
  86 #define STRERR_BUFSIZE  128
  87
  88 #define CHECK_ERR(action, err, out) do {        \
  89         err = action;                   \
  90         if (err)                        \
  91                 goto out;               \
  92 } while(0)
  93
  94
  95 /* Copied from tools/perf/util/util.h */
  96 #ifndef zfree
  97 # define zfree(ptr) ({ free(*ptr); *ptr = NULL; })
  98 #endif
  99
 100 #ifndef zclose
 101 # define zclose(fd) ({                  \
 102         int ___err = 0;                 \
 103         if ((fd) >= 0)                  \
 104                 ___err = close((fd));   \
 105         fd = -1;                        \
 106         ___err; })
 107 #endif
 108
 109 #ifdef HAVE_LIBELF_MMAP_SUPPORT
 110 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ_MMAP
 111 #else
 112 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ
 113 #endif
 114
 115 struct bpf_capabilities {
 116         /* v4.14: kernel support for program & map names. */
 117         __u32 name:1;
 118 };
 119
 120 /*
 121  * bpf_prog should be a better name but it has been used in
 122  * linux/filter.h.
 123  */
 124 struct bpf_program {
 125         /* Index in elf obj file, for relocation use. */
 126         int idx;
 127         char *name;
 128         int prog_ifindex;
 129         char *section_name;
 130         /* section_name with / replaced by _; makes recursive pinning
 131          * in bpf_object__pin_programs easier
 132          */
 133         char *pin_name;
 134         struct bpf_insn *insns;
 135         size_t insns_cnt, main_prog_cnt;
 136         enum bpf_prog_type type;
 137
 138         struct reloc_desc {
 139                 enum {
 140                         RELO_LD64,
 141                         RELO_CALL,
 142                 } type;
 143                 int insn_idx;
 144                 union {
 145                         int map_idx;
 146                         int text_off;
 147                 };
 148         } *reloc_desc;
 149         int nr_reloc;
 150
 151         struct {
 152                 int nr;
 153                 int *fds;
 154         } instances;
 155         bpf_program_prep_t preprocessor;
 156
 157         struct bpf_object *obj;
 158         void *priv;
 159         bpf_program_clear_priv_t clear_priv;
 160
 161         enum bpf_attach_type expected_attach_type;
 162         int btf_fd;
 163         void *func_info;
 164         __u32 func_info_rec_size;
 165         __u32 func_info_cnt;
 166
 167         struct bpf_capabilities *caps;
 168
 169         void *line_info;
 170         __u32 line_info_rec_size;
 171         __u32 line_info_cnt;
 172 };
 173
 174 struct bpf_map {
 175         int fd;
 176         char *name;
 177         size_t offset;
 178         int map_ifindex;
 179         int inner_map_fd;
 180         struct bpf_map_def def;
 181         __u32 btf_key_type_id;
 182         __u32 btf_value_type_id;
 183         void *priv;
 184         bpf_map_clear_priv_t clear_priv;
 185 };
 186
 187 static LIST_HEAD(bpf_objects_list);
 188
 189 struct bpf_object {
 190         char license[64];
 191         __u32 kern_version;
 192
 193         struct bpf_program *programs;
 194         size_t nr_programs;
 195         struct bpf_map *maps;
 196         size_t nr_maps;
 197
 198         bool loaded;
 199         bool has_pseudo_calls;
 200
 201         /*
 202          * Information when doing elf related work. Only valid if fd
 203          * is valid.
 204          */
 205         struct {
 206                 int fd;
 207                 void *obj_buf;
 208                 size_t obj_buf_sz;
 209                 Elf *elf;
 210                 GElf_Ehdr ehdr;
 211                 Elf_Data *symbols;
 212                 size_t strtabidx;
 213                 struct {
 214                         GElf_Shdr shdr;
 215                         Elf_Data *data;
 216                 } *reloc;
 217                 int nr_reloc;
 218                 int maps_shndx;
 219                 int text_shndx;
 220         } efile;
 221         /*
 222          * All loaded bpf_object is linked in a list, which is
 223          * hidden to caller. bpf_objects__<func> handlers deal with
 224          * all objects.
 225          */
 226         struct list_head list;
 227
 228         struct btf *btf;
 229         struct btf_ext *btf_ext;
 230
 231         void *priv;
 232         bpf_object_clear_priv_t clear_priv;
 233
 234         struct bpf_capabilities caps;
 235
 236         char path[];
 237 };
 238 #define obj_elf_valid(o)        ((o)->efile.elf)
 239
 240 void bpf_program__unload(struct bpf_program *prog)
 241 {
 242         int i;
 243
 244         if (!prog)
 245                 return;
 246
 247         /*
 248          * If the object is opened but the program was never loaded,
 249          * it is possible that prog->instances.nr == -1.
 250          */
 251         if (prog->instances.nr > 0) {
 252                 for (i = 0; i < prog->instances.nr; i++)
 253                         zclose(prog->instances.fds[i]);
 254         } else if (prog->instances.nr != -1) {
 255                 pr_warning("Internal error: instances.nr is %d\n",
 256                            prog->instances.nr);
 257         }
 258
 259         prog->instances.nr = -1;
 260         zfree(&prog->instances.fds);
 261
 262         zclose(prog->btf_fd);
 263         zfree(&prog->func_info);
 264         zfree(&prog->line_info);
 265 }
 266
 267 static void bpf_program__exit(struct bpf_program *prog)
 268 {
 269         if (!prog)
 270                 return;
 271
 272         if (prog->clear_priv)
 273                 prog->clear_priv(prog, prog->priv);
 274
 275         prog->priv = NULL;
 276         prog->clear_priv = NULL;
 277
 278         bpf_program__unload(prog);
 279         zfree(&prog->name);
 280         zfree(&prog->section_name);
 281         zfree(&prog->pin_name);
 282         zfree(&prog->insns);
 283         zfree(&prog->reloc_desc);
 284
 285         prog->nr_reloc = 0;
 286         prog->insns_cnt = 0;
 287         prog->idx = -1;
 288 }
 289
 290 static char *__bpf_program__pin_name(struct bpf_program *prog)
 291 {
 292         char *name, *p;
 293
 294         name = p = strdup(prog->section_name);
 295         while ((p = strchr(p, '/')))
 296                 *p = '_';
 297
 298         return name;
 299 }
 300
 301 static int
 302 bpf_program__init(void *data, size_t size, char *section_name, int idx,
 303                   struct bpf_program *prog)
 304 {
 305         if (size < sizeof(struct bpf_insn)) {
 306                 pr_warning("corrupted section '%s'\n", section_name);
 307                 return -EINVAL;
 308         }
 309
 310         memset(prog, 0, sizeof(*prog));
 311
 312         prog->section_name = strdup(section_name);
 313         if (!prog->section_name) {
 314                 pr_warning("failed to alloc name for prog under section(%d) %s\n",
 315                            idx, section_name);
 316                 goto errout;
 317         }
 318
 319         prog->pin_name = __bpf_program__pin_name(prog);
 320         if (!prog->pin_name) {
 321                 pr_warning("failed to alloc pin name for prog under section(%d) %s\n",
 322                            idx, section_name);
 323                 goto errout;
 324         }
 325
 326         prog->insns = malloc(size);
 327         if (!prog->insns) {
 328                 pr_warning("failed to alloc insns for prog under section %s\n",
 329                            section_name);
 330                 goto errout;
 331         }
 332         prog->insns_cnt = size / sizeof(struct bpf_insn);
 333         memcpy(prog->insns, data,
 334                prog->insns_cnt * sizeof(struct bpf_insn));
 335         prog->idx = idx;
 336         prog->instances.fds = NULL;
 337         prog->instances.nr = -1;
 338         prog->type = BPF_PROG_TYPE_UNSPEC;
 339         prog->btf_fd = -1;
 340
 341         return 0;
 342 errout:
 343         bpf_program__exit(prog);
 344         return -ENOMEM;
 345 }
 346
 347 static int
 348 bpf_object__add_program(struct bpf_object *obj, void *data, size_t size,
 349                         char *section_name, int idx)
 350 {
 351         struct bpf_program prog, *progs;
 352         int nr_progs, err;
 353
 354         err = bpf_program__init(data, size, section_name, idx, &prog);
 355         if (err)
 356                 return err;
 357
 358         prog.caps = &obj->caps;
 359         progs = obj->programs;
 360         nr_progs = obj->nr_programs;
 361
 362         progs = reallocarray(progs, nr_progs + 1, sizeof(progs[0]));
 363         if (!progs) {
 364                 /*
 365                  * In this case the original obj->programs
 366                  * is still valid, so don't need special treat for
 367                  * bpf_close_object().
 368                  */
 369                 pr_warning("failed to alloc a new program under section '%s'\n",
 370                            section_name);
 371                 bpf_program__exit(&prog);
 372                 return -ENOMEM;
 373         }
 374
 375         pr_debug("found program %s\n", prog.section_name);
 376         obj->programs = progs;
 377         obj->nr_programs = nr_progs + 1;
 378         prog.obj = obj;
 379         progs[nr_progs] = prog;
 380         return 0;
 381 }
 382
 383 static int
 384 bpf_object__init_prog_names(struct bpf_object *obj)
 385 {
 386         Elf_Data *symbols = obj->efile.symbols;
 387         struct bpf_program *prog;
 388         size_t pi, si;
 389
 390         for (pi = 0; pi < obj->nr_programs; pi++) {
 391                 const char *name = NULL;
 392
 393                 prog = &obj->programs[pi];
 394
 395                 for (si = 0; si < symbols->d_size / sizeof(GElf_Sym) && !name;
 396                      si++) {
 397                         GElf_Sym sym;
 398
 399                         if (!gelf_getsym(symbols, si, &sym))
 400                                 continue;
 401                         if (sym.st_shndx != prog->idx)
 402                                 continue;
 403                         if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL)
 404                                 continue;
 405
 406                         name = elf_strptr(obj->efile.elf,
 407                                           obj->efile.strtabidx,
 408                                           sym.st_name);
 409                         if (!name) {
 410                                 pr_warning("failed to get sym name string for prog %s\n",
 411                                            prog->section_name);
 412                                 return -LIBBPF_ERRNO__LIBELF;
 413                         }
 414                 }
 415
 416                 if (!name && prog->idx == obj->efile.text_shndx)
 417                         name = ".text";
 418
 419                 if (!name) {
 420                         pr_warning("failed to find sym for prog %s\n",
 421                                    prog->section_name);
 422                         return -EINVAL;
 423                 }
 424
 425                 prog->name = strdup(name);
 426                 if (!prog->name) {
 427                         pr_warning("failed to allocate memory for prog sym %s\n",
 428                                    name);
 429                         return -ENOMEM;
 430                 }
 431         }
 432
 433         return 0;
 434 }
 435
 436 static struct bpf_object *bpf_object__new(const char *path,
 437                                           void *obj_buf,
 438                                           size_t obj_buf_sz)
 439 {
 440         struct bpf_object *obj;
 441
 442         obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1);
 443         if (!obj) {
 444                 pr_warning("alloc memory failed for %s\n", path);
 445                 return ERR_PTR(-ENOMEM);
 446         }
 447
 448         strcpy(obj->path, path);
 449         obj->efile.fd = -1;
 450
 451         /*
 452          * Caller of this function should also calls
 453          * bpf_object__elf_finish() after data collection to return
 454          * obj_buf to user. If not, we should duplicate the buffer to
 455          * avoid user freeing them before elf finish.
 456          */
 457         obj->efile.obj_buf = obj_buf;
 458         obj->efile.obj_buf_sz = obj_buf_sz;
 459         obj->efile.maps_shndx = -1;
 460
 461         obj->loaded = false;
 462
 463         INIT_LIST_HEAD(&obj->list);
 464         list_add(&obj->list, &bpf_objects_list);
 465         return obj;
 466 }
 467
 468 static void bpf_object__elf_finish(struct bpf_object *obj)
 469 {
 470         if (!obj_elf_valid(obj))
 471                 return;
 472
 473         if (obj->efile.elf) {
 474                 elf_end(obj->efile.elf);
 475                 obj->efile.elf = NULL;
 476         }
 477         obj->efile.symbols = NULL;
 478
 479         zfree(&obj->efile.reloc);
 480         obj->efile.nr_reloc = 0;
 481         zclose(obj->efile.fd);
 482         obj->efile.obj_buf = NULL;
 483         obj->efile.obj_buf_sz = 0;
 484 }
 485
 486 static int bpf_object__elf_init(struct bpf_object *obj)
 487 {
 488         int err = 0;
 489         GElf_Ehdr *ep;
 490
 491         if (obj_elf_valid(obj)) {
 492                 pr_warning("elf init: internal error\n");
 493                 return -LIBBPF_ERRNO__LIBELF;
 494         }
 495
 496         if (obj->efile.obj_buf_sz > 0) {
 497                 /*
 498                  * obj_buf should have been validated by
 499                  * bpf_object__open_buffer().
 500                  */
 501                 obj->efile.elf = elf_memory(obj->efile.obj_buf,
 502                                             obj->efile.obj_buf_sz);
 503         } else {
 504                 obj->efile.fd = open(obj->path, O_RDONLY);
 505                 if (obj->efile.fd < 0) {
 506                         char errmsg[STRERR_BUFSIZE];
 507                         char *cp = libbpf_strerror_r(errno, errmsg,
 508                                                      sizeof(errmsg));
 509
 510                         pr_warning("failed to open %s: %s\n", obj->path, cp);
 511                         return -errno;
 512                 }
 513
 514                 obj->efile.elf = elf_begin(obj->efile.fd,
 515                                 LIBBPF_ELF_C_READ_MMAP,
 516                                 NULL);
 517         }
 518
 519         if (!obj->efile.elf) {
 520                 pr_warning("failed to open %s as ELF file\n",
 521                                 obj->path);
 522                 err = -LIBBPF_ERRNO__LIBELF;
 523                 goto errout;
 524         }
 525
 526         if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) {
 527                 pr_warning("failed to get EHDR from %s\n",
 528                                 obj->path);
 529                 err = -LIBBPF_ERRNO__FORMAT;
 530                 goto errout;
 531         }
 532         ep = &obj->efile.ehdr;
 533
 534         /* Old LLVM set e_machine to EM_NONE */
 535         if ((ep->e_type != ET_REL) || (ep->e_machine && (ep->e_machine != EM_BPF))) {
 536                 pr_warning("%s is not an eBPF object file\n",
 537                         obj->path);
 538                 err = -LIBBPF_ERRNO__FORMAT;
 539                 goto errout;
 540         }
 541
 542         return 0;
 543 errout:
 544         bpf_object__elf_finish(obj);
 545         return err;
 546 }
 547
 548 static int
 549 bpf_object__check_endianness(struct bpf_object *obj)
 550 {
 551         static unsigned int const endian = 1;
 552
 553         switch (obj->efile.ehdr.e_ident[EI_DATA]) {
 554         case ELFDATA2LSB:
 555                 /* We are big endian, BPF obj is little endian. */
 556                 if (*(unsigned char const *)&endian != 1)
 557                         goto mismatch;
 558                 break;
 559
 560         case ELFDATA2MSB:
 561                 /* We are little endian, BPF obj is big endian. */
 562                 if (*(unsigned char const *)&endian != 0)
 563                         goto mismatch;
 564                 break;
 565         default:
 566                 return -LIBBPF_ERRNO__ENDIAN;
 567         }
 568
 569         return 0;
 570
 571 mismatch:
 572         pr_warning("Error: endianness mismatch.\n");
 573         return -LIBBPF_ERRNO__ENDIAN;
 574 }
 575
 576 static int
 577 bpf_object__init_license(struct bpf_object *obj,
 578                          void *data, size_t size)
 579 {
 580         memcpy(obj->license, data,
 581                min(size, sizeof(obj->license) - 1));
 582         pr_debug("license of %s is %s\n", obj->path, obj->license);
 583         return 0;
 584 }
 585
 586 static int
 587 bpf_object__init_kversion(struct bpf_object *obj,
 588                           void *data, size_t size)
 589 {
 590         __u32 kver;
 591
 592         if (size != sizeof(kver)) {
 593                 pr_warning("invalid kver section in %s\n", obj->path);
 594                 return -LIBBPF_ERRNO__FORMAT;
 595         }
 596         memcpy(&kver, data, sizeof(kver));
 597         obj->kern_version = kver;
 598         pr_debug("kernel version of %s is %x\n", obj->path,
 599                  obj->kern_version);
 600         return 0;
 601 }
 602
 603 static int compare_bpf_map(const void *_a, const void *_b)
 604 {
 605         const struct bpf_map *a = _a;
 606         const struct bpf_map *b = _b;
 607
 608         return a->offset - b->offset;
 609 }
 610
 611 static bool bpf_map_type__is_map_in_map(enum bpf_map_type type)
 612 {
 613         if (type == BPF_MAP_TYPE_ARRAY_OF_MAPS ||
 614             type == BPF_MAP_TYPE_HASH_OF_MAPS)
 615                 return true;
 616         return false;
 617 }
 618
 619 static int
 620 bpf_object__init_maps(struct bpf_object *obj, int flags)
 621 {
 622         bool strict = !(flags & MAPS_RELAX_COMPAT);
 623         int i, map_idx, map_def_sz, nr_maps = 0;
 624         Elf_Scn *scn;
 625         Elf_Data *data;
 626         Elf_Data *symbols = obj->efile.symbols;
 627
 628         if (obj->efile.maps_shndx < 0)
 629                 return -EINVAL;
 630         if (!symbols)
 631                 return -EINVAL;
 632
 633         scn = elf_getscn(obj->efile.elf, obj->efile.maps_shndx);
 634         if (scn)
 635                 data = elf_getdata(scn, NULL);
 636         if (!scn || !data) {
 637                 pr_warning("failed to get Elf_Data from map section %d\n",
 638                            obj->efile.maps_shndx);
 639                 return -EINVAL;
 640         }
 641
 642         /*
 643          * Count number of maps. Each map has a name.
 644          * Array of maps is not supported: only the first element is
 645          * considered.
 646          *
 647          * TODO: Detect array of map and report error.
 648          */
 649         for (i = 0; i < symbols->d_size / sizeof(GElf_Sym); i++) {
 650                 GElf_Sym sym;
 651
 652                 if (!gelf_getsym(symbols, i, &sym))
 653                         continue;
 654                 if (sym.st_shndx != obj->efile.maps_shndx)
 655                         continue;
 656                 nr_maps++;
 657         }
 658
 659         /* Alloc obj->maps and fill nr_maps. */
 660         pr_debug("maps in %s: %d maps in %zd bytes\n", obj->path,
 661                  nr_maps, data->d_size);
 662
 663         if (!nr_maps)
 664                 return 0;
 665
 666         /* Assume equally sized map definitions */
 667         map_def_sz = data->d_size / nr_maps;
 668         if (!data->d_size || (data->d_size % nr_maps) != 0) {
 669                 pr_warning("unable to determine map definition size "
 670                            "section %s, %d maps in %zd bytes\n",
 671                            obj->path, nr_maps, data->d_size);
 672                 return -EINVAL;
 673         }
 674
 675         obj->maps = calloc(nr_maps, sizeof(obj->maps[0]));
 676         if (!obj->maps) {
 677                 pr_warning("alloc maps for object failed\n");
 678                 return -ENOMEM;
 679         }
 680         obj->nr_maps = nr_maps;
 681
 682         for (i = 0; i < nr_maps; i++) {
 683                 /*
 684                  * fill all fd with -1 so won't close incorrect
 685                  * fd (fd=0 is stdin) when failure (zclose won't close
 686                  * negative fd)).
 687                  */
 688                 obj->maps[i].fd = -1;
 689                 obj->maps[i].inner_map_fd = -1;
 690         }
 691
 692         /*
 693          * Fill obj->maps using data in "maps" section.
 694          */
 695         for (i = 0, map_idx = 0; i < symbols->d_size / sizeof(GElf_Sym); i++) {
 696                 GElf_Sym sym;
 697                 const char *map_name;
 698                 struct bpf_map_def *def;
 699
 700                 if (!gelf_getsym(symbols, i, &sym))
 701                         continue;
 702                 if (sym.st_shndx != obj->efile.maps_shndx)
 703                         continue;
 704
 705                 map_name = elf_strptr(obj->efile.elf,
 706                                       obj->efile.strtabidx,
 707                                       sym.st_name);
 708                 obj->maps[map_idx].offset = sym.st_value;
 709                 if (sym.st_value + map_def_sz > data->d_size) {
 710                         pr_warning("corrupted maps section in %s: last map \"%s\" too small\n",
 711                                    obj->path, map_name);
 712                         return -EINVAL;
 713                 }
 714
 715                 obj->maps[map_idx].name = strdup(map_name);
 716                 if (!obj->maps[map_idx].name) {
 717                         pr_warning("failed to alloc map name\n");
 718                         return -ENOMEM;
 719                 }
 720                 pr_debug("map %d is \"%s\"\n", map_idx,
 721                          obj->maps[map_idx].name);
 722                 def = (struct bpf_map_def *)(data->d_buf + sym.st_value);
 723                 /*
 724                  * If the definition of the map in the object file fits in
 725                  * bpf_map_def, copy it.  Any extra fields in our version
 726                  * of bpf_map_def will default to zero as a result of the
 727                  * calloc above.
 728                  */
 729                 if (map_def_sz <= sizeof(struct bpf_map_def)) {
 730                         memcpy(&obj->maps[map_idx].def, def, map_def_sz);
 731                 } else {
 732                         /*
 733                          * Here the map structure being read is bigger than what
 734                          * we expect, truncate if the excess bits are all zero.
 735                          * If they are not zero, reject this map as
 736                          * incompatible.
 737                          */
 738                         char *b;
 739                         for (b = ((char *)def) + sizeof(struct bpf_map_def);
 740                              b < ((char *)def) + map_def_sz; b++) {
 741                                 if (*b != 0) {
 742                                         pr_warning("maps section in %s: \"%s\" "
 743                                                    "has unrecognized, non-zero "
 744                                                    "options\n",
 745                                                    obj->path, map_name);
 746                                         if (strict)
 747                                                 return -EINVAL;
 748                                 }
 749                         }
 750                         memcpy(&obj->maps[map_idx].def, def,
 751                                sizeof(struct bpf_map_def));
 752                 }
 753                 map_idx++;
 754         }
 755
 756         qsort(obj->maps, obj->nr_maps, sizeof(obj->maps[0]), compare_bpf_map);
 757         return 0;
 758 }
 759
 760 static bool section_have_execinstr(struct bpf_object *obj, int idx)
 761 {
 762         Elf_Scn *scn;
 763         GElf_Shdr sh;
 764
 765         scn = elf_getscn(obj->efile.elf, idx);
 766         if (!scn)
 767                 return false;
 768
 769         if (gelf_getshdr(scn, &sh) != &sh)
 770                 return false;
 771
 772         if (sh.sh_flags & SHF_EXECINSTR)
 773                 return true;
 774
 775         return false;
 776 }
 777
 778 static int bpf_object__elf_collect(struct bpf_object *obj, int flags)
 779 {
 780         Elf *elf = obj->efile.elf;
 781         GElf_Ehdr *ep = &obj->efile.ehdr;
 782         Elf_Data *btf_ext_data = NULL;
 783         Elf_Scn *scn = NULL;
 784         int idx = 0, err = 0;
 785
 786         /* Elf is corrupted/truncated, avoid calling elf_strptr. */
 787         if (!elf_rawdata(elf_getscn(elf, ep->e_shstrndx), NULL)) {
 788                 pr_warning("failed to get e_shstrndx from %s\n",
 789                            obj->path);
 790                 return -LIBBPF_ERRNO__FORMAT;
 791         }
 792
 793         while ((scn = elf_nextscn(elf, scn)) != NULL) {
 794                 char *name;
 795                 GElf_Shdr sh;
 796                 Elf_Data *data;
 797
 798                 idx++;
 799                 if (gelf_getshdr(scn, &sh) != &sh) {
 800                         pr_warning("failed to get section(%d) header from %s\n",
 801                                    idx, obj->path);
 802                         err = -LIBBPF_ERRNO__FORMAT;
 803                         goto out;
 804                 }
 805
 806                 name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name);
 807                 if (!name) {
 808                         pr_warning("failed to get section(%d) name from %s\n",
 809                                    idx, obj->path);
 810                         err = -LIBBPF_ERRNO__FORMAT;
 811                         goto out;
 812                 }
 813
 814                 data = elf_getdata(scn, 0);
 815                 if (!data) {
 816                         pr_warning("failed to get section(%d) data from %s(%s)\n",
 817                                    idx, name, obj->path);
 818                         err = -LIBBPF_ERRNO__FORMAT;
 819                         goto out;
 820                 }
 821                 pr_debug("section(%d) %s, size %ld, link %d, flags %lx, type=%d\n",
 822                          idx, name, (unsigned long)data->d_size,
 823                          (int)sh.sh_link, (unsigned long)sh.sh_flags,
 824                          (int)sh.sh_type);
 825
 826                 if (strcmp(name, "license") == 0)
 827                         err = bpf_object__init_license(obj,
 828                                                        data->d_buf,
 829                                                        data->d_size);
 830                 else if (strcmp(name, "version") == 0)
 831                         err = bpf_object__init_kversion(obj,
 832                                                         data->d_buf,
 833                                                         data->d_size);
 834                 else if (strcmp(name, "maps") == 0)
 835                         obj->efile.maps_shndx = idx;
 836                 else if (strcmp(name, BTF_ELF_SEC) == 0) {
 837                         obj->btf = btf__new(data->d_buf, data->d_size);
 838                         if (IS_ERR(obj->btf)) {
 839                                 pr_warning("Error loading ELF section %s: %ld. Ignored and continue.\n",
 840                                            BTF_ELF_SEC, PTR_ERR(obj->btf));
 841                                 obj->btf = NULL;
 842                                 continue;
 843                         }
 844                         err = btf__load(obj->btf);
 845                         if (err) {
 846                                 pr_warning("Error loading %s into kernel: %d. Ignored and continue.\n",
 847                                            BTF_ELF_SEC, err);
 848                                 btf__free(obj->btf);
 849                                 obj->btf = NULL;
 850                                 err = 0;
 851                         }
 852                 } else if (strcmp(name, BTF_EXT_ELF_SEC) == 0) {
 853                         btf_ext_data = data;
 854                 } else if (sh.sh_type == SHT_SYMTAB) {
 855                         if (obj->efile.symbols) {
 856                                 pr_warning("bpf: multiple SYMTAB in %s\n",
 857                                            obj->path);
 858                                 err = -LIBBPF_ERRNO__FORMAT;
 859                         } else {
 860                                 obj->efile.symbols = data;
 861                                 obj->efile.strtabidx = sh.sh_link;
 862                         }
 863                 } else if ((sh.sh_type == SHT_PROGBITS) &&
 864                            (sh.sh_flags & SHF_EXECINSTR) &&
 865                            (data->d_size > 0)) {
 866                         if (strcmp(name, ".text") == 0)
 867                                 obj->efile.text_shndx = idx;
 868                         err = bpf_object__add_program(obj, data->d_buf,
 869                                                       data->d_size, name, idx);
 870                         if (err) {
 871                                 char errmsg[STRERR_BUFSIZE];
 872                                 char *cp = libbpf_strerror_r(-err, errmsg,
 873                                                              sizeof(errmsg));
 874
 875                                 pr_warning("failed to alloc program %s (%s): %s",
 876                                            name, obj->path, cp);
 877                         }
 878                 } else if (sh.sh_type == SHT_REL) {
 879                         void *reloc = obj->efile.reloc;
 880                         int nr_reloc = obj->efile.nr_reloc + 1;
 881                         int sec = sh.sh_info; /* points to other section */
 882
 883                         /* Only do relo for section with exec instructions */
 884                         if (!section_have_execinstr(obj, sec)) {
 885                                 pr_debug("skip relo %s(%d) for section(%d)\n",
 886                                          name, idx, sec);
 887                                 continue;
 888                         }
 889
 890                         reloc = reallocarray(reloc, nr_reloc,
 891                                              sizeof(*obj->efile.reloc));
 892                         if (!reloc) {
 893                                 pr_warning("realloc failed\n");
 894                                 err = -ENOMEM;
 895                         } else {
 896                                 int n = nr_reloc - 1;
 897
 898                                 obj->efile.reloc = reloc;
 899                                 obj->efile.nr_reloc = nr_reloc;
 900
 901                                 obj->efile.reloc[n].shdr = sh;
 902                                 obj->efile.reloc[n].data = data;
 903                         }
 904                 } else {
 905                         pr_debug("skip section(%d) %s\n", idx, name);
 906                 }
 907                 if (err)
 908                         goto out;
 909         }
 910
 911         if (!obj->efile.strtabidx || obj->efile.strtabidx >= idx) {
 912                 pr_warning("Corrupted ELF file: index of strtab invalid\n");
 913                 return LIBBPF_ERRNO__FORMAT;
 914         }
 915         if (btf_ext_data) {
 916                 if (!obj->btf) {
 917                         pr_debug("Ignore ELF section %s because its depending ELF section %s is not found.\n",
 918                                  BTF_EXT_ELF_SEC, BTF_ELF_SEC);
 919                 } else {
 920                         obj->btf_ext = btf_ext__new(btf_ext_data->d_buf,
 921                                                     btf_ext_data->d_size);
 922                         if (IS_ERR(obj->btf_ext)) {
 923                                 pr_warning("Error loading ELF section %s: %ld. Ignored and continue.\n",
 924                                            BTF_EXT_ELF_SEC,
 925                                            PTR_ERR(obj->btf_ext));
 926                                 obj->btf_ext = NULL;
 927                         }
 928                 }
 929         }
 930         if (obj->efile.maps_shndx >= 0) {
 931                 err = bpf_object__init_maps(obj, flags);
 932                 if (err)
 933                         goto out;
 934         }
 935         err = bpf_object__init_prog_names(obj);
 936 out:
 937         return err;
 938 }
 939
 940 static struct bpf_program *
 941 bpf_object__find_prog_by_idx(struct bpf_object *obj, int idx)
 942 {
 943         struct bpf_program *prog;
 944         size_t i;
 945
 946         for (i = 0; i < obj->nr_programs; i++) {
 947                 prog = &obj->programs[i];
 948                 if (prog->idx == idx)
 949                         return prog;
 950         }
 951         return NULL;
 952 }
 953
 954 struct bpf_program *
 955 bpf_object__find_program_by_title(struct bpf_object *obj, const char *title)
 956 {
 957         struct bpf_program *pos;
 958
 959         bpf_object__for_each_program(pos, obj) {
 960                 if (pos->section_name && !strcmp(pos->section_name, title))
 961                         return pos;
 962         }
 963         return NULL;
 964 }
 965
 966 static int
 967 bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr,
 968                            Elf_Data *data, struct bpf_object *obj)
 969 {
 970         Elf_Data *symbols = obj->efile.symbols;
 971         int text_shndx = obj->efile.text_shndx;
 972         int maps_shndx = obj->efile.maps_shndx;
 973         struct bpf_map *maps = obj->maps;
 974         size_t nr_maps = obj->nr_maps;
 975         int i, nrels;
 976
 977         pr_debug("collecting relocating info for: '%s'\n",
 978                  prog->section_name);
 979         nrels = shdr->sh_size / shdr->sh_entsize;
 980
 981         prog->reloc_desc = malloc(sizeof(*prog->reloc_desc) * nrels);
 982         if (!prog->reloc_desc) {
 983                 pr_warning("failed to alloc memory in relocation\n");
 984                 return -ENOMEM;
 985         }
 986         prog->nr_reloc = nrels;
 987
 988         for (i = 0; i < nrels; i++) {
 989                 GElf_Sym sym;
 990                 GElf_Rel rel;
 991                 unsigned int insn_idx;
 992                 struct bpf_insn *insns = prog->insns;
 993                 size_t map_idx;
 994
 995                 if (!gelf_getrel(data, i, &rel)) {
 996                         pr_warning("relocation: failed to get %d reloc\n", i);
 997                         return -LIBBPF_ERRNO__FORMAT;
 998                 }
 999
1000                 if (!gelf_getsym(symbols,
1001                                  GELF_R_SYM(rel.r_info),
1002                                  &sym)) {
1003                         pr_warning("relocation: symbol %"PRIx64" not found\n",
1004                                    GELF_R_SYM(rel.r_info));
1005                         return -LIBBPF_ERRNO__FORMAT;
1006                 }
1007                 pr_debug("relo for %lld value %lld name %d\n",
1008                          (long long) (rel.r_info >> 32),
1009                          (long long) sym.st_value, sym.st_name);
1010
1011                 if (sym.st_shndx != maps_shndx && sym.st_shndx != text_shndx) {
1012                         pr_warning("Program '%s' contains non-map related relo data pointing to section %u\n",
1013                                    prog->section_name, sym.st_shndx);
1014                         return -LIBBPF_ERRNO__RELOC;
1015                 }
1016
1017                 insn_idx = rel.r_offset / sizeof(struct bpf_insn);
1018                 pr_debug("relocation: insn_idx=%u\n", insn_idx);
1019
1020                 if (insns[insn_idx].code == (BPF_JMP | BPF_CALL)) {
1021                         if (insns[insn_idx].src_reg != BPF_PSEUDO_CALL) {
1022                                 pr_warning("incorrect bpf_call opcode\n");
1023                                 return -LIBBPF_ERRNO__RELOC;
1024                         }
1025                         prog->reloc_desc[i].type = RELO_CALL;
1026                         prog->reloc_desc[i].insn_idx = insn_idx;
1027                         prog->reloc_desc[i].text_off = sym.st_value;
1028                         obj->has_pseudo_calls = true;
1029                         continue;
1030                 }
1031
1032                 if (insns[insn_idx].code != (BPF_LD | BPF_IMM | BPF_DW)) {
1033                         pr_warning("bpf: relocation: invalid relo for insns[%d].code 0x%x\n",
1034                                    insn_idx, insns[insn_idx].code);
1035                         return -LIBBPF_ERRNO__RELOC;
1036                 }
1037
1038                 /* TODO: 'maps' is sorted. We can use bsearch to make it faster. */
1039                 for (map_idx = 0; map_idx < nr_maps; map_idx++) {
1040                         if (maps[map_idx].offset == sym.st_value) {
1041                                 pr_debug("relocation: find map %zd (%s) for insn %u\n",
1042                                          map_idx, maps[map_idx].name, insn_idx);
1043                                 break;
1044                         }
1045                 }
1046
1047                 if (map_idx >= nr_maps) {
1048                         pr_warning("bpf relocation: map_idx %d large than %d\n",
1049                                    (int)map_idx, (int)nr_maps - 1);
1050                         return -LIBBPF_ERRNO__RELOC;
1051                 }
1052
1053                 prog->reloc_desc[i].type = RELO_LD64;
1054                 prog->reloc_desc[i].insn_idx = insn_idx;
1055                 prog->reloc_desc[i].map_idx = map_idx;
1056         }
1057         return 0;
1058 }
1059
1060 static int bpf_map_find_btf_info(struct bpf_map *map, const struct btf *btf)
1061 {
1062         struct bpf_map_def *def = &map->def;
1063         __u32 key_type_id, value_type_id;
1064         int ret;
1065
1066         ret = btf__get_map_kv_tids(btf, map->name, def->key_size,
1067                                    def->value_size, &key_type_id,
1068                                    &value_type_id);
1069         if (ret)
1070                 return ret;
1071
1072         map->btf_key_type_id = key_type_id;
1073         map->btf_value_type_id = value_type_id;
1074
1075         return 0;
1076 }
1077
1078 int bpf_map__reuse_fd(struct bpf_map *map, int fd)
1079 {
1080         struct bpf_map_info info = {};
1081         __u32 len = sizeof(info);
1082         int new_fd, err;
1083         char *new_name;
1084
1085         err = bpf_obj_get_info_by_fd(fd, &info, &len);
1086         if (err)
1087                 return err;
1088
1089         new_name = strdup(info.name);
1090         if (!new_name)
1091                 return -errno;
1092
1093         new_fd = open("/", O_RDONLY | O_CLOEXEC);
1094         if (new_fd < 0)
1095                 goto err_free_new_name;
1096
1097         new_fd = dup3(fd, new_fd, O_CLOEXEC);
1098         if (new_fd < 0)
1099                 goto err_close_new_fd;
1100
1101         err = zclose(map->fd);
1102         if (err)
1103                 goto err_close_new_fd;
1104         free(map->name);
1105
1106         map->fd = new_fd;
1107         map->name = new_name;
1108         map->def.type = info.type;
1109         map->def.key_size = info.key_size;
1110         map->def.value_size = info.value_size;
1111         map->def.max_entries = info.max_entries;
1112         map->def.map_flags = info.map_flags;
1113         map->btf_key_type_id = info.btf_key_type_id;
1114         map->btf_value_type_id = info.btf_value_type_id;
1115
1116         return 0;
1117
1118 err_close_new_fd:
1119         close(new_fd);
1120 err_free_new_name:
1121         free(new_name);
1122         return -errno;
1123 }
1124
1125 int bpf_map__resize(struct bpf_map *map, __u32 max_entries)
1126 {
1127         if (!map || !max_entries)
1128                 return -EINVAL;
1129
1130         /* If map already created, its attributes can't be changed. */
1131         if (map->fd >= 0)
1132                 return -EBUSY;
1133
1134         map->def.max_entries = max_entries;
1135
1136         return 0;
1137 }
1138
1139 static int
1140 bpf_object__probe_name(struct bpf_object *obj)
1141 {
1142         struct bpf_load_program_attr attr;
1143         char *cp, errmsg[STRERR_BUFSIZE];
1144         struct bpf_insn insns[] = {
1145                 BPF_MOV64_IMM(BPF_REG_0, 0),
1146                 BPF_EXIT_INSN(),
1147         };
1148         int ret;
1149
1150         /* make sure basic loading works */
1151
1152         memset(&attr, 0, sizeof(attr));
1153         attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER;
1154         attr.insns = insns;
1155         attr.insns_cnt = ARRAY_SIZE(insns);
1156         attr.license = "GPL";
1157
1158         ret = bpf_load_program_xattr(&attr, NULL, 0);
1159         if (ret < 0) {
1160                 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
1161                 pr_warning("Error in %s():%s(%d). Couldn't load basic 'r0 = 0' BPF program.\n",
1162                            __func__, cp, errno);
1163                 return -errno;
1164         }
1165         close(ret);
1166
1167         /* now try the same program, but with the name */
1168
1169         attr.name = "test";
1170         ret = bpf_load_program_xattr(&attr, NULL, 0);
1171         if (ret >= 0) {
1172                 obj->caps.name = 1;
1173                 close(ret);
1174         }
1175
1176         return 0;
1177 }
1178
1179 static int
1180 bpf_object__probe_caps(struct bpf_object *obj)
1181 {
1182         return bpf_object__probe_name(obj);
1183 }
1184
1185 static int
1186 bpf_object__create_maps(struct bpf_object *obj)
1187 {
1188         struct bpf_create_map_attr create_attr = {};
1189         unsigned int i;
1190         int err;
1191
1192         for (i = 0; i < obj->nr_maps; i++) {
1193                 struct bpf_map *map = &obj->maps[i];
1194                 struct bpf_map_def *def = &map->def;
1195                 char *cp, errmsg[STRERR_BUFSIZE];
1196                 int *pfd = &map->fd;
1197
1198                 if (map->fd >= 0) {
1199                         pr_debug("skip map create (preset) %s: fd=%d\n",
1200                                  map->name, map->fd);
1201                         continue;
1202                 }
1203
1204                 if (obj->caps.name)
1205                         create_attr.name = map->name;
1206                 create_attr.map_ifindex = map->map_ifindex;
1207                 create_attr.map_type = def->type;
1208                 create_attr.map_flags = def->map_flags;
1209                 create_attr.key_size = def->key_size;
1210                 create_attr.value_size = def->value_size;
1211                 create_attr.max_entries = def->max_entries;
1212                 create_attr.btf_fd = 0;
1213                 create_attr.btf_key_type_id = 0;
1214                 create_attr.btf_value_type_id = 0;
1215                 if (bpf_map_type__is_map_in_map(def->type) &&
1216                     map->inner_map_fd >= 0)
1217                         create_attr.inner_map_fd = map->inner_map_fd;
1218
1219                 if (obj->btf && !bpf_map_find_btf_info(map, obj->btf)) {
1220                         create_attr.btf_fd = btf__fd(obj->btf);
1221                         create_attr.btf_key_type_id = map->btf_key_type_id;
1222                         create_attr.btf_value_type_id = map->btf_value_type_id;
1223                 }
1224
1225                 *pfd = bpf_create_map_xattr(&create_attr);
1226                 if (*pfd < 0 && create_attr.btf_key_type_id) {
1227                         cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
1228                         pr_warning("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n",
1229                                    map->name, cp, errno);
1230                         create_attr.btf_fd = 0;
1231                         create_attr.btf_key_type_id = 0;
1232                         create_attr.btf_value_type_id = 0;
1233                         map->btf_key_type_id = 0;
1234                         map->btf_value_type_id = 0;
1235                         *pfd = bpf_create_map_xattr(&create_attr);
1236                 }
1237
1238                 if (*pfd < 0) {
1239                         size_t j;
1240
1241                         err = *pfd;
1242                         cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
1243                         pr_warning("failed to create map (name: '%s'): %s\n",
1244                                    map->name, cp);
1245                         for (j = 0; j < i; j++)
1246                                 zclose(obj->maps[j].fd);
1247                         return err;
1248                 }
1249                 pr_debug("create map %s: fd=%d\n", map->name, *pfd);
1250         }
1251
1252         return 0;
1253 }
1254
1255 static int
1256 check_btf_ext_reloc_err(struct bpf_program *prog, int err,
1257                         void *btf_prog_info, const char *info_name)
1258 {
1259         if (err != -ENOENT) {
1260                 pr_warning("Error in loading %s for sec %s.\n",
1261                            info_name, prog->section_name);
1262                 return err;
1263         }
1264
1265         /* err == -ENOENT (i.e. prog->section_name not found in btf_ext) */
1266
1267         if (btf_prog_info) {
1268                 /*
1269                  * Some info has already been found but has problem
1270                  * in the last btf_ext reloc.  Must have to error
1271                  * out.
1272                  */
1273                 pr_warning("Error in relocating %s for sec %s.\n",
1274                            info_name, prog->section_name);
1275                 return err;
1276         }
1277
1278         /*
1279          * Have problem loading the very first info.  Ignore
1280          * the rest.
1281          */
1282         pr_warning("Cannot find %s for main program sec %s. Ignore all %s.\n",
1283                    info_name, prog->section_name, info_name);
1284         return 0;
1285 }
1286
1287 static int
1288 bpf_program_reloc_btf_ext(struct bpf_program *prog, struct bpf_object *obj,
1289                           const char *section_name,  __u32 insn_offset)
1290 {
1291         int err;
1292
1293         if (!insn_offset || prog->func_info) {
1294                 /*
1295                  * !insn_offset => main program
1296                  *
1297                  * For sub prog, the main program's func_info has to
1298                  * be loaded first (i.e. prog->func_info != NULL)
1299                  */
1300                 err = btf_ext__reloc_func_info(obj->btf, obj->btf_ext,
1301                                                section_name, insn_offset,
1302                                                &prog->func_info,
1303                                                &prog->func_info_cnt);
1304                 if (err)
1305                         return check_btf_ext_reloc_err(prog, err,
1306                                                        prog->func_info,
1307                                                        "bpf_func_info");
1308
1309                 prog->func_info_rec_size = btf_ext__func_info_rec_size(obj->btf_ext);
1310         }
1311
1312         if (!insn_offset || prog->line_info) {
1313                 err = btf_ext__reloc_line_info(obj->btf, obj->btf_ext,
1314                                                section_name, insn_offset,
1315                                                &prog->line_info,
1316                                                &prog->line_info_cnt);
1317                 if (err)
1318                         return check_btf_ext_reloc_err(prog, err,
1319                                                        prog->line_info,
1320                                                        "bpf_line_info");
1321
1322                 prog->line_info_rec_size = btf_ext__line_info_rec_size(obj->btf_ext);
1323         }
1324
1325         if (!insn_offset)
1326                 prog->btf_fd = btf__fd(obj->btf);
1327
1328         return 0;
1329 }
1330
1331 static int
1332 bpf_program__reloc_text(struct bpf_program *prog, struct bpf_object *obj,
1333                         struct reloc_desc *relo)
1334 {
1335         struct bpf_insn *insn, *new_insn;
1336         struct bpf_program *text;
1337         size_t new_cnt;
1338         int err;
1339
1340         if (relo->type != RELO_CALL)
1341                 return -LIBBPF_ERRNO__RELOC;
1342
1343         if (prog->idx == obj->efile.text_shndx) {
1344                 pr_warning("relo in .text insn %d into off %d\n",
1345                            relo->insn_idx, relo->text_off);
1346                 return -LIBBPF_ERRNO__RELOC;
1347         }
1348
1349         if (prog->main_prog_cnt == 0) {
1350                 text = bpf_object__find_prog_by_idx(obj, obj->efile.text_shndx);
1351                 if (!text) {
1352                         pr_warning("no .text section found yet relo into text exist\n");
1353                         return -LIBBPF_ERRNO__RELOC;
1354                 }
1355                 new_cnt = prog->insns_cnt + text->insns_cnt;
1356                 new_insn = reallocarray(prog->insns, new_cnt, sizeof(*insn));
1357                 if (!new_insn) {
1358                         pr_warning("oom in prog realloc\n");
1359                         return -ENOMEM;
1360                 }
1361
1362                 if (obj->btf_ext) {
1363                         err = bpf_program_reloc_btf_ext(prog, obj,
1364                                                         text->section_name,
1365                                                         prog->insns_cnt);
1366                         if (err)
1367                                 return err;
1368                 }
1369
1370                 memcpy(new_insn + prog->insns_cnt, text->insns,
1371                        text->insns_cnt * sizeof(*insn));
1372                 prog->insns = new_insn;
1373                 prog->main_prog_cnt = prog->insns_cnt;
1374                 prog->insns_cnt = new_cnt;
1375                 pr_debug("added %zd insn from %s to prog %s\n",
1376                          text->insns_cnt, text->section_name,
1377                          prog->section_name);
1378         }
1379         insn = &prog->insns[relo->insn_idx];
1380         insn->imm += prog->main_prog_cnt - relo->insn_idx;
1381         return 0;
1382 }
1383
1384 static int
1385 bpf_program__relocate(struct bpf_program *prog, struct bpf_object *obj)
1386 {
1387         int i, err;
1388
1389         if (!prog)
1390                 return 0;
1391
1392         if (obj->btf_ext) {
1393                 err = bpf_program_reloc_btf_ext(prog, obj,
1394                                                 prog->section_name, 0);
1395                 if (err)
1396                         return err;
1397         }
1398
1399         if (!prog->reloc_desc)
1400                 return 0;
1401
1402         for (i = 0; i < prog->nr_reloc; i++) {
1403                 if (prog->reloc_desc[i].type == RELO_LD64) {
1404                         struct bpf_insn *insns = prog->insns;
1405                         int insn_idx, map_idx;
1406
1407                         insn_idx = prog->reloc_desc[i].insn_idx;
1408                         map_idx = prog->reloc_desc[i].map_idx;
1409
1410                         if (insn_idx >= (int)prog->insns_cnt) {
1411                                 pr_warning("relocation out of range: '%s'\n",
1412                                            prog->section_name);
1413                                 return -LIBBPF_ERRNO__RELOC;
1414                         }
1415                         insns[insn_idx].src_reg = BPF_PSEUDO_MAP_FD;
1416                         insns[insn_idx].imm = obj->maps[map_idx].fd;
1417                 } else {
1418                         err = bpf_program__reloc_text(prog, obj,
1419                                                       &prog->reloc_desc[i]);
1420                         if (err)
1421                                 return err;
1422                 }
1423         }
1424
1425         zfree(&prog->reloc_desc);
1426         prog->nr_reloc = 0;
1427         return 0;
1428 }
1429
1430
1431 static int
1432 bpf_object__relocate(struct bpf_object *obj)
1433 {
1434         struct bpf_program *prog;
1435         size_t i;
1436         int err;
1437
1438         for (i = 0; i < obj->nr_programs; i++) {
1439                 prog = &obj->programs[i];
1440
1441                 err = bpf_program__relocate(prog, obj);
1442                 if (err) {
1443                         pr_warning("failed to relocate '%s'\n",
1444                                    prog->section_name);
1445                         return err;
1446                 }
1447         }
1448         return 0;
1449 }
1450
1451 static int bpf_object__collect_reloc(struct bpf_object *obj)
1452 {
1453         int i, err;
1454
1455         if (!obj_elf_valid(obj)) {
1456                 pr_warning("Internal error: elf object is closed\n");
1457                 return -LIBBPF_ERRNO__INTERNAL;
1458         }
1459
1460         for (i = 0; i < obj->efile.nr_reloc; i++) {
1461                 GElf_Shdr *shdr = &obj->efile.reloc[i].shdr;
1462                 Elf_Data *data = obj->efile.reloc[i].data;
1463                 int idx = shdr->sh_info;
1464                 struct bpf_program *prog;
1465
1466                 if (shdr->sh_type != SHT_REL) {
1467                         pr_warning("internal error at %d\n", __LINE__);
1468                         return -LIBBPF_ERRNO__INTERNAL;
1469                 }
1470
1471                 prog = bpf_object__find_prog_by_idx(obj, idx);
1472                 if (!prog) {
1473                         pr_warning("relocation failed: no section(%d)\n", idx);
1474                         return -LIBBPF_ERRNO__RELOC;
1475                 }
1476
1477                 err = bpf_program__collect_reloc(prog,
1478                                                  shdr, data,
1479                                                  obj);
1480                 if (err)
1481                         return err;
1482         }
1483         return 0;
1484 }
1485
1486 static int
1487 load_program(struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt,
1488              char *license, __u32 kern_version, int *pfd)
1489 {
1490         struct bpf_load_program_attr load_attr;
1491         char *cp, errmsg[STRERR_BUFSIZE];
1492         char *log_buf;
1493         int ret;
1494
1495         memset(&load_attr, 0, sizeof(struct bpf_load_program_attr));
1496         load_attr.prog_type = prog->type;
1497         load_attr.expected_attach_type = prog->expected_attach_type;
1498         if (prog->caps->name)
1499                 load_attr.name = prog->name;
1500         load_attr.insns = insns;
1501         load_attr.insns_cnt = insns_cnt;
1502         load_attr.license = license;
1503         load_attr.kern_version = kern_version;
1504         load_attr.prog_ifindex = prog->prog_ifindex;
1505         load_attr.prog_btf_fd = prog->btf_fd >= 0 ? prog->btf_fd : 0;
1506         load_attr.func_info = prog->func_info;
1507         load_attr.func_info_rec_size = prog->func_info_rec_size;
1508         load_attr.func_info_cnt = prog->func_info_cnt;
1509         load_attr.line_info = prog->line_info;
1510         load_attr.line_info_rec_size = prog->line_info_rec_size;
1511         load_attr.line_info_cnt = prog->line_info_cnt;
1512         if (!load_attr.insns || !load_attr.insns_cnt)
1513                 return -EINVAL;
1514
1515         log_buf = malloc(BPF_LOG_BUF_SIZE);
1516         if (!log_buf)
1517                 pr_warning("Alloc log buffer for bpf loader error, continue without log\n");
1518
1519         ret = bpf_load_program_xattr(&load_attr, log_buf, BPF_LOG_BUF_SIZE);
1520
1521         if (ret >= 0) {
1522                 *pfd = ret;
1523                 ret = 0;
1524                 goto out;
1525         }
1526
1527         ret = -LIBBPF_ERRNO__LOAD;
1528         cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
1529         pr_warning("load bpf program failed: %s\n", cp);
1530
1531         if (log_buf && log_buf[0] != '\0') {
1532                 ret = -LIBBPF_ERRNO__VERIFY;
1533                 pr_warning("-- BEGIN DUMP LOG ---\n");
1534                 pr_warning("\n%s\n", log_buf);
1535                 pr_warning("-- END LOG --\n");
1536         } else if (load_attr.insns_cnt >= BPF_MAXINSNS) {
1537                 pr_warning("Program too large (%zu insns), at most %d insns\n",
1538                            load_attr.insns_cnt, BPF_MAXINSNS);
1539                 ret = -LIBBPF_ERRNO__PROG2BIG;
1540         } else {
1541                 /* Wrong program type? */
1542                 if (load_attr.prog_type != BPF_PROG_TYPE_KPROBE) {
1543                         int fd;
1544
1545                         load_attr.prog_type = BPF_PROG_TYPE_KPROBE;
1546                         load_attr.expected_attach_type = 0;
1547                         fd = bpf_load_program_xattr(&load_attr, NULL, 0);
1548                         if (fd >= 0) {
1549                                 close(fd);
1550                                 ret = -LIBBPF_ERRNO__PROGTYPE;
1551                                 goto out;
1552                         }
1553                 }
1554
1555                 if (log_buf)
1556                         ret = -LIBBPF_ERRNO__KVER;
1557         }
1558
1559 out:
1560         free(log_buf);
1561         return ret;
1562 }
1563
1564 int
1565 bpf_program__load(struct bpf_program *prog,
1566                   char *license, __u32 kern_version)
1567 {
1568         int err = 0, fd, i;
1569
1570         if (prog->instances.nr < 0 || !prog->instances.fds) {
1571                 if (prog->preprocessor) {
1572                         pr_warning("Internal error: can't load program '%s'\n",
1573                                    prog->section_name);
1574                         return -LIBBPF_ERRNO__INTERNAL;
1575                 }
1576
1577                 prog->instances.fds = malloc(sizeof(int));
1578                 if (!prog->instances.fds) {
1579                         pr_warning("Not enough memory for BPF fds\n");
1580                         return -ENOMEM;
1581                 }
1582                 prog->instances.nr = 1;
1583                 prog->instances.fds[0] = -1;
1584         }
1585
1586         if (!prog->preprocessor) {
1587                 if (prog->instances.nr != 1) {
1588                         pr_warning("Program '%s' is inconsistent: nr(%d) != 1\n",
1589                                    prog->section_name, prog->instances.nr);
1590                 }
1591                 err = load_program(prog, prog->insns, prog->insns_cnt,
1592                                    license, kern_version, &fd);
1593                 if (!err)
1594                         prog->instances.fds[0] = fd;
1595                 goto out;
1596         }
1597
1598         for (i = 0; i < prog->instances.nr; i++) {
1599                 struct bpf_prog_prep_result result;
1600                 bpf_program_prep_t preprocessor = prog->preprocessor;
1601
1602                 memset(&result, 0, sizeof(result));
1603                 err = preprocessor(prog, i, prog->insns,
1604                                    prog->insns_cnt, &result);
1605                 if (err) {
1606                         pr_warning("Preprocessing the %dth instance of program '%s' failed\n",
1607                                    i, prog->section_name);
1608                         goto out;
1609                 }
1610
1611                 if (!result.new_insn_ptr || !result.new_insn_cnt) {
1612                         pr_debug("Skip loading the %dth instance of program '%s'\n",
1613                                  i, prog->section_name);
1614                         prog->instances.fds[i] = -1;
1615                         if (result.pfd)
1616                                 *result.pfd = -1;
1617                         continue;
1618                 }
1619
1620                 err = load_program(prog, result.new_insn_ptr,
1621                                    result.new_insn_cnt,
1622                                    license, kern_version, &fd);
1623
1624                 if (err) {
1625                         pr_warning("Loading the %dth instance of program '%s' failed\n",
1626                                         i, prog->section_name);
1627                         goto out;
1628                 }
1629
1630                 if (result.pfd)
1631                         *result.pfd = fd;
1632                 prog->instances.fds[i] = fd;
1633         }
1634 out:
1635         if (err)
1636                 pr_warning("failed to load program '%s'\n",
1637                            prog->section_name);
1638         zfree(&prog->insns);
1639         prog->insns_cnt = 0;
1640         return err;
1641 }
1642
1643 static bool bpf_program__is_function_storage(struct bpf_program *prog,
1644                                              struct bpf_object *obj)
1645 {
1646         return prog->idx == obj->efile.text_shndx && obj->has_pseudo_calls;
1647 }
1648
1649 static int
1650 bpf_object__load_progs(struct bpf_object *obj)
1651 {
1652         size_t i;
1653         int err;
1654
1655         for (i = 0; i < obj->nr_programs; i++) {
1656                 if (bpf_program__is_function_storage(&obj->programs[i], obj))
1657                         continue;
1658                 err = bpf_program__load(&obj->programs[i],
1659                                         obj->license,
1660                                         obj->kern_version);
1661                 if (err)
1662                         return err;
1663         }
1664         return 0;
1665 }
1666
1667 static bool bpf_prog_type__needs_kver(enum bpf_prog_type type)
1668 {
1669         switch (type) {
1670         case BPF_PROG_TYPE_SOCKET_FILTER:
1671         case BPF_PROG_TYPE_SCHED_CLS:
1672         case BPF_PROG_TYPE_SCHED_ACT:
1673         case BPF_PROG_TYPE_XDP:
1674         case BPF_PROG_TYPE_CGROUP_SKB:
1675         case BPF_PROG_TYPE_CGROUP_SOCK:
1676         case BPF_PROG_TYPE_LWT_IN:
1677         case BPF_PROG_TYPE_LWT_OUT:
1678         case BPF_PROG_TYPE_LWT_XMIT:
1679         case BPF_PROG_TYPE_LWT_SEG6LOCAL:
1680         case BPF_PROG_TYPE_SOCK_OPS:
1681         case BPF_PROG_TYPE_SK_SKB:
1682         case BPF_PROG_TYPE_CGROUP_DEVICE:
1683         case BPF_PROG_TYPE_SK_MSG:
1684         case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
1685         case BPF_PROG_TYPE_LIRC_MODE2:
1686         case BPF_PROG_TYPE_SK_REUSEPORT:
1687         case BPF_PROG_TYPE_FLOW_DISSECTOR:
1688         case BPF_PROG_TYPE_UNSPEC:
1689         case BPF_PROG_TYPE_TRACEPOINT:
1690         case BPF_PROG_TYPE_RAW_TRACEPOINT:
1691         case BPF_PROG_TYPE_PERF_EVENT:
1692                 return false;
1693         case BPF_PROG_TYPE_KPROBE:
1694         default:
1695                 return true;
1696         }
1697 }
1698
1699 static int bpf_object__validate(struct bpf_object *obj, bool needs_kver)
1700 {
1701         if (needs_kver && obj->kern_version == 0) {
1702                 pr_warning("%s doesn't provide kernel version\n",
1703                            obj->path);
1704                 return -LIBBPF_ERRNO__KVERSION;
1705         }
1706         return 0;
1707 }
1708
1709 static struct bpf_object *
1710 __bpf_object__open(const char *path, void *obj_buf, size_t obj_buf_sz,
1711                    bool needs_kver, int flags)
1712 {
1713         struct bpf_object *obj;
1714         int err;
1715
1716         if (elf_version(EV_CURRENT) == EV_NONE) {
1717                 pr_warning("failed to init libelf for %s\n", path);
1718                 return ERR_PTR(-LIBBPF_ERRNO__LIBELF);
1719         }
1720
1721         obj = bpf_object__new(path, obj_buf, obj_buf_sz);
1722         if (IS_ERR(obj))
1723                 return obj;
1724
1725         CHECK_ERR(bpf_object__elf_init(obj), err, out);
1726         CHECK_ERR(bpf_object__check_endianness(obj), err, out);
1727         CHECK_ERR(bpf_object__elf_collect(obj, flags), err, out);
1728         CHECK_ERR(bpf_object__collect_reloc(obj), err, out);
1729         CHECK_ERR(bpf_object__validate(obj, needs_kver), err, out);
1730
1731         bpf_object__elf_finish(obj);
1732         return obj;
1733 out:
1734         bpf_object__close(obj);
1735         return ERR_PTR(err);
1736 }
1737
1738 struct bpf_object *__bpf_object__open_xattr(struct bpf_object_open_attr *attr,
1739                                             int flags)
1740 {
1741         /* param validation */
1742         if (!attr->file)
1743                 return NULL;
1744
1745         pr_debug("loading %s\n", attr->file);
1746
1747         return __bpf_object__open(attr->file, NULL, 0,
1748                                   bpf_prog_type__needs_kver(attr->prog_type),
1749                                   flags);
1750 }
1751
1752 struct bpf_object *bpf_object__open_xattr(struct bpf_object_open_attr *attr)
1753 {
1754         return __bpf_object__open_xattr(attr, 0);
1755 }
1756
1757 struct bpf_object *bpf_object__open(const char *path)
1758 {
1759         struct bpf_object_open_attr attr = {
1760                 .file           = path,
1761                 .prog_type      = BPF_PROG_TYPE_UNSPEC,
1762         };
1763
1764         return bpf_object__open_xattr(&attr);
1765 }
1766
1767 struct bpf_object *bpf_object__open_buffer(void *obj_buf,
1768                                            size_t obj_buf_sz,
1769                                            const char *name)
1770 {
1771         char tmp_name[64];
1772
1773         /* param validation */
1774         if (!obj_buf || obj_buf_sz <= 0)
1775                 return NULL;
1776
1777         if (!name) {
1778                 snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx",
1779                          (unsigned long)obj_buf,
1780                          (unsigned long)obj_buf_sz);
1781                 tmp_name[sizeof(tmp_name) - 1] = '\0';
1782                 name = tmp_name;
1783         }
1784         pr_debug("loading object '%s' from buffer\n",
1785                  name);
1786
1787         return __bpf_object__open(name, obj_buf, obj_buf_sz, true, true);
1788 }
1789
1790 int bpf_object__unload(struct bpf_object *obj)
1791 {
1792         size_t i;
1793
1794         if (!obj)
1795                 return -EINVAL;
1796
1797         for (i = 0; i < obj->nr_maps; i++)
1798                 zclose(obj->maps[i].fd);
1799
1800         for (i = 0; i < obj->nr_programs; i++)
1801                 bpf_program__unload(&obj->programs[i]);
1802
1803         return 0;
1804 }
1805
1806 int bpf_object__load(struct bpf_object *obj)
1807 {
1808         int err;
1809
1810         if (!obj)
1811                 return -EINVAL;
1812
1813         if (obj->loaded) {
1814                 pr_warning("object should not be loaded twice\n");
1815                 return -EINVAL;
1816         }
1817
1818         obj->loaded = true;
1819
1820         CHECK_ERR(bpf_object__probe_caps(obj), err, out);
1821         CHECK_ERR(bpf_object__create_maps(obj), err, out);
1822         CHECK_ERR(bpf_object__relocate(obj), err, out);
1823         CHECK_ERR(bpf_object__load_progs(obj), err, out);
1824
1825         return 0;
1826 out:
1827         bpf_object__unload(obj);
1828         pr_warning("failed to load object '%s'\n", obj->path);
1829         return err;
1830 }
1831
1832 static int check_path(const char *path)
1833 {
1834         char *cp, errmsg[STRERR_BUFSIZE];
1835         struct statfs st_fs;
1836         char *dname, *dir;
1837         int err = 0;
1838
1839         if (path == NULL)
1840                 return -EINVAL;
1841
1842         dname = strdup(path);
1843         if (dname == NULL)
1844                 return -ENOMEM;
1845
1846         dir = dirname(dname);
1847         if (statfs(dir, &st_fs)) {
1848                 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
1849                 pr_warning("failed to statfs %s: %s\n", dir, cp);
1850                 err = -errno;
1851         }
1852         free(dname);
1853
1854         if (!err && st_fs.f_type != BPF_FS_MAGIC) {
1855                 pr_warning("specified path %s is not on BPF FS\n", path);
1856                 err = -EINVAL;
1857         }
1858
1859         return err;
1860 }
1861
1862 int bpf_program__pin_instance(struct bpf_program *prog, const char *path,
1863                               int instance)
1864 {
1865         char *cp, errmsg[STRERR_BUFSIZE];
1866         int err;
1867
1868         err = check_path(path);
1869         if (err)
1870                 return err;
1871
1872         if (prog == NULL) {
1873                 pr_warning("invalid program pointer\n");
1874                 return -EINVAL;
1875         }
1876
1877         if (instance < 0 || instance >= prog->instances.nr) {
1878                 pr_warning("invalid prog instance %d of prog %s (max %d)\n",
1879                            instance, prog->section_name, prog->instances.nr);
1880                 return -EINVAL;
1881         }
1882
1883         if (bpf_obj_pin(prog->instances.fds[instance], path)) {
1884                 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
1885                 pr_warning("failed to pin program: %s\n", cp);
1886                 return -errno;
1887         }
1888         pr_debug("pinned program '%s'\n", path);
1889
1890         return 0;
1891 }
1892
1893 int bpf_program__unpin_instance(struct bpf_program *prog, const char *path,
1894                                 int instance)
1895 {
1896         int err;
1897
1898         err = check_path(path);
1899         if (err)
1900                 return err;
1901
1902         if (prog == NULL) {
1903                 pr_warning("invalid program pointer\n");
1904                 return -EINVAL;
1905         }
1906
1907         if (instance < 0 || instance >= prog->instances.nr) {
1908                 pr_warning("invalid prog instance %d of prog %s (max %d)\n",
1909                            instance, prog->section_name, prog->instances.nr);
1910                 return -EINVAL;
1911         }
1912
1913         err = unlink(path);
1914         if (err != 0)
1915                 return -errno;
1916         pr_debug("unpinned program '%s'\n", path);
1917
1918         return 0;
1919 }
1920
1921 static int make_dir(const char *path)
1922 {
1923         char *cp, errmsg[STRERR_BUFSIZE];
1924         int err = 0;
1925
1926         if (mkdir(path, 0700) && errno != EEXIST)
1927                 err = -errno;
1928
1929         if (err) {
1930                 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
1931                 pr_warning("failed to mkdir %s: %s\n", path, cp);
1932         }
1933         return err;
1934 }
1935
1936 int bpf_program__pin(struct bpf_program *prog, const char *path)
1937 {
1938         int i, err;
1939
1940         err = check_path(path);
1941         if (err)
1942                 return err;
1943
1944         if (prog == NULL) {
1945                 pr_warning("invalid program pointer\n");
1946                 return -EINVAL;
1947         }
1948
1949         if (prog->instances.nr <= 0) {
1950                 pr_warning("no instances of prog %s to pin\n",
1951                            prog->section_name);
1952                 return -EINVAL;
1953         }
1954
1955         if (prog->instances.nr == 1) {
1956                 /* don't create subdirs when pinning single instance */
1957                 return bpf_program__pin_instance(prog, path, 0);
1958         }
1959
1960         err = make_dir(path);
1961         if (err)
1962                 return err;
1963
1964         for (i = 0; i < prog->instances.nr; i++) {
1965                 char buf[PATH_MAX];
1966                 int len;
1967
1968                 len = snprintf(buf, PATH_MAX, "%s/%d", path, i);
1969                 if (len < 0) {
1970                         err = -EINVAL;
1971                         goto err_unpin;
1972                 } else if (len >= PATH_MAX) {
1973                         err = -ENAMETOOLONG;
1974                         goto err_unpin;
1975                 }
1976
1977                 err = bpf_program__pin_instance(prog, buf, i);
1978                 if (err)
1979                         goto err_unpin;
1980         }
1981
1982         return 0;
1983
1984 err_unpin:
1985         for (i = i - 1; i >= 0; i--) {
1986                 char buf[PATH_MAX];
1987                 int len;
1988
1989                 len = snprintf(buf, PATH_MAX, "%s/%d", path, i);
1990                 if (len < 0)
1991                         continue;
1992                 else if (len >= PATH_MAX)
1993                         continue;
1994
1995                 bpf_program__unpin_instance(prog, buf, i);
1996         }
1997
1998         rmdir(path);
1999
2000         return err;
2001 }
2002
2003 int bpf_program__unpin(struct bpf_program *prog, const char *path)
2004 {
2005         int i, err;
2006
2007         err = check_path(path);
2008         if (err)
2009                 return err;
2010
2011         if (prog == NULL) {
2012                 pr_warning("invalid program pointer\n");
2013                 return -EINVAL;
2014         }
2015
2016         if (prog->instances.nr <= 0) {
2017                 pr_warning("no instances of prog %s to pin\n",
2018                            prog->section_name);
2019                 return -EINVAL;
2020         }
2021
2022         if (prog->instances.nr == 1) {
2023                 /* don't create subdirs when pinning single instance */
2024                 return bpf_program__unpin_instance(prog, path, 0);
2025         }
2026
2027         for (i = 0; i < prog->instances.nr; i++) {
2028                 char buf[PATH_MAX];
2029                 int len;
2030
2031                 len = snprintf(buf, PATH_MAX, "%s/%d", path, i);
2032                 if (len < 0)
2033                         return -EINVAL;
2034                 else if (len >= PATH_MAX)
2035                         return -ENAMETOOLONG;
2036
2037                 err = bpf_program__unpin_instance(prog, buf, i);
2038                 if (err)
2039                         return err;
2040         }
2041
2042         err = rmdir(path);
2043         if (err)
2044                 return -errno;
2045
2046         return 0;
2047 }
2048
2049 int bpf_map__pin(struct bpf_map *map, const char *path)
2050 {
2051         char *cp, errmsg[STRERR_BUFSIZE];
2052         int err;
2053
2054         err = check_path(path);
2055         if (err)
2056                 return err;
2057
2058         if (map == NULL) {
2059                 pr_warning("invalid map pointer\n");
2060                 return -EINVAL;
2061         }
2062
2063         if (bpf_obj_pin(map->fd, path)) {
2064                 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
2065                 pr_warning("failed to pin map: %s\n", cp);
2066                 return -errno;
2067         }
2068
2069         pr_debug("pinned map '%s'\n", path);
2070
2071         return 0;
2072 }
2073
2074 int bpf_map__unpin(struct bpf_map *map, const char *path)
2075 {
2076         int err;
2077
2078         err = check_path(path);
2079         if (err)
2080                 return err;
2081
2082         if (map == NULL) {
2083                 pr_warning("invalid map pointer\n");
2084                 return -EINVAL;
2085         }
2086
2087         err = unlink(path);
2088         if (err != 0)
2089                 return -errno;
2090         pr_debug("unpinned map '%s'\n", path);
2091
2092         return 0;
2093 }
2094
2095 int bpf_object__pin_maps(struct bpf_object *obj, const char *path)
2096 {
2097         struct bpf_map *map;
2098         int err;
2099
2100         if (!obj)
2101                 return -ENOENT;
2102
2103         if (!obj->loaded) {
2104                 pr_warning("object not yet loaded; load it first\n");
2105                 return -ENOENT;
2106         }
2107
2108         err = make_dir(path);
2109         if (err)
2110                 return err;
2111
2112         bpf_object__for_each_map(map, obj) {
2113                 char buf[PATH_MAX];
2114                 int len;
2115
2116                 len = snprintf(buf, PATH_MAX, "%s/%s", path,
2117                                bpf_map__name(map));
2118                 if (len < 0) {
2119                         err = -EINVAL;
2120                         goto err_unpin_maps;
2121                 } else if (len >= PATH_MAX) {
2122                         err = -ENAMETOOLONG;
2123                         goto err_unpin_maps;
2124                 }
2125
2126                 err = bpf_map__pin(map, buf);
2127                 if (err)
2128                         goto err_unpin_maps;
2129         }
2130
2131         return 0;
2132
2133 err_unpin_maps:
2134         while ((map = bpf_map__prev(map, obj))) {
2135                 char buf[PATH_MAX];
2136                 int len;
2137
2138                 len = snprintf(buf, PATH_MAX, "%s/%s", path,
2139                                bpf_map__name(map));
2140                 if (len < 0)
2141                         continue;
2142                 else if (len >= PATH_MAX)
2143                         continue;
2144
2145                 bpf_map__unpin(map, buf);
2146         }
2147
2148         return err;
2149 }
2150
2151 int bpf_object__unpin_maps(struct bpf_object *obj, const char *path)
2152 {
2153         struct bpf_map *map;
2154         int err;
2155
2156         if (!obj)
2157                 return -ENOENT;
2158
2159         bpf_object__for_each_map(map, obj) {
2160                 char buf[PATH_MAX];
2161                 int len;
2162
2163                 len = snprintf(buf, PATH_MAX, "%s/%s", path,
2164                                bpf_map__name(map));
2165                 if (len < 0)
2166                         return -EINVAL;
2167                 else if (len >= PATH_MAX)
2168                         return -ENAMETOOLONG;
2169
2170                 err = bpf_map__unpin(map, buf);
2171                 if (err)
2172                         return err;
2173         }
2174
2175         return 0;
2176 }
2177
2178 int bpf_object__pin_programs(struct bpf_object *obj, const char *path)
2179 {
2180         struct bpf_program *prog;
2181         int err;
2182
2183         if (!obj)
2184                 return -ENOENT;
2185
2186         if (!obj->loaded) {
2187                 pr_warning("object not yet loaded; load it first\n");
2188                 return -ENOENT;
2189         }
2190
2191         err = make_dir(path);
2192         if (err)
2193                 return err;
2194
2195         bpf_object__for_each_program(prog, obj) {
2196                 char buf[PATH_MAX];
2197                 int len;
2198
2199                 len = snprintf(buf, PATH_MAX, "%s/%s", path,
2200                                prog->pin_name);
2201                 if (len < 0) {
2202                         err = -EINVAL;
2203                         goto err_unpin_programs;
2204                 } else if (len >= PATH_MAX) {
2205                         err = -ENAMETOOLONG;
2206                         goto err_unpin_programs;
2207                 }
2208
2209                 err = bpf_program__pin(prog, buf);
2210                 if (err)
2211                         goto err_unpin_programs;
2212         }
2213
2214         return 0;
2215
2216 err_unpin_programs:
2217         while ((prog = bpf_program__prev(prog, obj))) {
2218                 char buf[PATH_MAX];
2219                 int len;
2220
2221                 len = snprintf(buf, PATH_MAX, "%s/%s", path,
2222                                prog->pin_name);
2223                 if (len < 0)
2224                         continue;
2225                 else if (len >= PATH_MAX)
2226                         continue;
2227
2228                 bpf_program__unpin(prog, buf);
2229         }
2230
2231         return err;
2232 }
2233
2234 int bpf_object__unpin_programs(struct bpf_object *obj, const char *path)
2235 {
2236         struct bpf_program *prog;
2237         int err;
2238
2239         if (!obj)
2240                 return -ENOENT;
2241
2242         bpf_object__for_each_program(prog, obj) {
2243                 char buf[PATH_MAX];
2244                 int len;
2245
2246                 len = snprintf(buf, PATH_MAX, "%s/%s", path,
2247                                prog->pin_name);
2248                 if (len < 0)
2249                         return -EINVAL;
2250                 else if (len >= PATH_MAX)
2251                         return -ENAMETOOLONG;
2252
2253                 err = bpf_program__unpin(prog, buf);
2254                 if (err)
2255                         return err;
2256         }
2257
2258         return 0;
2259 }
2260
2261 int bpf_object__pin(struct bpf_object *obj, const char *path)
2262 {
2263         int err;
2264
2265         err = bpf_object__pin_maps(obj, path);
2266         if (err)
2267                 return err;
2268
2269         err = bpf_object__pin_programs(obj, path);
2270         if (err) {
2271                 bpf_object__unpin_maps(obj, path);
2272                 return err;
2273         }
2274
2275         return 0;
2276 }
2277
2278 void bpf_object__close(struct bpf_object *obj)
2279 {
2280         size_t i;
2281
2282         if (!obj)
2283                 return;
2284
2285         if (obj->clear_priv)
2286                 obj->clear_priv(obj, obj->priv);
2287
2288         bpf_object__elf_finish(obj);
2289         bpf_object__unload(obj);
2290         btf__free(obj->btf);
2291         btf_ext__free(obj->btf_ext);
2292
2293         for (i = 0; i < obj->nr_maps; i++) {
2294                 zfree(&obj->maps[i].name);
2295                 if (obj->maps[i].clear_priv)
2296                         obj->maps[i].clear_priv(&obj->maps[i],
2297                                                 obj->maps[i].priv);
2298                 obj->maps[i].priv = NULL;
2299                 obj->maps[i].clear_priv = NULL;
2300         }
2301         zfree(&obj->maps);
2302         obj->nr_maps = 0;
2303
2304         if (obj->programs && obj->nr_programs) {
2305                 for (i = 0; i < obj->nr_programs; i++)
2306                         bpf_program__exit(&obj->programs[i]);
2307         }
2308         zfree(&obj->programs);
2309
2310         list_del(&obj->list);
2311         free(obj);
2312 }
2313
2314 struct bpf_object *
2315 bpf_object__next(struct bpf_object *prev)
2316 {
2317         struct bpf_object *next;
2318
2319         if (!prev)
2320                 next = list_first_entry(&bpf_objects_list,
2321                                         struct bpf_object,
2322                                         list);
2323         else
2324                 next = list_next_entry(prev, list);
2325
2326         /* Empty list is noticed here so don't need checking on entry. */
2327         if (&next->list == &bpf_objects_list)
2328                 return NULL;
2329
2330         return next;
2331 }
2332
2333 const char *bpf_object__name(struct bpf_object *obj)
2334 {
2335         return obj ? obj->path : ERR_PTR(-EINVAL);
2336 }
2337
2338 unsigned int bpf_object__kversion(struct bpf_object *obj)
2339 {
2340         return obj ? obj->kern_version : 0;
2341 }
2342
2343 struct btf *bpf_object__btf(struct bpf_object *obj)
2344 {
2345         return obj ? obj->btf : NULL;
2346 }
2347
2348 int bpf_object__btf_fd(const struct bpf_object *obj)
2349 {
2350         return obj->btf ? btf__fd(obj->btf) : -1;
2351 }
2352
2353 int bpf_object__set_priv(struct bpf_object *obj, void *priv,
2354                          bpf_object_clear_priv_t clear_priv)
2355 {
2356         if (obj->priv && obj->clear_priv)
2357                 obj->clear_priv(obj, obj->priv);
2358
2359         obj->priv = priv;
2360         obj->clear_priv = clear_priv;
2361         return 0;
2362 }
2363
2364 void *bpf_object__priv(struct bpf_object *obj)
2365 {
2366         return obj ? obj->priv : ERR_PTR(-EINVAL);
2367 }
2368
2369 static struct bpf_program *
2370 __bpf_program__iter(struct bpf_program *p, struct bpf_object *obj, bool forward)
2371 {
2372         size_t nr_programs = obj->nr_programs;
2373         ssize_t idx;
2374
2375         if (!nr_programs)
2376                 return NULL;
2377
2378         if (!p)
2379                 /* Iter from the beginning */
2380                 return forward ? &obj->programs[0] :
2381                         &obj->programs[nr_programs - 1];
2382
2383         if (p->obj != obj) {
2384                 pr_warning("error: program handler doesn't match object\n");
2385                 return NULL;
2386         }
2387
2388         idx = (p - obj->programs) + (forward ? 1 : -1);
2389         if (idx >= obj->nr_programs || idx < 0)
2390                 return NULL;
2391         return &obj->programs[idx];
2392 }
2393
2394 struct bpf_program *
2395 bpf_program__next(struct bpf_program *prev, struct bpf_object *obj)
2396 {
2397         struct bpf_program *prog = prev;
2398
2399         do {
2400                 prog = __bpf_program__iter(prog, obj, true);
2401         } while (prog && bpf_program__is_function_storage(prog, obj));
2402
2403         return prog;
2404 }
2405
2406 struct bpf_program *
2407 bpf_program__prev(struct bpf_program *next, struct bpf_object *obj)
2408 {
2409         struct bpf_program *prog = next;
2410
2411         do {
2412                 prog = __bpf_program__iter(prog, obj, false);
2413         } while (prog && bpf_program__is_function_storage(prog, obj));
2414
2415         return prog;
2416 }
2417
2418 int bpf_program__set_priv(struct bpf_program *prog, void *priv,
2419                           bpf_program_clear_priv_t clear_priv)
2420 {
2421         if (prog->priv && prog->clear_priv)
2422                 prog->clear_priv(prog, prog->priv);
2423
2424         prog->priv = priv;
2425         prog->clear_priv = clear_priv;
2426         return 0;
2427 }
2428
2429 void *bpf_program__priv(struct bpf_program *prog)
2430 {
2431         return prog ? prog->priv : ERR_PTR(-EINVAL);
2432 }
2433
2434 void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex)
2435 {
2436         prog->prog_ifindex = ifindex;
2437 }
2438
2439 const char *bpf_program__title(struct bpf_program *prog, bool needs_copy)
2440 {
2441         const char *title;
2442
2443         title = prog->section_name;
2444         if (needs_copy) {
2445                 title = strdup(title);
2446                 if (!title) {
2447                         pr_warning("failed to strdup program title\n");
2448                         return ERR_PTR(-ENOMEM);
2449                 }
2450         }
2451
2452         return title;
2453 }
2454
2455 int bpf_program__fd(struct bpf_program *prog)
2456 {
2457         return bpf_program__nth_fd(prog, 0);
2458 }
2459
2460 int bpf_program__set_prep(struct bpf_program *prog, int nr_instances,
2461                           bpf_program_prep_t prep)
2462 {
2463         int *instances_fds;
2464
2465         if (nr_instances <= 0 || !prep)
2466                 return -EINVAL;
2467
2468         if (prog->instances.nr > 0 || prog->instances.fds) {
2469                 pr_warning("Can't set pre-processor after loading\n");
2470                 return -EINVAL;
2471         }
2472
2473         instances_fds = malloc(sizeof(int) * nr_instances);
2474         if (!instances_fds) {
2475                 pr_warning("alloc memory failed for fds\n");
2476                 return -ENOMEM;
2477         }
2478
2479         /* fill all fd with -1 */
2480         memset(instances_fds, -1, sizeof(int) * nr_instances);
2481
2482         prog->instances.nr = nr_instances;
2483         prog->instances.fds = instances_fds;
2484         prog->preprocessor = prep;
2485         return 0;
2486 }
2487
2488 int bpf_program__nth_fd(struct bpf_program *prog, int n)
2489 {
2490         int fd;
2491
2492         if (!prog)
2493                 return -EINVAL;
2494
2495         if (n >= prog->instances.nr || n < 0) {
2496                 pr_warning("Can't get the %dth fd from program %s: only %d instances\n",
2497                            n, prog->section_name, prog->instances.nr);
2498                 return -EINVAL;
2499         }
2500
2501         fd = prog->instances.fds[n];
2502         if (fd < 0) {
2503                 pr_warning("%dth instance of program '%s' is invalid\n",
2504                            n, prog->section_name);
2505                 return -ENOENT;
2506         }
2507
2508         return fd;
2509 }
2510
2511 void bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type)
2512 {
2513         prog->type = type;
2514 }
2515
2516 static bool bpf_program__is_type(struct bpf_program *prog,
2517                                  enum bpf_prog_type type)
2518 {
2519         return prog ? (prog->type == type) : false;
2520 }
2521
2522 #define BPF_PROG_TYPE_FNS(NAME, TYPE)                   \
2523 int bpf_program__set_##NAME(struct bpf_program *prog)   \
2524 {                                                       \
2525         if (!prog)                                      \
2526                 return -EINVAL;                         \
2527         bpf_program__set_type(prog, TYPE);              \
2528         return 0;                                       \
2529 }                                                       \
2530                                                         \
2531 bool bpf_program__is_##NAME(struct bpf_program *prog)   \
2532 {                                                       \
2533         return bpf_program__is_type(prog, TYPE);        \
2534 }                                                       \
2535
2536 BPF_PROG_TYPE_FNS(socket_filter, BPF_PROG_TYPE_SOCKET_FILTER);
2537 BPF_PROG_TYPE_FNS(kprobe, BPF_PROG_TYPE_KPROBE);
2538 BPF_PROG_TYPE_FNS(sched_cls, BPF_PROG_TYPE_SCHED_CLS);
2539 BPF_PROG_TYPE_FNS(sched_act, BPF_PROG_TYPE_SCHED_ACT);
2540 BPF_PROG_TYPE_FNS(tracepoint, BPF_PROG_TYPE_TRACEPOINT);
2541 BPF_PROG_TYPE_FNS(raw_tracepoint, BPF_PROG_TYPE_RAW_TRACEPOINT);
2542 BPF_PROG_TYPE_FNS(xdp, BPF_PROG_TYPE_XDP);
2543 BPF_PROG_TYPE_FNS(perf_event, BPF_PROG_TYPE_PERF_EVENT);
2544
2545 void bpf_program__set_expected_attach_type(struct bpf_program *prog,
2546                                            enum bpf_attach_type type)
2547 {
2548         prog->expected_attach_type = type;
2549 }
2550
2551 #define BPF_PROG_SEC_IMPL(string, ptype, eatype, is_attachable, atype) \
2552         { string, sizeof(string) - 1, ptype, eatype, is_attachable, atype }
2553
2554 /* Programs that can NOT be attached. */
2555 #define BPF_PROG_SEC(string, ptype) BPF_PROG_SEC_IMPL(string, ptype, 0, 0, 0)
2556
2557 /* Programs that can be attached. */
2558 #define BPF_APROG_SEC(string, ptype, atype) \
2559         BPF_PROG_SEC_IMPL(string, ptype, 0, 1, atype)
2560
2561 /* Programs that must specify expected attach type at load time. */
2562 #define BPF_EAPROG_SEC(string, ptype, eatype) \
2563         BPF_PROG_SEC_IMPL(string, ptype, eatype, 1, eatype)
2564
2565 /* Programs that can be attached but attach type can't be identified by section
2566  * name. Kept for backward compatibility.
2567  */
2568 #define BPF_APROG_COMPAT(string, ptype) BPF_PROG_SEC(string, ptype)
2569
2570 static const struct {
2571         const char *sec;
2572         size_t len;
2573         enum bpf_prog_type prog_type;
2574         enum bpf_attach_type expected_attach_type;
2575         int is_attachable;
2576         enum bpf_attach_type attach_type;
2577 } section_names[] = {
2578         BPF_PROG_SEC("socket",                  BPF_PROG_TYPE_SOCKET_FILTER),
2579         BPF_PROG_SEC("kprobe/",                 BPF_PROG_TYPE_KPROBE),
2580         BPF_PROG_SEC("kretprobe/",              BPF_PROG_TYPE_KPROBE),
2581         BPF_PROG_SEC("classifier",              BPF_PROG_TYPE_SCHED_CLS),
2582         BPF_PROG_SEC("action",                  BPF_PROG_TYPE_SCHED_ACT),
2583         BPF_PROG_SEC("tracepoint/",             BPF_PROG_TYPE_TRACEPOINT),
2584         BPF_PROG_SEC("raw_tracepoint/",         BPF_PROG_TYPE_RAW_TRACEPOINT),
2585         BPF_PROG_SEC("xdp",                     BPF_PROG_TYPE_XDP),
2586         BPF_PROG_SEC("perf_event",              BPF_PROG_TYPE_PERF_EVENT),
2587         BPF_PROG_SEC("lwt_in",                  BPF_PROG_TYPE_LWT_IN),
2588         BPF_PROG_SEC("lwt_out",                 BPF_PROG_TYPE_LWT_OUT),
2589         BPF_PROG_SEC("lwt_xmit",                BPF_PROG_TYPE_LWT_XMIT),
2590         BPF_PROG_SEC("lwt_seg6local",           BPF_PROG_TYPE_LWT_SEG6LOCAL),
2591         BPF_APROG_SEC("cgroup_skb/ingress",     BPF_PROG_TYPE_CGROUP_SKB,
2592                                                 BPF_CGROUP_INET_INGRESS),
2593         BPF_APROG_SEC("cgroup_skb/egress",      BPF_PROG_TYPE_CGROUP_SKB,
2594                                                 BPF_CGROUP_INET_EGRESS),
2595         BPF_APROG_COMPAT("cgroup/skb",          BPF_PROG_TYPE_CGROUP_SKB),
2596         BPF_APROG_SEC("cgroup/sock",            BPF_PROG_TYPE_CGROUP_SOCK,
2597                                                 BPF_CGROUP_INET_SOCK_CREATE),
2598         BPF_EAPROG_SEC("cgroup/post_bind4",     BPF_PROG_TYPE_CGROUP_SOCK,
2599                                                 BPF_CGROUP_INET4_POST_BIND),
2600         BPF_EAPROG_SEC("cgroup/post_bind6",     BPF_PROG_TYPE_CGROUP_SOCK,
2601                                                 BPF_CGROUP_INET6_POST_BIND),
2602         BPF_APROG_SEC("cgroup/dev",             BPF_PROG_TYPE_CGROUP_DEVICE,
2603                                                 BPF_CGROUP_DEVICE),
2604         BPF_APROG_SEC("sockops",                BPF_PROG_TYPE_SOCK_OPS,
2605                                                 BPF_CGROUP_SOCK_OPS),
2606         BPF_APROG_SEC("sk_skb/stream_parser",   BPF_PROG_TYPE_SK_SKB,
2607                                                 BPF_SK_SKB_STREAM_PARSER),
2608         BPF_APROG_SEC("sk_skb/stream_verdict",  BPF_PROG_TYPE_SK_SKB,
2609                                                 BPF_SK_SKB_STREAM_VERDICT),
2610         BPF_APROG_COMPAT("sk_skb",              BPF_PROG_TYPE_SK_SKB),
2611         BPF_APROG_SEC("sk_msg",                 BPF_PROG_TYPE_SK_MSG,
2612                                                 BPF_SK_MSG_VERDICT),
2613         BPF_APROG_SEC("lirc_mode2",             BPF_PROG_TYPE_LIRC_MODE2,
2614                                                 BPF_LIRC_MODE2),
2615         BPF_APROG_SEC("flow_dissector",         BPF_PROG_TYPE_FLOW_DISSECTOR,
2616                                                 BPF_FLOW_DISSECTOR),
2617         BPF_EAPROG_SEC("cgroup/bind4",          BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
2618                                                 BPF_CGROUP_INET4_BIND),
2619         BPF_EAPROG_SEC("cgroup/bind6",          BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
2620                                                 BPF_CGROUP_INET6_BIND),
2621         BPF_EAPROG_SEC("cgroup/connect4",       BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
2622                                                 BPF_CGROUP_INET4_CONNECT),
2623         BPF_EAPROG_SEC("cgroup/connect6",       BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
2624                                                 BPF_CGROUP_INET6_CONNECT),
2625         BPF_EAPROG_SEC("cgroup/sendmsg4",       BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
2626                                                 BPF_CGROUP_UDP4_SENDMSG),
2627         BPF_EAPROG_SEC("cgroup/sendmsg6",       BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
2628                                                 BPF_CGROUP_UDP6_SENDMSG),
2629 };
2630
2631 #undef BPF_PROG_SEC_IMPL
2632 #undef BPF_PROG_SEC
2633 #undef BPF_APROG_SEC
2634 #undef BPF_EAPROG_SEC
2635 #undef BPF_APROG_COMPAT
2636
2637 #define MAX_TYPE_NAME_SIZE 32
2638
2639 static char *libbpf_get_type_names(bool attach_type)
2640 {
2641         int i, len = ARRAY_SIZE(section_names) * MAX_TYPE_NAME_SIZE;
2642         char *buf;
2643
2644         buf = malloc(len);
2645         if (!buf)
2646                 return NULL;
2647
2648         buf[0] = '\0';
2649         /* Forge string buf with all available names */
2650         for (i = 0; i < ARRAY_SIZE(section_names); i++) {
2651                 if (attach_type && !section_names[i].is_attachable)
2652                         continue;
2653
2654                 if (strlen(buf) + strlen(section_names[i].sec) + 2 > len) {
2655                         free(buf);
2656                         return NULL;
2657                 }
2658                 strcat(buf, " ");
2659                 strcat(buf, section_names[i].sec);
2660         }
2661
2662         return buf;
2663 }
2664
2665 int libbpf_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type,
2666                              enum bpf_attach_type *expected_attach_type)
2667 {
2668         char *type_names;
2669         int i;
2670
2671         if (!name)
2672                 return -EINVAL;
2673
2674         for (i = 0; i < ARRAY_SIZE(section_names); i++) {
2675                 if (strncmp(name, section_names[i].sec, section_names[i].len))
2676                         continue;
2677                 *prog_type = section_names[i].prog_type;
2678                 *expected_attach_type = section_names[i].expected_attach_type;
2679                 return 0;
2680         }
2681         pr_warning("failed to guess program type based on ELF section name '%s'\n", name);
2682         type_names = libbpf_get_type_names(false);
2683         if (type_names != NULL) {
2684                 pr_info("supported section(type) names are:%s\n", type_names);
2685                 free(type_names);
2686         }
2687
2688         return -EINVAL;
2689 }
2690
2691 int libbpf_attach_type_by_name(const char *name,
2692                                enum bpf_attach_type *attach_type)
2693 {
2694         char *type_names;
2695         int i;
2696
2697         if (!name)
2698                 return -EINVAL;
2699
2700         for (i = 0; i < ARRAY_SIZE(section_names); i++) {
2701                 if (strncmp(name, section_names[i].sec, section_names[i].len))
2702                         continue;
2703                 if (!section_names[i].is_attachable)
2704                         return -EINVAL;
2705                 *attach_type = section_names[i].attach_type;
2706                 return 0;
2707         }
2708         pr_warning("failed to guess attach type based on ELF section name '%s'\n", name);
2709         type_names = libbpf_get_type_names(true);
2710         if (type_names != NULL) {
2711                 pr_info("attachable section(type) names are:%s\n", type_names);
2712                 free(type_names);
2713         }
2714
2715         return -EINVAL;
2716 }
2717
2718 static int
2719 bpf_program__identify_section(struct bpf_program *prog,
2720                               enum bpf_prog_type *prog_type,
2721                               enum bpf_attach_type *expected_attach_type)
2722 {
2723         return libbpf_prog_type_by_name(prog->section_name, prog_type,
2724                                         expected_attach_type);
2725 }
2726
2727 int bpf_map__fd(struct bpf_map *map)
2728 {
2729         return map ? map->fd : -EINVAL;
2730 }
2731
2732 const struct bpf_map_def *bpf_map__def(struct bpf_map *map)
2733 {
2734         return map ? &map->def : ERR_PTR(-EINVAL);
2735 }
2736
2737 const char *bpf_map__name(struct bpf_map *map)
2738 {
2739         return map ? map->name : NULL;
2740 }
2741
2742 __u32 bpf_map__btf_key_type_id(const struct bpf_map *map)
2743 {
2744         return map ? map->btf_key_type_id : 0;
2745 }
2746
2747 __u32 bpf_map__btf_value_type_id(const struct bpf_map *map)
2748 {
2749         return map ? map->btf_value_type_id : 0;
2750 }
2751
2752 int bpf_map__set_priv(struct bpf_map *map, void *priv,
2753                      bpf_map_clear_priv_t clear_priv)
2754 {
2755         if (!map)
2756                 return -EINVAL;
2757
2758         if (map->priv) {
2759                 if (map->clear_priv)
2760                         map->clear_priv(map, map->priv);
2761         }
2762
2763         map->priv = priv;
2764         map->clear_priv = clear_priv;
2765         return 0;
2766 }
2767
2768 void *bpf_map__priv(struct bpf_map *map)
2769 {
2770         return map ? map->priv : ERR_PTR(-EINVAL);
2771 }
2772
2773 bool bpf_map__is_offload_neutral(struct bpf_map *map)
2774 {
2775         return map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY;
2776 }
2777
2778 void bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex)
2779 {
2780         map->map_ifindex = ifindex;
2781 }
2782
2783 int bpf_map__set_inner_map_fd(struct bpf_map *map, int fd)
2784 {
2785         if (!bpf_map_type__is_map_in_map(map->def.type)) {
2786                 pr_warning("error: unsupported map type\n");
2787                 return -EINVAL;
2788         }
2789         if (map->inner_map_fd != -1) {
2790                 pr_warning("error: inner_map_fd already specified\n");
2791                 return -EINVAL;
2792         }
2793         map->inner_map_fd = fd;
2794         return 0;
2795 }
2796
2797 static struct bpf_map *
2798 __bpf_map__iter(struct bpf_map *m, struct bpf_object *obj, int i)
2799 {
2800         ssize_t idx;
2801         struct bpf_map *s, *e;
2802
2803         if (!obj || !obj->maps)
2804                 return NULL;
2805
2806         s = obj->maps;
2807         e = obj->maps + obj->nr_maps;
2808
2809         if ((m < s) || (m >= e)) {
2810                 pr_warning("error in %s: map handler doesn't belong to object\n",
2811                            __func__);
2812                 return NULL;
2813         }
2814
2815         idx = (m - obj->maps) + i;
2816         if (idx >= obj->nr_maps || idx < 0)
2817                 return NULL;
2818         return &obj->maps[idx];
2819 }
2820
2821 struct bpf_map *
2822 bpf_map__next(struct bpf_map *prev, struct bpf_object *obj)
2823 {
2824         if (prev == NULL)
2825                 return obj->maps;
2826
2827         return __bpf_map__iter(prev, obj, 1);
2828 }
2829
2830 struct bpf_map *
2831 bpf_map__prev(struct bpf_map *next, struct bpf_object *obj)
2832 {
2833         if (next == NULL) {
2834                 if (!obj->nr_maps)
2835                         return NULL;
2836                 return obj->maps + obj->nr_maps - 1;
2837         }
2838
2839         return __bpf_map__iter(next, obj, -1);
2840 }
2841
2842 struct bpf_map *
2843 bpf_object__find_map_by_name(struct bpf_object *obj, const char *name)
2844 {
2845         struct bpf_map *pos;
2846
2847         bpf_object__for_each_map(pos, obj) {
2848                 if (pos->name && !strcmp(pos->name, name))
2849                         return pos;
2850         }
2851         return NULL;
2852 }
2853
2854 int
2855 bpf_object__find_map_fd_by_name(struct bpf_object *obj, const char *name)
2856 {
2857         return bpf_map__fd(bpf_object__find_map_by_name(obj, name));
2858 }
2859
2860 struct bpf_map *
2861 bpf_object__find_map_by_offset(struct bpf_object *obj, size_t offset)
2862 {
2863         int i;
2864
2865         for (i = 0; i < obj->nr_maps; i++) {
2866                 if (obj->maps[i].offset == offset)
2867                         return &obj->maps[i];
2868         }
2869         return ERR_PTR(-ENOENT);
2870 }
2871
2872 long libbpf_get_error(const void *ptr)
2873 {
2874         if (IS_ERR(ptr))
2875                 return PTR_ERR(ptr);
2876         return 0;
2877 }
2878
2879 int bpf_prog_load(const char *file, enum bpf_prog_type type,
2880                   struct bpf_object **pobj, int *prog_fd)
2881 {
2882         struct bpf_prog_load_attr attr;
2883
2884         memset(&attr, 0, sizeof(struct bpf_prog_load_attr));
2885         attr.file = file;
2886         attr.prog_type = type;
2887         attr.expected_attach_type = 0;
2888
2889         return bpf_prog_load_xattr(&attr, pobj, prog_fd);
2890 }
2891
2892 int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr,
2893                         struct bpf_object **pobj, int *prog_fd)
2894 {
2895         struct bpf_object_open_attr open_attr = {
2896                 .file           = attr->file,
2897                 .prog_type      = attr->prog_type,
2898         };
2899         struct bpf_program *prog, *first_prog = NULL;
2900         enum bpf_attach_type expected_attach_type;
2901         enum bpf_prog_type prog_type;
2902         struct bpf_object *obj;
2903         struct bpf_map *map;
2904         int err;
2905
2906         if (!attr)
2907                 return -EINVAL;
2908         if (!attr->file)
2909                 return -EINVAL;
2910
2911         obj = bpf_object__open_xattr(&open_attr);
2912         if (IS_ERR_OR_NULL(obj))
2913                 return -ENOENT;
2914
2915         bpf_object__for_each_program(prog, obj) {
2916                 /*
2917                  * If type is not specified, try to guess it based on
2918                  * section name.
2919                  */
2920                 prog_type = attr->prog_type;
2921                 prog->prog_ifindex = attr->ifindex;
2922                 expected_attach_type = attr->expected_attach_type;
2923                 if (prog_type == BPF_PROG_TYPE_UNSPEC) {
2924                         err = bpf_program__identify_section(prog, &prog_type,
2925                                                             &expected_attach_type);
2926                         if (err < 0) {
2927                                 bpf_object__close(obj);
2928                                 return -EINVAL;
2929                         }
2930                 }
2931
2932                 bpf_program__set_type(prog, prog_type);
2933                 bpf_program__set_expected_attach_type(prog,
2934                                                       expected_attach_type);
2935
2936                 if (!first_prog)
2937                         first_prog = prog;
2938         }
2939
2940         bpf_object__for_each_map(map, obj) {
2941                 if (!bpf_map__is_offload_neutral(map))
2942                         map->map_ifindex = attr->ifindex;
2943         }
2944
2945         if (!first_prog) {
2946                 pr_warning("object file doesn't contain bpf program\n");
2947                 bpf_object__close(obj);
2948                 return -ENOENT;
2949         }
2950
2951         err = bpf_object__load(obj);
2952         if (err) {
2953                 bpf_object__close(obj);
2954                 return -EINVAL;
2955         }
2956
2957         *pobj = obj;
2958         *prog_fd = bpf_program__fd(first_prog);
2959         return 0;
2960 }
2961
2962 enum bpf_perf_event_ret
2963 bpf_perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size,
2964                            void **copy_mem, size_t *copy_size,
2965                            bpf_perf_event_print_t fn, void *private_data)
2966 {
2967         struct perf_event_mmap_page *header = mmap_mem;
2968         __u64 data_head = ring_buffer_read_head(header);
2969         __u64 data_tail = header->data_tail;
2970         void *base = ((__u8 *)header) + page_size;
2971         int ret = LIBBPF_PERF_EVENT_CONT;
2972         struct perf_event_header *ehdr;
2973         size_t ehdr_size;
2974
2975         while (data_head != data_tail) {
2976                 ehdr = base + (data_tail & (mmap_size - 1));
2977                 ehdr_size = ehdr->size;
2978
2979                 if (((void *)ehdr) + ehdr_size > base + mmap_size) {
2980                         void *copy_start = ehdr;
2981                         size_t len_first = base + mmap_size - copy_start;
2982                         size_t len_secnd = ehdr_size - len_first;
2983
2984                         if (*copy_size < ehdr_size) {
2985                                 free(*copy_mem);
2986                                 *copy_mem = malloc(ehdr_size);
2987                                 if (!*copy_mem) {
2988                                         *copy_size = 0;
2989                                         ret = LIBBPF_PERF_EVENT_ERROR;
2990                                         break;
2991                                 }
2992                                 *copy_size = ehdr_size;
2993                         }
2994
2995                         memcpy(*copy_mem, copy_start, len_first);
2996                         memcpy(*copy_mem + len_first, base, len_secnd);
2997                         ehdr = *copy_mem;
2998                 }
2999
3000                 ret = fn(ehdr, private_data);
3001                 data_tail += ehdr_size;
3002                 if (ret != LIBBPF_PERF_EVENT_CONT)
3003                         break;
3004         }
3005
3006         ring_buffer_write_tail(header, data_tail);
3007         return ret;
3008 }