tools/lib/bpf/libbpf.c

   1 // SPDX-License-Identifier: LGPL-2.1
   2
   3 /*
   4  * Common eBPF ELF object loading operations.
   5  *
   6  * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org>
   7  * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com>
   8  * Copyright (C) 2015 Huawei Inc.
   9  * Copyright (C) 2017 Nicira, Inc.
  10  *
  11  * This program is free software; you can redistribute it and/or
  12  * modify it under the terms of the GNU Lesser General Public
  13  * License as published by the Free Software Foundation;
  14  * version 2.1 of the License (not later!)
  15  *
  16  * This program is distributed in the hope that it will be useful,
  17  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  18  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19  * GNU Lesser General Public License for more details.
  20  *
  21  * You should have received a copy of the GNU Lesser General Public
  22  * License along with this program; if not,  see <http://www.gnu.org/licenses>
  23  */
  24
  25 #define _GNU_SOURCE
  26 #include <stdlib.h>
  27 #include <stdio.h>
  28 #include <stdarg.h>
  29 #include <libgen.h>
  30 #include <inttypes.h>
  31 #include <string.h>
  32 #include <unistd.h>
  33 #include <fcntl.h>
  34 #include <errno.h>
  35 #include <perf-sys.h>
  36 #include <asm/unistd.h>
  37 #include <linux/err.h>
  38 #include <linux/kernel.h>
  39 #include <linux/bpf.h>
  40 #include <linux/btf.h>
  41 #include <linux/list.h>
  42 #include <linux/limits.h>
  43 #include <sys/stat.h>
  44 #include <sys/types.h>
  45 #include <sys/vfs.h>
  46 #include <tools/libc_compat.h>
  47 #include <libelf.h>
  48 #include <gelf.h>
  49
  50 #include "libbpf.h"
  51 #include "bpf.h"
  52 #include "btf.h"
  53
  54 #ifndef EM_BPF
  55 #define EM_BPF 247
  56 #endif
  57
  58 #ifndef BPF_FS_MAGIC
  59 #define BPF_FS_MAGIC            0xcafe4a11
  60 #endif
  61
  62 #define __printf(a, b)  __attribute__((format(printf, a, b)))
  63
  64 __printf(1, 2)
  65 static int __base_pr(const char *format, ...)
  66 {
  67         va_list args;
  68         int err;
  69
  70         va_start(args, format);
  71         err = vfprintf(stderr, format, args);
  72         va_end(args);
  73         return err;
  74 }
  75
  76 static __printf(1, 2) libbpf_print_fn_t __pr_warning = __base_pr;
  77 static __printf(1, 2) libbpf_print_fn_t __pr_info = __base_pr;
  78 static __printf(1, 2) libbpf_print_fn_t __pr_debug;
  79
  80 #define __pr(func, fmt, ...)    \
  81 do {                            \
  82         if ((func))             \
  83                 (func)("libbpf: " fmt, ##__VA_ARGS__); \
  84 } while (0)
  85
  86 #define pr_warning(fmt, ...)    __pr(__pr_warning, fmt, ##__VA_ARGS__)
  87 #define pr_info(fmt, ...)       __pr(__pr_info, fmt, ##__VA_ARGS__)
  88 #define pr_debug(fmt, ...)      __pr(__pr_debug, fmt, ##__VA_ARGS__)
  89
  90 void libbpf_set_print(libbpf_print_fn_t warn,
  91                       libbpf_print_fn_t info,
  92                       libbpf_print_fn_t debug)
  93 {
  94         __pr_warning = warn;
  95         __pr_info = info;
  96         __pr_debug = debug;
  97 }
  98
  99 #define STRERR_BUFSIZE  128
 100
 101 #define CHECK_ERR(action, err, out) do {        \
 102         err = action;                   \
 103         if (err)                        \
 104                 goto out;               \
 105 } while(0)
 106
 107
 108 /* Copied from tools/perf/util/util.h */
 109 #ifndef zfree
 110 # define zfree(ptr) ({ free(*ptr); *ptr = NULL; })
 111 #endif
 112
 113 #ifndef zclose
 114 # define zclose(fd) ({                  \
 115         int ___err = 0;                 \
 116         if ((fd) >= 0)                  \
 117                 ___err = close((fd));   \
 118         fd = -1;                        \
 119         ___err; })
 120 #endif
 121
 122 #ifdef HAVE_LIBELF_MMAP_SUPPORT
 123 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ_MMAP
 124 #else
 125 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ
 126 #endif
 127
 128 /*
 129  * bpf_prog should be a better name but it has been used in
 130  * linux/filter.h.
 131  */
 132 struct bpf_program {
 133         /* Index in elf obj file, for relocation use. */
 134         int idx;
 135         char *name;
 136         int prog_ifindex;
 137         char *section_name;
 138         struct bpf_insn *insns;
 139         size_t insns_cnt, main_prog_cnt;
 140         enum bpf_prog_type type;
 141
 142         struct reloc_desc {
 143                 enum {
 144                         RELO_LD64,
 145                         RELO_CALL,
 146                 } type;
 147                 int insn_idx;
 148                 union {
 149                         int map_idx;
 150                         int text_off;
 151                 };
 152         } *reloc_desc;
 153         int nr_reloc;
 154
 155         struct {
 156                 int nr;
 157                 int *fds;
 158         } instances;
 159         bpf_program_prep_t preprocessor;
 160
 161         struct bpf_object *obj;
 162         void *priv;
 163         bpf_program_clear_priv_t clear_priv;
 164
 165         enum bpf_attach_type expected_attach_type;
 166 };
 167
 168 struct bpf_map {
 169         int fd;
 170         char *name;
 171         size_t offset;
 172         int map_ifindex;
 173         struct bpf_map_def def;
 174         __u32 btf_key_type_id;
 175         __u32 btf_value_type_id;
 176         void *priv;
 177         bpf_map_clear_priv_t clear_priv;
 178 };
 179
 180 static LIST_HEAD(bpf_objects_list);
 181
 182 struct bpf_object {
 183         char license[64];
 184         u32 kern_version;
 185
 186         struct bpf_program *programs;
 187         size_t nr_programs;
 188         struct bpf_map *maps;
 189         size_t nr_maps;
 190
 191         bool loaded;
 192         bool has_pseudo_calls;
 193
 194         /*
 195          * Information when doing elf related work. Only valid if fd
 196          * is valid.
 197          */
 198         struct {
 199                 int fd;
 200                 void *obj_buf;
 201                 size_t obj_buf_sz;
 202                 Elf *elf;
 203                 GElf_Ehdr ehdr;
 204                 Elf_Data *symbols;
 205                 size_t strtabidx;
 206                 struct {
 207                         GElf_Shdr shdr;
 208                         Elf_Data *data;
 209                 } *reloc;
 210                 int nr_reloc;
 211                 int maps_shndx;
 212                 int text_shndx;
 213         } efile;
 214         /*
 215          * All loaded bpf_object is linked in a list, which is
 216          * hidden to caller. bpf_objects__<func> handlers deal with
 217          * all objects.
 218          */
 219         struct list_head list;
 220
 221         struct btf *btf;
 222
 223         void *priv;
 224         bpf_object_clear_priv_t clear_priv;
 225
 226         char path[];
 227 };
 228 #define obj_elf_valid(o)        ((o)->efile.elf)
 229
 230 static void bpf_program__unload(struct bpf_program *prog)
 231 {
 232         int i;
 233
 234         if (!prog)
 235                 return;
 236
 237         /*
 238          * If the object is opened but the program was never loaded,
 239          * it is possible that prog->instances.nr == -1.
 240          */
 241         if (prog->instances.nr > 0) {
 242                 for (i = 0; i < prog->instances.nr; i++)
 243                         zclose(prog->instances.fds[i]);
 244         } else if (prog->instances.nr != -1) {
 245                 pr_warning("Internal error: instances.nr is %d\n",
 246                            prog->instances.nr);
 247         }
 248
 249         prog->instances.nr = -1;
 250         zfree(&prog->instances.fds);
 251 }
 252
 253 static void bpf_program__exit(struct bpf_program *prog)
 254 {
 255         if (!prog)
 256                 return;
 257
 258         if (prog->clear_priv)
 259                 prog->clear_priv(prog, prog->priv);
 260
 261         prog->priv = NULL;
 262         prog->clear_priv = NULL;
 263
 264         bpf_program__unload(prog);
 265         zfree(&prog->name);
 266         zfree(&prog->section_name);
 267         zfree(&prog->insns);
 268         zfree(&prog->reloc_desc);
 269
 270         prog->nr_reloc = 0;
 271         prog->insns_cnt = 0;
 272         prog->idx = -1;
 273 }
 274
 275 static int
 276 bpf_program__init(void *data, size_t size, char *section_name, int idx,
 277                   struct bpf_program *prog)
 278 {
 279         if (size < sizeof(struct bpf_insn)) {
 280                 pr_warning("corrupted section '%s'\n", section_name);
 281                 return -EINVAL;
 282         }
 283
 284         bzero(prog, sizeof(*prog));
 285
 286         prog->section_name = strdup(section_name);
 287         if (!prog->section_name) {
 288                 pr_warning("failed to alloc name for prog under section(%d) %s\n",
 289                            idx, section_name);
 290                 goto errout;
 291         }
 292
 293         prog->insns = malloc(size);
 294         if (!prog->insns) {
 295                 pr_warning("failed to alloc insns for prog under section %s\n",
 296                            section_name);
 297                 goto errout;
 298         }
 299         prog->insns_cnt = size / sizeof(struct bpf_insn);
 300         memcpy(prog->insns, data,
 301                prog->insns_cnt * sizeof(struct bpf_insn));
 302         prog->idx = idx;
 303         prog->instances.fds = NULL;
 304         prog->instances.nr = -1;
 305         prog->type = BPF_PROG_TYPE_KPROBE;
 306
 307         return 0;
 308 errout:
 309         bpf_program__exit(prog);
 310         return -ENOMEM;
 311 }
 312
 313 static int
 314 bpf_object__add_program(struct bpf_object *obj, void *data, size_t size,
 315                         char *section_name, int idx)
 316 {
 317         struct bpf_program prog, *progs;
 318         int nr_progs, err;
 319
 320         err = bpf_program__init(data, size, section_name, idx, &prog);
 321         if (err)
 322                 return err;
 323
 324         progs = obj->programs;
 325         nr_progs = obj->nr_programs;
 326
 327         progs = reallocarray(progs, nr_progs + 1, sizeof(progs[0]));
 328         if (!progs) {
 329                 /*
 330                  * In this case the original obj->programs
 331                  * is still valid, so don't need special treat for
 332                  * bpf_close_object().
 333                  */
 334                 pr_warning("failed to alloc a new program under section '%s'\n",
 335                            section_name);
 336                 bpf_program__exit(&prog);
 337                 return -ENOMEM;
 338         }
 339
 340         pr_debug("found program %s\n", prog.section_name);
 341         obj->programs = progs;
 342         obj->nr_programs = nr_progs + 1;
 343         prog.obj = obj;
 344         progs[nr_progs] = prog;
 345         return 0;
 346 }
 347
 348 static int
 349 bpf_object__init_prog_names(struct bpf_object *obj)
 350 {
 351         Elf_Data *symbols = obj->efile.symbols;
 352         struct bpf_program *prog;
 353         size_t pi, si;
 354
 355         for (pi = 0; pi < obj->nr_programs; pi++) {
 356                 const char *name = NULL;
 357
 358                 prog = &obj->programs[pi];
 359
 360                 for (si = 0; si < symbols->d_size / sizeof(GElf_Sym) && !name;
 361                      si++) {
 362                         GElf_Sym sym;
 363
 364                         if (!gelf_getsym(symbols, si, &sym))
 365                                 continue;
 366                         if (sym.st_shndx != prog->idx)
 367                                 continue;
 368                         if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL)
 369                                 continue;
 370
 371                         name = elf_strptr(obj->efile.elf,
 372                                           obj->efile.strtabidx,
 373                                           sym.st_name);
 374                         if (!name) {
 375                                 pr_warning("failed to get sym name string for prog %s\n",
 376                                            prog->section_name);
 377                                 return -LIBBPF_ERRNO__LIBELF;
 378                         }
 379                 }
 380
 381                 if (!name && prog->idx == obj->efile.text_shndx)
 382                         name = ".text";
 383
 384                 if (!name) {
 385                         pr_warning("failed to find sym for prog %s\n",
 386                                    prog->section_name);
 387                         return -EINVAL;
 388                 }
 389
 390                 prog->name = strdup(name);
 391                 if (!prog->name) {
 392                         pr_warning("failed to allocate memory for prog sym %s\n",
 393                                    name);
 394                         return -ENOMEM;
 395                 }
 396         }
 397
 398         return 0;
 399 }
 400
 401 static struct bpf_object *bpf_object__new(const char *path,
 402                                           void *obj_buf,
 403                                           size_t obj_buf_sz)
 404 {
 405         struct bpf_object *obj;
 406
 407         obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1);
 408         if (!obj) {
 409                 pr_warning("alloc memory failed for %s\n", path);
 410                 return ERR_PTR(-ENOMEM);
 411         }
 412
 413         strcpy(obj->path, path);
 414         obj->efile.fd = -1;
 415
 416         /*
 417          * Caller of this function should also calls
 418          * bpf_object__elf_finish() after data collection to return
 419          * obj_buf to user. If not, we should duplicate the buffer to
 420          * avoid user freeing them before elf finish.
 421          */
 422         obj->efile.obj_buf = obj_buf;
 423         obj->efile.obj_buf_sz = obj_buf_sz;
 424         obj->efile.maps_shndx = -1;
 425
 426         obj->loaded = false;
 427
 428         INIT_LIST_HEAD(&obj->list);
 429         list_add(&obj->list, &bpf_objects_list);
 430         return obj;
 431 }
 432
 433 static void bpf_object__elf_finish(struct bpf_object *obj)
 434 {
 435         if (!obj_elf_valid(obj))
 436                 return;
 437
 438         if (obj->efile.elf) {
 439                 elf_end(obj->efile.elf);
 440                 obj->efile.elf = NULL;
 441         }
 442         obj->efile.symbols = NULL;
 443
 444         zfree(&obj->efile.reloc);
 445         obj->efile.nr_reloc = 0;
 446         zclose(obj->efile.fd);
 447         obj->efile.obj_buf = NULL;
 448         obj->efile.obj_buf_sz = 0;
 449 }
 450
 451 static int bpf_object__elf_init(struct bpf_object *obj)
 452 {
 453         int err = 0;
 454         GElf_Ehdr *ep;
 455
 456         if (obj_elf_valid(obj)) {
 457                 pr_warning("elf init: internal error\n");
 458                 return -LIBBPF_ERRNO__LIBELF;
 459         }
 460
 461         if (obj->efile.obj_buf_sz > 0) {
 462                 /*
 463                  * obj_buf should have been validated by
 464                  * bpf_object__open_buffer().
 465                  */
 466                 obj->efile.elf = elf_memory(obj->efile.obj_buf,
 467                                             obj->efile.obj_buf_sz);
 468         } else {
 469                 obj->efile.fd = open(obj->path, O_RDONLY);
 470                 if (obj->efile.fd < 0) {
 471                         pr_warning("failed to open %s: %s\n", obj->path,
 472                                         strerror(errno));
 473                         return -errno;
 474                 }
 475
 476                 obj->efile.elf = elf_begin(obj->efile.fd,
 477                                 LIBBPF_ELF_C_READ_MMAP,
 478                                 NULL);
 479         }
 480
 481         if (!obj->efile.elf) {
 482                 pr_warning("failed to open %s as ELF file\n",
 483                                 obj->path);
 484                 err = -LIBBPF_ERRNO__LIBELF;
 485                 goto errout;
 486         }
 487
 488         if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) {
 489                 pr_warning("failed to get EHDR from %s\n",
 490                                 obj->path);
 491                 err = -LIBBPF_ERRNO__FORMAT;
 492                 goto errout;
 493         }
 494         ep = &obj->efile.ehdr;
 495
 496         /* Old LLVM set e_machine to EM_NONE */
 497         if ((ep->e_type != ET_REL) || (ep->e_machine && (ep->e_machine != EM_BPF))) {
 498                 pr_warning("%s is not an eBPF object file\n",
 499                         obj->path);
 500                 err = -LIBBPF_ERRNO__FORMAT;
 501                 goto errout;
 502         }
 503
 504         return 0;
 505 errout:
 506         bpf_object__elf_finish(obj);
 507         return err;
 508 }
 509
 510 static int
 511 bpf_object__check_endianness(struct bpf_object *obj)
 512 {
 513         static unsigned int const endian = 1;
 514
 515         switch (obj->efile.ehdr.e_ident[EI_DATA]) {
 516         case ELFDATA2LSB:
 517                 /* We are big endian, BPF obj is little endian. */
 518                 if (*(unsigned char const *)&endian != 1)
 519                         goto mismatch;
 520                 break;
 521
 522         case ELFDATA2MSB:
 523                 /* We are little endian, BPF obj is big endian. */
 524                 if (*(unsigned char const *)&endian != 0)
 525                         goto mismatch;
 526                 break;
 527         default:
 528                 return -LIBBPF_ERRNO__ENDIAN;
 529         }
 530
 531         return 0;
 532
 533 mismatch:
 534         pr_warning("Error: endianness mismatch.\n");
 535         return -LIBBPF_ERRNO__ENDIAN;
 536 }
 537
 538 static int
 539 bpf_object__init_license(struct bpf_object *obj,
 540                          void *data, size_t size)
 541 {
 542         memcpy(obj->license, data,
 543                min(size, sizeof(obj->license) - 1));
 544         pr_debug("license of %s is %s\n", obj->path, obj->license);
 545         return 0;
 546 }
 547
 548 static int
 549 bpf_object__init_kversion(struct bpf_object *obj,
 550                           void *data, size_t size)
 551 {
 552         u32 kver;
 553
 554         if (size != sizeof(kver)) {
 555                 pr_warning("invalid kver section in %s\n", obj->path);
 556                 return -LIBBPF_ERRNO__FORMAT;
 557         }
 558         memcpy(&kver, data, sizeof(kver));
 559         obj->kern_version = kver;
 560         pr_debug("kernel version of %s is %x\n", obj->path,
 561                  obj->kern_version);
 562         return 0;
 563 }
 564
 565 static int compare_bpf_map(const void *_a, const void *_b)
 566 {
 567         const struct bpf_map *a = _a;
 568         const struct bpf_map *b = _b;
 569
 570         return a->offset - b->offset;
 571 }
 572
 573 static int
 574 bpf_object__init_maps(struct bpf_object *obj)
 575 {
 576         int i, map_idx, map_def_sz, nr_maps = 0;
 577         Elf_Scn *scn;
 578         Elf_Data *data;
 579         Elf_Data *symbols = obj->efile.symbols;
 580
 581         if (obj->efile.maps_shndx < 0)
 582                 return -EINVAL;
 583         if (!symbols)
 584                 return -EINVAL;
 585
 586         scn = elf_getscn(obj->efile.elf, obj->efile.maps_shndx);
 587         if (scn)
 588                 data = elf_getdata(scn, NULL);
 589         if (!scn || !data) {
 590                 pr_warning("failed to get Elf_Data from map section %d\n",
 591                            obj->efile.maps_shndx);
 592                 return -EINVAL;
 593         }
 594
 595         /*
 596          * Count number of maps. Each map has a name.
 597          * Array of maps is not supported: only the first element is
 598          * considered.
 599          *
 600          * TODO: Detect array of map and report error.
 601          */
 602         for (i = 0; i < symbols->d_size / sizeof(GElf_Sym); i++) {
 603                 GElf_Sym sym;
 604
 605                 if (!gelf_getsym(symbols, i, &sym))
 606                         continue;
 607                 if (sym.st_shndx != obj->efile.maps_shndx)
 608                         continue;
 609                 nr_maps++;
 610         }
 611
 612         /* Alloc obj->maps and fill nr_maps. */
 613         pr_debug("maps in %s: %d maps in %zd bytes\n", obj->path,
 614                  nr_maps, data->d_size);
 615
 616         if (!nr_maps)
 617                 return 0;
 618
 619         /* Assume equally sized map definitions */
 620         map_def_sz = data->d_size / nr_maps;
 621         if (!data->d_size || (data->d_size % nr_maps) != 0) {
 622                 pr_warning("unable to determine map definition size "
 623                            "section %s, %d maps in %zd bytes\n",
 624                            obj->path, nr_maps, data->d_size);
 625                 return -EINVAL;
 626         }
 627
 628         obj->maps = calloc(nr_maps, sizeof(obj->maps[0]));
 629         if (!obj->maps) {
 630                 pr_warning("alloc maps for object failed\n");
 631                 return -ENOMEM;
 632         }
 633         obj->nr_maps = nr_maps;
 634
 635         /*
 636          * fill all fd with -1 so won't close incorrect
 637          * fd (fd=0 is stdin) when failure (zclose won't close
 638          * negative fd)).
 639          */
 640         for (i = 0; i < nr_maps; i++)
 641                 obj->maps[i].fd = -1;
 642
 643         /*
 644          * Fill obj->maps using data in "maps" section.
 645          */
 646         for (i = 0, map_idx = 0; i < symbols->d_size / sizeof(GElf_Sym); i++) {
 647                 GElf_Sym sym;
 648                 const char *map_name;
 649                 struct bpf_map_def *def;
 650
 651                 if (!gelf_getsym(symbols, i, &sym))
 652                         continue;
 653                 if (sym.st_shndx != obj->efile.maps_shndx)
 654                         continue;
 655
 656                 map_name = elf_strptr(obj->efile.elf,
 657                                       obj->efile.strtabidx,
 658                                       sym.st_name);
 659                 obj->maps[map_idx].offset = sym.st_value;
 660                 if (sym.st_value + map_def_sz > data->d_size) {
 661                         pr_warning("corrupted maps section in %s: last map \"%s\" too small\n",
 662                                    obj->path, map_name);
 663                         return -EINVAL;
 664                 }
 665
 666                 obj->maps[map_idx].name = strdup(map_name);
 667                 if (!obj->maps[map_idx].name) {
 668                         pr_warning("failed to alloc map name\n");
 669                         return -ENOMEM;
 670                 }
 671                 pr_debug("map %d is \"%s\"\n", map_idx,
 672                          obj->maps[map_idx].name);
 673                 def = (struct bpf_map_def *)(data->d_buf + sym.st_value);
 674                 /*
 675                  * If the definition of the map in the object file fits in
 676                  * bpf_map_def, copy it.  Any extra fields in our version
 677                  * of bpf_map_def will default to zero as a result of the
 678                  * calloc above.
 679                  */
 680                 if (map_def_sz <= sizeof(struct bpf_map_def)) {
 681                         memcpy(&obj->maps[map_idx].def, def, map_def_sz);
 682                 } else {
 683                         /*
 684                          * Here the map structure being read is bigger than what
 685                          * we expect, truncate if the excess bits are all zero.
 686                          * If they are not zero, reject this map as
 687                          * incompatible.
 688                          */
 689                         char *b;
 690                         for (b = ((char *)def) + sizeof(struct bpf_map_def);
 691                              b < ((char *)def) + map_def_sz; b++) {
 692                                 if (*b != 0) {
 693                                         pr_warning("maps section in %s: \"%s\" "
 694                                                    "has unrecognized, non-zero "
 695                                                    "options\n",
 696                                                    obj->path, map_name);
 697                                         return -EINVAL;
 698                                 }
 699                         }
 700                         memcpy(&obj->maps[map_idx].def, def,
 701                                sizeof(struct bpf_map_def));
 702                 }
 703                 map_idx++;
 704         }
 705
 706         qsort(obj->maps, obj->nr_maps, sizeof(obj->maps[0]), compare_bpf_map);
 707         return 0;
 708 }
 709
 710 static bool section_have_execinstr(struct bpf_object *obj, int idx)
 711 {
 712         Elf_Scn *scn;
 713         GElf_Shdr sh;
 714
 715         scn = elf_getscn(obj->efile.elf, idx);
 716         if (!scn)
 717                 return false;
 718
 719         if (gelf_getshdr(scn, &sh) != &sh)
 720                 return false;
 721
 722         if (sh.sh_flags & SHF_EXECINSTR)
 723                 return true;
 724
 725         return false;
 726 }
 727
 728 static int bpf_object__elf_collect(struct bpf_object *obj)
 729 {
 730         Elf *elf = obj->efile.elf;
 731         GElf_Ehdr *ep = &obj->efile.ehdr;
 732         Elf_Scn *scn = NULL;
 733         int idx = 0, err = 0;
 734
 735         /* Elf is corrupted/truncated, avoid calling elf_strptr. */
 736         if (!elf_rawdata(elf_getscn(elf, ep->e_shstrndx), NULL)) {
 737                 pr_warning("failed to get e_shstrndx from %s\n",
 738                            obj->path);
 739                 return -LIBBPF_ERRNO__FORMAT;
 740         }
 741
 742         while ((scn = elf_nextscn(elf, scn)) != NULL) {
 743                 char *name;
 744                 GElf_Shdr sh;
 745                 Elf_Data *data;
 746
 747                 idx++;
 748                 if (gelf_getshdr(scn, &sh) != &sh) {
 749                         pr_warning("failed to get section(%d) header from %s\n",
 750                                    idx, obj->path);
 751                         err = -LIBBPF_ERRNO__FORMAT;
 752                         goto out;
 753                 }
 754
 755                 name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name);
 756                 if (!name) {
 757                         pr_warning("failed to get section(%d) name from %s\n",
 758                                    idx, obj->path);
 759                         err = -LIBBPF_ERRNO__FORMAT;
 760                         goto out;
 761                 }
 762
 763                 data = elf_getdata(scn, 0);
 764                 if (!data) {
 765                         pr_warning("failed to get section(%d) data from %s(%s)\n",
 766                                    idx, name, obj->path);
 767                         err = -LIBBPF_ERRNO__FORMAT;
 768                         goto out;
 769                 }
 770                 pr_debug("section(%d) %s, size %ld, link %d, flags %lx, type=%d\n",
 771                          idx, name, (unsigned long)data->d_size,
 772                          (int)sh.sh_link, (unsigned long)sh.sh_flags,
 773                          (int)sh.sh_type);
 774
 775                 if (strcmp(name, "license") == 0)
 776                         err = bpf_object__init_license(obj,
 777                                                        data->d_buf,
 778                                                        data->d_size);
 779                 else if (strcmp(name, "version") == 0)
 780                         err = bpf_object__init_kversion(obj,
 781                                                         data->d_buf,
 782                                                         data->d_size);
 783                 else if (strcmp(name, "maps") == 0)
 784                         obj->efile.maps_shndx = idx;
 785                 else if (strcmp(name, BTF_ELF_SEC) == 0) {
 786                         obj->btf = btf__new(data->d_buf, data->d_size,
 787                                             __pr_debug);
 788                         if (IS_ERR(obj->btf)) {
 789                                 pr_warning("Error loading ELF section %s: %ld. Ignored and continue.\n",
 790                                            BTF_ELF_SEC, PTR_ERR(obj->btf));
 791                                 obj->btf = NULL;
 792                         }
 793                 } else if (sh.sh_type == SHT_SYMTAB) {
 794                         if (obj->efile.symbols) {
 795                                 pr_warning("bpf: multiple SYMTAB in %s\n",
 796                                            obj->path);
 797                                 err = -LIBBPF_ERRNO__FORMAT;
 798                         } else {
 799                                 obj->efile.symbols = data;
 800                                 obj->efile.strtabidx = sh.sh_link;
 801                         }
 802                 } else if ((sh.sh_type == SHT_PROGBITS) &&
 803                            (sh.sh_flags & SHF_EXECINSTR) &&
 804                            (data->d_size > 0)) {
 805                         if (strcmp(name, ".text") == 0)
 806                                 obj->efile.text_shndx = idx;
 807                         err = bpf_object__add_program(obj, data->d_buf,
 808                                                       data->d_size, name, idx);
 809                         if (err) {
 810                                 char errmsg[STRERR_BUFSIZE];
 811
 812                                 strerror_r(-err, errmsg, sizeof(errmsg));
 813                                 pr_warning("failed to alloc program %s (%s): %s",
 814                                            name, obj->path, errmsg);
 815                         }
 816                 } else if (sh.sh_type == SHT_REL) {
 817                         void *reloc = obj->efile.reloc;
 818                         int nr_reloc = obj->efile.nr_reloc + 1;
 819                         int sec = sh.sh_info; /* points to other section */
 820
 821                         /* Only do relo for section with exec instructions */
 822                         if (!section_have_execinstr(obj, sec)) {
 823                                 pr_debug("skip relo %s(%d) for section(%d)\n",
 824                                          name, idx, sec);
 825                                 continue;
 826                         }
 827
 828                         reloc = reallocarray(reloc, nr_reloc,
 829                                              sizeof(*obj->efile.reloc));
 830                         if (!reloc) {
 831                                 pr_warning("realloc failed\n");
 832                                 err = -ENOMEM;
 833                         } else {
 834                                 int n = nr_reloc - 1;
 835
 836                                 obj->efile.reloc = reloc;
 837                                 obj->efile.nr_reloc = nr_reloc;
 838
 839                                 obj->efile.reloc[n].shdr = sh;
 840                                 obj->efile.reloc[n].data = data;
 841                         }
 842                 } else {
 843                         pr_debug("skip section(%d) %s\n", idx, name);
 844                 }
 845                 if (err)
 846                         goto out;
 847         }
 848
 849         if (!obj->efile.strtabidx || obj->efile.strtabidx >= idx) {
 850                 pr_warning("Corrupted ELF file: index of strtab invalid\n");
 851                 return LIBBPF_ERRNO__FORMAT;
 852         }
 853         if (obj->efile.maps_shndx >= 0) {
 854                 err = bpf_object__init_maps(obj);
 855                 if (err)
 856                         goto out;
 857         }
 858         err = bpf_object__init_prog_names(obj);
 859 out:
 860         return err;
 861 }
 862
 863 static struct bpf_program *
 864 bpf_object__find_prog_by_idx(struct bpf_object *obj, int idx)
 865 {
 866         struct bpf_program *prog;
 867         size_t i;
 868
 869         for (i = 0; i < obj->nr_programs; i++) {
 870                 prog = &obj->programs[i];
 871                 if (prog->idx == idx)
 872                         return prog;
 873         }
 874         return NULL;
 875 }
 876
 877 static int
 878 bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr,
 879                            Elf_Data *data, struct bpf_object *obj)
 880 {
 881         Elf_Data *symbols = obj->efile.symbols;
 882         int text_shndx = obj->efile.text_shndx;
 883         int maps_shndx = obj->efile.maps_shndx;
 884         struct bpf_map *maps = obj->maps;
 885         size_t nr_maps = obj->nr_maps;
 886         int i, nrels;
 887
 888         pr_debug("collecting relocating info for: '%s'\n",
 889                  prog->section_name);
 890         nrels = shdr->sh_size / shdr->sh_entsize;
 891
 892         prog->reloc_desc = malloc(sizeof(*prog->reloc_desc) * nrels);
 893         if (!prog->reloc_desc) {
 894                 pr_warning("failed to alloc memory in relocation\n");
 895                 return -ENOMEM;
 896         }
 897         prog->nr_reloc = nrels;
 898
 899         for (i = 0; i < nrels; i++) {
 900                 GElf_Sym sym;
 901                 GElf_Rel rel;
 902                 unsigned int insn_idx;
 903                 struct bpf_insn *insns = prog->insns;
 904                 size_t map_idx;
 905
 906                 if (!gelf_getrel(data, i, &rel)) {
 907                         pr_warning("relocation: failed to get %d reloc\n", i);
 908                         return -LIBBPF_ERRNO__FORMAT;
 909                 }
 910
 911                 if (!gelf_getsym(symbols,
 912                                  GELF_R_SYM(rel.r_info),
 913                                  &sym)) {
 914                         pr_warning("relocation: symbol %"PRIx64" not found\n",
 915                                    GELF_R_SYM(rel.r_info));
 916                         return -LIBBPF_ERRNO__FORMAT;
 917                 }
 918                 pr_debug("relo for %lld value %lld name %d\n",
 919                          (long long) (rel.r_info >> 32),
 920                          (long long) sym.st_value, sym.st_name);
 921
 922                 if (sym.st_shndx != maps_shndx && sym.st_shndx != text_shndx) {
 923                         pr_warning("Program '%s' contains non-map related relo data pointing to section %u\n",
 924                                    prog->section_name, sym.st_shndx);
 925                         return -LIBBPF_ERRNO__RELOC;
 926                 }
 927
 928                 insn_idx = rel.r_offset / sizeof(struct bpf_insn);
 929                 pr_debug("relocation: insn_idx=%u\n", insn_idx);
 930
 931                 if (insns[insn_idx].code == (BPF_JMP | BPF_CALL)) {
 932                         if (insns[insn_idx].src_reg != BPF_PSEUDO_CALL) {
 933                                 pr_warning("incorrect bpf_call opcode\n");
 934                                 return -LIBBPF_ERRNO__RELOC;
 935                         }
 936                         prog->reloc_desc[i].type = RELO_CALL;
 937                         prog->reloc_desc[i].insn_idx = insn_idx;
 938                         prog->reloc_desc[i].text_off = sym.st_value;
 939                         obj->has_pseudo_calls = true;
 940                         continue;
 941                 }
 942
 943                 if (insns[insn_idx].code != (BPF_LD | BPF_IMM | BPF_DW)) {
 944                         pr_warning("bpf: relocation: invalid relo for insns[%d].code 0x%x\n",
 945                                    insn_idx, insns[insn_idx].code);
 946                         return -LIBBPF_ERRNO__RELOC;
 947                 }
 948
 949                 /* TODO: 'maps' is sorted. We can use bsearch to make it faster. */
 950                 for (map_idx = 0; map_idx < nr_maps; map_idx++) {
 951                         if (maps[map_idx].offset == sym.st_value) {
 952                                 pr_debug("relocation: find map %zd (%s) for insn %u\n",
 953                                          map_idx, maps[map_idx].name, insn_idx);
 954                                 break;
 955                         }
 956                 }
 957
 958                 if (map_idx >= nr_maps) {
 959                         pr_warning("bpf relocation: map_idx %d large than %d\n",
 960                                    (int)map_idx, (int)nr_maps - 1);
 961                         return -LIBBPF_ERRNO__RELOC;
 962                 }
 963
 964                 prog->reloc_desc[i].type = RELO_LD64;
 965                 prog->reloc_desc[i].insn_idx = insn_idx;
 966                 prog->reloc_desc[i].map_idx = map_idx;
 967         }
 968         return 0;
 969 }
 970
 971 static int bpf_map_find_btf_info(struct bpf_map *map, const struct btf *btf)
 972 {
 973         const struct btf_type *container_type;
 974         const struct btf_member *key, *value;
 975         struct bpf_map_def *def = &map->def;
 976         const size_t max_name = 256;
 977         char container_name[max_name];
 978         __s64 key_size, value_size;
 979         __s32 container_id;
 980
 981         if (snprintf(container_name, max_name, "____btf_map_%s", map->name) ==
 982             max_name) {
 983                 pr_warning("map:%s length of '____btf_map_%s' is too long\n",
 984                            map->name, map->name);
 985                 return -EINVAL;
 986         }
 987
 988         container_id = btf__find_by_name(btf, container_name);
 989         if (container_id < 0) {
 990                 pr_debug("map:%s container_name:%s cannot be found in BTF. Missing BPF_ANNOTATE_KV_PAIR?\n",
 991                          map->name, container_name);
 992                 return container_id;
 993         }
 994
 995         container_type = btf__type_by_id(btf, container_id);
 996         if (!container_type) {
 997                 pr_warning("map:%s cannot find BTF type for container_id:%u\n",
 998                            map->name, container_id);
 999                 return -EINVAL;
1000         }
1001
1002         if (BTF_INFO_KIND(container_type->info) != BTF_KIND_STRUCT ||
1003             BTF_INFO_VLEN(container_type->info) < 2) {
1004                 pr_warning("map:%s container_name:%s is an invalid container struct\n",
1005                            map->name, container_name);
1006                 return -EINVAL;
1007         }
1008
1009         key = (struct btf_member *)(container_type + 1);
1010         value = key + 1;
1011
1012         key_size = btf__resolve_size(btf, key->type);
1013         if (key_size < 0) {
1014                 pr_warning("map:%s invalid BTF key_type_size\n",
1015                            map->name);
1016                 return key_size;
1017         }
1018
1019         if (def->key_size != key_size) {
1020                 pr_warning("map:%s btf_key_type_size:%u != map_def_key_size:%u\n",
1021                            map->name, (__u32)key_size, def->key_size);
1022                 return -EINVAL;
1023         }
1024
1025         value_size = btf__resolve_size(btf, value->type);
1026         if (value_size < 0) {
1027                 pr_warning("map:%s invalid BTF value_type_size\n", map->name);
1028                 return value_size;
1029         }
1030
1031         if (def->value_size != value_size) {
1032                 pr_warning("map:%s btf_value_type_size:%u != map_def_value_size:%u\n",
1033                            map->name, (__u32)value_size, def->value_size);
1034                 return -EINVAL;
1035         }
1036
1037         map->btf_key_type_id = key->type;
1038         map->btf_value_type_id = value->type;
1039
1040         return 0;
1041 }
1042
1043 int bpf_map__reuse_fd(struct bpf_map *map, int fd)
1044 {
1045         struct bpf_map_info info = {};
1046         __u32 len = sizeof(info);
1047         int new_fd, err;
1048         char *new_name;
1049
1050         err = bpf_obj_get_info_by_fd(fd, &info, &len);
1051         if (err)
1052                 return err;
1053
1054         new_name = strdup(info.name);
1055         if (!new_name)
1056                 return -errno;
1057
1058         new_fd = open("/", O_RDONLY | O_CLOEXEC);
1059         if (new_fd < 0)
1060                 goto err_free_new_name;
1061
1062         new_fd = dup3(fd, new_fd, O_CLOEXEC);
1063         if (new_fd < 0)
1064                 goto err_close_new_fd;
1065
1066         err = zclose(map->fd);
1067         if (err)
1068                 goto err_close_new_fd;
1069         free(map->name);
1070
1071         map->fd = new_fd;
1072         map->name = new_name;
1073         map->def.type = info.type;
1074         map->def.key_size = info.key_size;
1075         map->def.value_size = info.value_size;
1076         map->def.max_entries = info.max_entries;
1077         map->def.map_flags = info.map_flags;
1078         map->btf_key_type_id = info.btf_key_type_id;
1079         map->btf_value_type_id = info.btf_value_type_id;
1080
1081         return 0;
1082
1083 err_close_new_fd:
1084         close(new_fd);
1085 err_free_new_name:
1086         free(new_name);
1087         return -errno;
1088 }
1089
1090 static int
1091 bpf_object__create_maps(struct bpf_object *obj)
1092 {
1093         struct bpf_create_map_attr create_attr = {};
1094         unsigned int i;
1095         int err;
1096
1097         for (i = 0; i < obj->nr_maps; i++) {
1098                 struct bpf_map *map = &obj->maps[i];
1099                 struct bpf_map_def *def = &map->def;
1100                 int *pfd = &map->fd;
1101
1102                 if (map->fd >= 0) {
1103                         pr_debug("skip map create (preset) %s: fd=%d\n",
1104                                  map->name, map->fd);
1105                         continue;
1106                 }
1107
1108                 create_attr.name = map->name;
1109                 create_attr.map_ifindex = map->map_ifindex;
1110                 create_attr.map_type = def->type;
1111                 create_attr.map_flags = def->map_flags;
1112                 create_attr.key_size = def->key_size;
1113                 create_attr.value_size = def->value_size;
1114                 create_attr.max_entries = def->max_entries;
1115                 create_attr.btf_fd = 0;
1116                 create_attr.btf_key_type_id = 0;
1117                 create_attr.btf_value_type_id = 0;
1118
1119                 if (obj->btf && !bpf_map_find_btf_info(map, obj->btf)) {
1120                         create_attr.btf_fd = btf__fd(obj->btf);
1121                         create_attr.btf_key_type_id = map->btf_key_type_id;
1122                         create_attr.btf_value_type_id = map->btf_value_type_id;
1123                 }
1124
1125                 *pfd = bpf_create_map_xattr(&create_attr);
1126                 if (*pfd < 0 && create_attr.btf_key_type_id) {
1127                         pr_warning("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n",
1128                                    map->name, strerror(errno), errno);
1129                         create_attr.btf_fd = 0;
1130                         create_attr.btf_key_type_id = 0;
1131                         create_attr.btf_value_type_id = 0;
1132                         map->btf_key_type_id = 0;
1133                         map->btf_value_type_id = 0;
1134                         *pfd = bpf_create_map_xattr(&create_attr);
1135                 }
1136
1137                 if (*pfd < 0) {
1138                         size_t j;
1139
1140                         err = *pfd;
1141                         pr_warning("failed to create map (name: '%s'): %s\n",
1142                                    map->name,
1143                                    strerror(errno));
1144                         for (j = 0; j < i; j++)
1145                                 zclose(obj->maps[j].fd);
1146                         return err;
1147                 }
1148                 pr_debug("create map %s: fd=%d\n", map->name, *pfd);
1149         }
1150
1151         return 0;
1152 }
1153
1154 static int
1155 bpf_program__reloc_text(struct bpf_program *prog, struct bpf_object *obj,
1156                         struct reloc_desc *relo)
1157 {
1158         struct bpf_insn *insn, *new_insn;
1159         struct bpf_program *text;
1160         size_t new_cnt;
1161
1162         if (relo->type != RELO_CALL)
1163                 return -LIBBPF_ERRNO__RELOC;
1164
1165         if (prog->idx == obj->efile.text_shndx) {
1166                 pr_warning("relo in .text insn %d into off %d\n",
1167                            relo->insn_idx, relo->text_off);
1168                 return -LIBBPF_ERRNO__RELOC;
1169         }
1170
1171         if (prog->main_prog_cnt == 0) {
1172                 text = bpf_object__find_prog_by_idx(obj, obj->efile.text_shndx);
1173                 if (!text) {
1174                         pr_warning("no .text section found yet relo into text exist\n");
1175                         return -LIBBPF_ERRNO__RELOC;
1176                 }
1177                 new_cnt = prog->insns_cnt + text->insns_cnt;
1178                 new_insn = reallocarray(prog->insns, new_cnt, sizeof(*insn));
1179                 if (!new_insn) {
1180                         pr_warning("oom in prog realloc\n");
1181                         return -ENOMEM;
1182                 }
1183                 memcpy(new_insn + prog->insns_cnt, text->insns,
1184                        text->insns_cnt * sizeof(*insn));
1185                 prog->insns = new_insn;
1186                 prog->main_prog_cnt = prog->insns_cnt;
1187                 prog->insns_cnt = new_cnt;
1188                 pr_debug("added %zd insn from %s to prog %s\n",
1189                          text->insns_cnt, text->section_name,
1190                          prog->section_name);
1191         }
1192         insn = &prog->insns[relo->insn_idx];
1193         insn->imm += prog->main_prog_cnt - relo->insn_idx;
1194         return 0;
1195 }
1196
1197 static int
1198 bpf_program__relocate(struct bpf_program *prog, struct bpf_object *obj)
1199 {
1200         int i, err;
1201
1202         if (!prog || !prog->reloc_desc)
1203                 return 0;
1204
1205         for (i = 0; i < prog->nr_reloc; i++) {
1206                 if (prog->reloc_desc[i].type == RELO_LD64) {
1207                         struct bpf_insn *insns = prog->insns;
1208                         int insn_idx, map_idx;
1209
1210                         insn_idx = prog->reloc_desc[i].insn_idx;
1211                         map_idx = prog->reloc_desc[i].map_idx;
1212
1213                         if (insn_idx >= (int)prog->insns_cnt) {
1214                                 pr_warning("relocation out of range: '%s'\n",
1215                                            prog->section_name);
1216                                 return -LIBBPF_ERRNO__RELOC;
1217                         }
1218                         insns[insn_idx].src_reg = BPF_PSEUDO_MAP_FD;
1219                         insns[insn_idx].imm = obj->maps[map_idx].fd;
1220                 } else {
1221                         err = bpf_program__reloc_text(prog, obj,
1222                                                       &prog->reloc_desc[i]);
1223                         if (err)
1224                                 return err;
1225                 }
1226         }
1227
1228         zfree(&prog->reloc_desc);
1229         prog->nr_reloc = 0;
1230         return 0;
1231 }
1232
1233
1234 static int
1235 bpf_object__relocate(struct bpf_object *obj)
1236 {
1237         struct bpf_program *prog;
1238         size_t i;
1239         int err;
1240
1241         for (i = 0; i < obj->nr_programs; i++) {
1242                 prog = &obj->programs[i];
1243
1244                 err = bpf_program__relocate(prog, obj);
1245                 if (err) {
1246                         pr_warning("failed to relocate '%s'\n",
1247                                    prog->section_name);
1248                         return err;
1249                 }
1250         }
1251         return 0;
1252 }
1253
1254 static int bpf_object__collect_reloc(struct bpf_object *obj)
1255 {
1256         int i, err;
1257
1258         if (!obj_elf_valid(obj)) {
1259                 pr_warning("Internal error: elf object is closed\n");
1260                 return -LIBBPF_ERRNO__INTERNAL;
1261         }
1262
1263         for (i = 0; i < obj->efile.nr_reloc; i++) {
1264                 GElf_Shdr *shdr = &obj->efile.reloc[i].shdr;
1265                 Elf_Data *data = obj->efile.reloc[i].data;
1266                 int idx = shdr->sh_info;
1267                 struct bpf_program *prog;
1268
1269                 if (shdr->sh_type != SHT_REL) {
1270                         pr_warning("internal error at %d\n", __LINE__);
1271                         return -LIBBPF_ERRNO__INTERNAL;
1272                 }
1273
1274                 prog = bpf_object__find_prog_by_idx(obj, idx);
1275                 if (!prog) {
1276                         pr_warning("relocation failed: no section(%d)\n", idx);
1277                         return -LIBBPF_ERRNO__RELOC;
1278                 }
1279
1280                 err = bpf_program__collect_reloc(prog,
1281                                                  shdr, data,
1282                                                  obj);
1283                 if (err)
1284                         return err;
1285         }
1286         return 0;
1287 }
1288
1289 static int
1290 load_program(enum bpf_prog_type type, enum bpf_attach_type expected_attach_type,
1291              const char *name, struct bpf_insn *insns, int insns_cnt,
1292              char *license, u32 kern_version, int *pfd, int prog_ifindex)
1293 {
1294         struct bpf_load_program_attr load_attr;
1295         char *log_buf;
1296         int ret;
1297
1298         memset(&load_attr, 0, sizeof(struct bpf_load_program_attr));
1299         load_attr.prog_type = type;
1300         load_attr.expected_attach_type = expected_attach_type;
1301         load_attr.name = name;
1302         load_attr.insns = insns;
1303         load_attr.insns_cnt = insns_cnt;
1304         load_attr.license = license;
1305         load_attr.kern_version = kern_version;
1306         load_attr.prog_ifindex = prog_ifindex;
1307
1308         if (!load_attr.insns || !load_attr.insns_cnt)
1309                 return -EINVAL;
1310
1311         log_buf = malloc(BPF_LOG_BUF_SIZE);
1312         if (!log_buf)
1313                 pr_warning("Alloc log buffer for bpf loader error, continue without log\n");
1314
1315         ret = bpf_load_program_xattr(&load_attr, log_buf, BPF_LOG_BUF_SIZE);
1316
1317         if (ret >= 0) {
1318                 *pfd = ret;
1319                 ret = 0;
1320                 goto out;
1321         }
1322
1323         ret = -LIBBPF_ERRNO__LOAD;
1324         pr_warning("load bpf program failed: %s\n", strerror(errno));
1325
1326         if (log_buf && log_buf[0] != '\0') {
1327                 ret = -LIBBPF_ERRNO__VERIFY;
1328                 pr_warning("-- BEGIN DUMP LOG ---\n");
1329                 pr_warning("\n%s\n", log_buf);
1330                 pr_warning("-- END LOG --\n");
1331         } else if (load_attr.insns_cnt >= BPF_MAXINSNS) {
1332                 pr_warning("Program too large (%zu insns), at most %d insns\n",
1333                            load_attr.insns_cnt, BPF_MAXINSNS);
1334                 ret = -LIBBPF_ERRNO__PROG2BIG;
1335         } else {
1336                 /* Wrong program type? */
1337                 if (load_attr.prog_type != BPF_PROG_TYPE_KPROBE) {
1338                         int fd;
1339
1340                         load_attr.prog_type = BPF_PROG_TYPE_KPROBE;
1341                         load_attr.expected_attach_type = 0;
1342                         fd = bpf_load_program_xattr(&load_attr, NULL, 0);
1343                         if (fd >= 0) {
1344                                 close(fd);
1345                                 ret = -LIBBPF_ERRNO__PROGTYPE;
1346                                 goto out;
1347                         }
1348                 }
1349
1350                 if (log_buf)
1351                         ret = -LIBBPF_ERRNO__KVER;
1352         }
1353
1354 out:
1355         free(log_buf);
1356         return ret;
1357 }
1358
1359 static int
1360 bpf_program__load(struct bpf_program *prog,
1361                   char *license, u32 kern_version)
1362 {
1363         int err = 0, fd, i;
1364
1365         if (prog->instances.nr < 0 || !prog->instances.fds) {
1366                 if (prog->preprocessor) {
1367                         pr_warning("Internal error: can't load program '%s'\n",
1368                                    prog->section_name);
1369                         return -LIBBPF_ERRNO__INTERNAL;
1370                 }
1371
1372                 prog->instances.fds = malloc(sizeof(int));
1373                 if (!prog->instances.fds) {
1374                         pr_warning("Not enough memory for BPF fds\n");
1375                         return -ENOMEM;
1376                 }
1377                 prog->instances.nr = 1;
1378                 prog->instances.fds[0] = -1;
1379         }
1380
1381         if (!prog->preprocessor) {
1382                 if (prog->instances.nr != 1) {
1383                         pr_warning("Program '%s' is inconsistent: nr(%d) != 1\n",
1384                                    prog->section_name, prog->instances.nr);
1385                 }
1386                 err = load_program(prog->type, prog->expected_attach_type,
1387                                    prog->name, prog->insns, prog->insns_cnt,
1388                                    license, kern_version, &fd,
1389                                    prog->prog_ifindex);
1390                 if (!err)
1391                         prog->instances.fds[0] = fd;
1392                 goto out;
1393         }
1394
1395         for (i = 0; i < prog->instances.nr; i++) {
1396                 struct bpf_prog_prep_result result;
1397                 bpf_program_prep_t preprocessor = prog->preprocessor;
1398
1399                 bzero(&result, sizeof(result));
1400                 err = preprocessor(prog, i, prog->insns,
1401                                    prog->insns_cnt, &result);
1402                 if (err) {
1403                         pr_warning("Preprocessing the %dth instance of program '%s' failed\n",
1404                                    i, prog->section_name);
1405                         goto out;
1406                 }
1407
1408                 if (!result.new_insn_ptr || !result.new_insn_cnt) {
1409                         pr_debug("Skip loading the %dth instance of program '%s'\n",
1410                                  i, prog->section_name);
1411                         prog->instances.fds[i] = -1;
1412                         if (result.pfd)
1413                                 *result.pfd = -1;
1414                         continue;
1415                 }
1416
1417                 err = load_program(prog->type, prog->expected_attach_type,
1418                                    prog->name, result.new_insn_ptr,
1419                                    result.new_insn_cnt,
1420                                    license, kern_version, &fd,
1421                                    prog->prog_ifindex);
1422
1423                 if (err) {
1424                         pr_warning("Loading the %dth instance of program '%s' failed\n",
1425                                         i, prog->section_name);
1426                         goto out;
1427                 }
1428
1429                 if (result.pfd)
1430                         *result.pfd = fd;
1431                 prog->instances.fds[i] = fd;
1432         }
1433 out:
1434         if (err)
1435                 pr_warning("failed to load program '%s'\n",
1436                            prog->section_name);
1437         zfree(&prog->insns);
1438         prog->insns_cnt = 0;
1439         return err;
1440 }
1441
1442 static bool bpf_program__is_function_storage(struct bpf_program *prog,
1443                                              struct bpf_object *obj)
1444 {
1445         return prog->idx == obj->efile.text_shndx && obj->has_pseudo_calls;
1446 }
1447
1448 static int
1449 bpf_object__load_progs(struct bpf_object *obj)
1450 {
1451         size_t i;
1452         int err;
1453
1454         for (i = 0; i < obj->nr_programs; i++) {
1455                 if (bpf_program__is_function_storage(&obj->programs[i], obj))
1456                         continue;
1457                 err = bpf_program__load(&obj->programs[i],
1458                                         obj->license,
1459                                         obj->kern_version);
1460                 if (err)
1461                         return err;
1462         }
1463         return 0;
1464 }
1465
1466 static bool bpf_prog_type__needs_kver(enum bpf_prog_type type)
1467 {
1468         switch (type) {
1469         case BPF_PROG_TYPE_SOCKET_FILTER:
1470         case BPF_PROG_TYPE_SCHED_CLS:
1471         case BPF_PROG_TYPE_SCHED_ACT:
1472         case BPF_PROG_TYPE_XDP:
1473         case BPF_PROG_TYPE_CGROUP_SKB:
1474         case BPF_PROG_TYPE_CGROUP_SOCK:
1475         case BPF_PROG_TYPE_LWT_IN:
1476         case BPF_PROG_TYPE_LWT_OUT:
1477         case BPF_PROG_TYPE_LWT_XMIT:
1478         case BPF_PROG_TYPE_LWT_SEG6LOCAL:
1479         case BPF_PROG_TYPE_SOCK_OPS:
1480         case BPF_PROG_TYPE_SK_SKB:
1481         case BPF_PROG_TYPE_CGROUP_DEVICE:
1482         case BPF_PROG_TYPE_SK_MSG:
1483         case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
1484         case BPF_PROG_TYPE_LIRC_MODE2:
1485                 return false;
1486         case BPF_PROG_TYPE_UNSPEC:
1487         case BPF_PROG_TYPE_KPROBE:
1488         case BPF_PROG_TYPE_TRACEPOINT:
1489         case BPF_PROG_TYPE_PERF_EVENT:
1490         case BPF_PROG_TYPE_RAW_TRACEPOINT:
1491         default:
1492                 return true;
1493         }
1494 }
1495
1496 static int bpf_object__validate(struct bpf_object *obj, bool needs_kver)
1497 {
1498         if (needs_kver && obj->kern_version == 0) {
1499                 pr_warning("%s doesn't provide kernel version\n",
1500                            obj->path);
1501                 return -LIBBPF_ERRNO__KVERSION;
1502         }
1503         return 0;
1504 }
1505
1506 static struct bpf_object *
1507 __bpf_object__open(const char *path, void *obj_buf, size_t obj_buf_sz,
1508                    bool needs_kver)
1509 {
1510         struct bpf_object *obj;
1511         int err;
1512
1513         if (elf_version(EV_CURRENT) == EV_NONE) {
1514                 pr_warning("failed to init libelf for %s\n", path);
1515                 return ERR_PTR(-LIBBPF_ERRNO__LIBELF);
1516         }
1517
1518         obj = bpf_object__new(path, obj_buf, obj_buf_sz);
1519         if (IS_ERR(obj))
1520                 return obj;
1521
1522         CHECK_ERR(bpf_object__elf_init(obj), err, out);
1523         CHECK_ERR(bpf_object__check_endianness(obj), err, out);
1524         CHECK_ERR(bpf_object__elf_collect(obj), err, out);
1525         CHECK_ERR(bpf_object__collect_reloc(obj), err, out);
1526         CHECK_ERR(bpf_object__validate(obj, needs_kver), err, out);
1527
1528         bpf_object__elf_finish(obj);
1529         return obj;
1530 out:
1531         bpf_object__close(obj);
1532         return ERR_PTR(err);
1533 }
1534
1535 struct bpf_object *bpf_object__open_xattr(struct bpf_object_open_attr *attr)
1536 {
1537         /* param validation */
1538         if (!attr->file)
1539                 return NULL;
1540
1541         pr_debug("loading %s\n", attr->file);
1542
1543         return __bpf_object__open(attr->file, NULL, 0,
1544                                   bpf_prog_type__needs_kver(attr->prog_type));
1545 }
1546
1547 struct bpf_object *bpf_object__open(const char *path)
1548 {
1549         struct bpf_object_open_attr attr = {
1550                 .file           = path,
1551                 .prog_type      = BPF_PROG_TYPE_UNSPEC,
1552         };
1553
1554         return bpf_object__open_xattr(&attr);
1555 }
1556
1557 struct bpf_object *bpf_object__open_buffer(void *obj_buf,
1558                                            size_t obj_buf_sz,
1559                                            const char *name)
1560 {
1561         char tmp_name[64];
1562
1563         /* param validation */
1564         if (!obj_buf || obj_buf_sz <= 0)
1565                 return NULL;
1566
1567         if (!name) {
1568                 snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx",
1569                          (unsigned long)obj_buf,
1570                          (unsigned long)obj_buf_sz);
1571                 tmp_name[sizeof(tmp_name) - 1] = '\0';
1572                 name = tmp_name;
1573         }
1574         pr_debug("loading object '%s' from buffer\n",
1575                  name);
1576
1577         return __bpf_object__open(name, obj_buf, obj_buf_sz, true);
1578 }
1579
1580 int bpf_object__unload(struct bpf_object *obj)
1581 {
1582         size_t i;
1583
1584         if (!obj)
1585                 return -EINVAL;
1586
1587         for (i = 0; i < obj->nr_maps; i++)
1588                 zclose(obj->maps[i].fd);
1589
1590         for (i = 0; i < obj->nr_programs; i++)
1591                 bpf_program__unload(&obj->programs[i]);
1592
1593         return 0;
1594 }
1595
1596 int bpf_object__load(struct bpf_object *obj)
1597 {
1598         int err;
1599
1600         if (!obj)
1601                 return -EINVAL;
1602
1603         if (obj->loaded) {
1604                 pr_warning("object should not be loaded twice\n");
1605                 return -EINVAL;
1606         }
1607
1608         obj->loaded = true;
1609
1610         CHECK_ERR(bpf_object__create_maps(obj), err, out);
1611         CHECK_ERR(bpf_object__relocate(obj), err, out);
1612         CHECK_ERR(bpf_object__load_progs(obj), err, out);
1613
1614         return 0;
1615 out:
1616         bpf_object__unload(obj);
1617         pr_warning("failed to load object '%s'\n", obj->path);
1618         return err;
1619 }
1620
1621 static int check_path(const char *path)
1622 {
1623         struct statfs st_fs;
1624         char *dname, *dir;
1625         int err = 0;
1626
1627         if (path == NULL)
1628                 return -EINVAL;
1629
1630         dname = strdup(path);
1631         if (dname == NULL)
1632                 return -ENOMEM;
1633
1634         dir = dirname(dname);
1635         if (statfs(dir, &st_fs)) {
1636                 pr_warning("failed to statfs %s: %s\n", dir, strerror(errno));
1637                 err = -errno;
1638         }
1639         free(dname);
1640
1641         if (!err && st_fs.f_type != BPF_FS_MAGIC) {
1642                 pr_warning("specified path %s is not on BPF FS\n", path);
1643                 err = -EINVAL;
1644         }
1645
1646         return err;
1647 }
1648
1649 int bpf_program__pin_instance(struct bpf_program *prog, const char *path,
1650                               int instance)
1651 {
1652         int err;
1653
1654         err = check_path(path);
1655         if (err)
1656                 return err;
1657
1658         if (prog == NULL) {
1659                 pr_warning("invalid program pointer\n");
1660                 return -EINVAL;
1661         }
1662
1663         if (instance < 0 || instance >= prog->instances.nr) {
1664                 pr_warning("invalid prog instance %d of prog %s (max %d)\n",
1665                            instance, prog->section_name, prog->instances.nr);
1666                 return -EINVAL;
1667         }
1668
1669         if (bpf_obj_pin(prog->instances.fds[instance], path)) {
1670                 pr_warning("failed to pin program: %s\n", strerror(errno));
1671                 return -errno;
1672         }
1673         pr_debug("pinned program '%s'\n", path);
1674
1675         return 0;
1676 }
1677
1678 static int make_dir(const char *path)
1679 {
1680         int err = 0;
1681
1682         if (mkdir(path, 0700) && errno != EEXIST)
1683                 err = -errno;
1684
1685         if (err)
1686                 pr_warning("failed to mkdir %s: %s\n", path, strerror(-err));
1687         return err;
1688 }
1689
1690 int bpf_program__pin(struct bpf_program *prog, const char *path)
1691 {
1692         int i, err;
1693
1694         err = check_path(path);
1695         if (err)
1696                 return err;
1697
1698         if (prog == NULL) {
1699                 pr_warning("invalid program pointer\n");
1700                 return -EINVAL;
1701         }
1702
1703         if (prog->instances.nr <= 0) {
1704                 pr_warning("no instances of prog %s to pin\n",
1705                            prog->section_name);
1706                 return -EINVAL;
1707         }
1708
1709         err = make_dir(path);
1710         if (err)
1711                 return err;
1712
1713         for (i = 0; i < prog->instances.nr; i++) {
1714                 char buf[PATH_MAX];
1715                 int len;
1716
1717                 len = snprintf(buf, PATH_MAX, "%s/%d", path, i);
1718                 if (len < 0)
1719                         return -EINVAL;
1720                 else if (len >= PATH_MAX)
1721                         return -ENAMETOOLONG;
1722
1723                 err = bpf_program__pin_instance(prog, buf, i);
1724                 if (err)
1725                         return err;
1726         }
1727
1728         return 0;
1729 }
1730
1731 int bpf_map__pin(struct bpf_map *map, const char *path)
1732 {
1733         int err;
1734
1735         err = check_path(path);
1736         if (err)
1737                 return err;
1738
1739         if (map == NULL) {
1740                 pr_warning("invalid map pointer\n");
1741                 return -EINVAL;
1742         }
1743
1744         if (bpf_obj_pin(map->fd, path)) {
1745                 pr_warning("failed to pin map: %s\n", strerror(errno));
1746                 return -errno;
1747         }
1748
1749         pr_debug("pinned map '%s'\n", path);
1750         return 0;
1751 }
1752
1753 int bpf_object__pin(struct bpf_object *obj, const char *path)
1754 {
1755         struct bpf_program *prog;
1756         struct bpf_map *map;
1757         int err;
1758
1759         if (!obj)
1760                 return -ENOENT;
1761
1762         if (!obj->loaded) {
1763                 pr_warning("object not yet loaded; load it first\n");
1764                 return -ENOENT;
1765         }
1766
1767         err = make_dir(path);
1768         if (err)
1769                 return err;
1770
1771         bpf_map__for_each(map, obj) {
1772                 char buf[PATH_MAX];
1773                 int len;
1774
1775                 len = snprintf(buf, PATH_MAX, "%s/%s", path,
1776                                bpf_map__name(map));
1777                 if (len < 0)
1778                         return -EINVAL;
1779                 else if (len >= PATH_MAX)
1780                         return -ENAMETOOLONG;
1781
1782                 err = bpf_map__pin(map, buf);
1783                 if (err)
1784                         return err;
1785         }
1786
1787         bpf_object__for_each_program(prog, obj) {
1788                 char buf[PATH_MAX];
1789                 int len;
1790
1791                 len = snprintf(buf, PATH_MAX, "%s/%s", path,
1792                                prog->section_name);
1793                 if (len < 0)
1794                         return -EINVAL;
1795                 else if (len >= PATH_MAX)
1796                         return -ENAMETOOLONG;
1797
1798                 err = bpf_program__pin(prog, buf);
1799                 if (err)
1800                         return err;
1801         }
1802
1803         return 0;
1804 }
1805
1806 void bpf_object__close(struct bpf_object *obj)
1807 {
1808         size_t i;
1809
1810         if (!obj)
1811                 return;
1812
1813         if (obj->clear_priv)
1814                 obj->clear_priv(obj, obj->priv);
1815
1816         bpf_object__elf_finish(obj);
1817         bpf_object__unload(obj);
1818         btf__free(obj->btf);
1819
1820         for (i = 0; i < obj->nr_maps; i++) {
1821                 zfree(&obj->maps[i].name);
1822                 if (obj->maps[i].clear_priv)
1823                         obj->maps[i].clear_priv(&obj->maps[i],
1824                                                 obj->maps[i].priv);
1825                 obj->maps[i].priv = NULL;
1826                 obj->maps[i].clear_priv = NULL;
1827         }
1828         zfree(&obj->maps);
1829         obj->nr_maps = 0;
1830
1831         if (obj->programs && obj->nr_programs) {
1832                 for (i = 0; i < obj->nr_programs; i++)
1833                         bpf_program__exit(&obj->programs[i]);
1834         }
1835         zfree(&obj->programs);
1836
1837         list_del(&obj->list);
1838         free(obj);
1839 }
1840
1841 struct bpf_object *
1842 bpf_object__next(struct bpf_object *prev)
1843 {
1844         struct bpf_object *next;
1845
1846         if (!prev)
1847                 next = list_first_entry(&bpf_objects_list,
1848                                         struct bpf_object,
1849                                         list);
1850         else
1851                 next = list_next_entry(prev, list);
1852
1853         /* Empty list is noticed here so don't need checking on entry. */
1854         if (&next->list == &bpf_objects_list)
1855                 return NULL;
1856
1857         return next;
1858 }
1859
1860 const char *bpf_object__name(struct bpf_object *obj)
1861 {
1862         return obj ? obj->path : ERR_PTR(-EINVAL);
1863 }
1864
1865 unsigned int bpf_object__kversion(struct bpf_object *obj)
1866 {
1867         return obj ? obj->kern_version : 0;
1868 }
1869
1870 int bpf_object__btf_fd(const struct bpf_object *obj)
1871 {
1872         return obj->btf ? btf__fd(obj->btf) : -1;
1873 }
1874
1875 int bpf_object__set_priv(struct bpf_object *obj, void *priv,
1876                          bpf_object_clear_priv_t clear_priv)
1877 {
1878         if (obj->priv && obj->clear_priv)
1879                 obj->clear_priv(obj, obj->priv);
1880
1881         obj->priv = priv;
1882         obj->clear_priv = clear_priv;
1883         return 0;
1884 }
1885
1886 void *bpf_object__priv(struct bpf_object *obj)
1887 {
1888         return obj ? obj->priv : ERR_PTR(-EINVAL);
1889 }
1890
1891 static struct bpf_program *
1892 __bpf_program__next(struct bpf_program *prev, struct bpf_object *obj)
1893 {
1894         size_t idx;
1895
1896         if (!obj->programs)
1897                 return NULL;
1898         /* First handler */
1899         if (prev == NULL)
1900                 return &obj->programs[0];
1901
1902         if (prev->obj != obj) {
1903                 pr_warning("error: program handler doesn't match object\n");
1904                 return NULL;
1905         }
1906
1907         idx = (prev - obj->programs) + 1;
1908         if (idx >= obj->nr_programs)
1909                 return NULL;
1910         return &obj->programs[idx];
1911 }
1912
1913 struct bpf_program *
1914 bpf_program__next(struct bpf_program *prev, struct bpf_object *obj)
1915 {
1916         struct bpf_program *prog = prev;
1917
1918         do {
1919                 prog = __bpf_program__next(prog, obj);
1920         } while (prog && bpf_program__is_function_storage(prog, obj));
1921
1922         return prog;
1923 }
1924
1925 int bpf_program__set_priv(struct bpf_program *prog, void *priv,
1926                           bpf_program_clear_priv_t clear_priv)
1927 {
1928         if (prog->priv && prog->clear_priv)
1929                 prog->clear_priv(prog, prog->priv);
1930
1931         prog->priv = priv;
1932         prog->clear_priv = clear_priv;
1933         return 0;
1934 }
1935
1936 void *bpf_program__priv(struct bpf_program *prog)
1937 {
1938         return prog ? prog->priv : ERR_PTR(-EINVAL);
1939 }
1940
1941 void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex)
1942 {
1943         prog->prog_ifindex = ifindex;
1944 }
1945
1946 const char *bpf_program__title(struct bpf_program *prog, bool needs_copy)
1947 {
1948         const char *title;
1949
1950         title = prog->section_name;
1951         if (needs_copy) {
1952                 title = strdup(title);
1953                 if (!title) {
1954                         pr_warning("failed to strdup program title\n");
1955                         return ERR_PTR(-ENOMEM);
1956                 }
1957         }
1958
1959         return title;
1960 }
1961
1962 int bpf_program__fd(struct bpf_program *prog)
1963 {
1964         return bpf_program__nth_fd(prog, 0);
1965 }
1966
1967 int bpf_program__set_prep(struct bpf_program *prog, int nr_instances,
1968                           bpf_program_prep_t prep)
1969 {
1970         int *instances_fds;
1971
1972         if (nr_instances <= 0 || !prep)
1973                 return -EINVAL;
1974
1975         if (prog->instances.nr > 0 || prog->instances.fds) {
1976                 pr_warning("Can't set pre-processor after loading\n");
1977                 return -EINVAL;
1978         }
1979
1980         instances_fds = malloc(sizeof(int) * nr_instances);
1981         if (!instances_fds) {
1982                 pr_warning("alloc memory failed for fds\n");
1983                 return -ENOMEM;
1984         }
1985
1986         /* fill all fd with -1 */
1987         memset(instances_fds, -1, sizeof(int) * nr_instances);
1988
1989         prog->instances.nr = nr_instances;
1990         prog->instances.fds = instances_fds;
1991         prog->preprocessor = prep;
1992         return 0;
1993 }
1994
1995 int bpf_program__nth_fd(struct bpf_program *prog, int n)
1996 {
1997         int fd;
1998
1999         if (n >= prog->instances.nr || n < 0) {
2000                 pr_warning("Can't get the %dth fd from program %s: only %d instances\n",
2001                            n, prog->section_name, prog->instances.nr);
2002                 return -EINVAL;
2003         }
2004
2005         fd = prog->instances.fds[n];
2006         if (fd < 0) {
2007                 pr_warning("%dth instance of program '%s' is invalid\n",
2008                            n, prog->section_name);
2009                 return -ENOENT;
2010         }
2011
2012         return fd;
2013 }
2014
2015 void bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type)
2016 {
2017         prog->type = type;
2018 }
2019
2020 static bool bpf_program__is_type(struct bpf_program *prog,
2021                                  enum bpf_prog_type type)
2022 {
2023         return prog ? (prog->type == type) : false;
2024 }
2025
2026 #define BPF_PROG_TYPE_FNS(NAME, TYPE)                   \
2027 int bpf_program__set_##NAME(struct bpf_program *prog)   \
2028 {                                                       \
2029         if (!prog)                                      \
2030                 return -EINVAL;                         \
2031         bpf_program__set_type(prog, TYPE);              \
2032         return 0;                                       \
2033 }                                                       \
2034                                                         \
2035 bool bpf_program__is_##NAME(struct bpf_program *prog)   \
2036 {                                                       \
2037         return bpf_program__is_type(prog, TYPE);        \
2038 }                                                       \
2039
2040 BPF_PROG_TYPE_FNS(socket_filter, BPF_PROG_TYPE_SOCKET_FILTER);
2041 BPF_PROG_TYPE_FNS(kprobe, BPF_PROG_TYPE_KPROBE);
2042 BPF_PROG_TYPE_FNS(sched_cls, BPF_PROG_TYPE_SCHED_CLS);
2043 BPF_PROG_TYPE_FNS(sched_act, BPF_PROG_TYPE_SCHED_ACT);
2044 BPF_PROG_TYPE_FNS(tracepoint, BPF_PROG_TYPE_TRACEPOINT);
2045 BPF_PROG_TYPE_FNS(raw_tracepoint, BPF_PROG_TYPE_RAW_TRACEPOINT);
2046 BPF_PROG_TYPE_FNS(xdp, BPF_PROG_TYPE_XDP);
2047 BPF_PROG_TYPE_FNS(perf_event, BPF_PROG_TYPE_PERF_EVENT);
2048
2049 void bpf_program__set_expected_attach_type(struct bpf_program *prog,
2050                                            enum bpf_attach_type type)
2051 {
2052         prog->expected_attach_type = type;
2053 }
2054
2055 #define BPF_PROG_SEC_FULL(string, ptype, atype) \
2056         { string, sizeof(string) - 1, ptype, atype }
2057
2058 #define BPF_PROG_SEC(string, ptype) BPF_PROG_SEC_FULL(string, ptype, 0)
2059
2060 #define BPF_S_PROG_SEC(string, ptype) \
2061         BPF_PROG_SEC_FULL(string, BPF_PROG_TYPE_CGROUP_SOCK, ptype)
2062
2063 #define BPF_SA_PROG_SEC(string, ptype) \
2064         BPF_PROG_SEC_FULL(string, BPF_PROG_TYPE_CGROUP_SOCK_ADDR, ptype)
2065
2066 static const struct {
2067         const char *sec;
2068         size_t len;
2069         enum bpf_prog_type prog_type;
2070         enum bpf_attach_type expected_attach_type;
2071 } section_names[] = {
2072         BPF_PROG_SEC("socket",          BPF_PROG_TYPE_SOCKET_FILTER),
2073         BPF_PROG_SEC("kprobe/",         BPF_PROG_TYPE_KPROBE),
2074         BPF_PROG_SEC("kretprobe/",      BPF_PROG_TYPE_KPROBE),
2075         BPF_PROG_SEC("classifier",      BPF_PROG_TYPE_SCHED_CLS),
2076         BPF_PROG_SEC("action",          BPF_PROG_TYPE_SCHED_ACT),
2077         BPF_PROG_SEC("tracepoint/",     BPF_PROG_TYPE_TRACEPOINT),
2078         BPF_PROG_SEC("raw_tracepoint/", BPF_PROG_TYPE_RAW_TRACEPOINT),
2079         BPF_PROG_SEC("xdp",             BPF_PROG_TYPE_XDP),
2080         BPF_PROG_SEC("perf_event",      BPF_PROG_TYPE_PERF_EVENT),
2081         BPF_PROG_SEC("cgroup/skb",      BPF_PROG_TYPE_CGROUP_SKB),
2082         BPF_PROG_SEC("cgroup/sock",     BPF_PROG_TYPE_CGROUP_SOCK),
2083         BPF_PROG_SEC("cgroup/dev",      BPF_PROG_TYPE_CGROUP_DEVICE),
2084         BPF_PROG_SEC("lwt_in",          BPF_PROG_TYPE_LWT_IN),
2085         BPF_PROG_SEC("lwt_out",         BPF_PROG_TYPE_LWT_OUT),
2086         BPF_PROG_SEC("lwt_xmit",        BPF_PROG_TYPE_LWT_XMIT),
2087         BPF_PROG_SEC("lwt_seg6local",   BPF_PROG_TYPE_LWT_SEG6LOCAL),
2088         BPF_PROG_SEC("sockops",         BPF_PROG_TYPE_SOCK_OPS),
2089         BPF_PROG_SEC("sk_skb",          BPF_PROG_TYPE_SK_SKB),
2090         BPF_PROG_SEC("sk_msg",          BPF_PROG_TYPE_SK_MSG),
2091         BPF_PROG_SEC("lirc_mode2",      BPF_PROG_TYPE_LIRC_MODE2),
2092         BPF_SA_PROG_SEC("cgroup/bind4", BPF_CGROUP_INET4_BIND),
2093         BPF_SA_PROG_SEC("cgroup/bind6", BPF_CGROUP_INET6_BIND),
2094         BPF_SA_PROG_SEC("cgroup/connect4", BPF_CGROUP_INET4_CONNECT),
2095         BPF_SA_PROG_SEC("cgroup/connect6", BPF_CGROUP_INET6_CONNECT),
2096         BPF_SA_PROG_SEC("cgroup/sendmsg4", BPF_CGROUP_UDP4_SENDMSG),
2097         BPF_SA_PROG_SEC("cgroup/sendmsg6", BPF_CGROUP_UDP6_SENDMSG),
2098         BPF_S_PROG_SEC("cgroup/post_bind4", BPF_CGROUP_INET4_POST_BIND),
2099         BPF_S_PROG_SEC("cgroup/post_bind6", BPF_CGROUP_INET6_POST_BIND),
2100 };
2101
2102 #undef BPF_PROG_SEC
2103 #undef BPF_PROG_SEC_FULL
2104 #undef BPF_S_PROG_SEC
2105 #undef BPF_SA_PROG_SEC
2106
2107 int libbpf_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type,
2108                              enum bpf_attach_type *expected_attach_type)
2109 {
2110         int i;
2111
2112         if (!name)
2113                 return -EINVAL;
2114
2115         for (i = 0; i < ARRAY_SIZE(section_names); i++) {
2116                 if (strncmp(name, section_names[i].sec, section_names[i].len))
2117                         continue;
2118                 *prog_type = section_names[i].prog_type;
2119                 *expected_attach_type = section_names[i].expected_attach_type;
2120                 return 0;
2121         }
2122         return -EINVAL;
2123 }
2124
2125 static int
2126 bpf_program__identify_section(struct bpf_program *prog,
2127                               enum bpf_prog_type *prog_type,
2128                               enum bpf_attach_type *expected_attach_type)
2129 {
2130         return libbpf_prog_type_by_name(prog->section_name, prog_type,
2131                                         expected_attach_type);
2132 }
2133
2134 int bpf_map__fd(struct bpf_map *map)
2135 {
2136         return map ? map->fd : -EINVAL;
2137 }
2138
2139 const struct bpf_map_def *bpf_map__def(struct bpf_map *map)
2140 {
2141         return map ? &map->def : ERR_PTR(-EINVAL);
2142 }
2143
2144 const char *bpf_map__name(struct bpf_map *map)
2145 {
2146         return map ? map->name : NULL;
2147 }
2148
2149 __u32 bpf_map__btf_key_type_id(const struct bpf_map *map)
2150 {
2151         return map ? map->btf_key_type_id : 0;
2152 }
2153
2154 __u32 bpf_map__btf_value_type_id(const struct bpf_map *map)
2155 {
2156         return map ? map->btf_value_type_id : 0;
2157 }
2158
2159 int bpf_map__set_priv(struct bpf_map *map, void *priv,
2160                      bpf_map_clear_priv_t clear_priv)
2161 {
2162         if (!map)
2163                 return -EINVAL;
2164
2165         if (map->priv) {
2166                 if (map->clear_priv)
2167                         map->clear_priv(map, map->priv);
2168         }
2169
2170         map->priv = priv;
2171         map->clear_priv = clear_priv;
2172         return 0;
2173 }
2174
2175 void *bpf_map__priv(struct bpf_map *map)
2176 {
2177         return map ? map->priv : ERR_PTR(-EINVAL);
2178 }
2179
2180 bool bpf_map__is_offload_neutral(struct bpf_map *map)
2181 {
2182         return map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY;
2183 }
2184
2185 void bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex)
2186 {
2187         map->map_ifindex = ifindex;
2188 }
2189
2190 struct bpf_map *
2191 bpf_map__next(struct bpf_map *prev, struct bpf_object *obj)
2192 {
2193         size_t idx;
2194         struct bpf_map *s, *e;
2195
2196         if (!obj || !obj->maps)
2197                 return NULL;
2198
2199         s = obj->maps;
2200         e = obj->maps + obj->nr_maps;
2201
2202         if (prev == NULL)
2203                 return s;
2204
2205         if ((prev < s) || (prev >= e)) {
2206                 pr_warning("error in %s: map handler doesn't belong to object\n",
2207                            __func__);
2208                 return NULL;
2209         }
2210
2211         idx = (prev - obj->maps) + 1;
2212         if (idx >= obj->nr_maps)
2213                 return NULL;
2214         return &obj->maps[idx];
2215 }
2216
2217 struct bpf_map *
2218 bpf_object__find_map_by_name(struct bpf_object *obj, const char *name)
2219 {
2220         struct bpf_map *pos;
2221
2222         bpf_map__for_each(pos, obj) {
2223                 if (pos->name && !strcmp(pos->name, name))
2224                         return pos;
2225         }
2226         return NULL;
2227 }
2228
2229 struct bpf_map *
2230 bpf_object__find_map_by_offset(struct bpf_object *obj, size_t offset)
2231 {
2232         int i;
2233
2234         for (i = 0; i < obj->nr_maps; i++) {
2235                 if (obj->maps[i].offset == offset)
2236                         return &obj->maps[i];
2237         }
2238         return ERR_PTR(-ENOENT);
2239 }
2240
2241 long libbpf_get_error(const void *ptr)
2242 {
2243         if (IS_ERR(ptr))
2244                 return PTR_ERR(ptr);
2245         return 0;
2246 }
2247
2248 int bpf_prog_load(const char *file, enum bpf_prog_type type,
2249                   struct bpf_object **pobj, int *prog_fd)
2250 {
2251         struct bpf_prog_load_attr attr;
2252
2253         memset(&attr, 0, sizeof(struct bpf_prog_load_attr));
2254         attr.file = file;
2255         attr.prog_type = type;
2256         attr.expected_attach_type = 0;
2257
2258         return bpf_prog_load_xattr(&attr, pobj, prog_fd);
2259 }
2260
2261 int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr,
2262                         struct bpf_object **pobj, int *prog_fd)
2263 {
2264         struct bpf_object_open_attr open_attr = {
2265                 .file           = attr->file,
2266                 .prog_type      = attr->prog_type,
2267         };
2268         struct bpf_program *prog, *first_prog = NULL;
2269         enum bpf_attach_type expected_attach_type;
2270         enum bpf_prog_type prog_type;
2271         struct bpf_object *obj;
2272         struct bpf_map *map;
2273         int err;
2274
2275         if (!attr)
2276                 return -EINVAL;
2277         if (!attr->file)
2278                 return -EINVAL;
2279
2280         obj = bpf_object__open_xattr(&open_attr);
2281         if (IS_ERR_OR_NULL(obj))
2282                 return -ENOENT;
2283
2284         bpf_object__for_each_program(prog, obj) {
2285                 /*
2286                  * If type is not specified, try to guess it based on
2287                  * section name.
2288                  */
2289                 prog_type = attr->prog_type;
2290                 prog->prog_ifindex = attr->ifindex;
2291                 expected_attach_type = attr->expected_attach_type;
2292                 if (prog_type == BPF_PROG_TYPE_UNSPEC) {
2293                         err = bpf_program__identify_section(prog, &prog_type,
2294                                                             &expected_attach_type);
2295                         if (err < 0) {
2296                                 pr_warning("failed to guess program type based on section name %s\n",
2297                                            prog->section_name);
2298                                 bpf_object__close(obj);
2299                                 return -EINVAL;
2300                         }
2301                 }
2302
2303                 bpf_program__set_type(prog, prog_type);
2304                 bpf_program__set_expected_attach_type(prog,
2305                                                       expected_attach_type);
2306
2307                 if (!bpf_program__is_function_storage(prog, obj) && !first_prog)
2308                         first_prog = prog;
2309         }
2310
2311         bpf_map__for_each(map, obj) {
2312                 if (!bpf_map__is_offload_neutral(map))
2313                         map->map_ifindex = attr->ifindex;
2314         }
2315
2316         if (!first_prog) {
2317                 pr_warning("object file doesn't contain bpf program\n");
2318                 bpf_object__close(obj);
2319                 return -ENOENT;
2320         }
2321
2322         err = bpf_object__load(obj);
2323         if (err) {
2324                 bpf_object__close(obj);
2325                 return -EINVAL;
2326         }
2327
2328         *pobj = obj;
2329         *prog_fd = bpf_program__fd(first_prog);
2330         return 0;
2331 }
2332
2333 enum bpf_perf_event_ret
2334 bpf_perf_event_read_simple(void *mem, unsigned long size,
2335                            unsigned long page_size, void **buf, size_t *buf_len,
2336                            bpf_perf_event_print_t fn, void *priv)
2337 {
2338         volatile struct perf_event_mmap_page *header = mem;
2339         __u64 data_tail = header->data_tail;
2340         __u64 data_head = header->data_head;
2341         int ret = LIBBPF_PERF_EVENT_ERROR;
2342         void *base, *begin, *end;
2343
2344         asm volatile("" ::: "memory"); /* in real code it should be smp_rmb() */
2345         if (data_head == data_tail)
2346                 return LIBBPF_PERF_EVENT_CONT;
2347
2348         base = ((char *)header) + page_size;
2349
2350         begin = base + data_tail % size;
2351         end = base + data_head % size;
2352
2353         while (begin != end) {
2354                 struct perf_event_header *ehdr;
2355
2356                 ehdr = begin;
2357                 if (begin + ehdr->size > base + size) {
2358                         long len = base + size - begin;
2359
2360                         if (*buf_len < ehdr->size) {
2361                                 free(*buf);
2362                                 *buf = malloc(ehdr->size);
2363                                 if (!*buf) {
2364                                         ret = LIBBPF_PERF_EVENT_ERROR;
2365                                         break;
2366                                 }
2367                                 *buf_len = ehdr->size;
2368                         }
2369
2370                         memcpy(*buf, begin, len);
2371                         memcpy(*buf + len, base, ehdr->size - len);
2372                         ehdr = (void *)*buf;
2373                         begin = base + ehdr->size - len;
2374                 } else if (begin + ehdr->size == base + size) {
2375                         begin = base;
2376                 } else {
2377                         begin += ehdr->size;
2378                 }
2379
2380                 ret = fn(ehdr, priv);
2381                 if (ret != LIBBPF_PERF_EVENT_CONT)
2382                         break;
2383
2384                 data_tail += ehdr->size;
2385         }
2386
2387         __sync_synchronize(); /* smp_mb() */
2388         header->data_tail = data_tail;
2389
2390         return ret;
2391 }