tools/testing/selftests/bpf/test_progs.c

   1 /* Copyright (c) 2017 Facebook
   2  *
   3  * This program is free software; you can redistribute it and/or
   4  * modify it under the terms of version 2 of the GNU General Public
   5  * License as published by the Free Software Foundation.
   6  */
   7 #include <stdio.h>
   8 #include <unistd.h>
   9 #include <errno.h>
  10 #include <string.h>
  11 #include <assert.h>
  12 #include <stdlib.h>
  13 #include <time.h>
  14
  15 #include <linux/types.h>
  16 typedef __u16 __sum16;
  17 #include <arpa/inet.h>
  18 #include <linux/if_ether.h>
  19 #include <linux/if_packet.h>
  20 #include <linux/ip.h>
  21 #include <linux/ipv6.h>
  22 #include <linux/tcp.h>
  23 #include <linux/filter.h>
  24 #include <linux/perf_event.h>
  25 #include <linux/unistd.h>
  26
  27 #include <sys/ioctl.h>
  28 #include <sys/wait.h>
  29 #include <sys/types.h>
  30 #include <fcntl.h>
  31
  32 #include <linux/bpf.h>
  33 #include <linux/err.h>
  34 #include <bpf/bpf.h>
  35 #include <bpf/libbpf.h>
  36
  37 #include "test_iptunnel_common.h"
  38 #include "bpf_util.h"
  39 #include "bpf_endian.h"
  40 #include "bpf_rlimit.h"
  41 #include "trace_helpers.h"
  42
  43 static int error_cnt, pass_cnt;
  44 static bool jit_enabled;
  45
  46 #define MAGIC_BYTES 123
  47
  48 /* ipv4 test vector */
  49 static struct {
  50         struct ethhdr eth;
  51         struct iphdr iph;
  52         struct tcphdr tcp;
  53 } __packed pkt_v4 = {
  54         .eth.h_proto = __bpf_constant_htons(ETH_P_IP),
  55         .iph.ihl = 5,
  56         .iph.protocol = 6,
  57         .iph.tot_len = __bpf_constant_htons(MAGIC_BYTES),
  58         .tcp.urg_ptr = 123,
  59 };
  60
  61 /* ipv6 test vector */
  62 static struct {
  63         struct ethhdr eth;
  64         struct ipv6hdr iph;
  65         struct tcphdr tcp;
  66 } __packed pkt_v6 = {
  67         .eth.h_proto = __bpf_constant_htons(ETH_P_IPV6),
  68         .iph.nexthdr = 6,
  69         .iph.payload_len = __bpf_constant_htons(MAGIC_BYTES),
  70         .tcp.urg_ptr = 123,
  71 };
  72
  73 #define _CHECK(condition, tag, duration, format...) ({                  \
  74         int __ret = !!(condition);                                      \
  75         if (__ret) {                                                    \
  76                 error_cnt++;                                            \
  77                 printf("%s:FAIL:%s ", __func__, tag);                   \
  78                 printf(format);                                         \
  79         } else {                                                        \
  80                 pass_cnt++;                                             \
  81                 printf("%s:PASS:%s %d nsec\n", __func__, tag, duration);\
  82         }                                                               \
  83         __ret;                                                          \
  84 })
  85
  86 #define CHECK(condition, tag, format...) \
  87         _CHECK(condition, tag, duration, format)
  88 #define CHECK_ATTR(condition, tag, format...) \
  89         _CHECK(condition, tag, tattr.duration, format)
  90
  91 static int bpf_find_map(const char *test, struct bpf_object *obj,
  92                         const char *name)
  93 {
  94         struct bpf_map *map;
  95
  96         map = bpf_object__find_map_by_name(obj, name);
  97         if (!map) {
  98                 printf("%s:FAIL:map '%s' not found\n", test, name);
  99                 error_cnt++;
 100                 return -1;
 101         }
 102         return bpf_map__fd(map);
 103 }
 104
 105 static void test_pkt_access(void)
 106 {
 107         const char *file = "./test_pkt_access.o";
 108         struct bpf_object *obj;
 109         __u32 duration, retval;
 110         int err, prog_fd;
 111
 112         err = bpf_prog_load(file, BPF_PROG_TYPE_SCHED_CLS, &obj, &prog_fd);
 113         if (err) {
 114                 error_cnt++;
 115                 return;
 116         }
 117
 118         err = bpf_prog_test_run(prog_fd, 100000, &pkt_v4, sizeof(pkt_v4),
 119                                 NULL, NULL, &retval, &duration);
 120         CHECK(err || retval, "ipv4",
 121               "err %d errno %d retval %d duration %d\n",
 122               err, errno, retval, duration);
 123
 124         err = bpf_prog_test_run(prog_fd, 100000, &pkt_v6, sizeof(pkt_v6),
 125                                 NULL, NULL, &retval, &duration);
 126         CHECK(err || retval, "ipv6",
 127               "err %d errno %d retval %d duration %d\n",
 128               err, errno, retval, duration);
 129         bpf_object__close(obj);
 130 }
 131
 132 static void test_prog_run_xattr(void)
 133 {
 134         const char *file = "./test_pkt_access.o";
 135         struct bpf_object *obj;
 136         char buf[10];
 137         int err;
 138         struct bpf_prog_test_run_attr tattr = {
 139                 .repeat = 1,
 140                 .data_in = &pkt_v4,
 141                 .data_size_in = sizeof(pkt_v4),
 142                 .data_out = buf,
 143                 .data_size_out = 5,
 144         };
 145
 146         err = bpf_prog_load(file, BPF_PROG_TYPE_SCHED_CLS, &obj,
 147                             &tattr.prog_fd);
 148         if (CHECK_ATTR(err, "load", "err %d errno %d\n", err, errno))
 149                 return;
 150
 151         memset(buf, 0, sizeof(buf));
 152
 153         err = bpf_prog_test_run_xattr(&tattr);
 154         CHECK_ATTR(err != -1 || errno != ENOSPC || tattr.retval, "run",
 155               "err %d errno %d retval %d\n", err, errno, tattr.retval);
 156
 157         CHECK_ATTR(tattr.data_size_out != sizeof(pkt_v4), "data_size_out",
 158               "incorrect output size, want %lu have %u\n",
 159               sizeof(pkt_v4), tattr.data_size_out);
 160
 161         CHECK_ATTR(buf[5] != 0, "overflow",
 162               "BPF_PROG_TEST_RUN ignored size hint\n");
 163
 164         tattr.data_out = NULL;
 165         tattr.data_size_out = 0;
 166         errno = 0;
 167
 168         err = bpf_prog_test_run_xattr(&tattr);
 169         CHECK_ATTR(err || errno || tattr.retval, "run_no_output",
 170               "err %d errno %d retval %d\n", err, errno, tattr.retval);
 171
 172         tattr.data_size_out = 1;
 173         err = bpf_prog_test_run_xattr(&tattr);
 174         CHECK_ATTR(err != -EINVAL, "run_wrong_size_out", "err %d\n", err);
 175
 176         bpf_object__close(obj);
 177 }
 178
 179 static void test_xdp(void)
 180 {
 181         struct vip key4 = {.protocol = 6, .family = AF_INET};
 182         struct vip key6 = {.protocol = 6, .family = AF_INET6};
 183         struct iptnl_info value4 = {.family = AF_INET};
 184         struct iptnl_info value6 = {.family = AF_INET6};
 185         const char *file = "./test_xdp.o";
 186         struct bpf_object *obj;
 187         char buf[128];
 188         struct ipv6hdr *iph6 = (void *)buf + sizeof(struct ethhdr);
 189         struct iphdr *iph = (void *)buf + sizeof(struct ethhdr);
 190         __u32 duration, retval, size;
 191         int err, prog_fd, map_fd;
 192
 193         err = bpf_prog_load(file, BPF_PROG_TYPE_XDP, &obj, &prog_fd);
 194         if (err) {
 195                 error_cnt++;
 196                 return;
 197         }
 198
 199         map_fd = bpf_find_map(__func__, obj, "vip2tnl");
 200         if (map_fd < 0)
 201                 goto out;
 202         bpf_map_update_elem(map_fd, &key4, &value4, 0);
 203         bpf_map_update_elem(map_fd, &key6, &value6, 0);
 204
 205         err = bpf_prog_test_run(prog_fd, 1, &pkt_v4, sizeof(pkt_v4),
 206                                 buf, &size, &retval, &duration);
 207
 208         CHECK(err || retval != XDP_TX || size != 74 ||
 209               iph->protocol != IPPROTO_IPIP, "ipv4",
 210               "err %d errno %d retval %d size %d\n",
 211               err, errno, retval, size);
 212
 213         err = bpf_prog_test_run(prog_fd, 1, &pkt_v6, sizeof(pkt_v6),
 214                                 buf, &size, &retval, &duration);
 215         CHECK(err || retval != XDP_TX || size != 114 ||
 216               iph6->nexthdr != IPPROTO_IPV6, "ipv6",
 217               "err %d errno %d retval %d size %d\n",
 218               err, errno, retval, size);
 219 out:
 220         bpf_object__close(obj);
 221 }
 222
 223 static void test_xdp_adjust_tail(void)
 224 {
 225         const char *file = "./test_adjust_tail.o";
 226         struct bpf_object *obj;
 227         char buf[128];
 228         __u32 duration, retval, size;
 229         int err, prog_fd;
 230
 231         err = bpf_prog_load(file, BPF_PROG_TYPE_XDP, &obj, &prog_fd);
 232         if (err) {
 233                 error_cnt++;
 234                 return;
 235         }
 236
 237         err = bpf_prog_test_run(prog_fd, 1, &pkt_v4, sizeof(pkt_v4),
 238                                 buf, &size, &retval, &duration);
 239
 240         CHECK(err || retval != XDP_DROP,
 241               "ipv4", "err %d errno %d retval %d size %d\n",
 242               err, errno, retval, size);
 243
 244         err = bpf_prog_test_run(prog_fd, 1, &pkt_v6, sizeof(pkt_v6),
 245                                 buf, &size, &retval, &duration);
 246         CHECK(err || retval != XDP_TX || size != 54,
 247               "ipv6", "err %d errno %d retval %d size %d\n",
 248               err, errno, retval, size);
 249         bpf_object__close(obj);
 250 }
 251
 252
 253
 254 #define MAGIC_VAL 0x1234
 255 #define NUM_ITER 100000
 256 #define VIP_NUM 5
 257
 258 static void test_l4lb(const char *file)
 259 {
 260         unsigned int nr_cpus = bpf_num_possible_cpus();
 261         struct vip key = {.protocol = 6};
 262         struct vip_meta {
 263                 __u32 flags;
 264                 __u32 vip_num;
 265         } value = {.vip_num = VIP_NUM};
 266         __u32 stats_key = VIP_NUM;
 267         struct vip_stats {
 268                 __u64 bytes;
 269                 __u64 pkts;
 270         } stats[nr_cpus];
 271         struct real_definition {
 272                 union {
 273                         __be32 dst;
 274                         __be32 dstv6[4];
 275                 };
 276                 __u8 flags;
 277         } real_def = {.dst = MAGIC_VAL};
 278         __u32 ch_key = 11, real_num = 3;
 279         __u32 duration, retval, size;
 280         int err, i, prog_fd, map_fd;
 281         __u64 bytes = 0, pkts = 0;
 282         struct bpf_object *obj;
 283         char buf[128];
 284         u32 *magic = (u32 *)buf;
 285
 286         err = bpf_prog_load(file, BPF_PROG_TYPE_SCHED_CLS, &obj, &prog_fd);
 287         if (err) {
 288                 error_cnt++;
 289                 return;
 290         }
 291
 292         map_fd = bpf_find_map(__func__, obj, "vip_map");
 293         if (map_fd < 0)
 294                 goto out;
 295         bpf_map_update_elem(map_fd, &key, &value, 0);
 296
 297         map_fd = bpf_find_map(__func__, obj, "ch_rings");
 298         if (map_fd < 0)
 299                 goto out;
 300         bpf_map_update_elem(map_fd, &ch_key, &real_num, 0);
 301
 302         map_fd = bpf_find_map(__func__, obj, "reals");
 303         if (map_fd < 0)
 304                 goto out;
 305         bpf_map_update_elem(map_fd, &real_num, &real_def, 0);
 306
 307         err = bpf_prog_test_run(prog_fd, NUM_ITER, &pkt_v4, sizeof(pkt_v4),
 308                                 buf, &size, &retval, &duration);
 309         CHECK(err || retval != 7/*TC_ACT_REDIRECT*/ || size != 54 ||
 310               *magic != MAGIC_VAL, "ipv4",
 311               "err %d errno %d retval %d size %d magic %x\n",
 312               err, errno, retval, size, *magic);
 313
 314         err = bpf_prog_test_run(prog_fd, NUM_ITER, &pkt_v6, sizeof(pkt_v6),
 315                                 buf, &size, &retval, &duration);
 316         CHECK(err || retval != 7/*TC_ACT_REDIRECT*/ || size != 74 ||
 317               *magic != MAGIC_VAL, "ipv6",
 318               "err %d errno %d retval %d size %d magic %x\n",
 319               err, errno, retval, size, *magic);
 320
 321         map_fd = bpf_find_map(__func__, obj, "stats");
 322         if (map_fd < 0)
 323                 goto out;
 324         bpf_map_lookup_elem(map_fd, &stats_key, stats);
 325         for (i = 0; i < nr_cpus; i++) {
 326                 bytes += stats[i].bytes;
 327                 pkts += stats[i].pkts;
 328         }
 329         if (bytes != MAGIC_BYTES * NUM_ITER * 2 || pkts != NUM_ITER * 2) {
 330                 error_cnt++;
 331                 printf("test_l4lb:FAIL:stats %lld %lld\n", bytes, pkts);
 332         }
 333 out:
 334         bpf_object__close(obj);
 335 }
 336
 337 static void test_l4lb_all(void)
 338 {
 339         const char *file1 = "./test_l4lb.o";
 340         const char *file2 = "./test_l4lb_noinline.o";
 341
 342         test_l4lb(file1);
 343         test_l4lb(file2);
 344 }
 345
 346 static void test_xdp_noinline(void)
 347 {
 348         const char *file = "./test_xdp_noinline.o";
 349         unsigned int nr_cpus = bpf_num_possible_cpus();
 350         struct vip key = {.protocol = 6};
 351         struct vip_meta {
 352                 __u32 flags;
 353                 __u32 vip_num;
 354         } value = {.vip_num = VIP_NUM};
 355         __u32 stats_key = VIP_NUM;
 356         struct vip_stats {
 357                 __u64 bytes;
 358                 __u64 pkts;
 359         } stats[nr_cpus];
 360         struct real_definition {
 361                 union {
 362                         __be32 dst;
 363                         __be32 dstv6[4];
 364                 };
 365                 __u8 flags;
 366         } real_def = {.dst = MAGIC_VAL};
 367         __u32 ch_key = 11, real_num = 3;
 368         __u32 duration, retval, size;
 369         int err, i, prog_fd, map_fd;
 370         __u64 bytes = 0, pkts = 0;
 371         struct bpf_object *obj;
 372         char buf[128];
 373         u32 *magic = (u32 *)buf;
 374
 375         err = bpf_prog_load(file, BPF_PROG_TYPE_XDP, &obj, &prog_fd);
 376         if (err) {
 377                 error_cnt++;
 378                 return;
 379         }
 380
 381         map_fd = bpf_find_map(__func__, obj, "vip_map");
 382         if (map_fd < 0)
 383                 goto out;
 384         bpf_map_update_elem(map_fd, &key, &value, 0);
 385
 386         map_fd = bpf_find_map(__func__, obj, "ch_rings");
 387         if (map_fd < 0)
 388                 goto out;
 389         bpf_map_update_elem(map_fd, &ch_key, &real_num, 0);
 390
 391         map_fd = bpf_find_map(__func__, obj, "reals");
 392         if (map_fd < 0)
 393                 goto out;
 394         bpf_map_update_elem(map_fd, &real_num, &real_def, 0);
 395
 396         err = bpf_prog_test_run(prog_fd, NUM_ITER, &pkt_v4, sizeof(pkt_v4),
 397                                 buf, &size, &retval, &duration);
 398         CHECK(err || retval != 1 || size != 54 ||
 399               *magic != MAGIC_VAL, "ipv4",
 400               "err %d errno %d retval %d size %d magic %x\n",
 401               err, errno, retval, size, *magic);
 402
 403         err = bpf_prog_test_run(prog_fd, NUM_ITER, &pkt_v6, sizeof(pkt_v6),
 404                                 buf, &size, &retval, &duration);
 405         CHECK(err || retval != 1 || size != 74 ||
 406               *magic != MAGIC_VAL, "ipv6",
 407               "err %d errno %d retval %d size %d magic %x\n",
 408               err, errno, retval, size, *magic);
 409
 410         map_fd = bpf_find_map(__func__, obj, "stats");
 411         if (map_fd < 0)
 412                 goto out;
 413         bpf_map_lookup_elem(map_fd, &stats_key, stats);
 414         for (i = 0; i < nr_cpus; i++) {
 415                 bytes += stats[i].bytes;
 416                 pkts += stats[i].pkts;
 417         }
 418         if (bytes != MAGIC_BYTES * NUM_ITER * 2 || pkts != NUM_ITER * 2) {
 419                 error_cnt++;
 420                 printf("test_xdp_noinline:FAIL:stats %lld %lld\n", bytes, pkts);
 421         }
 422 out:
 423         bpf_object__close(obj);
 424 }
 425
 426 static void test_tcp_estats(void)
 427 {
 428         const char *file = "./test_tcp_estats.o";
 429         int err, prog_fd;
 430         struct bpf_object *obj;
 431         __u32 duration = 0;
 432
 433         err = bpf_prog_load(file, BPF_PROG_TYPE_TRACEPOINT, &obj, &prog_fd);
 434         CHECK(err, "", "err %d errno %d\n", err, errno);
 435         if (err) {
 436                 error_cnt++;
 437                 return;
 438         }
 439
 440         bpf_object__close(obj);
 441 }
 442
 443 static inline __u64 ptr_to_u64(const void *ptr)
 444 {
 445         return (__u64) (unsigned long) ptr;
 446 }
 447
 448 static bool is_jit_enabled(void)
 449 {
 450         const char *jit_sysctl = "/proc/sys/net/core/bpf_jit_enable";
 451         bool enabled = false;
 452         int sysctl_fd;
 453
 454         sysctl_fd = open(jit_sysctl, 0, O_RDONLY);
 455         if (sysctl_fd != -1) {
 456                 char tmpc;
 457
 458                 if (read(sysctl_fd, &tmpc, sizeof(tmpc)) == 1)
 459                         enabled = (tmpc != '0');
 460                 close(sysctl_fd);
 461         }
 462
 463         return enabled;
 464 }
 465
 466 static void test_bpf_obj_id(void)
 467 {
 468         const __u64 array_magic_value = 0xfaceb00c;
 469         const __u32 array_key = 0;
 470         const int nr_iters = 2;
 471         const char *file = "./test_obj_id.o";
 472         const char *expected_prog_name = "test_obj_id";
 473         const char *expected_map_name = "test_map_id";
 474         const __u64 nsec_per_sec = 1000000000;
 475
 476         struct bpf_object *objs[nr_iters];
 477         int prog_fds[nr_iters], map_fds[nr_iters];
 478         /* +1 to test for the info_len returned by kernel */
 479         struct bpf_prog_info prog_infos[nr_iters + 1];
 480         struct bpf_map_info map_infos[nr_iters + 1];
 481         /* Each prog only uses one map. +1 to test nr_map_ids
 482          * returned by kernel.
 483          */
 484         __u32 map_ids[nr_iters + 1];
 485         char jited_insns[128], xlated_insns[128], zeros[128];
 486         __u32 i, next_id, info_len, nr_id_found, duration = 0;
 487         struct timespec real_time_ts, boot_time_ts;
 488         int err = 0;
 489         __u64 array_value;
 490         uid_t my_uid = getuid();
 491         time_t now, load_time;
 492
 493         err = bpf_prog_get_fd_by_id(0);
 494         CHECK(err >= 0 || errno != ENOENT,
 495               "get-fd-by-notexist-prog-id", "err %d errno %d\n", err, errno);
 496
 497         err = bpf_map_get_fd_by_id(0);
 498         CHECK(err >= 0 || errno != ENOENT,
 499               "get-fd-by-notexist-map-id", "err %d errno %d\n", err, errno);
 500
 501         for (i = 0; i < nr_iters; i++)
 502                 objs[i] = NULL;
 503
 504         /* Check bpf_obj_get_info_by_fd() */
 505         bzero(zeros, sizeof(zeros));
 506         for (i = 0; i < nr_iters; i++) {
 507                 now = time(NULL);
 508                 err = bpf_prog_load(file, BPF_PROG_TYPE_SOCKET_FILTER,
 509                                     &objs[i], &prog_fds[i]);
 510                 /* test_obj_id.o is a dumb prog. It should never fail
 511                  * to load.
 512                  */
 513                 if (err)
 514                         error_cnt++;
 515                 assert(!err);
 516
 517                 /* Insert a magic value to the map */
 518                 map_fds[i] = bpf_find_map(__func__, objs[i], "test_map_id");
 519                 assert(map_fds[i] >= 0);
 520                 err = bpf_map_update_elem(map_fds[i], &array_key,
 521                                           &array_magic_value, 0);
 522                 assert(!err);
 523
 524                 /* Check getting map info */
 525                 info_len = sizeof(struct bpf_map_info) * 2;
 526                 bzero(&map_infos[i], info_len);
 527                 err = bpf_obj_get_info_by_fd(map_fds[i], &map_infos[i],
 528                                              &info_len);
 529                 if (CHECK(err ||
 530                           map_infos[i].type != BPF_MAP_TYPE_ARRAY ||
 531                           map_infos[i].key_size != sizeof(__u32) ||
 532                           map_infos[i].value_size != sizeof(__u64) ||
 533                           map_infos[i].max_entries != 1 ||
 534                           map_infos[i].map_flags != 0 ||
 535                           info_len != sizeof(struct bpf_map_info) ||
 536                           strcmp((char *)map_infos[i].name, expected_map_name),
 537                           "get-map-info(fd)",
 538                           "err %d errno %d type %d(%d) info_len %u(%Zu) key_size %u value_size %u max_entries %u map_flags %X name %s(%s)\n",
 539                           err, errno,
 540                           map_infos[i].type, BPF_MAP_TYPE_ARRAY,
 541                           info_len, sizeof(struct bpf_map_info),
 542                           map_infos[i].key_size,
 543                           map_infos[i].value_size,
 544                           map_infos[i].max_entries,
 545                           map_infos[i].map_flags,
 546                           map_infos[i].name, expected_map_name))
 547                         goto done;
 548
 549                 /* Check getting prog info */
 550                 info_len = sizeof(struct bpf_prog_info) * 2;
 551                 bzero(&prog_infos[i], info_len);
 552                 bzero(jited_insns, sizeof(jited_insns));
 553                 bzero(xlated_insns, sizeof(xlated_insns));
 554                 prog_infos[i].jited_prog_insns = ptr_to_u64(jited_insns);
 555                 prog_infos[i].jited_prog_len = sizeof(jited_insns);
 556                 prog_infos[i].xlated_prog_insns = ptr_to_u64(xlated_insns);
 557                 prog_infos[i].xlated_prog_len = sizeof(xlated_insns);
 558                 prog_infos[i].map_ids = ptr_to_u64(map_ids + i);
 559                 prog_infos[i].nr_map_ids = 2;
 560                 err = clock_gettime(CLOCK_REALTIME, &real_time_ts);
 561                 assert(!err);
 562                 err = clock_gettime(CLOCK_BOOTTIME, &boot_time_ts);
 563                 assert(!err);
 564                 err = bpf_obj_get_info_by_fd(prog_fds[i], &prog_infos[i],
 565                                              &info_len);
 566                 load_time = (real_time_ts.tv_sec - boot_time_ts.tv_sec)
 567                         + (prog_infos[i].load_time / nsec_per_sec);
 568                 if (CHECK(err ||
 569                           prog_infos[i].type != BPF_PROG_TYPE_SOCKET_FILTER ||
 570                           info_len != sizeof(struct bpf_prog_info) ||
 571                           (jit_enabled && !prog_infos[i].jited_prog_len) ||
 572                           (jit_enabled &&
 573                            !memcmp(jited_insns, zeros, sizeof(zeros))) ||
 574                           !prog_infos[i].xlated_prog_len ||
 575                           !memcmp(xlated_insns, zeros, sizeof(zeros)) ||
 576                           load_time < now - 60 || load_time > now + 60 ||
 577                           prog_infos[i].created_by_uid != my_uid ||
 578                           prog_infos[i].nr_map_ids != 1 ||
 579                           *(int *)(long)prog_infos[i].map_ids != map_infos[i].id ||
 580                           strcmp((char *)prog_infos[i].name, expected_prog_name),
 581                           "get-prog-info(fd)",
 582                           "err %d errno %d i %d type %d(%d) info_len %u(%Zu) jit_enabled %d jited_prog_len %u xlated_prog_len %u jited_prog %d xlated_prog %d load_time %lu(%lu) uid %u(%u) nr_map_ids %u(%u) map_id %u(%u) name %s(%s)\n",
 583                           err, errno, i,
 584                           prog_infos[i].type, BPF_PROG_TYPE_SOCKET_FILTER,
 585                           info_len, sizeof(struct bpf_prog_info),
 586                           jit_enabled,
 587                           prog_infos[i].jited_prog_len,
 588                           prog_infos[i].xlated_prog_len,
 589                           !!memcmp(jited_insns, zeros, sizeof(zeros)),
 590                           !!memcmp(xlated_insns, zeros, sizeof(zeros)),
 591                           load_time, now,
 592                           prog_infos[i].created_by_uid, my_uid,
 593                           prog_infos[i].nr_map_ids, 1,
 594                           *(int *)(long)prog_infos[i].map_ids, map_infos[i].id,
 595                           prog_infos[i].name, expected_prog_name))
 596                         goto done;
 597         }
 598
 599         /* Check bpf_prog_get_next_id() */
 600         nr_id_found = 0;
 601         next_id = 0;
 602         while (!bpf_prog_get_next_id(next_id, &next_id)) {
 603                 struct bpf_prog_info prog_info = {};
 604                 __u32 saved_map_id;
 605                 int prog_fd;
 606
 607                 info_len = sizeof(prog_info);
 608
 609                 prog_fd = bpf_prog_get_fd_by_id(next_id);
 610                 if (prog_fd < 0 && errno == ENOENT)
 611                         /* The bpf_prog is in the dead row */
 612                         continue;
 613                 if (CHECK(prog_fd < 0, "get-prog-fd(next_id)",
 614                           "prog_fd %d next_id %d errno %d\n",
 615                           prog_fd, next_id, errno))
 616                         break;
 617
 618                 for (i = 0; i < nr_iters; i++)
 619                         if (prog_infos[i].id == next_id)
 620                                 break;
 621
 622                 if (i == nr_iters)
 623                         continue;
 624
 625                 nr_id_found++;
 626
 627                 /* Negative test:
 628                  * prog_info.nr_map_ids = 1
 629                  * prog_info.map_ids = NULL
 630                  */
 631                 prog_info.nr_map_ids = 1;
 632                 err = bpf_obj_get_info_by_fd(prog_fd, &prog_info, &info_len);
 633                 if (CHECK(!err || errno != EFAULT,
 634                           "get-prog-fd-bad-nr-map-ids", "err %d errno %d(%d)",
 635                           err, errno, EFAULT))
 636                         break;
 637                 bzero(&prog_info, sizeof(prog_info));
 638                 info_len = sizeof(prog_info);
 639
 640                 saved_map_id = *(int *)((long)prog_infos[i].map_ids);
 641                 prog_info.map_ids = prog_infos[i].map_ids;
 642                 prog_info.nr_map_ids = 2;
 643                 err = bpf_obj_get_info_by_fd(prog_fd, &prog_info, &info_len);
 644                 prog_infos[i].jited_prog_insns = 0;
 645                 prog_infos[i].xlated_prog_insns = 0;
 646                 CHECK(err || info_len != sizeof(struct bpf_prog_info) ||
 647                       memcmp(&prog_info, &prog_infos[i], info_len) ||
 648                       *(int *)(long)prog_info.map_ids != saved_map_id,
 649                       "get-prog-info(next_id->fd)",
 650                       "err %d errno %d info_len %u(%Zu) memcmp %d map_id %u(%u)\n",
 651                       err, errno, info_len, sizeof(struct bpf_prog_info),
 652                       memcmp(&prog_info, &prog_infos[i], info_len),
 653                       *(int *)(long)prog_info.map_ids, saved_map_id);
 654                 close(prog_fd);
 655         }
 656         CHECK(nr_id_found != nr_iters,
 657               "check total prog id found by get_next_id",
 658               "nr_id_found %u(%u)\n",
 659               nr_id_found, nr_iters);
 660
 661         /* Check bpf_map_get_next_id() */
 662         nr_id_found = 0;
 663         next_id = 0;
 664         while (!bpf_map_get_next_id(next_id, &next_id)) {
 665                 struct bpf_map_info map_info = {};
 666                 int map_fd;
 667
 668                 info_len = sizeof(map_info);
 669
 670                 map_fd = bpf_map_get_fd_by_id(next_id);
 671                 if (map_fd < 0 && errno == ENOENT)
 672                         /* The bpf_map is in the dead row */
 673                         continue;
 674                 if (CHECK(map_fd < 0, "get-map-fd(next_id)",
 675                           "map_fd %d next_id %u errno %d\n",
 676                           map_fd, next_id, errno))
 677                         break;
 678
 679                 for (i = 0; i < nr_iters; i++)
 680                         if (map_infos[i].id == next_id)
 681                                 break;
 682
 683                 if (i == nr_iters)
 684                         continue;
 685
 686                 nr_id_found++;
 687
 688                 err = bpf_map_lookup_elem(map_fd, &array_key, &array_value);
 689                 assert(!err);
 690
 691                 err = bpf_obj_get_info_by_fd(map_fd, &map_info, &info_len);
 692                 CHECK(err || info_len != sizeof(struct bpf_map_info) ||
 693                       memcmp(&map_info, &map_infos[i], info_len) ||
 694                       array_value != array_magic_value,
 695                       "check get-map-info(next_id->fd)",
 696                       "err %d errno %d info_len %u(%Zu) memcmp %d array_value %llu(%llu)\n",
 697                       err, errno, info_len, sizeof(struct bpf_map_info),
 698                       memcmp(&map_info, &map_infos[i], info_len),
 699                       array_value, array_magic_value);
 700
 701                 close(map_fd);
 702         }
 703         CHECK(nr_id_found != nr_iters,
 704               "check total map id found by get_next_id",
 705               "nr_id_found %u(%u)\n",
 706               nr_id_found, nr_iters);
 707
 708 done:
 709         for (i = 0; i < nr_iters; i++)
 710                 bpf_object__close(objs[i]);
 711 }
 712
 713 static void test_pkt_md_access(void)
 714 {
 715         const char *file = "./test_pkt_md_access.o";
 716         struct bpf_object *obj;
 717         __u32 duration, retval;
 718         int err, prog_fd;
 719
 720         err = bpf_prog_load(file, BPF_PROG_TYPE_SCHED_CLS, &obj, &prog_fd);
 721         if (err) {
 722                 error_cnt++;
 723                 return;
 724         }
 725
 726         err = bpf_prog_test_run(prog_fd, 10, &pkt_v4, sizeof(pkt_v4),
 727                                 NULL, NULL, &retval, &duration);
 728         CHECK(err || retval, "",
 729               "err %d errno %d retval %d duration %d\n",
 730               err, errno, retval, duration);
 731
 732         bpf_object__close(obj);
 733 }
 734
 735 static void test_obj_name(void)
 736 {
 737         struct {
 738                 const char *name;
 739                 int success;
 740                 int expected_errno;
 741         } tests[] = {
 742                 { "", 1, 0 },
 743                 { "_123456789ABCDE", 1, 0 },
 744                 { "_123456789ABCDEF", 0, EINVAL },
 745                 { "_123456789ABCD\n", 0, EINVAL },
 746         };
 747         struct bpf_insn prog[] = {
 748                 BPF_ALU64_IMM(BPF_MOV, BPF_REG_0, 0),
 749                 BPF_EXIT_INSN(),
 750         };
 751         __u32 duration = 0;
 752         int i;
 753
 754         for (i = 0; i < sizeof(tests) / sizeof(tests[0]); i++) {
 755                 size_t name_len = strlen(tests[i].name) + 1;
 756                 union bpf_attr attr;
 757                 size_t ncopy;
 758                 int fd;
 759
 760                 /* test different attr.prog_name during BPF_PROG_LOAD */
 761                 ncopy = name_len < sizeof(attr.prog_name) ?
 762                         name_len : sizeof(attr.prog_name);
 763                 bzero(&attr, sizeof(attr));
 764                 attr.prog_type = BPF_PROG_TYPE_SCHED_CLS;
 765                 attr.insn_cnt = 2;
 766                 attr.insns = ptr_to_u64(prog);
 767                 attr.license = ptr_to_u64("");
 768                 memcpy(attr.prog_name, tests[i].name, ncopy);
 769
 770                 fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, sizeof(attr));
 771                 CHECK((tests[i].success && fd < 0) ||
 772                       (!tests[i].success && fd != -1) ||
 773                       (!tests[i].success && errno != tests[i].expected_errno),
 774                       "check-bpf-prog-name",
 775                       "fd %d(%d) errno %d(%d)\n",
 776                        fd, tests[i].success, errno, tests[i].expected_errno);
 777
 778                 if (fd != -1)
 779                         close(fd);
 780
 781                 /* test different attr.map_name during BPF_MAP_CREATE */
 782                 ncopy = name_len < sizeof(attr.map_name) ?
 783                         name_len : sizeof(attr.map_name);
 784                 bzero(&attr, sizeof(attr));
 785                 attr.map_type = BPF_MAP_TYPE_ARRAY;
 786                 attr.key_size = 4;
 787                 attr.value_size = 4;
 788                 attr.max_entries = 1;
 789                 attr.map_flags = 0;
 790                 memcpy(attr.map_name, tests[i].name, ncopy);
 791                 fd = syscall(__NR_bpf, BPF_MAP_CREATE, &attr, sizeof(attr));
 792                 CHECK((tests[i].success && fd < 0) ||
 793                       (!tests[i].success && fd != -1) ||
 794                       (!tests[i].success && errno != tests[i].expected_errno),
 795                       "check-bpf-map-name",
 796                       "fd %d(%d) errno %d(%d)\n",
 797                       fd, tests[i].success, errno, tests[i].expected_errno);
 798
 799                 if (fd != -1)
 800                         close(fd);
 801         }
 802 }
 803
 804 static void test_tp_attach_query(void)
 805 {
 806         const int num_progs = 3;
 807         int i, j, bytes, efd, err, prog_fd[num_progs], pmu_fd[num_progs];
 808         __u32 duration = 0, info_len, saved_prog_ids[num_progs];
 809         const char *file = "./test_tracepoint.o";
 810         struct perf_event_query_bpf *query;
 811         struct perf_event_attr attr = {};
 812         struct bpf_object *obj[num_progs];
 813         struct bpf_prog_info prog_info;
 814         char buf[256];
 815
 816         snprintf(buf, sizeof(buf),
 817                  "/sys/kernel/debug/tracing/events/sched/sched_switch/id");
 818         efd = open(buf, O_RDONLY, 0);
 819         if (CHECK(efd < 0, "open", "err %d errno %d\n", efd, errno))
 820                 return;
 821         bytes = read(efd, buf, sizeof(buf));
 822         close(efd);
 823         if (CHECK(bytes <= 0 || bytes >= sizeof(buf),
 824                   "read", "bytes %d errno %d\n", bytes, errno))
 825                 return;
 826
 827         attr.config = strtol(buf, NULL, 0);
 828         attr.type = PERF_TYPE_TRACEPOINT;
 829         attr.sample_type = PERF_SAMPLE_RAW | PERF_SAMPLE_CALLCHAIN;
 830         attr.sample_period = 1;
 831         attr.wakeup_events = 1;
 832
 833         query = malloc(sizeof(*query) + sizeof(__u32) * num_progs);
 834         for (i = 0; i < num_progs; i++) {
 835                 err = bpf_prog_load(file, BPF_PROG_TYPE_TRACEPOINT, &obj[i],
 836                                     &prog_fd[i]);
 837                 if (CHECK(err, "prog_load", "err %d errno %d\n", err, errno))
 838                         goto cleanup1;
 839
 840                 bzero(&prog_info, sizeof(prog_info));
 841                 prog_info.jited_prog_len = 0;
 842                 prog_info.xlated_prog_len = 0;
 843                 prog_info.nr_map_ids = 0;
 844                 info_len = sizeof(prog_info);
 845                 err = bpf_obj_get_info_by_fd(prog_fd[i], &prog_info, &info_len);
 846                 if (CHECK(err, "bpf_obj_get_info_by_fd", "err %d errno %d\n",
 847                           err, errno))
 848                         goto cleanup1;
 849                 saved_prog_ids[i] = prog_info.id;
 850
 851                 pmu_fd[i] = syscall(__NR_perf_event_open, &attr, -1 /* pid */,
 852                                     0 /* cpu 0 */, -1 /* group id */,
 853                                     0 /* flags */);
 854                 if (CHECK(pmu_fd[i] < 0, "perf_event_open", "err %d errno %d\n",
 855                           pmu_fd[i], errno))
 856                         goto cleanup2;
 857                 err = ioctl(pmu_fd[i], PERF_EVENT_IOC_ENABLE, 0);
 858                 if (CHECK(err, "perf_event_ioc_enable", "err %d errno %d\n",
 859                           err, errno))
 860                         goto cleanup3;
 861
 862                 if (i == 0) {
 863                         /* check NULL prog array query */
 864                         query->ids_len = num_progs;
 865                         err = ioctl(pmu_fd[i], PERF_EVENT_IOC_QUERY_BPF, query);
 866                         if (CHECK(err || query->prog_cnt != 0,
 867                                   "perf_event_ioc_query_bpf",
 868                                   "err %d errno %d query->prog_cnt %u\n",
 869                                   err, errno, query->prog_cnt))
 870                                 goto cleanup3;
 871                 }
 872
 873                 err = ioctl(pmu_fd[i], PERF_EVENT_IOC_SET_BPF, prog_fd[i]);
 874                 if (CHECK(err, "perf_event_ioc_set_bpf", "err %d errno %d\n",
 875                           err, errno))
 876                         goto cleanup3;
 877
 878                 if (i == 1) {
 879                         /* try to get # of programs only */
 880                         query->ids_len = 0;
 881                         err = ioctl(pmu_fd[i], PERF_EVENT_IOC_QUERY_BPF, query);
 882                         if (CHECK(err || query->prog_cnt != 2,
 883                                   "perf_event_ioc_query_bpf",
 884                                   "err %d errno %d query->prog_cnt %u\n",
 885                                   err, errno, query->prog_cnt))
 886                                 goto cleanup3;
 887
 888                         /* try a few negative tests */
 889                         /* invalid query pointer */
 890                         err = ioctl(pmu_fd[i], PERF_EVENT_IOC_QUERY_BPF,
 891                                     (struct perf_event_query_bpf *)0x1);
 892                         if (CHECK(!err || errno != EFAULT,
 893                                   "perf_event_ioc_query_bpf",
 894                                   "err %d errno %d\n", err, errno))
 895                                 goto cleanup3;
 896
 897                         /* no enough space */
 898                         query->ids_len = 1;
 899                         err = ioctl(pmu_fd[i], PERF_EVENT_IOC_QUERY_BPF, query);
 900                         if (CHECK(!err || errno != ENOSPC || query->prog_cnt != 2,
 901                                   "perf_event_ioc_query_bpf",
 902                                   "err %d errno %d query->prog_cnt %u\n",
 903                                   err, errno, query->prog_cnt))
 904                                 goto cleanup3;
 905                 }
 906
 907                 query->ids_len = num_progs;
 908                 err = ioctl(pmu_fd[i], PERF_EVENT_IOC_QUERY_BPF, query);
 909                 if (CHECK(err || query->prog_cnt != (i + 1),
 910                           "perf_event_ioc_query_bpf",
 911                           "err %d errno %d query->prog_cnt %u\n",
 912                           err, errno, query->prog_cnt))
 913                         goto cleanup3;
 914                 for (j = 0; j < i + 1; j++)
 915                         if (CHECK(saved_prog_ids[j] != query->ids[j],
 916                                   "perf_event_ioc_query_bpf",
 917                                   "#%d saved_prog_id %x query prog_id %x\n",
 918                                   j, saved_prog_ids[j], query->ids[j]))
 919                                 goto cleanup3;
 920         }
 921
 922         i = num_progs - 1;
 923         for (; i >= 0; i--) {
 924  cleanup3:
 925                 ioctl(pmu_fd[i], PERF_EVENT_IOC_DISABLE);
 926  cleanup2:
 927                 close(pmu_fd[i]);
 928  cleanup1:
 929                 bpf_object__close(obj[i]);
 930         }
 931         free(query);
 932 }
 933
 934 static int compare_map_keys(int map1_fd, int map2_fd)
 935 {
 936         __u32 key, next_key;
 937         char val_buf[PERF_MAX_STACK_DEPTH *
 938                      sizeof(struct bpf_stack_build_id)];
 939         int err;
 940
 941         err = bpf_map_get_next_key(map1_fd, NULL, &key);
 942         if (err)
 943                 return err;
 944         err = bpf_map_lookup_elem(map2_fd, &key, val_buf);
 945         if (err)
 946                 return err;
 947
 948         while (bpf_map_get_next_key(map1_fd, &key, &next_key) == 0) {
 949                 err = bpf_map_lookup_elem(map2_fd, &next_key, val_buf);
 950                 if (err)
 951                         return err;
 952
 953                 key = next_key;
 954         }
 955         if (errno != ENOENT)
 956                 return -1;
 957
 958         return 0;
 959 }
 960
 961 static int compare_stack_ips(int smap_fd, int amap_fd, int stack_trace_len)
 962 {
 963         __u32 key, next_key, *cur_key_p, *next_key_p;
 964         char *val_buf1, *val_buf2;
 965         int i, err = 0;
 966
 967         val_buf1 = malloc(stack_trace_len);
 968         val_buf2 = malloc(stack_trace_len);
 969         cur_key_p = NULL;
 970         next_key_p = &key;
 971         while (bpf_map_get_next_key(smap_fd, cur_key_p, next_key_p) == 0) {
 972                 err = bpf_map_lookup_elem(smap_fd, next_key_p, val_buf1);
 973                 if (err)
 974                         goto out;
 975                 err = bpf_map_lookup_elem(amap_fd, next_key_p, val_buf2);
 976                 if (err)
 977                         goto out;
 978                 for (i = 0; i < stack_trace_len; i++) {
 979                         if (val_buf1[i] != val_buf2[i]) {
 980                                 err = -1;
 981                                 goto out;
 982                         }
 983                 }
 984                 key = *next_key_p;
 985                 cur_key_p = &key;
 986                 next_key_p = &next_key;
 987         }
 988         if (errno != ENOENT)
 989                 err = -1;
 990
 991 out:
 992         free(val_buf1);
 993         free(val_buf2);
 994         return err;
 995 }
 996
 997 static void test_stacktrace_map()
 998 {
 999         int control_map_fd, stackid_hmap_fd, stackmap_fd, stack_amap_fd;
1000         const char *file = "./test_stacktrace_map.o";
1001         int bytes, efd, err, pmu_fd, prog_fd, stack_trace_len;
1002         struct perf_event_attr attr = {};
1003         __u32 key, val, duration = 0;
1004         struct bpf_object *obj;
1005         char buf[256];
1006
1007         err = bpf_prog_load(file, BPF_PROG_TYPE_TRACEPOINT, &obj, &prog_fd);
1008         if (CHECK(err, "prog_load", "err %d errno %d\n", err, errno))
1009                 return;
1010
1011         /* Get the ID for the sched/sched_switch tracepoint */
1012         snprintf(buf, sizeof(buf),
1013                  "/sys/kernel/debug/tracing/events/sched/sched_switch/id");
1014         efd = open(buf, O_RDONLY, 0);
1015         if (CHECK(efd < 0, "open", "err %d errno %d\n", efd, errno))
1016                 goto close_prog;
1017
1018         bytes = read(efd, buf, sizeof(buf));
1019         close(efd);
1020         if (bytes <= 0 || bytes >= sizeof(buf))
1021                 goto close_prog;
1022
1023         /* Open the perf event and attach bpf progrram */
1024         attr.config = strtol(buf, NULL, 0);
1025         attr.type = PERF_TYPE_TRACEPOINT;
1026         attr.sample_type = PERF_SAMPLE_RAW | PERF_SAMPLE_CALLCHAIN;
1027         attr.sample_period = 1;
1028         attr.wakeup_events = 1;
1029         pmu_fd = syscall(__NR_perf_event_open, &attr, -1 /* pid */,
1030                          0 /* cpu 0 */, -1 /* group id */,
1031                          0 /* flags */);
1032         if (CHECK(pmu_fd < 0, "perf_event_open", "err %d errno %d\n",
1033                   pmu_fd, errno))
1034                 goto close_prog;
1035
1036         err = ioctl(pmu_fd, PERF_EVENT_IOC_ENABLE, 0);
1037         if (err)
1038                 goto disable_pmu;
1039
1040         err = ioctl(pmu_fd, PERF_EVENT_IOC_SET_BPF, prog_fd);
1041         if (err)
1042                 goto disable_pmu;
1043
1044         /* find map fds */
1045         control_map_fd = bpf_find_map(__func__, obj, "control_map");
1046         if (control_map_fd < 0)
1047                 goto disable_pmu;
1048
1049         stackid_hmap_fd = bpf_find_map(__func__, obj, "stackid_hmap");
1050         if (stackid_hmap_fd < 0)
1051                 goto disable_pmu;
1052
1053         stackmap_fd = bpf_find_map(__func__, obj, "stackmap");
1054         if (stackmap_fd < 0)
1055                 goto disable_pmu;
1056
1057         stack_amap_fd = bpf_find_map(__func__, obj, "stack_amap");
1058         if (stack_amap_fd < 0)
1059                 goto disable_pmu;
1060
1061         /* give some time for bpf program run */
1062         sleep(1);
1063
1064         /* disable stack trace collection */
1065         key = 0;
1066         val = 1;
1067         bpf_map_update_elem(control_map_fd, &key, &val, 0);
1068
1069         /* for every element in stackid_hmap, we can find a corresponding one
1070          * in stackmap, and vise versa.
1071          */
1072         err = compare_map_keys(stackid_hmap_fd, stackmap_fd);
1073         if (CHECK(err, "compare_map_keys stackid_hmap vs. stackmap",
1074                   "err %d errno %d\n", err, errno))
1075                 goto disable_pmu_noerr;
1076
1077         err = compare_map_keys(stackmap_fd, stackid_hmap_fd);
1078         if (CHECK(err, "compare_map_keys stackmap vs. stackid_hmap",
1079                   "err %d errno %d\n", err, errno))
1080                 goto disable_pmu_noerr;
1081
1082         stack_trace_len = PERF_MAX_STACK_DEPTH * sizeof(__u64);
1083         err = compare_stack_ips(stackmap_fd, stack_amap_fd, stack_trace_len);
1084         if (CHECK(err, "compare_stack_ips stackmap vs. stack_amap",
1085                   "err %d errno %d\n", err, errno))
1086                 goto disable_pmu_noerr;
1087
1088         goto disable_pmu_noerr;
1089 disable_pmu:
1090         error_cnt++;
1091 disable_pmu_noerr:
1092         ioctl(pmu_fd, PERF_EVENT_IOC_DISABLE);
1093         close(pmu_fd);
1094 close_prog:
1095         bpf_object__close(obj);
1096 }
1097
1098 static void test_stacktrace_map_raw_tp()
1099 {
1100         int control_map_fd, stackid_hmap_fd, stackmap_fd;
1101         const char *file = "./test_stacktrace_map.o";
1102         int efd, err, prog_fd;
1103         __u32 key, val, duration = 0;
1104         struct bpf_object *obj;
1105
1106         err = bpf_prog_load(file, BPF_PROG_TYPE_RAW_TRACEPOINT, &obj, &prog_fd);
1107         if (CHECK(err, "prog_load raw tp", "err %d errno %d\n", err, errno))
1108                 return;
1109
1110         efd = bpf_raw_tracepoint_open("sched_switch", prog_fd);
1111         if (CHECK(efd < 0, "raw_tp_open", "err %d errno %d\n", efd, errno))
1112                 goto close_prog;
1113
1114         /* find map fds */
1115         control_map_fd = bpf_find_map(__func__, obj, "control_map");
1116         if (control_map_fd < 0)
1117                 goto close_prog;
1118
1119         stackid_hmap_fd = bpf_find_map(__func__, obj, "stackid_hmap");
1120         if (stackid_hmap_fd < 0)
1121                 goto close_prog;
1122
1123         stackmap_fd = bpf_find_map(__func__, obj, "stackmap");
1124         if (stackmap_fd < 0)
1125                 goto close_prog;
1126
1127         /* give some time for bpf program run */
1128         sleep(1);
1129
1130         /* disable stack trace collection */
1131         key = 0;
1132         val = 1;
1133         bpf_map_update_elem(control_map_fd, &key, &val, 0);
1134
1135         /* for every element in stackid_hmap, we can find a corresponding one
1136          * in stackmap, and vise versa.
1137          */
1138         err = compare_map_keys(stackid_hmap_fd, stackmap_fd);
1139         if (CHECK(err, "compare_map_keys stackid_hmap vs. stackmap",
1140                   "err %d errno %d\n", err, errno))
1141                 goto close_prog;
1142
1143         err = compare_map_keys(stackmap_fd, stackid_hmap_fd);
1144         if (CHECK(err, "compare_map_keys stackmap vs. stackid_hmap",
1145                   "err %d errno %d\n", err, errno))
1146                 goto close_prog;
1147
1148         goto close_prog_noerr;
1149 close_prog:
1150         error_cnt++;
1151 close_prog_noerr:
1152         bpf_object__close(obj);
1153 }
1154
1155 static int extract_build_id(char *build_id, size_t size)
1156 {
1157         FILE *fp;
1158         char *line = NULL;
1159         size_t len = 0;
1160
1161         fp = popen("readelf -n ./urandom_read | grep 'Build ID'", "r");
1162         if (fp == NULL)
1163                 return -1;
1164
1165         if (getline(&line, &len, fp) == -1)
1166                 goto err;
1167         fclose(fp);
1168
1169         if (len > size)
1170                 len = size;
1171         memcpy(build_id, line, len);
1172         build_id[len] = '\0';
1173         return 0;
1174 err:
1175         fclose(fp);
1176         return -1;
1177 }
1178
1179 static void test_stacktrace_build_id(void)
1180 {
1181         int control_map_fd, stackid_hmap_fd, stackmap_fd, stack_amap_fd;
1182         const char *file = "./test_stacktrace_build_id.o";
1183         int bytes, efd, err, pmu_fd, prog_fd, stack_trace_len;
1184         struct perf_event_attr attr = {};
1185         __u32 key, previous_key, val, duration = 0;
1186         struct bpf_object *obj;
1187         char buf[256];
1188         int i, j;
1189         struct bpf_stack_build_id id_offs[PERF_MAX_STACK_DEPTH];
1190         int build_id_matches = 0;
1191         int retry = 1;
1192
1193 retry:
1194         err = bpf_prog_load(file, BPF_PROG_TYPE_TRACEPOINT, &obj, &prog_fd);
1195         if (CHECK(err, "prog_load", "err %d errno %d\n", err, errno))
1196                 goto out;
1197
1198         /* Get the ID for the sched/sched_switch tracepoint */
1199         snprintf(buf, sizeof(buf),
1200                  "/sys/kernel/debug/tracing/events/random/urandom_read/id");
1201         efd = open(buf, O_RDONLY, 0);
1202         if (CHECK(efd < 0, "open", "err %d errno %d\n", efd, errno))
1203                 goto close_prog;
1204
1205         bytes = read(efd, buf, sizeof(buf));
1206         close(efd);
1207         if (CHECK(bytes <= 0 || bytes >= sizeof(buf),
1208                   "read", "bytes %d errno %d\n", bytes, errno))
1209                 goto close_prog;
1210
1211         /* Open the perf event and attach bpf progrram */
1212         attr.config = strtol(buf, NULL, 0);
1213         attr.type = PERF_TYPE_TRACEPOINT;
1214         attr.sample_type = PERF_SAMPLE_RAW | PERF_SAMPLE_CALLCHAIN;
1215         attr.sample_period = 1;
1216         attr.wakeup_events = 1;
1217         pmu_fd = syscall(__NR_perf_event_open, &attr, -1 /* pid */,
1218                          0 /* cpu 0 */, -1 /* group id */,
1219                          0 /* flags */);
1220         if (CHECK(pmu_fd < 0, "perf_event_open", "err %d errno %d\n",
1221                   pmu_fd, errno))
1222                 goto close_prog;
1223
1224         err = ioctl(pmu_fd, PERF_EVENT_IOC_ENABLE, 0);
1225         if (CHECK(err, "perf_event_ioc_enable", "err %d errno %d\n",
1226                   err, errno))
1227                 goto close_pmu;
1228
1229         err = ioctl(pmu_fd, PERF_EVENT_IOC_SET_BPF, prog_fd);
1230         if (CHECK(err, "perf_event_ioc_set_bpf", "err %d errno %d\n",
1231                   err, errno))
1232                 goto disable_pmu;
1233
1234         /* find map fds */
1235         control_map_fd = bpf_find_map(__func__, obj, "control_map");
1236         if (CHECK(control_map_fd < 0, "bpf_find_map control_map",
1237                   "err %d errno %d\n", err, errno))
1238                 goto disable_pmu;
1239
1240         stackid_hmap_fd = bpf_find_map(__func__, obj, "stackid_hmap");
1241         if (CHECK(stackid_hmap_fd < 0, "bpf_find_map stackid_hmap",
1242                   "err %d errno %d\n", err, errno))
1243                 goto disable_pmu;
1244
1245         stackmap_fd = bpf_find_map(__func__, obj, "stackmap");
1246         if (CHECK(stackmap_fd < 0, "bpf_find_map stackmap", "err %d errno %d\n",
1247                   err, errno))
1248                 goto disable_pmu;
1249
1250         stack_amap_fd = bpf_find_map(__func__, obj, "stack_amap");
1251         if (CHECK(stack_amap_fd < 0, "bpf_find_map stack_amap",
1252                   "err %d errno %d\n", err, errno))
1253                 goto disable_pmu;
1254
1255         assert(system("dd if=/dev/urandom of=/dev/zero count=4 2> /dev/null")
1256                == 0);
1257         assert(system("./urandom_read") == 0);
1258         /* disable stack trace collection */
1259         key = 0;
1260         val = 1;
1261         bpf_map_update_elem(control_map_fd, &key, &val, 0);
1262
1263         /* for every element in stackid_hmap, we can find a corresponding one
1264          * in stackmap, and vise versa.
1265          */
1266         err = compare_map_keys(stackid_hmap_fd, stackmap_fd);
1267         if (CHECK(err, "compare_map_keys stackid_hmap vs. stackmap",
1268                   "err %d errno %d\n", err, errno))
1269                 goto disable_pmu;
1270
1271         err = compare_map_keys(stackmap_fd, stackid_hmap_fd);
1272         if (CHECK(err, "compare_map_keys stackmap vs. stackid_hmap",
1273                   "err %d errno %d\n", err, errno))
1274                 goto disable_pmu;
1275
1276         err = extract_build_id(buf, 256);
1277
1278         if (CHECK(err, "get build_id with readelf",
1279                   "err %d errno %d\n", err, errno))
1280                 goto disable_pmu;
1281
1282         err = bpf_map_get_next_key(stackmap_fd, NULL, &key);
1283         if (CHECK(err, "get_next_key from stackmap",
1284                   "err %d, errno %d\n", err, errno))
1285                 goto disable_pmu;
1286
1287         do {
1288                 char build_id[64];
1289
1290                 err = bpf_map_lookup_elem(stackmap_fd, &key, id_offs);
1291                 if (CHECK(err, "lookup_elem from stackmap",
1292                           "err %d, errno %d\n", err, errno))
1293                         goto disable_pmu;
1294                 for (i = 0; i < PERF_MAX_STACK_DEPTH; ++i)
1295                         if (id_offs[i].status == BPF_STACK_BUILD_ID_VALID &&
1296                             id_offs[i].offset != 0) {
1297                                 for (j = 0; j < 20; ++j)
1298                                         sprintf(build_id + 2 * j, "%02x",
1299                                                 id_offs[i].build_id[j] & 0xff);
1300                                 if (strstr(buf, build_id) != NULL)
1301                                         build_id_matches = 1;
1302                         }
1303                 previous_key = key;
1304         } while (bpf_map_get_next_key(stackmap_fd, &previous_key, &key) == 0);
1305
1306         /* stack_map_get_build_id_offset() is racy and sometimes can return
1307          * BPF_STACK_BUILD_ID_IP instead of BPF_STACK_BUILD_ID_VALID;
1308          * try it one more time.
1309          */
1310         if (build_id_matches < 1 && retry--) {
1311                 ioctl(pmu_fd, PERF_EVENT_IOC_DISABLE);
1312                 close(pmu_fd);
1313                 bpf_object__close(obj);
1314                 printf("%s:WARN:Didn't find expected build ID from the map, retrying\n",
1315                        __func__);
1316                 goto retry;
1317         }
1318
1319         if (CHECK(build_id_matches < 1, "build id match",
1320                   "Didn't find expected build ID from the map\n"))
1321                 goto disable_pmu;
1322
1323         stack_trace_len = PERF_MAX_STACK_DEPTH
1324                 * sizeof(struct bpf_stack_build_id);
1325         err = compare_stack_ips(stackmap_fd, stack_amap_fd, stack_trace_len);
1326         CHECK(err, "compare_stack_ips stackmap vs. stack_amap",
1327               "err %d errno %d\n", err, errno);
1328
1329 disable_pmu:
1330         ioctl(pmu_fd, PERF_EVENT_IOC_DISABLE);
1331
1332 close_pmu:
1333         close(pmu_fd);
1334
1335 close_prog:
1336         bpf_object__close(obj);
1337
1338 out:
1339         return;
1340 }
1341
1342 static void test_stacktrace_build_id_nmi(void)
1343 {
1344         int control_map_fd, stackid_hmap_fd, stackmap_fd, stack_amap_fd;
1345         const char *file = "./test_stacktrace_build_id.o";
1346         int err, pmu_fd, prog_fd;
1347         struct perf_event_attr attr = {
1348                 .sample_freq = 5000,
1349                 .freq = 1,
1350                 .type = PERF_TYPE_HARDWARE,
1351                 .config = PERF_COUNT_HW_CPU_CYCLES,
1352         };
1353         __u32 key, previous_key, val, duration = 0;
1354         struct bpf_object *obj;
1355         char buf[256];
1356         int i, j;
1357         struct bpf_stack_build_id id_offs[PERF_MAX_STACK_DEPTH];
1358         int build_id_matches = 0;
1359         int retry = 1;
1360
1361 retry:
1362         err = bpf_prog_load(file, BPF_PROG_TYPE_PERF_EVENT, &obj, &prog_fd);
1363         if (CHECK(err, "prog_load", "err %d errno %d\n", err, errno))
1364                 return;
1365
1366         pmu_fd = syscall(__NR_perf_event_open, &attr, -1 /* pid */,
1367                          0 /* cpu 0 */, -1 /* group id */,
1368                          0 /* flags */);
1369         if (CHECK(pmu_fd < 0, "perf_event_open",
1370                   "err %d errno %d. Does the test host support PERF_COUNT_HW_CPU_CYCLES?\n",
1371                   pmu_fd, errno))
1372                 goto close_prog;
1373
1374         err = ioctl(pmu_fd, PERF_EVENT_IOC_ENABLE, 0);
1375         if (CHECK(err, "perf_event_ioc_enable", "err %d errno %d\n",
1376                   err, errno))
1377                 goto close_pmu;
1378
1379         err = ioctl(pmu_fd, PERF_EVENT_IOC_SET_BPF, prog_fd);
1380         if (CHECK(err, "perf_event_ioc_set_bpf", "err %d errno %d\n",
1381                   err, errno))
1382                 goto disable_pmu;
1383
1384         /* find map fds */
1385         control_map_fd = bpf_find_map(__func__, obj, "control_map");
1386         if (CHECK(control_map_fd < 0, "bpf_find_map control_map",
1387                   "err %d errno %d\n", err, errno))
1388                 goto disable_pmu;
1389
1390         stackid_hmap_fd = bpf_find_map(__func__, obj, "stackid_hmap");
1391         if (CHECK(stackid_hmap_fd < 0, "bpf_find_map stackid_hmap",
1392                   "err %d errno %d\n", err, errno))
1393                 goto disable_pmu;
1394
1395         stackmap_fd = bpf_find_map(__func__, obj, "stackmap");
1396         if (CHECK(stackmap_fd < 0, "bpf_find_map stackmap", "err %d errno %d\n",
1397                   err, errno))
1398                 goto disable_pmu;
1399
1400         stack_amap_fd = bpf_find_map(__func__, obj, "stack_amap");
1401         if (CHECK(stack_amap_fd < 0, "bpf_find_map stack_amap",
1402                   "err %d errno %d\n", err, errno))
1403                 goto disable_pmu;
1404
1405         assert(system("dd if=/dev/urandom of=/dev/zero count=4 2> /dev/null")
1406                == 0);
1407         assert(system("taskset 0x1 ./urandom_read 100000") == 0);
1408         /* disable stack trace collection */
1409         key = 0;
1410         val = 1;
1411         bpf_map_update_elem(control_map_fd, &key, &val, 0);
1412
1413         /* for every element in stackid_hmap, we can find a corresponding one
1414          * in stackmap, and vise versa.
1415          */
1416         err = compare_map_keys(stackid_hmap_fd, stackmap_fd);
1417         if (CHECK(err, "compare_map_keys stackid_hmap vs. stackmap",
1418                   "err %d errno %d\n", err, errno))
1419                 goto disable_pmu;
1420
1421         err = compare_map_keys(stackmap_fd, stackid_hmap_fd);
1422         if (CHECK(err, "compare_map_keys stackmap vs. stackid_hmap",
1423                   "err %d errno %d\n", err, errno))
1424                 goto disable_pmu;
1425
1426         err = extract_build_id(buf, 256);
1427
1428         if (CHECK(err, "get build_id with readelf",
1429                   "err %d errno %d\n", err, errno))
1430                 goto disable_pmu;
1431
1432         err = bpf_map_get_next_key(stackmap_fd, NULL, &key);
1433         if (CHECK(err, "get_next_key from stackmap",
1434                   "err %d, errno %d\n", err, errno))
1435                 goto disable_pmu;
1436
1437         do {
1438                 char build_id[64];
1439
1440                 err = bpf_map_lookup_elem(stackmap_fd, &key, id_offs);
1441                 if (CHECK(err, "lookup_elem from stackmap",
1442                           "err %d, errno %d\n", err, errno))
1443                         goto disable_pmu;
1444                 for (i = 0; i < PERF_MAX_STACK_DEPTH; ++i)
1445                         if (id_offs[i].status == BPF_STACK_BUILD_ID_VALID &&
1446                             id_offs[i].offset != 0) {
1447                                 for (j = 0; j < 20; ++j)
1448                                         sprintf(build_id + 2 * j, "%02x",
1449                                                 id_offs[i].build_id[j] & 0xff);
1450                                 if (strstr(buf, build_id) != NULL)
1451                                         build_id_matches = 1;
1452                         }
1453                 previous_key = key;
1454         } while (bpf_map_get_next_key(stackmap_fd, &previous_key, &key) == 0);
1455
1456         /* stack_map_get_build_id_offset() is racy and sometimes can return
1457          * BPF_STACK_BUILD_ID_IP instead of BPF_STACK_BUILD_ID_VALID;
1458          * try it one more time.
1459          */
1460         if (build_id_matches < 1 && retry--) {
1461                 ioctl(pmu_fd, PERF_EVENT_IOC_DISABLE);
1462                 close(pmu_fd);
1463                 bpf_object__close(obj);
1464                 printf("%s:WARN:Didn't find expected build ID from the map, retrying\n",
1465                        __func__);
1466                 goto retry;
1467         }
1468
1469         if (CHECK(build_id_matches < 1, "build id match",
1470                   "Didn't find expected build ID from the map\n"))
1471                 goto disable_pmu;
1472
1473         /*
1474          * We intentionally skip compare_stack_ips(). This is because we
1475          * only support one in_nmi() ips-to-build_id translation per cpu
1476          * at any time, thus stack_amap here will always fallback to
1477          * BPF_STACK_BUILD_ID_IP;
1478          */
1479
1480 disable_pmu:
1481         ioctl(pmu_fd, PERF_EVENT_IOC_DISABLE);
1482
1483 close_pmu:
1484         close(pmu_fd);
1485
1486 close_prog:
1487         bpf_object__close(obj);
1488 }
1489
1490 #define MAX_CNT_RAWTP   10ull
1491 #define MAX_STACK_RAWTP 100
1492 struct get_stack_trace_t {
1493         int pid;
1494         int kern_stack_size;
1495         int user_stack_size;
1496         int user_stack_buildid_size;
1497         __u64 kern_stack[MAX_STACK_RAWTP];
1498         __u64 user_stack[MAX_STACK_RAWTP];
1499         struct bpf_stack_build_id user_stack_buildid[MAX_STACK_RAWTP];
1500 };
1501
1502 static int get_stack_print_output(void *data, int size)
1503 {
1504         bool good_kern_stack = false, good_user_stack = false;
1505         const char *nonjit_func = "___bpf_prog_run";
1506         struct get_stack_trace_t *e = data;
1507         int i, num_stack;
1508         static __u64 cnt;
1509         struct ksym *ks;
1510
1511         cnt++;
1512
1513         if (size < sizeof(struct get_stack_trace_t)) {
1514                 __u64 *raw_data = data;
1515                 bool found = false;
1516
1517                 num_stack = size / sizeof(__u64);
1518                 /* If jit is enabled, we do not have a good way to
1519                  * verify the sanity of the kernel stack. So we
1520                  * just assume it is good if the stack is not empty.
1521                  * This could be improved in the future.
1522                  */
1523                 if (jit_enabled) {
1524                         found = num_stack > 0;
1525                 } else {
1526                         for (i = 0; i < num_stack; i++) {
1527                                 ks = ksym_search(raw_data[i]);
1528                                 if (strcmp(ks->name, nonjit_func) == 0) {
1529                                         found = true;
1530                                         break;
1531                                 }
1532                         }
1533                 }
1534                 if (found) {
1535                         good_kern_stack = true;
1536                         good_user_stack = true;
1537                 }
1538         } else {
1539                 num_stack = e->kern_stack_size / sizeof(__u64);
1540                 if (jit_enabled) {
1541                         good_kern_stack = num_stack > 0;
1542                 } else {
1543                         for (i = 0; i < num_stack; i++) {
1544                                 ks = ksym_search(e->kern_stack[i]);
1545                                 if (strcmp(ks->name, nonjit_func) == 0) {
1546                                         good_kern_stack = true;
1547                                         break;
1548                                 }
1549                         }
1550                 }
1551                 if (e->user_stack_size > 0 && e->user_stack_buildid_size > 0)
1552                         good_user_stack = true;
1553         }
1554         if (!good_kern_stack || !good_user_stack)
1555                 return LIBBPF_PERF_EVENT_ERROR;
1556
1557         if (cnt == MAX_CNT_RAWTP)
1558                 return LIBBPF_PERF_EVENT_DONE;
1559
1560         return LIBBPF_PERF_EVENT_CONT;
1561 }
1562
1563 static void test_get_stack_raw_tp(void)
1564 {
1565         const char *file = "./test_get_stack_rawtp.o";
1566         int i, efd, err, prog_fd, pmu_fd, perfmap_fd;
1567         struct perf_event_attr attr = {};
1568         struct timespec tv = {0, 10};
1569         __u32 key = 0, duration = 0;
1570         struct bpf_object *obj;
1571
1572         err = bpf_prog_load(file, BPF_PROG_TYPE_RAW_TRACEPOINT, &obj, &prog_fd);
1573         if (CHECK(err, "prog_load raw tp", "err %d errno %d\n", err, errno))
1574                 return;
1575
1576         efd = bpf_raw_tracepoint_open("sys_enter", prog_fd);
1577         if (CHECK(efd < 0, "raw_tp_open", "err %d errno %d\n", efd, errno))
1578                 goto close_prog;
1579
1580         perfmap_fd = bpf_find_map(__func__, obj, "perfmap");
1581         if (CHECK(perfmap_fd < 0, "bpf_find_map", "err %d errno %d\n",
1582                   perfmap_fd, errno))
1583                 goto close_prog;
1584
1585         err = load_kallsyms();
1586         if (CHECK(err < 0, "load_kallsyms", "err %d errno %d\n", err, errno))
1587                 goto close_prog;
1588
1589         attr.sample_type = PERF_SAMPLE_RAW;
1590         attr.type = PERF_TYPE_SOFTWARE;
1591         attr.config = PERF_COUNT_SW_BPF_OUTPUT;
1592         pmu_fd = syscall(__NR_perf_event_open, &attr, getpid()/*pid*/, -1/*cpu*/,
1593                          -1/*group_fd*/, 0);
1594         if (CHECK(pmu_fd < 0, "perf_event_open", "err %d errno %d\n", pmu_fd,
1595                   errno))
1596                 goto close_prog;
1597
1598         err = bpf_map_update_elem(perfmap_fd, &key, &pmu_fd, BPF_ANY);
1599         if (CHECK(err < 0, "bpf_map_update_elem", "err %d errno %d\n", err,
1600                   errno))
1601                 goto close_prog;
1602
1603         err = ioctl(pmu_fd, PERF_EVENT_IOC_ENABLE, 0);
1604         if (CHECK(err < 0, "ioctl PERF_EVENT_IOC_ENABLE", "err %d errno %d\n",
1605                   err, errno))
1606                 goto close_prog;
1607
1608         err = perf_event_mmap(pmu_fd);
1609         if (CHECK(err < 0, "perf_event_mmap", "err %d errno %d\n", err, errno))
1610                 goto close_prog;
1611
1612         /* trigger some syscall action */
1613         for (i = 0; i < MAX_CNT_RAWTP; i++)
1614                 nanosleep(&tv, NULL);
1615
1616         err = perf_event_poller(pmu_fd, get_stack_print_output);
1617         if (CHECK(err < 0, "perf_event_poller", "err %d errno %d\n", err, errno))
1618                 goto close_prog;
1619
1620         goto close_prog_noerr;
1621 close_prog:
1622         error_cnt++;
1623 close_prog_noerr:
1624         bpf_object__close(obj);
1625 }
1626
1627 static void test_task_fd_query_rawtp(void)
1628 {
1629         const char *file = "./test_get_stack_rawtp.o";
1630         __u64 probe_offset, probe_addr;
1631         __u32 len, prog_id, fd_type;
1632         struct bpf_object *obj;
1633         int efd, err, prog_fd;
1634         __u32 duration = 0;
1635         char buf[256];
1636
1637         err = bpf_prog_load(file, BPF_PROG_TYPE_RAW_TRACEPOINT, &obj, &prog_fd);
1638         if (CHECK(err, "prog_load raw tp", "err %d errno %d\n", err, errno))
1639                 return;
1640
1641         efd = bpf_raw_tracepoint_open("sys_enter", prog_fd);
1642         if (CHECK(efd < 0, "raw_tp_open", "err %d errno %d\n", efd, errno))
1643                 goto close_prog;
1644
1645         /* query (getpid(), efd) */
1646         len = sizeof(buf);
1647         err = bpf_task_fd_query(getpid(), efd, 0, buf, &len, &prog_id,
1648                                 &fd_type, &probe_offset, &probe_addr);
1649         if (CHECK(err < 0, "bpf_task_fd_query", "err %d errno %d\n", err,
1650                   errno))
1651                 goto close_prog;
1652
1653         err = fd_type == BPF_FD_TYPE_RAW_TRACEPOINT &&
1654               strcmp(buf, "sys_enter") == 0;
1655         if (CHECK(!err, "check_results", "fd_type %d tp_name %s\n",
1656                   fd_type, buf))
1657                 goto close_prog;
1658
1659         /* test zero len */
1660         len = 0;
1661         err = bpf_task_fd_query(getpid(), efd, 0, buf, &len, &prog_id,
1662                                 &fd_type, &probe_offset, &probe_addr);
1663         if (CHECK(err < 0, "bpf_task_fd_query (len = 0)", "err %d errno %d\n",
1664                   err, errno))
1665                 goto close_prog;
1666         err = fd_type == BPF_FD_TYPE_RAW_TRACEPOINT &&
1667               len == strlen("sys_enter");
1668         if (CHECK(!err, "check_results", "fd_type %d len %u\n", fd_type, len))
1669                 goto close_prog;
1670
1671         /* test empty buffer */
1672         len = sizeof(buf);
1673         err = bpf_task_fd_query(getpid(), efd, 0, 0, &len, &prog_id,
1674                                 &fd_type, &probe_offset, &probe_addr);
1675         if (CHECK(err < 0, "bpf_task_fd_query (buf = 0)", "err %d errno %d\n",
1676                   err, errno))
1677                 goto close_prog;
1678         err = fd_type == BPF_FD_TYPE_RAW_TRACEPOINT &&
1679               len == strlen("sys_enter");
1680         if (CHECK(!err, "check_results", "fd_type %d len %u\n", fd_type, len))
1681                 goto close_prog;
1682
1683         /* test smaller buffer */
1684         len = 3;
1685         err = bpf_task_fd_query(getpid(), efd, 0, buf, &len, &prog_id,
1686                                 &fd_type, &probe_offset, &probe_addr);
1687         if (CHECK(err >= 0 || errno != ENOSPC, "bpf_task_fd_query (len = 3)",
1688                   "err %d errno %d\n", err, errno))
1689                 goto close_prog;
1690         err = fd_type == BPF_FD_TYPE_RAW_TRACEPOINT &&
1691               len == strlen("sys_enter") &&
1692               strcmp(buf, "sy") == 0;
1693         if (CHECK(!err, "check_results", "fd_type %d len %u\n", fd_type, len))
1694                 goto close_prog;
1695
1696         goto close_prog_noerr;
1697 close_prog:
1698         error_cnt++;
1699 close_prog_noerr:
1700         bpf_object__close(obj);
1701 }
1702
1703 static void test_task_fd_query_tp_core(const char *probe_name,
1704                                        const char *tp_name)
1705 {
1706         const char *file = "./test_tracepoint.o";
1707         int err, bytes, efd, prog_fd, pmu_fd;
1708         struct perf_event_attr attr = {};
1709         __u64 probe_offset, probe_addr;
1710         __u32 len, prog_id, fd_type;
1711         struct bpf_object *obj;
1712         __u32 duration = 0;
1713         char buf[256];
1714
1715         err = bpf_prog_load(file, BPF_PROG_TYPE_TRACEPOINT, &obj, &prog_fd);
1716         if (CHECK(err, "bpf_prog_load", "err %d errno %d\n", err, errno))
1717                 goto close_prog;
1718
1719         snprintf(buf, sizeof(buf),
1720                  "/sys/kernel/debug/tracing/events/%s/id", probe_name);
1721         efd = open(buf, O_RDONLY, 0);
1722         if (CHECK(efd < 0, "open", "err %d errno %d\n", efd, errno))
1723                 goto close_prog;
1724         bytes = read(efd, buf, sizeof(buf));
1725         close(efd);
1726         if (CHECK(bytes <= 0 || bytes >= sizeof(buf), "read",
1727                   "bytes %d errno %d\n", bytes, errno))
1728                 goto close_prog;
1729
1730         attr.config = strtol(buf, NULL, 0);
1731         attr.type = PERF_TYPE_TRACEPOINT;
1732         attr.sample_type = PERF_SAMPLE_RAW;
1733         attr.sample_period = 1;
1734         attr.wakeup_events = 1;
1735         pmu_fd = syscall(__NR_perf_event_open, &attr, -1 /* pid */,
1736                          0 /* cpu 0 */, -1 /* group id */,
1737                          0 /* flags */);
1738         if (CHECK(err, "perf_event_open", "err %d errno %d\n", err, errno))
1739                 goto close_pmu;
1740
1741         err = ioctl(pmu_fd, PERF_EVENT_IOC_ENABLE, 0);
1742         if (CHECK(err, "perf_event_ioc_enable", "err %d errno %d\n", err,
1743                   errno))
1744                 goto close_pmu;
1745
1746         err = ioctl(pmu_fd, PERF_EVENT_IOC_SET_BPF, prog_fd);
1747         if (CHECK(err, "perf_event_ioc_set_bpf", "err %d errno %d\n", err,
1748                   errno))
1749                 goto close_pmu;
1750
1751         /* query (getpid(), pmu_fd) */
1752         len = sizeof(buf);
1753         err = bpf_task_fd_query(getpid(), pmu_fd, 0, buf, &len, &prog_id,
1754                                 &fd_type, &probe_offset, &probe_addr);
1755         if (CHECK(err < 0, "bpf_task_fd_query", "err %d errno %d\n", err,
1756                   errno))
1757                 goto close_pmu;
1758
1759         err = (fd_type == BPF_FD_TYPE_TRACEPOINT) && !strcmp(buf, tp_name);
1760         if (CHECK(!err, "check_results", "fd_type %d tp_name %s\n",
1761                   fd_type, buf))
1762                 goto close_pmu;
1763
1764         close(pmu_fd);
1765         goto close_prog_noerr;
1766
1767 close_pmu:
1768         close(pmu_fd);
1769 close_prog:
1770         error_cnt++;
1771 close_prog_noerr:
1772         bpf_object__close(obj);
1773 }
1774
1775 static void test_task_fd_query_tp(void)
1776 {
1777         test_task_fd_query_tp_core("sched/sched_switch",
1778                                    "sched_switch");
1779         test_task_fd_query_tp_core("syscalls/sys_enter_read",
1780                                    "sys_enter_read");
1781 }
1782
1783 static void test_reference_tracking()
1784 {
1785         const char *file = "./test_sk_lookup_kern.o";
1786         struct bpf_object *obj;
1787         struct bpf_program *prog;
1788         __u32 duration = 0;
1789         int err = 0;
1790
1791         obj = bpf_object__open(file);
1792         if (IS_ERR(obj)) {
1793                 error_cnt++;
1794                 return;
1795         }
1796
1797         bpf_object__for_each_program(prog, obj) {
1798                 const char *title;
1799
1800                 /* Ignore .text sections */
1801                 title = bpf_program__title(prog, false);
1802                 if (strstr(title, ".text") != NULL)
1803                         continue;
1804
1805                 bpf_program__set_type(prog, BPF_PROG_TYPE_SCHED_CLS);
1806
1807                 /* Expect verifier failure if test name has 'fail' */
1808                 if (strstr(title, "fail") != NULL) {
1809                         libbpf_set_print(NULL, NULL, NULL);
1810                         err = !bpf_program__load(prog, "GPL", 0);
1811                         libbpf_set_print(printf, printf, NULL);
1812                 } else {
1813                         err = bpf_program__load(prog, "GPL", 0);
1814                 }
1815                 CHECK(err, title, "\n");
1816         }
1817         bpf_object__close(obj);
1818 }
1819
1820 enum {
1821         QUEUE,
1822         STACK,
1823 };
1824
1825 static void test_queue_stack_map(int type)
1826 {
1827         const int MAP_SIZE = 32;
1828         __u32 vals[MAP_SIZE], duration, retval, size, val;
1829         int i, err, prog_fd, map_in_fd, map_out_fd;
1830         char file[32], buf[128];
1831         struct bpf_object *obj;
1832         struct iphdr *iph = (void *)buf + sizeof(struct ethhdr);
1833
1834         /* Fill test values to be used */
1835         for (i = 0; i < MAP_SIZE; i++)
1836                 vals[i] = rand();
1837
1838         if (type == QUEUE)
1839                 strncpy(file, "./test_queue_map.o", sizeof(file));
1840         else if (type == STACK)
1841                 strncpy(file, "./test_stack_map.o", sizeof(file));
1842         else
1843                 return;
1844
1845         err = bpf_prog_load(file, BPF_PROG_TYPE_SCHED_CLS, &obj, &prog_fd);
1846         if (err) {
1847                 error_cnt++;
1848                 return;
1849         }
1850
1851         map_in_fd = bpf_find_map(__func__, obj, "map_in");
1852         if (map_in_fd < 0)
1853                 goto out;
1854
1855         map_out_fd = bpf_find_map(__func__, obj, "map_out");
1856         if (map_out_fd < 0)
1857                 goto out;
1858
1859         /* Push 32 elements to the input map */
1860         for (i = 0; i < MAP_SIZE; i++) {
1861                 err = bpf_map_update_elem(map_in_fd, NULL, &vals[i], 0);
1862                 if (err) {
1863                         error_cnt++;
1864                         goto out;
1865                 }
1866         }
1867
1868         /* The eBPF program pushes iph.saddr in the output map,
1869          * pops the input map and saves this value in iph.daddr
1870          */
1871         for (i = 0; i < MAP_SIZE; i++) {
1872                 if (type == QUEUE) {
1873                         val = vals[i];
1874                         pkt_v4.iph.saddr = vals[i] * 5;
1875                 } else if (type == STACK) {
1876                         val = vals[MAP_SIZE - 1 - i];
1877                         pkt_v4.iph.saddr = vals[MAP_SIZE - 1 - i] * 5;
1878                 }
1879
1880                 err = bpf_prog_test_run(prog_fd, 1, &pkt_v4, sizeof(pkt_v4),
1881                                         buf, &size, &retval, &duration);
1882                 if (err || retval || size != sizeof(pkt_v4) ||
1883                     iph->daddr != val)
1884                         break;
1885         }
1886
1887         CHECK(err || retval || size != sizeof(pkt_v4) || iph->daddr != val,
1888               "bpf_map_pop_elem",
1889               "err %d errno %d retval %d size %d iph->daddr %u\n",
1890               err, errno, retval, size, iph->daddr);
1891
1892         /* Queue is empty, program should return TC_ACT_SHOT */
1893         err = bpf_prog_test_run(prog_fd, 1, &pkt_v4, sizeof(pkt_v4),
1894                                 buf, &size, &retval, &duration);
1895         CHECK(err || retval != 2 /* TC_ACT_SHOT */|| size != sizeof(pkt_v4),
1896               "check-queue-stack-map-empty",
1897               "err %d errno %d retval %d size %d\n",
1898               err, errno, retval, size);
1899
1900         /* Check that the program pushed elements correctly */
1901         for (i = 0; i < MAP_SIZE; i++) {
1902                 err = bpf_map_lookup_and_delete_elem(map_out_fd, NULL, &val);
1903                 if (err || val != vals[i] * 5)
1904                         break;
1905         }
1906
1907         CHECK(i != MAP_SIZE && (err || val != vals[i] * 5),
1908               "bpf_map_push_elem", "err %d value %u\n", err, val);
1909
1910 out:
1911         pkt_v4.iph.saddr = 0;
1912         bpf_object__close(obj);
1913 }
1914
1915 int main(void)
1916 {
1917         srand(time(NULL));
1918
1919         jit_enabled = is_jit_enabled();
1920
1921         test_pkt_access();
1922         test_prog_run_xattr();
1923         test_xdp();
1924         test_xdp_adjust_tail();
1925         test_l4lb_all();
1926         test_xdp_noinline();
1927         test_tcp_estats();
1928         test_bpf_obj_id();
1929         test_pkt_md_access();
1930         test_obj_name();
1931         test_tp_attach_query();
1932         test_stacktrace_map();
1933         test_stacktrace_build_id();
1934         test_stacktrace_build_id_nmi();
1935         test_stacktrace_map_raw_tp();
1936         test_get_stack_raw_tp();
1937         test_task_fd_query_rawtp();
1938         test_task_fd_query_tp();
1939         test_reference_tracking();
1940         test_queue_stack_map(QUEUE);
1941         test_queue_stack_map(STACK);
1942
1943         printf("Summary: %d PASSED, %d FAILED\n", pass_cnt, error_cnt);
1944         return error_cnt ? EXIT_FAILURE : EXIT_SUCCESS;
1945 }