1 // SPDX-License-Identifier: GPL-2.0
6 #include <bpf/libbpf.h>
9 #include <linux/string.h>
10 #include <internal/lib.h>
11 #include <symbol/kallsyms.h>
12 #include "bpf-event.h"
13 #include "bpf-utils.h"
23 #include "util/synthetic-events.h"
25 #ifndef HAVE_LIBBPF_BTF__LOAD_FROM_KERNEL_BY_ID
26 struct btf *btf__load_from_kernel_by_id(__u32 id)
29 #pragma GCC diagnostic push
30 #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
31 int err = btf__get_from_id(id, &btf);
32 #pragma GCC diagnostic pop
34 return err ? ERR_PTR(err) : btf;
38 int __weak bpf_prog_load(enum bpf_prog_type prog_type,
39 const char *prog_name __maybe_unused,
41 const struct bpf_insn *insns, size_t insn_cnt,
42 const struct bpf_prog_load_opts *opts)
44 #pragma GCC diagnostic push
45 #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
46 return bpf_load_program(prog_type, insns, insn_cnt, license,
47 opts->kern_version, opts->log_buf, opts->log_size);
48 #pragma GCC diagnostic pop
51 struct bpf_program * __weak
52 bpf_object__next_program(const struct bpf_object *obj, struct bpf_program *prev)
54 #pragma GCC diagnostic push
55 #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
56 return bpf_program__next(prev, obj);
57 #pragma GCC diagnostic pop
60 struct bpf_map * __weak
61 bpf_object__next_map(const struct bpf_object *obj, const struct bpf_map *prev)
63 #pragma GCC diagnostic push
64 #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
65 return bpf_map__next(prev, obj);
66 #pragma GCC diagnostic pop
70 btf__raw_data(const struct btf *btf_ro, __u32 *size)
72 #pragma GCC diagnostic push
73 #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
74 return btf__get_raw_data(btf_ro, size);
75 #pragma GCC diagnostic pop
78 static int snprintf_hex(char *buf, size_t size, unsigned char *data, size_t len)
83 for (i = 0; i < len; i++)
84 ret += snprintf(buf + ret, size - ret, "%02x", data[i]);
88 static int machine__process_bpf_event_load(struct machine *machine,
89 union perf_event *event,
90 struct perf_sample *sample __maybe_unused)
92 struct bpf_prog_info_node *info_node;
93 struct perf_env *env = machine->env;
94 struct perf_bpil *info_linear;
95 int id = event->bpf.id;
98 /* perf-record, no need to handle bpf-event */
102 info_node = perf_env__find_bpf_prog_info(env, id);
105 info_linear = info_node->info_linear;
107 for (i = 0; i < info_linear->info.nr_jited_ksyms; i++) {
108 u64 *addrs = (u64 *)(uintptr_t)(info_linear->info.jited_ksyms);
110 struct map *map = maps__find(machine__kernel_maps(machine), addr);
113 map->dso->binary_type = DSO_BINARY_TYPE__BPF_PROG_INFO;
114 map->dso->bpf_prog.id = id;
115 map->dso->bpf_prog.sub_id = i;
116 map->dso->bpf_prog.env = env;
122 int machine__process_bpf(struct machine *machine, union perf_event *event,
123 struct perf_sample *sample)
126 perf_event__fprintf_bpf(event, stdout);
128 switch (event->bpf.type) {
129 case PERF_BPF_EVENT_PROG_LOAD:
130 return machine__process_bpf_event_load(machine, event, sample);
132 case PERF_BPF_EVENT_PROG_UNLOAD:
134 * Do not free bpf_prog_info and btf of the program here,
135 * as annotation still need them. They will be freed at
136 * the end of the session.
140 pr_debug("unexpected bpf event type of %d\n", event->bpf.type);
146 static int perf_env__fetch_btf(struct perf_env *env,
150 struct btf_node *node;
154 data = btf__raw_data(btf, &data_size);
156 node = malloc(data_size + sizeof(struct btf_node));
161 node->data_size = data_size;
162 memcpy(node->data, data, data_size);
164 if (!perf_env__insert_btf(env, node)) {
165 /* Insertion failed because of a duplicate. */
172 static int synthesize_bpf_prog_name(char *buf, int size,
173 struct bpf_prog_info *info,
177 u8 (*prog_tags)[BPF_TAG_SIZE] = (void *)(uintptr_t)(info->prog_tags);
178 void *func_infos = (void *)(uintptr_t)(info->func_info);
179 u32 sub_prog_cnt = info->nr_jited_ksyms;
180 const struct bpf_func_info *finfo;
181 const char *short_name = NULL;
182 const struct btf_type *t;
185 name_len = snprintf(buf, size, "bpf_prog_");
186 name_len += snprintf_hex(buf + name_len, size - name_len,
187 prog_tags[sub_id], BPF_TAG_SIZE);
189 finfo = func_infos + sub_id * info->func_info_rec_size;
190 t = btf__type_by_id(btf, finfo->type_id);
191 short_name = btf__name_by_offset(btf, t->name_off);
192 } else if (sub_id == 0 && sub_prog_cnt == 1) {
195 short_name = info->name;
199 name_len += snprintf(buf + name_len, size - name_len,
205 * Synthesize PERF_RECORD_KSYMBOL and PERF_RECORD_BPF_EVENT for one bpf
206 * program. One PERF_RECORD_BPF_EVENT is generated for the program. And
207 * one PERF_RECORD_KSYMBOL is generated for each sub program.
212 * -2 for lack of kernel support.
214 static int perf_event__synthesize_one_bpf_prog(struct perf_session *session,
215 perf_event__handler_t process,
216 struct machine *machine,
218 union perf_event *event,
219 struct record_opts *opts)
221 struct perf_record_ksymbol *ksymbol_event = &event->ksymbol;
222 struct perf_record_bpf_event *bpf_event = &event->bpf;
223 struct perf_tool *tool = session->tool;
224 struct bpf_prog_info_node *info_node;
225 struct perf_bpil *info_linear;
226 struct bpf_prog_info *info;
227 struct btf *btf = NULL;
228 struct perf_env *env;
234 * for perf-record and perf-report use header.env;
235 * otherwise, use global perf_env.
237 env = session->data ? &session->header.env : &perf_env;
239 arrays = 1UL << PERF_BPIL_JITED_KSYMS;
240 arrays |= 1UL << PERF_BPIL_JITED_FUNC_LENS;
241 arrays |= 1UL << PERF_BPIL_FUNC_INFO;
242 arrays |= 1UL << PERF_BPIL_PROG_TAGS;
243 arrays |= 1UL << PERF_BPIL_JITED_INSNS;
244 arrays |= 1UL << PERF_BPIL_LINE_INFO;
245 arrays |= 1UL << PERF_BPIL_JITED_LINE_INFO;
247 info_linear = get_bpf_prog_info_linear(fd, arrays);
248 if (IS_ERR_OR_NULL(info_linear)) {
250 pr_debug("%s: failed to get BPF program info. aborting\n", __func__);
254 if (info_linear->info_len < offsetof(struct bpf_prog_info, prog_tags)) {
256 pr_debug("%s: the kernel is too old, aborting\n", __func__);
260 info = &info_linear->info;
261 if (!info->jited_ksyms) {
266 /* number of ksyms, func_lengths, and tags should match */
267 sub_prog_cnt = info->nr_jited_ksyms;
268 if (sub_prog_cnt != info->nr_prog_tags ||
269 sub_prog_cnt != info->nr_jited_func_lens) {
274 /* check BTF func info support */
275 if (info->btf_id && info->nr_func_info && info->func_info_rec_size) {
276 /* btf func info number should be same as sub_prog_cnt */
277 if (sub_prog_cnt != info->nr_func_info) {
278 pr_debug("%s: mismatch in BPF sub program count and BTF function info count, aborting\n", __func__);
282 btf = btf__load_from_kernel_by_id(info->btf_id);
283 if (libbpf_get_error(btf)) {
284 pr_debug("%s: failed to get BTF of id %u, aborting\n", __func__, info->btf_id);
288 perf_env__fetch_btf(env, info->btf_id, btf);
291 /* Synthesize PERF_RECORD_KSYMBOL */
292 for (i = 0; i < sub_prog_cnt; i++) {
293 __u32 *prog_lens = (__u32 *)(uintptr_t)(info->jited_func_lens);
294 __u64 *prog_addrs = (__u64 *)(uintptr_t)(info->jited_ksyms);
297 *ksymbol_event = (struct perf_record_ksymbol) {
299 .type = PERF_RECORD_KSYMBOL,
300 .size = offsetof(struct perf_record_ksymbol, name),
302 .addr = prog_addrs[i],
304 .ksym_type = PERF_RECORD_KSYMBOL_TYPE_BPF,
308 name_len = synthesize_bpf_prog_name(ksymbol_event->name,
309 KSYM_NAME_LEN, info, btf, i);
310 ksymbol_event->header.size += PERF_ALIGN(name_len + 1,
313 memset((void *)event + event->header.size, 0, machine->id_hdr_size);
314 event->header.size += machine->id_hdr_size;
315 err = perf_tool__process_synth_event(tool, event,
319 if (!opts->no_bpf_event) {
320 /* Synthesize PERF_RECORD_BPF_EVENT */
321 *bpf_event = (struct perf_record_bpf_event) {
323 .type = PERF_RECORD_BPF_EVENT,
324 .size = sizeof(struct perf_record_bpf_event),
326 .type = PERF_BPF_EVENT_PROG_LOAD,
330 memcpy(bpf_event->tag, info->tag, BPF_TAG_SIZE);
331 memset((void *)event + event->header.size, 0, machine->id_hdr_size);
332 event->header.size += machine->id_hdr_size;
334 /* save bpf_prog_info to env */
335 info_node = malloc(sizeof(struct bpf_prog_info_node));
341 info_node->info_linear = info_linear;
342 perf_env__insert_bpf_prog_info(env, info_node);
346 * process after saving bpf_prog_info to env, so that
347 * required information is ready for look up
349 err = perf_tool__process_synth_event(tool, event,
359 struct kallsyms_parse {
360 union perf_event *event;
361 perf_event__handler_t process;
362 struct machine *machine;
363 struct perf_tool *tool;
367 process_bpf_image(char *name, u64 addr, struct kallsyms_parse *data)
369 struct machine *machine = data->machine;
370 union perf_event *event = data->event;
371 struct perf_record_ksymbol *ksymbol;
374 ksymbol = &event->ksymbol;
376 *ksymbol = (struct perf_record_ksymbol) {
378 .type = PERF_RECORD_KSYMBOL,
379 .size = offsetof(struct perf_record_ksymbol, name),
383 .ksym_type = PERF_RECORD_KSYMBOL_TYPE_BPF,
387 len = scnprintf(ksymbol->name, KSYM_NAME_LEN, "%s", name);
388 ksymbol->header.size += PERF_ALIGN(len + 1, sizeof(u64));
389 memset((void *) event + event->header.size, 0, machine->id_hdr_size);
390 event->header.size += machine->id_hdr_size;
392 return perf_tool__process_synth_event(data->tool, event, machine,
397 kallsyms_process_symbol(void *data, const char *_name,
398 char type __maybe_unused, u64 start)
400 char disp[KSYM_NAME_LEN];
405 module = strchr(_name, '\t');
409 /* We are going after [bpf] module ... */
410 if (strcmp(module + 1, "[bpf]"))
413 name = memdup(_name, (module - _name) + 1);
417 name[module - _name] = 0;
419 /* .. and only for trampolines and dispatchers */
420 if ((sscanf(name, "bpf_trampoline_%lu", &id) == 1) ||
421 (sscanf(name, "bpf_dispatcher_%s", disp) == 1))
422 err = process_bpf_image(name, start, data);
428 int perf_event__synthesize_bpf_events(struct perf_session *session,
429 perf_event__handler_t process,
430 struct machine *machine,
431 struct record_opts *opts)
433 const char *kallsyms_filename = "/proc/kallsyms";
434 struct kallsyms_parse arg;
435 union perf_event *event;
440 event = malloc(sizeof(event->bpf) + KSYM_NAME_LEN + machine->id_hdr_size);
444 /* Synthesize all the bpf programs in system. */
446 err = bpf_prog_get_next_id(id, &id);
448 if (errno == ENOENT) {
452 pr_debug("%s: can't get next program: %s%s\n",
453 __func__, strerror(errno),
454 errno == EINVAL ? " -- kernel too old?" : "");
455 /* don't report error on old kernel or EPERM */
456 err = (errno == EINVAL || errno == EPERM) ? 0 : -1;
459 fd = bpf_prog_get_fd_by_id(id);
461 pr_debug("%s: failed to get fd for prog_id %u\n",
466 err = perf_event__synthesize_one_bpf_prog(session, process,
471 /* do not return error for old kernel */
478 /* Synthesize all the bpf images - trampolines/dispatchers. */
479 if (symbol_conf.kallsyms_name != NULL)
480 kallsyms_filename = symbol_conf.kallsyms_name;
482 arg = (struct kallsyms_parse) {
486 .tool = session->tool,
489 if (kallsyms__parse(kallsyms_filename, &arg, kallsyms_process_symbol)) {
490 pr_err("%s: failed to synthesize bpf images: %s\n",
491 __func__, strerror(errno));
498 static void perf_env__add_bpf_info(struct perf_env *env, u32 id)
500 struct bpf_prog_info_node *info_node;
501 struct perf_bpil *info_linear;
502 struct btf *btf = NULL;
507 fd = bpf_prog_get_fd_by_id(id);
511 arrays = 1UL << PERF_BPIL_JITED_KSYMS;
512 arrays |= 1UL << PERF_BPIL_JITED_FUNC_LENS;
513 arrays |= 1UL << PERF_BPIL_FUNC_INFO;
514 arrays |= 1UL << PERF_BPIL_PROG_TAGS;
515 arrays |= 1UL << PERF_BPIL_JITED_INSNS;
516 arrays |= 1UL << PERF_BPIL_LINE_INFO;
517 arrays |= 1UL << PERF_BPIL_JITED_LINE_INFO;
519 info_linear = get_bpf_prog_info_linear(fd, arrays);
520 if (IS_ERR_OR_NULL(info_linear)) {
521 pr_debug("%s: failed to get BPF program info. aborting\n", __func__);
525 btf_id = info_linear->info.btf_id;
527 info_node = malloc(sizeof(struct bpf_prog_info_node));
529 info_node->info_linear = info_linear;
530 perf_env__insert_bpf_prog_info(env, info_node);
537 btf = btf__load_from_kernel_by_id(btf_id);
538 if (libbpf_get_error(btf)) {
539 pr_debug("%s: failed to get BTF of id %u, aborting\n",
543 perf_env__fetch_btf(env, btf_id, btf);
550 static int bpf_event__sb_cb(union perf_event *event, void *data)
552 struct perf_env *env = data;
554 if (event->header.type != PERF_RECORD_BPF_EVENT)
557 switch (event->bpf.type) {
558 case PERF_BPF_EVENT_PROG_LOAD:
559 perf_env__add_bpf_info(env, event->bpf.id);
561 case PERF_BPF_EVENT_PROG_UNLOAD:
563 * Do not free bpf_prog_info and btf of the program here,
564 * as annotation still need them. They will be freed at
565 * the end of the session.
569 pr_debug("unexpected bpf event type of %d\n", event->bpf.type);
576 int evlist__add_bpf_sb_event(struct evlist *evlist, struct perf_env *env)
578 struct perf_event_attr attr = {
579 .type = PERF_TYPE_SOFTWARE,
580 .config = PERF_COUNT_SW_DUMMY,
584 .size = sizeof(attr), /* to capture ABI version */
588 * Older gcc versions don't support designated initializers, like above,
589 * for unnamed union members, such as the following:
591 attr.wakeup_watermark = 1;
593 return evlist__add_sb_event(evlist, &attr, bpf_event__sb_cb, env);
596 void bpf_event__print_bpf_prog_info(struct bpf_prog_info *info,
597 struct perf_env *env,
600 __u32 *prog_lens = (__u32 *)(uintptr_t)(info->jited_func_lens);
601 __u64 *prog_addrs = (__u64 *)(uintptr_t)(info->jited_ksyms);
602 char name[KSYM_NAME_LEN];
603 struct btf *btf = NULL;
606 sub_prog_cnt = info->nr_jited_ksyms;
607 if (sub_prog_cnt != info->nr_prog_tags ||
608 sub_prog_cnt != info->nr_jited_func_lens)
612 struct btf_node *node;
614 node = perf_env__find_btf(env, info->btf_id);
616 btf = btf__new((__u8 *)(node->data),
620 if (sub_prog_cnt == 1) {
621 synthesize_bpf_prog_name(name, KSYM_NAME_LEN, info, btf, 0);
622 fprintf(fp, "# bpf_prog_info %u: %s addr 0x%llx size %u\n",
623 info->id, name, prog_addrs[0], prog_lens[0]);
627 fprintf(fp, "# bpf_prog_info %u:\n", info->id);
628 for (i = 0; i < sub_prog_cnt; i++) {
629 synthesize_bpf_prog_name(name, KSYM_NAME_LEN, info, btf, i);
631 fprintf(fp, "# \tsub_prog %u: %s addr 0x%llx size %u\n",
632 i, name, prog_addrs[i], prog_lens[i]);