tools/perf/util/bpf_off_cpu.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 #include "util/bpf_counter.h"
   3 #include "util/debug.h"
   4 #include "util/evsel.h"
   5 #include "util/evlist.h"
   6 #include "util/off_cpu.h"
   7 #include "util/perf-hooks.h"
   8 #include "util/record.h"
   9 #include "util/session.h"
  10 #include "util/target.h"
  11 #include "util/cpumap.h"
  12 #include "util/thread_map.h"
  13 #include "util/cgroup.h"
  14 #include <bpf/bpf.h>
  15
  16 #include "bpf_skel/off_cpu.skel.h"
  17
  18 #define MAX_STACKS  32
  19 /* we don't need actual timestamp, just want to put the samples at last */
  20 #define OFF_CPU_TIMESTAMP  (~0ull << 32)
  21
  22 static struct off_cpu_bpf *skel;
  23
  24 struct off_cpu_key {
  25         u32 pid;
  26         u32 tgid;
  27         u32 stack_id;
  28         u32 state;
  29         u64 cgroup_id;
  30 };
  31
  32 union off_cpu_data {
  33         struct perf_event_header hdr;
  34         u64 array[1024 / sizeof(u64)];
  35 };
  36
  37 static int off_cpu_config(struct evlist *evlist)
  38 {
  39         struct evsel *evsel;
  40         struct perf_event_attr attr = {
  41                 .type   = PERF_TYPE_SOFTWARE,
  42                 .config = PERF_COUNT_SW_BPF_OUTPUT,
  43                 .size   = sizeof(attr), /* to capture ABI version */
  44         };
  45         char *evname = strdup(OFFCPU_EVENT);
  46
  47         if (evname == NULL)
  48                 return -ENOMEM;
  49
  50         evsel = evsel__new(&attr);
  51         if (!evsel) {
  52                 free(evname);
  53                 return -ENOMEM;
  54         }
  55
  56         evsel->core.attr.freq = 1;
  57         evsel->core.attr.sample_period = 1;
  58         /* off-cpu analysis depends on stack trace */
  59         evsel->core.attr.sample_type = PERF_SAMPLE_CALLCHAIN;
  60
  61         evlist__add(evlist, evsel);
  62
  63         free(evsel->name);
  64         evsel->name = evname;
  65
  66         return 0;
  67 }
  68
  69 static void off_cpu_start(void *arg)
  70 {
  71         struct evlist *evlist = arg;
  72
  73         /* update task filter for the given workload */
  74         if (!skel->bss->has_cpu && !skel->bss->has_task &&
  75             perf_thread_map__pid(evlist->core.threads, 0) != -1) {
  76                 int fd;
  77                 u32 pid;
  78                 u8 val = 1;
  79
  80                 skel->bss->has_task = 1;
  81                 fd = bpf_map__fd(skel->maps.task_filter);
  82                 pid = perf_thread_map__pid(evlist->core.threads, 0);
  83                 bpf_map_update_elem(fd, &pid, &val, BPF_ANY);
  84         }
  85
  86         skel->bss->enabled = 1;
  87 }
  88
  89 static void off_cpu_finish(void *arg __maybe_unused)
  90 {
  91         skel->bss->enabled = 0;
  92         off_cpu_bpf__destroy(skel);
  93 }
  94
  95 /* v5.18 kernel added prev_state arg, so it needs to check the signature */
  96 static void check_sched_switch_args(void)
  97 {
  98         const struct btf *btf = bpf_object__btf(skel->obj);
  99         const struct btf_type *t1, *t2, *t3;
 100         u32 type_id;
 101
 102         type_id = btf__find_by_name_kind(btf, "bpf_trace_sched_switch",
 103                                          BTF_KIND_TYPEDEF);
 104         if ((s32)type_id < 0)
 105                 return;
 106
 107         t1 = btf__type_by_id(btf, type_id);
 108         if (t1 == NULL)
 109                 return;
 110
 111         t2 = btf__type_by_id(btf, t1->type);
 112         if (t2 == NULL || !btf_is_ptr(t2))
 113                 return;
 114
 115         t3 = btf__type_by_id(btf, t2->type);
 116         if (t3 && btf_is_func_proto(t3) && btf_vlen(t3) == 4) {
 117                 /* new format: pass prev_state as 4th arg */
 118                 skel->rodata->has_prev_state = true;
 119         }
 120 }
 121
 122 int off_cpu_prepare(struct evlist *evlist, struct target *target,
 123                     struct record_opts *opts)
 124 {
 125         int err, fd, i;
 126         int ncpus = 1, ntasks = 1, ncgrps = 1;
 127
 128         if (off_cpu_config(evlist) < 0) {
 129                 pr_err("Failed to config off-cpu BPF event\n");
 130                 return -1;
 131         }
 132
 133         skel = off_cpu_bpf__open();
 134         if (!skel) {
 135                 pr_err("Failed to open off-cpu BPF skeleton\n");
 136                 return -1;
 137         }
 138
 139         /* don't need to set cpu filter for system-wide mode */
 140         if (target->cpu_list) {
 141                 ncpus = perf_cpu_map__nr(evlist->core.user_requested_cpus);
 142                 bpf_map__set_max_entries(skel->maps.cpu_filter, ncpus);
 143         }
 144
 145         if (target__has_task(target)) {
 146                 ntasks = perf_thread_map__nr(evlist->core.threads);
 147                 bpf_map__set_max_entries(skel->maps.task_filter, ntasks);
 148         }
 149
 150         if (evlist__first(evlist)->cgrp) {
 151                 ncgrps = evlist->core.nr_entries - 1; /* excluding a dummy */
 152                 bpf_map__set_max_entries(skel->maps.cgroup_filter, ncgrps);
 153
 154                 if (!cgroup_is_v2("perf_event"))
 155                         skel->rodata->uses_cgroup_v1 = true;
 156         }
 157
 158         if (opts->record_cgroup) {
 159                 skel->rodata->needs_cgroup = true;
 160
 161                 if (!cgroup_is_v2("perf_event"))
 162                         skel->rodata->uses_cgroup_v1 = true;
 163         }
 164
 165         set_max_rlimit();
 166         check_sched_switch_args();
 167
 168         err = off_cpu_bpf__load(skel);
 169         if (err) {
 170                 pr_err("Failed to load off-cpu skeleton\n");
 171                 goto out;
 172         }
 173
 174         if (target->cpu_list) {
 175                 u32 cpu;
 176                 u8 val = 1;
 177
 178                 skel->bss->has_cpu = 1;
 179                 fd = bpf_map__fd(skel->maps.cpu_filter);
 180
 181                 for (i = 0; i < ncpus; i++) {
 182                         cpu = perf_cpu_map__cpu(evlist->core.user_requested_cpus, i).cpu;
 183                         bpf_map_update_elem(fd, &cpu, &val, BPF_ANY);
 184                 }
 185         }
 186
 187         if (target__has_task(target)) {
 188                 u32 pid;
 189                 u8 val = 1;
 190
 191                 skel->bss->has_task = 1;
 192                 fd = bpf_map__fd(skel->maps.task_filter);
 193
 194                 for (i = 0; i < ntasks; i++) {
 195                         pid = perf_thread_map__pid(evlist->core.threads, i);
 196                         bpf_map_update_elem(fd, &pid, &val, BPF_ANY);
 197                 }
 198         }
 199
 200         if (evlist__first(evlist)->cgrp) {
 201                 struct evsel *evsel;
 202                 u8 val = 1;
 203
 204                 skel->bss->has_cgroup = 1;
 205                 fd = bpf_map__fd(skel->maps.cgroup_filter);
 206
 207                 evlist__for_each_entry(evlist, evsel) {
 208                         struct cgroup *cgrp = evsel->cgrp;
 209
 210                         if (cgrp == NULL)
 211                                 continue;
 212
 213                         if (!cgrp->id && read_cgroup_id(cgrp) < 0) {
 214                                 pr_err("Failed to read cgroup id of %s\n",
 215                                        cgrp->name);
 216                                 goto out;
 217                         }
 218
 219                         bpf_map_update_elem(fd, &cgrp->id, &val, BPF_ANY);
 220                 }
 221         }
 222
 223         err = off_cpu_bpf__attach(skel);
 224         if (err) {
 225                 pr_err("Failed to attach off-cpu BPF skeleton\n");
 226                 goto out;
 227         }
 228
 229         if (perf_hooks__set_hook("record_start", off_cpu_start, evlist) ||
 230             perf_hooks__set_hook("record_end", off_cpu_finish, evlist)) {
 231                 pr_err("Failed to attach off-cpu skeleton\n");
 232                 goto out;
 233         }
 234
 235         return 0;
 236
 237 out:
 238         off_cpu_bpf__destroy(skel);
 239         return -1;
 240 }
 241
 242 int off_cpu_write(struct perf_session *session)
 243 {
 244         int bytes = 0, size;
 245         int fd, stack;
 246         u64 sample_type, val, sid = 0;
 247         struct evsel *evsel;
 248         struct perf_data_file *file = &session->data->file;
 249         struct off_cpu_key prev, key;
 250         union off_cpu_data data = {
 251                 .hdr = {
 252                         .type = PERF_RECORD_SAMPLE,
 253                         .misc = PERF_RECORD_MISC_USER,
 254                 },
 255         };
 256         u64 tstamp = OFF_CPU_TIMESTAMP;
 257
 258         skel->bss->enabled = 0;
 259
 260         evsel = evlist__find_evsel_by_str(session->evlist, OFFCPU_EVENT);
 261         if (evsel == NULL) {
 262                 pr_err("%s evsel not found\n", OFFCPU_EVENT);
 263                 return 0;
 264         }
 265
 266         sample_type = evsel->core.attr.sample_type;
 267
 268         if (sample_type & ~OFFCPU_SAMPLE_TYPES) {
 269                 pr_err("not supported sample type: %llx\n",
 270                        (unsigned long long)sample_type);
 271                 return -1;
 272         }
 273
 274         if (sample_type & (PERF_SAMPLE_ID | PERF_SAMPLE_IDENTIFIER)) {
 275                 if (evsel->core.id)
 276                         sid = evsel->core.id[0];
 277         }
 278
 279         fd = bpf_map__fd(skel->maps.off_cpu);
 280         stack = bpf_map__fd(skel->maps.stacks);
 281         memset(&prev, 0, sizeof(prev));
 282
 283         while (!bpf_map_get_next_key(fd, &prev, &key)) {
 284                 int n = 1;  /* start from perf_event_header */
 285                 int ip_pos = -1;
 286
 287                 bpf_map_lookup_elem(fd, &key, &val);
 288
 289                 if (sample_type & PERF_SAMPLE_IDENTIFIER)
 290                         data.array[n++] = sid;
 291                 if (sample_type & PERF_SAMPLE_IP) {
 292                         ip_pos = n;
 293                         data.array[n++] = 0;  /* will be updated */
 294                 }
 295                 if (sample_type & PERF_SAMPLE_TID)
 296                         data.array[n++] = (u64)key.pid << 32 | key.tgid;
 297                 if (sample_type & PERF_SAMPLE_TIME)
 298                         data.array[n++] = tstamp;
 299                 if (sample_type & PERF_SAMPLE_ID)
 300                         data.array[n++] = sid;
 301                 if (sample_type & PERF_SAMPLE_CPU)
 302                         data.array[n++] = 0;
 303                 if (sample_type & PERF_SAMPLE_PERIOD)
 304                         data.array[n++] = val;
 305                 if (sample_type & PERF_SAMPLE_CALLCHAIN) {
 306                         int len = 0;
 307
 308                         /* data.array[n] is callchain->nr (updated later) */
 309                         data.array[n + 1] = PERF_CONTEXT_USER;
 310                         data.array[n + 2] = 0;
 311
 312                         bpf_map_lookup_elem(stack, &key.stack_id, &data.array[n + 2]);
 313                         while (data.array[n + 2 + len])
 314                                 len++;
 315
 316                         /* update length of callchain */
 317                         data.array[n] = len + 1;
 318
 319                         /* update sample ip with the first callchain entry */
 320                         if (ip_pos >= 0)
 321                                 data.array[ip_pos] = data.array[n + 2];
 322
 323                         /* calculate sample callchain data array length */
 324                         n += len + 2;
 325                 }
 326                 if (sample_type & PERF_SAMPLE_CGROUP)
 327                         data.array[n++] = key.cgroup_id;
 328
 329                 size = n * sizeof(u64);
 330                 data.hdr.size = size;
 331                 bytes += size;
 332
 333                 if (perf_data_file__write(file, &data, size) < 0) {
 334                         pr_err("failed to write perf data, error: %m\n");
 335                         return bytes;
 336                 }
 337
 338                 prev = key;
 339                 /* increase dummy timestamp to sort later samples */
 340                 tstamp++;
 341         }
 342         return bytes;
 343 }