1 // SPDX-License-Identifier: GPL-2.0
2 #include <perf/evlist.h>
3 #include <perf/evsel.h>
4 #include <linux/bitops.h>
5 #include <linux/list.h>
6 #include <linux/hash.h>
8 #include <internal/evlist.h>
9 #include <internal/evsel.h>
10 #include <internal/xyarray.h>
11 #include <internal/mmap.h>
12 #include <internal/cpumap.h>
13 #include <internal/threadmap.h>
14 #include <internal/lib.h>
15 #include <linux/zalloc.h>
23 #include <perf/cpumap.h>
24 #include <perf/threadmap.h>
25 #include <api/fd/array.h>
27 void perf_evlist__init(struct perf_evlist *evlist)
31 for (i = 0; i < PERF_EVLIST__HLIST_SIZE; ++i)
32 INIT_HLIST_HEAD(&evlist->heads[i]);
33 INIT_LIST_HEAD(&evlist->entries);
34 evlist->nr_entries = 0;
35 fdarray__init(&evlist->pollfd, 64);
38 static void __perf_evlist__propagate_maps(struct perf_evlist *evlist,
39 struct perf_evsel *evsel)
42 * We already have cpus for evsel (via PMU sysfs) so
43 * keep it, if there's no target cpu list defined.
45 if (!evsel->own_cpus || evlist->has_user_cpus) {
46 perf_cpu_map__put(evsel->cpus);
47 evsel->cpus = perf_cpu_map__get(evlist->cpus);
48 } else if (!evsel->system_wide && perf_cpu_map__empty(evlist->cpus)) {
49 perf_cpu_map__put(evsel->cpus);
50 evsel->cpus = perf_cpu_map__get(evlist->cpus);
51 } else if (evsel->cpus != evsel->own_cpus) {
52 perf_cpu_map__put(evsel->cpus);
53 evsel->cpus = perf_cpu_map__get(evsel->own_cpus);
56 perf_thread_map__put(evsel->threads);
57 evsel->threads = perf_thread_map__get(evlist->threads);
58 evlist->all_cpus = perf_cpu_map__merge(evlist->all_cpus, evsel->cpus);
61 static void perf_evlist__propagate_maps(struct perf_evlist *evlist)
63 struct perf_evsel *evsel;
65 perf_evlist__for_each_evsel(evlist, evsel)
66 __perf_evlist__propagate_maps(evlist, evsel);
69 void perf_evlist__add(struct perf_evlist *evlist,
70 struct perf_evsel *evsel)
72 list_add_tail(&evsel->node, &evlist->entries);
73 evlist->nr_entries += 1;
74 __perf_evlist__propagate_maps(evlist, evsel);
77 void perf_evlist__remove(struct perf_evlist *evlist,
78 struct perf_evsel *evsel)
80 list_del_init(&evsel->node);
81 evlist->nr_entries -= 1;
84 struct perf_evlist *perf_evlist__new(void)
86 struct perf_evlist *evlist = zalloc(sizeof(*evlist));
89 perf_evlist__init(evlist);
95 perf_evlist__next(struct perf_evlist *evlist, struct perf_evsel *prev)
97 struct perf_evsel *next;
100 next = list_first_entry(&evlist->entries,
104 next = list_next_entry(prev, node);
107 /* Empty list is noticed here so don't need checking on entry. */
108 if (&next->node == &evlist->entries)
114 static void perf_evlist__purge(struct perf_evlist *evlist)
116 struct perf_evsel *pos, *n;
118 perf_evlist__for_each_entry_safe(evlist, n, pos) {
119 list_del_init(&pos->node);
120 perf_evsel__delete(pos);
123 evlist->nr_entries = 0;
126 void perf_evlist__exit(struct perf_evlist *evlist)
128 perf_cpu_map__put(evlist->cpus);
129 perf_cpu_map__put(evlist->all_cpus);
130 perf_thread_map__put(evlist->threads);
132 evlist->all_cpus = NULL;
133 evlist->threads = NULL;
134 fdarray__exit(&evlist->pollfd);
137 void perf_evlist__delete(struct perf_evlist *evlist)
142 perf_evlist__munmap(evlist);
143 perf_evlist__close(evlist);
144 perf_evlist__purge(evlist);
145 perf_evlist__exit(evlist);
149 void perf_evlist__set_maps(struct perf_evlist *evlist,
150 struct perf_cpu_map *cpus,
151 struct perf_thread_map *threads)
154 * Allow for the possibility that one or another of the maps isn't being
155 * changed i.e. don't put it. Note we are assuming the maps that are
156 * being applied are brand new and evlist is taking ownership of the
157 * original reference count of 1. If that is not the case it is up to
158 * the caller to increase the reference count.
160 if (cpus != evlist->cpus) {
161 perf_cpu_map__put(evlist->cpus);
162 evlist->cpus = perf_cpu_map__get(cpus);
165 if (threads != evlist->threads) {
166 perf_thread_map__put(evlist->threads);
167 evlist->threads = perf_thread_map__get(threads);
170 if (!evlist->all_cpus && cpus)
171 evlist->all_cpus = perf_cpu_map__get(cpus);
173 perf_evlist__propagate_maps(evlist);
176 int perf_evlist__open(struct perf_evlist *evlist)
178 struct perf_evsel *evsel;
181 perf_evlist__for_each_entry(evlist, evsel) {
182 err = perf_evsel__open(evsel, evsel->cpus, evsel->threads);
190 perf_evlist__close(evlist);
194 void perf_evlist__close(struct perf_evlist *evlist)
196 struct perf_evsel *evsel;
198 perf_evlist__for_each_entry_reverse(evlist, evsel)
199 perf_evsel__close(evsel);
202 void perf_evlist__enable(struct perf_evlist *evlist)
204 struct perf_evsel *evsel;
206 perf_evlist__for_each_entry(evlist, evsel)
207 perf_evsel__enable(evsel);
210 void perf_evlist__disable(struct perf_evlist *evlist)
212 struct perf_evsel *evsel;
214 perf_evlist__for_each_entry(evlist, evsel)
215 perf_evsel__disable(evsel);
218 u64 perf_evlist__read_format(struct perf_evlist *evlist)
220 struct perf_evsel *first = perf_evlist__first(evlist);
222 return first->attr.read_format;
225 #define SID(e, x, y) xyarray__entry(e->sample_id, x, y)
227 static void perf_evlist__id_hash(struct perf_evlist *evlist,
228 struct perf_evsel *evsel,
229 int cpu, int thread, u64 id)
232 struct perf_sample_id *sid = SID(evsel, cpu, thread);
236 hash = hash_64(sid->id, PERF_EVLIST__HLIST_BITS);
237 hlist_add_head(&sid->node, &evlist->heads[hash]);
240 void perf_evlist__id_add(struct perf_evlist *evlist,
241 struct perf_evsel *evsel,
242 int cpu, int thread, u64 id)
244 perf_evlist__id_hash(evlist, evsel, cpu, thread, id);
245 evsel->id[evsel->ids++] = id;
248 int perf_evlist__id_add_fd(struct perf_evlist *evlist,
249 struct perf_evsel *evsel,
250 int cpu, int thread, int fd)
252 u64 read_data[4] = { 0, };
253 int id_idx = 1; /* The first entry is the counter value */
257 ret = ioctl(fd, PERF_EVENT_IOC_ID, &id);
264 /* Legacy way to get event id.. All hail to old kernels! */
267 * This way does not work with group format read, so bail
270 if (perf_evlist__read_format(evlist) & PERF_FORMAT_GROUP)
273 if (!(evsel->attr.read_format & PERF_FORMAT_ID) ||
274 read(fd, &read_data, sizeof(read_data)) == -1)
277 if (evsel->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
279 if (evsel->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
282 id = read_data[id_idx];
285 perf_evlist__id_add(evlist, evsel, cpu, thread, id);
289 int perf_evlist__alloc_pollfd(struct perf_evlist *evlist)
291 int nr_cpus = perf_cpu_map__nr(evlist->cpus);
292 int nr_threads = perf_thread_map__nr(evlist->threads);
294 struct perf_evsel *evsel;
296 perf_evlist__for_each_entry(evlist, evsel) {
297 if (evsel->system_wide)
300 nfds += nr_cpus * nr_threads;
303 if (fdarray__available_entries(&evlist->pollfd) < nfds &&
304 fdarray__grow(&evlist->pollfd, nfds) < 0)
310 int perf_evlist__add_pollfd(struct perf_evlist *evlist, int fd,
311 void *ptr, short revent, enum fdarray_flags flags)
313 int pos = fdarray__add(&evlist->pollfd, fd, revent | POLLERR | POLLHUP, flags);
316 evlist->pollfd.priv[pos].ptr = ptr;
317 fcntl(fd, F_SETFL, O_NONBLOCK);
323 static void perf_evlist__munmap_filtered(struct fdarray *fda, int fd,
324 void *arg __maybe_unused)
326 struct perf_mmap *map = fda->priv[fd].ptr;
332 int perf_evlist__filter_pollfd(struct perf_evlist *evlist, short revents_and_mask)
334 return fdarray__filter(&evlist->pollfd, revents_and_mask,
335 perf_evlist__munmap_filtered, NULL);
338 int perf_evlist__poll(struct perf_evlist *evlist, int timeout)
340 return fdarray__poll(&evlist->pollfd, timeout);
343 static struct perf_mmap* perf_evlist__alloc_mmap(struct perf_evlist *evlist, bool overwrite)
346 struct perf_mmap *map;
348 map = zalloc(evlist->nr_mmaps * sizeof(struct perf_mmap));
352 for (i = 0; i < evlist->nr_mmaps; i++) {
353 struct perf_mmap *prev = i ? &map[i - 1] : NULL;
356 * When the perf_mmap() call is made we grab one refcount, plus
357 * one extra to let perf_mmap__consume() get the last
358 * events after all real references (perf_mmap__get()) are
361 * Each PERF_EVENT_IOC_SET_OUTPUT points to this mmap and
362 * thus does perf_mmap__get() on it.
364 perf_mmap__init(&map[i], prev, overwrite, NULL);
370 static void perf_evsel__set_sid_idx(struct perf_evsel *evsel, int idx, int cpu, int thread)
372 struct perf_sample_id *sid = SID(evsel, cpu, thread);
375 sid->cpu = perf_cpu_map__cpu(evsel->cpus, cpu);
376 sid->tid = perf_thread_map__pid(evsel->threads, thread);
379 static struct perf_mmap*
380 perf_evlist__mmap_cb_get(struct perf_evlist *evlist, bool overwrite, int idx)
382 struct perf_mmap *maps;
384 maps = overwrite ? evlist->mmap_ovw : evlist->mmap;
387 maps = perf_evlist__alloc_mmap(evlist, overwrite);
392 evlist->mmap_ovw = maps;
400 #define FD(e, x, y) (*(int *) xyarray__entry(e->fd, x, y))
403 perf_evlist__mmap_cb_mmap(struct perf_mmap *map, struct perf_mmap_param *mp,
406 return perf_mmap__mmap(map, mp, output, cpu);
409 static void perf_evlist__set_mmap_first(struct perf_evlist *evlist, struct perf_mmap *map,
413 evlist->mmap_ovw_first = map;
415 evlist->mmap_first = map;
419 mmap_per_evsel(struct perf_evlist *evlist, struct perf_evlist_mmap_ops *ops,
420 int idx, struct perf_mmap_param *mp, int cpu_idx,
421 int thread, int *_output, int *_output_overwrite)
423 int evlist_cpu = perf_cpu_map__cpu(evlist->cpus, cpu_idx);
424 struct perf_evsel *evsel;
427 perf_evlist__for_each_entry(evlist, evsel) {
428 bool overwrite = evsel->attr.write_backward;
429 struct perf_mmap *map;
430 int *output, fd, cpu;
432 if (evsel->system_wide && thread)
435 cpu = perf_cpu_map__idx(evsel->cpus, evlist_cpu);
439 map = ops->get(evlist, overwrite, idx);
444 mp->prot = PROT_READ;
445 output = _output_overwrite;
447 mp->prot = PROT_READ | PROT_WRITE;
451 fd = FD(evsel, cpu, thread);
457 * The last one will be done at perf_mmap__consume(), so that we
458 * make sure we don't prevent tools from consuming every last event in
461 * I.e. we can get the POLLHUP meaning that the fd doesn't exist
462 * anymore, but the last events for it are still in the ring buffer,
463 * waiting to be consumed.
465 * Tools can chose to ignore this at their own discretion, but the
466 * evlist layer can't just drop it when filtering events in
467 * perf_evlist__filter_pollfd().
469 refcount_set(&map->refcnt, 2);
471 if (ops->mmap(map, mp, *output, evlist_cpu) < 0)
475 perf_evlist__set_mmap_first(evlist, map, overwrite);
477 if (ioctl(fd, PERF_EVENT_IOC_SET_OUTPUT, *output) != 0)
483 revent = !overwrite ? POLLIN : 0;
485 if (!evsel->system_wide &&
486 perf_evlist__add_pollfd(evlist, fd, map, revent, fdarray_flag__default) < 0) {
491 if (evsel->attr.read_format & PERF_FORMAT_ID) {
492 if (perf_evlist__id_add_fd(evlist, evsel, cpu, thread,
495 perf_evsel__set_sid_idx(evsel, idx, cpu, thread);
503 mmap_per_thread(struct perf_evlist *evlist, struct perf_evlist_mmap_ops *ops,
504 struct perf_mmap_param *mp)
507 int nr_threads = perf_thread_map__nr(evlist->threads);
509 for (thread = 0; thread < nr_threads; thread++) {
511 int output_overwrite = -1;
514 ops->idx(evlist, mp, thread, false);
516 if (mmap_per_evsel(evlist, ops, thread, mp, 0, thread,
517 &output, &output_overwrite))
524 perf_evlist__munmap(evlist);
529 mmap_per_cpu(struct perf_evlist *evlist, struct perf_evlist_mmap_ops *ops,
530 struct perf_mmap_param *mp)
532 int nr_threads = perf_thread_map__nr(evlist->threads);
533 int nr_cpus = perf_cpu_map__nr(evlist->cpus);
536 for (cpu = 0; cpu < nr_cpus; cpu++) {
538 int output_overwrite = -1;
541 ops->idx(evlist, mp, cpu, true);
543 for (thread = 0; thread < nr_threads; thread++) {
544 if (mmap_per_evsel(evlist, ops, cpu, mp, cpu,
545 thread, &output, &output_overwrite))
553 perf_evlist__munmap(evlist);
557 static int perf_evlist__nr_mmaps(struct perf_evlist *evlist)
561 nr_mmaps = perf_cpu_map__nr(evlist->cpus);
562 if (perf_cpu_map__empty(evlist->cpus))
563 nr_mmaps = perf_thread_map__nr(evlist->threads);
568 int perf_evlist__mmap_ops(struct perf_evlist *evlist,
569 struct perf_evlist_mmap_ops *ops,
570 struct perf_mmap_param *mp)
572 struct perf_evsel *evsel;
573 const struct perf_cpu_map *cpus = evlist->cpus;
574 const struct perf_thread_map *threads = evlist->threads;
576 if (!ops || !ops->get || !ops->mmap)
579 mp->mask = evlist->mmap_len - page_size - 1;
581 evlist->nr_mmaps = perf_evlist__nr_mmaps(evlist);
583 perf_evlist__for_each_entry(evlist, evsel) {
584 if ((evsel->attr.read_format & PERF_FORMAT_ID) &&
585 evsel->sample_id == NULL &&
586 perf_evsel__alloc_id(evsel, perf_cpu_map__nr(cpus), threads->nr) < 0)
590 if (evlist->pollfd.entries == NULL && perf_evlist__alloc_pollfd(evlist) < 0)
593 if (perf_cpu_map__empty(cpus))
594 return mmap_per_thread(evlist, ops, mp);
596 return mmap_per_cpu(evlist, ops, mp);
599 int perf_evlist__mmap(struct perf_evlist *evlist, int pages)
601 struct perf_mmap_param mp;
602 struct perf_evlist_mmap_ops ops = {
603 .get = perf_evlist__mmap_cb_get,
604 .mmap = perf_evlist__mmap_cb_mmap,
607 evlist->mmap_len = (pages + 1) * page_size;
609 return perf_evlist__mmap_ops(evlist, &ops, &mp);
612 void perf_evlist__munmap(struct perf_evlist *evlist)
617 for (i = 0; i < evlist->nr_mmaps; i++)
618 perf_mmap__munmap(&evlist->mmap[i]);
621 if (evlist->mmap_ovw) {
622 for (i = 0; i < evlist->nr_mmaps; i++)
623 perf_mmap__munmap(&evlist->mmap_ovw[i]);
626 zfree(&evlist->mmap);
627 zfree(&evlist->mmap_ovw);
631 perf_evlist__next_mmap(struct perf_evlist *evlist, struct perf_mmap *map,
637 return overwrite ? evlist->mmap_ovw_first : evlist->mmap_first;