tools/perf/util/evlist.c

   1 /*
   2  * Copyright (C) 2011, Red Hat Inc, Arnaldo Carvalho de Melo <acme@redhat.com>
   3  *
   4  * Parts came from builtin-{top,stat,record}.c, see those files for further
   5  * copyright notes.
   6  *
   7  * Released under the GPL v2. (and only v2, not any later version)
   8  */
   9 #include "util.h"
  10 #include <api/fs/fs.h>
  11 #include <poll.h>
  12 #include "cpumap.h"
  13 #include "thread_map.h"
  14 #include "target.h"
  15 #include "evlist.h"
  16 #include "evsel.h"
  17 #include "debug.h"
  18 #include <unistd.h>
  19
  20 #include "parse-events.h"
  21 #include "parse-options.h"
  22
  23 #include <sys/mman.h>
  24
  25 #include <linux/bitops.h>
  26 #include <linux/hash.h>
  27 #include <linux/log2.h>
  28 #include <linux/err.h>
  29
  30 static void perf_evlist__mmap_put(struct perf_evlist *evlist, int idx);
  31 static void __perf_evlist__munmap(struct perf_evlist *evlist, int idx);
  32
  33 #define FD(e, x, y) (*(int *)xyarray__entry(e->fd, x, y))
  34 #define SID(e, x, y) xyarray__entry(e->sample_id, x, y)
  35
  36 void perf_evlist__init(struct perf_evlist *evlist, struct cpu_map *cpus,
  37                        struct thread_map *threads)
  38 {
  39         int i;
  40
  41         for (i = 0; i < PERF_EVLIST__HLIST_SIZE; ++i)
  42                 INIT_HLIST_HEAD(&evlist->heads[i]);
  43         INIT_LIST_HEAD(&evlist->entries);
  44         perf_evlist__set_maps(evlist, cpus, threads);
  45         fdarray__init(&evlist->pollfd, 64);
  46         evlist->workload.pid = -1;
  47 }
  48
  49 struct perf_evlist *perf_evlist__new(void)
  50 {
  51         struct perf_evlist *evlist = zalloc(sizeof(*evlist));
  52
  53         if (evlist != NULL)
  54                 perf_evlist__init(evlist, NULL, NULL);
  55
  56         return evlist;
  57 }
  58
  59 struct perf_evlist *perf_evlist__new_default(void)
  60 {
  61         struct perf_evlist *evlist = perf_evlist__new();
  62
  63         if (evlist && perf_evlist__add_default(evlist)) {
  64                 perf_evlist__delete(evlist);
  65                 evlist = NULL;
  66         }
  67
  68         return evlist;
  69 }
  70
  71 /**
  72  * perf_evlist__set_id_pos - set the positions of event ids.
  73  * @evlist: selected event list
  74  *
  75  * Events with compatible sample types all have the same id_pos
  76  * and is_pos.  For convenience, put a copy on evlist.
  77  */
  78 void perf_evlist__set_id_pos(struct perf_evlist *evlist)
  79 {
  80         struct perf_evsel *first = perf_evlist__first(evlist);
  81
  82         evlist->id_pos = first->id_pos;
  83         evlist->is_pos = first->is_pos;
  84 }
  85
  86 static void perf_evlist__update_id_pos(struct perf_evlist *evlist)
  87 {
  88         struct perf_evsel *evsel;
  89
  90         evlist__for_each(evlist, evsel)
  91                 perf_evsel__calc_id_pos(evsel);
  92
  93         perf_evlist__set_id_pos(evlist);
  94 }
  95
  96 static void perf_evlist__purge(struct perf_evlist *evlist)
  97 {
  98         struct perf_evsel *pos, *n;
  99
 100         evlist__for_each_safe(evlist, n, pos) {
 101                 list_del_init(&pos->node);
 102                 pos->evlist = NULL;
 103                 perf_evsel__delete(pos);
 104         }
 105
 106         evlist->nr_entries = 0;
 107 }
 108
 109 void perf_evlist__exit(struct perf_evlist *evlist)
 110 {
 111         zfree(&evlist->mmap);
 112         fdarray__exit(&evlist->pollfd);
 113 }
 114
 115 void perf_evlist__delete(struct perf_evlist *evlist)
 116 {
 117         perf_evlist__munmap(evlist);
 118         perf_evlist__close(evlist);
 119         cpu_map__put(evlist->cpus);
 120         thread_map__put(evlist->threads);
 121         evlist->cpus = NULL;
 122         evlist->threads = NULL;
 123         perf_evlist__purge(evlist);
 124         perf_evlist__exit(evlist);
 125         free(evlist);
 126 }
 127
 128 void perf_evlist__add(struct perf_evlist *evlist, struct perf_evsel *entry)
 129 {
 130         entry->evlist = evlist;
 131         list_add_tail(&entry->node, &evlist->entries);
 132         entry->idx = evlist->nr_entries;
 133         entry->tracking = !entry->idx;
 134
 135         if (!evlist->nr_entries++)
 136                 perf_evlist__set_id_pos(evlist);
 137 }
 138
 139 void perf_evlist__splice_list_tail(struct perf_evlist *evlist,
 140                                    struct list_head *list,
 141                                    int nr_entries)
 142 {
 143         bool set_id_pos = !evlist->nr_entries;
 144
 145         list_splice_tail(list, &evlist->entries);
 146         evlist->nr_entries += nr_entries;
 147         if (set_id_pos)
 148                 perf_evlist__set_id_pos(evlist);
 149 }
 150
 151 void __perf_evlist__set_leader(struct list_head *list)
 152 {
 153         struct perf_evsel *evsel, *leader;
 154
 155         leader = list_entry(list->next, struct perf_evsel, node);
 156         evsel = list_entry(list->prev, struct perf_evsel, node);
 157
 158         leader->nr_members = evsel->idx - leader->idx + 1;
 159
 160         __evlist__for_each(list, evsel) {
 161                 evsel->leader = leader;
 162         }
 163 }
 164
 165 void perf_evlist__set_leader(struct perf_evlist *evlist)
 166 {
 167         if (evlist->nr_entries) {
 168                 evlist->nr_groups = evlist->nr_entries > 1 ? 1 : 0;
 169                 __perf_evlist__set_leader(&evlist->entries);
 170         }
 171 }
 172
 173 int perf_evlist__add_default(struct perf_evlist *evlist)
 174 {
 175         struct perf_event_attr attr = {
 176                 .type = PERF_TYPE_HARDWARE,
 177                 .config = PERF_COUNT_HW_CPU_CYCLES,
 178         };
 179         struct perf_evsel *evsel;
 180
 181         event_attr_init(&attr);
 182
 183         evsel = perf_evsel__new(&attr);
 184         if (evsel == NULL)
 185                 goto error;
 186
 187         /* use strdup() because free(evsel) assumes name is allocated */
 188         evsel->name = strdup("cycles");
 189         if (!evsel->name)
 190                 goto error_free;
 191
 192         perf_evlist__add(evlist, evsel);
 193         return 0;
 194 error_free:
 195         perf_evsel__delete(evsel);
 196 error:
 197         return -ENOMEM;
 198 }
 199
 200 static int perf_evlist__add_attrs(struct perf_evlist *evlist,
 201                                   struct perf_event_attr *attrs, size_t nr_attrs)
 202 {
 203         struct perf_evsel *evsel, *n;
 204         LIST_HEAD(head);
 205         size_t i;
 206
 207         for (i = 0; i < nr_attrs; i++) {
 208                 evsel = perf_evsel__new_idx(attrs + i, evlist->nr_entries + i);
 209                 if (evsel == NULL)
 210                         goto out_delete_partial_list;
 211                 list_add_tail(&evsel->node, &head);
 212         }
 213
 214         perf_evlist__splice_list_tail(evlist, &head, nr_attrs);
 215
 216         return 0;
 217
 218 out_delete_partial_list:
 219         __evlist__for_each_safe(&head, n, evsel)
 220                 perf_evsel__delete(evsel);
 221         return -1;
 222 }
 223
 224 int __perf_evlist__add_default_attrs(struct perf_evlist *evlist,
 225                                      struct perf_event_attr *attrs, size_t nr_attrs)
 226 {
 227         size_t i;
 228
 229         for (i = 0; i < nr_attrs; i++)
 230                 event_attr_init(attrs + i);
 231
 232         return perf_evlist__add_attrs(evlist, attrs, nr_attrs);
 233 }
 234
 235 struct perf_evsel *
 236 perf_evlist__find_tracepoint_by_id(struct perf_evlist *evlist, int id)
 237 {
 238         struct perf_evsel *evsel;
 239
 240         evlist__for_each(evlist, evsel) {
 241                 if (evsel->attr.type   == PERF_TYPE_TRACEPOINT &&
 242                     (int)evsel->attr.config == id)
 243                         return evsel;
 244         }
 245
 246         return NULL;
 247 }
 248
 249 struct perf_evsel *
 250 perf_evlist__find_tracepoint_by_name(struct perf_evlist *evlist,
 251                                      const char *name)
 252 {
 253         struct perf_evsel *evsel;
 254
 255         evlist__for_each(evlist, evsel) {
 256                 if ((evsel->attr.type == PERF_TYPE_TRACEPOINT) &&
 257                     (strcmp(evsel->name, name) == 0))
 258                         return evsel;
 259         }
 260
 261         return NULL;
 262 }
 263
 264 int perf_evlist__add_newtp(struct perf_evlist *evlist,
 265                            const char *sys, const char *name, void *handler)
 266 {
 267         struct perf_evsel *evsel = perf_evsel__newtp(sys, name);
 268
 269         if (IS_ERR(evsel))
 270                 return -1;
 271
 272         evsel->handler = handler;
 273         perf_evlist__add(evlist, evsel);
 274         return 0;
 275 }
 276
 277 static int perf_evlist__nr_threads(struct perf_evlist *evlist,
 278                                    struct perf_evsel *evsel)
 279 {
 280         if (evsel->system_wide)
 281                 return 1;
 282         else
 283                 return thread_map__nr(evlist->threads);
 284 }
 285
 286 void perf_evlist__disable(struct perf_evlist *evlist)
 287 {
 288         int cpu, thread;
 289         struct perf_evsel *pos;
 290         int nr_cpus = cpu_map__nr(evlist->cpus);
 291         int nr_threads;
 292
 293         for (cpu = 0; cpu < nr_cpus; cpu++) {
 294                 evlist__for_each(evlist, pos) {
 295                         if (!perf_evsel__is_group_leader(pos) || !pos->fd)
 296                                 continue;
 297                         nr_threads = perf_evlist__nr_threads(evlist, pos);
 298                         for (thread = 0; thread < nr_threads; thread++)
 299                                 ioctl(FD(pos, cpu, thread),
 300                                       PERF_EVENT_IOC_DISABLE, 0);
 301                 }
 302         }
 303
 304         evlist->enabled = false;
 305 }
 306
 307 void perf_evlist__enable(struct perf_evlist *evlist)
 308 {
 309         int cpu, thread;
 310         struct perf_evsel *pos;
 311         int nr_cpus = cpu_map__nr(evlist->cpus);
 312         int nr_threads;
 313
 314         for (cpu = 0; cpu < nr_cpus; cpu++) {
 315                 evlist__for_each(evlist, pos) {
 316                         if (!perf_evsel__is_group_leader(pos) || !pos->fd)
 317                                 continue;
 318                         nr_threads = perf_evlist__nr_threads(evlist, pos);
 319                         for (thread = 0; thread < nr_threads; thread++)
 320                                 ioctl(FD(pos, cpu, thread),
 321                                       PERF_EVENT_IOC_ENABLE, 0);
 322                 }
 323         }
 324
 325         evlist->enabled = true;
 326 }
 327
 328 void perf_evlist__toggle_enable(struct perf_evlist *evlist)
 329 {
 330         (evlist->enabled ? perf_evlist__disable : perf_evlist__enable)(evlist);
 331 }
 332
 333 int perf_evlist__disable_event(struct perf_evlist *evlist,
 334                                struct perf_evsel *evsel)
 335 {
 336         int cpu, thread, err;
 337         int nr_cpus = cpu_map__nr(evlist->cpus);
 338         int nr_threads = perf_evlist__nr_threads(evlist, evsel);
 339
 340         if (!evsel->fd)
 341                 return 0;
 342
 343         for (cpu = 0; cpu < nr_cpus; cpu++) {
 344                 for (thread = 0; thread < nr_threads; thread++) {
 345                         err = ioctl(FD(evsel, cpu, thread),
 346                                     PERF_EVENT_IOC_DISABLE, 0);
 347                         if (err)
 348                                 return err;
 349                 }
 350         }
 351         return 0;
 352 }
 353
 354 int perf_evlist__enable_event(struct perf_evlist *evlist,
 355                               struct perf_evsel *evsel)
 356 {
 357         int cpu, thread, err;
 358         int nr_cpus = cpu_map__nr(evlist->cpus);
 359         int nr_threads = perf_evlist__nr_threads(evlist, evsel);
 360
 361         if (!evsel->fd)
 362                 return -EINVAL;
 363
 364         for (cpu = 0; cpu < nr_cpus; cpu++) {
 365                 for (thread = 0; thread < nr_threads; thread++) {
 366                         err = ioctl(FD(evsel, cpu, thread),
 367                                     PERF_EVENT_IOC_ENABLE, 0);
 368                         if (err)
 369                                 return err;
 370                 }
 371         }
 372         return 0;
 373 }
 374
 375 static int perf_evlist__enable_event_cpu(struct perf_evlist *evlist,
 376                                          struct perf_evsel *evsel, int cpu)
 377 {
 378         int thread, err;
 379         int nr_threads = perf_evlist__nr_threads(evlist, evsel);
 380
 381         if (!evsel->fd)
 382                 return -EINVAL;
 383
 384         for (thread = 0; thread < nr_threads; thread++) {
 385                 err = ioctl(FD(evsel, cpu, thread),
 386                             PERF_EVENT_IOC_ENABLE, 0);
 387                 if (err)
 388                         return err;
 389         }
 390         return 0;
 391 }
 392
 393 static int perf_evlist__enable_event_thread(struct perf_evlist *evlist,
 394                                             struct perf_evsel *evsel,
 395                                             int thread)
 396 {
 397         int cpu, err;
 398         int nr_cpus = cpu_map__nr(evlist->cpus);
 399
 400         if (!evsel->fd)
 401                 return -EINVAL;
 402
 403         for (cpu = 0; cpu < nr_cpus; cpu++) {
 404                 err = ioctl(FD(evsel, cpu, thread), PERF_EVENT_IOC_ENABLE, 0);
 405                 if (err)
 406                         return err;
 407         }
 408         return 0;
 409 }
 410
 411 int perf_evlist__enable_event_idx(struct perf_evlist *evlist,
 412                                   struct perf_evsel *evsel, int idx)
 413 {
 414         bool per_cpu_mmaps = !cpu_map__empty(evlist->cpus);
 415
 416         if (per_cpu_mmaps)
 417                 return perf_evlist__enable_event_cpu(evlist, evsel, idx);
 418         else
 419                 return perf_evlist__enable_event_thread(evlist, evsel, idx);
 420 }
 421
 422 int perf_evlist__alloc_pollfd(struct perf_evlist *evlist)
 423 {
 424         int nr_cpus = cpu_map__nr(evlist->cpus);
 425         int nr_threads = thread_map__nr(evlist->threads);
 426         int nfds = 0;
 427         struct perf_evsel *evsel;
 428
 429         evlist__for_each(evlist, evsel) {
 430                 if (evsel->system_wide)
 431                         nfds += nr_cpus;
 432                 else
 433                         nfds += nr_cpus * nr_threads;
 434         }
 435
 436         if (fdarray__available_entries(&evlist->pollfd) < nfds &&
 437             fdarray__grow(&evlist->pollfd, nfds) < 0)
 438                 return -ENOMEM;
 439
 440         return 0;
 441 }
 442
 443 static int __perf_evlist__add_pollfd(struct perf_evlist *evlist, int fd, int idx)
 444 {
 445         int pos = fdarray__add(&evlist->pollfd, fd, POLLIN | POLLERR | POLLHUP);
 446         /*
 447          * Save the idx so that when we filter out fds POLLHUP'ed we can
 448          * close the associated evlist->mmap[] entry.
 449          */
 450         if (pos >= 0) {
 451                 evlist->pollfd.priv[pos].idx = idx;
 452
 453                 fcntl(fd, F_SETFL, O_NONBLOCK);
 454         }
 455
 456         return pos;
 457 }
 458
 459 int perf_evlist__add_pollfd(struct perf_evlist *evlist, int fd)
 460 {
 461         return __perf_evlist__add_pollfd(evlist, fd, -1);
 462 }
 463
 464 static void perf_evlist__munmap_filtered(struct fdarray *fda, int fd)
 465 {
 466         struct perf_evlist *evlist = container_of(fda, struct perf_evlist, pollfd);
 467
 468         perf_evlist__mmap_put(evlist, fda->priv[fd].idx);
 469 }
 470
 471 int perf_evlist__filter_pollfd(struct perf_evlist *evlist, short revents_and_mask)
 472 {
 473         return fdarray__filter(&evlist->pollfd, revents_and_mask,
 474                                perf_evlist__munmap_filtered);
 475 }
 476
 477 int perf_evlist__poll(struct perf_evlist *evlist, int timeout)
 478 {
 479         return fdarray__poll(&evlist->pollfd, timeout);
 480 }
 481
 482 static void perf_evlist__id_hash(struct perf_evlist *evlist,
 483                                  struct perf_evsel *evsel,
 484                                  int cpu, int thread, u64 id)
 485 {
 486         int hash;
 487         struct perf_sample_id *sid = SID(evsel, cpu, thread);
 488
 489         sid->id = id;
 490         sid->evsel = evsel;
 491         hash = hash_64(sid->id, PERF_EVLIST__HLIST_BITS);
 492         hlist_add_head(&sid->node, &evlist->heads[hash]);
 493 }
 494
 495 void perf_evlist__id_add(struct perf_evlist *evlist, struct perf_evsel *evsel,
 496                          int cpu, int thread, u64 id)
 497 {
 498         perf_evlist__id_hash(evlist, evsel, cpu, thread, id);
 499         evsel->id[evsel->ids++] = id;
 500 }
 501
 502 static int perf_evlist__id_add_fd(struct perf_evlist *evlist,
 503                                   struct perf_evsel *evsel,
 504                                   int cpu, int thread, int fd)
 505 {
 506         u64 read_data[4] = { 0, };
 507         int id_idx = 1; /* The first entry is the counter value */
 508         u64 id;
 509         int ret;
 510
 511         ret = ioctl(fd, PERF_EVENT_IOC_ID, &id);
 512         if (!ret)
 513                 goto add;
 514
 515         if (errno != ENOTTY)
 516                 return -1;
 517
 518         /* Legacy way to get event id.. All hail to old kernels! */
 519
 520         /*
 521          * This way does not work with group format read, so bail
 522          * out in that case.
 523          */
 524         if (perf_evlist__read_format(evlist) & PERF_FORMAT_GROUP)
 525                 return -1;
 526
 527         if (!(evsel->attr.read_format & PERF_FORMAT_ID) ||
 528             read(fd, &read_data, sizeof(read_data)) == -1)
 529                 return -1;
 530
 531         if (evsel->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
 532                 ++id_idx;
 533         if (evsel->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
 534                 ++id_idx;
 535
 536         id = read_data[id_idx];
 537
 538  add:
 539         perf_evlist__id_add(evlist, evsel, cpu, thread, id);
 540         return 0;
 541 }
 542
 543 static void perf_evlist__set_sid_idx(struct perf_evlist *evlist,
 544                                      struct perf_evsel *evsel, int idx, int cpu,
 545                                      int thread)
 546 {
 547         struct perf_sample_id *sid = SID(evsel, cpu, thread);
 548         sid->idx = idx;
 549         if (evlist->cpus && cpu >= 0)
 550                 sid->cpu = evlist->cpus->map[cpu];
 551         else
 552                 sid->cpu = -1;
 553         if (!evsel->system_wide && evlist->threads && thread >= 0)
 554                 sid->tid = thread_map__pid(evlist->threads, thread);
 555         else
 556                 sid->tid = -1;
 557 }
 558
 559 struct perf_sample_id *perf_evlist__id2sid(struct perf_evlist *evlist, u64 id)
 560 {
 561         struct hlist_head *head;
 562         struct perf_sample_id *sid;
 563         int hash;
 564
 565         hash = hash_64(id, PERF_EVLIST__HLIST_BITS);
 566         head = &evlist->heads[hash];
 567
 568         hlist_for_each_entry(sid, head, node)
 569                 if (sid->id == id)
 570                         return sid;
 571
 572         return NULL;
 573 }
 574
 575 struct perf_evsel *perf_evlist__id2evsel(struct perf_evlist *evlist, u64 id)
 576 {
 577         struct perf_sample_id *sid;
 578
 579         if (evlist->nr_entries == 1 || !id)
 580                 return perf_evlist__first(evlist);
 581
 582         sid = perf_evlist__id2sid(evlist, id);
 583         if (sid)
 584                 return sid->evsel;
 585
 586         if (!perf_evlist__sample_id_all(evlist))
 587                 return perf_evlist__first(evlist);
 588
 589         return NULL;
 590 }
 591
 592 static int perf_evlist__event2id(struct perf_evlist *evlist,
 593                                  union perf_event *event, u64 *id)
 594 {
 595         const u64 *array = event->sample.array;
 596         ssize_t n;
 597
 598         n = (event->header.size - sizeof(event->header)) >> 3;
 599
 600         if (event->header.type == PERF_RECORD_SAMPLE) {
 601                 if (evlist->id_pos >= n)
 602                         return -1;
 603                 *id = array[evlist->id_pos];
 604         } else {
 605                 if (evlist->is_pos > n)
 606                         return -1;
 607                 n -= evlist->is_pos;
 608                 *id = array[n];
 609         }
 610         return 0;
 611 }
 612
 613 static struct perf_evsel *perf_evlist__event2evsel(struct perf_evlist *evlist,
 614                                                    union perf_event *event)
 615 {
 616         struct perf_evsel *first = perf_evlist__first(evlist);
 617         struct hlist_head *head;
 618         struct perf_sample_id *sid;
 619         int hash;
 620         u64 id;
 621
 622         if (evlist->nr_entries == 1)
 623                 return first;
 624
 625         if (!first->attr.sample_id_all &&
 626             event->header.type != PERF_RECORD_SAMPLE)
 627                 return first;
 628
 629         if (perf_evlist__event2id(evlist, event, &id))
 630                 return NULL;
 631
 632         /* Synthesized events have an id of zero */
 633         if (!id)
 634                 return first;
 635
 636         hash = hash_64(id, PERF_EVLIST__HLIST_BITS);
 637         head = &evlist->heads[hash];
 638
 639         hlist_for_each_entry(sid, head, node) {
 640                 if (sid->id == id)
 641                         return sid->evsel;
 642         }
 643         return NULL;
 644 }
 645
 646 union perf_event *perf_evlist__mmap_read(struct perf_evlist *evlist, int idx)
 647 {
 648         struct perf_mmap *md = &evlist->mmap[idx];
 649         u64 head;
 650         u64 old = md->prev;
 651         unsigned char *data = md->base + page_size;
 652         union perf_event *event = NULL;
 653
 654         /*
 655          * Check if event was unmapped due to a POLLHUP/POLLERR.
 656          */
 657         if (!atomic_read(&md->refcnt))
 658                 return NULL;
 659
 660         head = perf_mmap__read_head(md);
 661         if (evlist->overwrite) {
 662                 /*
 663                  * If we're further behind than half the buffer, there's a chance
 664                  * the writer will bite our tail and mess up the samples under us.
 665                  *
 666                  * If we somehow ended up ahead of the head, we got messed up.
 667                  *
 668                  * In either case, truncate and restart at head.
 669                  */
 670                 int diff = head - old;
 671                 if (diff > md->mask / 2 || diff < 0) {
 672                         fprintf(stderr, "WARNING: failed to keep up with mmap data.\n");
 673
 674                         /*
 675                          * head points to a known good entry, start there.
 676                          */
 677                         old = head;
 678                 }
 679         }
 680
 681         if (old != head) {
 682                 size_t size;
 683
 684                 event = (union perf_event *)&data[old & md->mask];
 685                 size = event->header.size;
 686
 687                 /*
 688                  * Event straddles the mmap boundary -- header should always
 689                  * be inside due to u64 alignment of output.
 690                  */
 691                 if ((old & md->mask) + size != ((old + size) & md->mask)) {
 692                         unsigned int offset = old;
 693                         unsigned int len = min(sizeof(*event), size), cpy;
 694                         void *dst = md->event_copy;
 695
 696                         do {
 697                                 cpy = min(md->mask + 1 - (offset & md->mask), len);
 698                                 memcpy(dst, &data[offset & md->mask], cpy);
 699                                 offset += cpy;
 700                                 dst += cpy;
 701                                 len -= cpy;
 702                         } while (len);
 703
 704                         event = (union perf_event *) md->event_copy;
 705                 }
 706
 707                 old += size;
 708         }
 709
 710         md->prev = old;
 711
 712         return event;
 713 }
 714
 715 static bool perf_mmap__empty(struct perf_mmap *md)
 716 {
 717         return perf_mmap__read_head(md) == md->prev && !md->auxtrace_mmap.base;
 718 }
 719
 720 static void perf_evlist__mmap_get(struct perf_evlist *evlist, int idx)
 721 {
 722         atomic_inc(&evlist->mmap[idx].refcnt);
 723 }
 724
 725 static void perf_evlist__mmap_put(struct perf_evlist *evlist, int idx)
 726 {
 727         BUG_ON(atomic_read(&evlist->mmap[idx].refcnt) == 0);
 728
 729         if (atomic_dec_and_test(&evlist->mmap[idx].refcnt))
 730                 __perf_evlist__munmap(evlist, idx);
 731 }
 732
 733 void perf_evlist__mmap_consume(struct perf_evlist *evlist, int idx)
 734 {
 735         struct perf_mmap *md = &evlist->mmap[idx];
 736
 737         if (!evlist->overwrite) {
 738                 u64 old = md->prev;
 739
 740                 perf_mmap__write_tail(md, old);
 741         }
 742
 743         if (atomic_read(&md->refcnt) == 1 && perf_mmap__empty(md))
 744                 perf_evlist__mmap_put(evlist, idx);
 745 }
 746
 747 int __weak auxtrace_mmap__mmap(struct auxtrace_mmap *mm __maybe_unused,
 748                                struct auxtrace_mmap_params *mp __maybe_unused,
 749                                void *userpg __maybe_unused,
 750                                int fd __maybe_unused)
 751 {
 752         return 0;
 753 }
 754
 755 void __weak auxtrace_mmap__munmap(struct auxtrace_mmap *mm __maybe_unused)
 756 {
 757 }
 758
 759 void __weak auxtrace_mmap_params__init(
 760                         struct auxtrace_mmap_params *mp __maybe_unused,
 761                         off_t auxtrace_offset __maybe_unused,
 762                         unsigned int auxtrace_pages __maybe_unused,
 763                         bool auxtrace_overwrite __maybe_unused)
 764 {
 765 }
 766
 767 void __weak auxtrace_mmap_params__set_idx(
 768                         struct auxtrace_mmap_params *mp __maybe_unused,
 769                         struct perf_evlist *evlist __maybe_unused,
 770                         int idx __maybe_unused,
 771                         bool per_cpu __maybe_unused)
 772 {
 773 }
 774
 775 static void __perf_evlist__munmap(struct perf_evlist *evlist, int idx)
 776 {
 777         if (evlist->mmap[idx].base != NULL) {
 778                 munmap(evlist->mmap[idx].base, evlist->mmap_len);
 779                 evlist->mmap[idx].base = NULL;
 780                 atomic_set(&evlist->mmap[idx].refcnt, 0);
 781         }
 782         auxtrace_mmap__munmap(&evlist->mmap[idx].auxtrace_mmap);
 783 }
 784
 785 void perf_evlist__munmap(struct perf_evlist *evlist)
 786 {
 787         int i;
 788
 789         if (evlist->mmap == NULL)
 790                 return;
 791
 792         for (i = 0; i < evlist->nr_mmaps; i++)
 793                 __perf_evlist__munmap(evlist, i);
 794
 795         zfree(&evlist->mmap);
 796 }
 797
 798 static int perf_evlist__alloc_mmap(struct perf_evlist *evlist)
 799 {
 800         evlist->nr_mmaps = cpu_map__nr(evlist->cpus);
 801         if (cpu_map__empty(evlist->cpus))
 802                 evlist->nr_mmaps = thread_map__nr(evlist->threads);
 803         evlist->mmap = zalloc(evlist->nr_mmaps * sizeof(struct perf_mmap));
 804         return evlist->mmap != NULL ? 0 : -ENOMEM;
 805 }
 806
 807 struct mmap_params {
 808         int prot;
 809         int mask;
 810         struct auxtrace_mmap_params auxtrace_mp;
 811 };
 812
 813 static int __perf_evlist__mmap(struct perf_evlist *evlist, int idx,
 814                                struct mmap_params *mp, int fd)
 815 {
 816         /*
 817          * The last one will be done at perf_evlist__mmap_consume(), so that we
 818          * make sure we don't prevent tools from consuming every last event in
 819          * the ring buffer.
 820          *
 821          * I.e. we can get the POLLHUP meaning that the fd doesn't exist
 822          * anymore, but the last events for it are still in the ring buffer,
 823          * waiting to be consumed.
 824          *
 825          * Tools can chose to ignore this at their own discretion, but the
 826          * evlist layer can't just drop it when filtering events in
 827          * perf_evlist__filter_pollfd().
 828          */
 829         atomic_set(&evlist->mmap[idx].refcnt, 2);
 830         evlist->mmap[idx].prev = 0;
 831         evlist->mmap[idx].mask = mp->mask;
 832         evlist->mmap[idx].base = mmap(NULL, evlist->mmap_len, mp->prot,
 833                                       MAP_SHARED, fd, 0);
 834         if (evlist->mmap[idx].base == MAP_FAILED) {
 835                 pr_debug2("failed to mmap perf event ring buffer, error %d\n",
 836                           errno);
 837                 evlist->mmap[idx].base = NULL;
 838                 return -1;
 839         }
 840
 841         if (auxtrace_mmap__mmap(&evlist->mmap[idx].auxtrace_mmap,
 842                                 &mp->auxtrace_mp, evlist->mmap[idx].base, fd))
 843                 return -1;
 844
 845         return 0;
 846 }
 847
 848 static int perf_evlist__mmap_per_evsel(struct perf_evlist *evlist, int idx,
 849                                        struct mmap_params *mp, int cpu,
 850                                        int thread, int *output)
 851 {
 852         struct perf_evsel *evsel;
 853
 854         evlist__for_each(evlist, evsel) {
 855                 int fd;
 856
 857                 if (evsel->system_wide && thread)
 858                         continue;
 859
 860                 fd = FD(evsel, cpu, thread);
 861
 862                 if (*output == -1) {
 863                         *output = fd;
 864                         if (__perf_evlist__mmap(evlist, idx, mp, *output) < 0)
 865                                 return -1;
 866                 } else {
 867                         if (ioctl(fd, PERF_EVENT_IOC_SET_OUTPUT, *output) != 0)
 868                                 return -1;
 869
 870                         perf_evlist__mmap_get(evlist, idx);
 871                 }
 872
 873                 /*
 874                  * The system_wide flag causes a selected event to be opened
 875                  * always without a pid.  Consequently it will never get a
 876                  * POLLHUP, but it is used for tracking in combination with
 877                  * other events, so it should not need to be polled anyway.
 878                  * Therefore don't add it for polling.
 879                  */
 880                 if (!evsel->system_wide &&
 881                     __perf_evlist__add_pollfd(evlist, fd, idx) < 0) {
 882                         perf_evlist__mmap_put(evlist, idx);
 883                         return -1;
 884                 }
 885
 886                 if (evsel->attr.read_format & PERF_FORMAT_ID) {
 887                         if (perf_evlist__id_add_fd(evlist, evsel, cpu, thread,
 888                                                    fd) < 0)
 889                                 return -1;
 890                         perf_evlist__set_sid_idx(evlist, evsel, idx, cpu,
 891                                                  thread);
 892                 }
 893         }
 894
 895         return 0;
 896 }
 897
 898 static int perf_evlist__mmap_per_cpu(struct perf_evlist *evlist,
 899                                      struct mmap_params *mp)
 900 {
 901         int cpu, thread;
 902         int nr_cpus = cpu_map__nr(evlist->cpus);
 903         int nr_threads = thread_map__nr(evlist->threads);
 904
 905         pr_debug2("perf event ring buffer mmapped per cpu\n");
 906         for (cpu = 0; cpu < nr_cpus; cpu++) {
 907                 int output = -1;
 908
 909                 auxtrace_mmap_params__set_idx(&mp->auxtrace_mp, evlist, cpu,
 910                                               true);
 911
 912                 for (thread = 0; thread < nr_threads; thread++) {
 913                         if (perf_evlist__mmap_per_evsel(evlist, cpu, mp, cpu,
 914                                                         thread, &output))
 915                                 goto out_unmap;
 916                 }
 917         }
 918
 919         return 0;
 920
 921 out_unmap:
 922         for (cpu = 0; cpu < nr_cpus; cpu++)
 923                 __perf_evlist__munmap(evlist, cpu);
 924         return -1;
 925 }
 926
 927 static int perf_evlist__mmap_per_thread(struct perf_evlist *evlist,
 928                                         struct mmap_params *mp)
 929 {
 930         int thread;
 931         int nr_threads = thread_map__nr(evlist->threads);
 932
 933         pr_debug2("perf event ring buffer mmapped per thread\n");
 934         for (thread = 0; thread < nr_threads; thread++) {
 935                 int output = -1;
 936
 937                 auxtrace_mmap_params__set_idx(&mp->auxtrace_mp, evlist, thread,
 938                                               false);
 939
 940                 if (perf_evlist__mmap_per_evsel(evlist, thread, mp, 0, thread,
 941                                                 &output))
 942                         goto out_unmap;
 943         }
 944
 945         return 0;
 946
 947 out_unmap:
 948         for (thread = 0; thread < nr_threads; thread++)
 949                 __perf_evlist__munmap(evlist, thread);
 950         return -1;
 951 }
 952
 953 static size_t perf_evlist__mmap_size(unsigned long pages)
 954 {
 955         if (pages == UINT_MAX) {
 956                 int max;
 957
 958                 if (sysctl__read_int("kernel/perf_event_mlock_kb", &max) < 0) {
 959                         /*
 960                          * Pick a once upon a time good value, i.e. things look
 961                          * strange since we can't read a sysctl value, but lets not
 962                          * die yet...
 963                          */
 964                         max = 512;
 965                 } else {
 966                         max -= (page_size / 1024);
 967                 }
 968
 969                 pages = (max * 1024) / page_size;
 970                 if (!is_power_of_2(pages))
 971                         pages = rounddown_pow_of_two(pages);
 972         } else if (!is_power_of_2(pages))
 973                 return 0;
 974
 975         return (pages + 1) * page_size;
 976 }
 977
 978 static long parse_pages_arg(const char *str, unsigned long min,
 979                             unsigned long max)
 980 {
 981         unsigned long pages, val;
 982         static struct parse_tag tags[] = {
 983                 { .tag  = 'B', .mult = 1       },
 984                 { .tag  = 'K', .mult = 1 << 10 },
 985                 { .tag  = 'M', .mult = 1 << 20 },
 986                 { .tag  = 'G', .mult = 1 << 30 },
 987                 { .tag  = 0 },
 988         };
 989
 990         if (str == NULL)
 991                 return -EINVAL;
 992
 993         val = parse_tag_value(str, tags);
 994         if (val != (unsigned long) -1) {
 995                 /* we got file size value */
 996                 pages = PERF_ALIGN(val, page_size) / page_size;
 997         } else {
 998                 /* we got pages count value */
 999                 char *eptr;
1000                 pages = strtoul(str, &eptr, 10);
1001                 if (*eptr != '\0')
1002                         return -EINVAL;
1003         }
1004
1005         if (pages == 0 && min == 0) {
1006                 /* leave number of pages at 0 */
1007         } else if (!is_power_of_2(pages)) {
1008                 /* round pages up to next power of 2 */
1009                 pages = roundup_pow_of_two(pages);
1010                 if (!pages)
1011                         return -EINVAL;
1012                 pr_info("rounding mmap pages size to %lu bytes (%lu pages)\n",
1013                         pages * page_size, pages);
1014         }
1015
1016         if (pages > max)
1017                 return -EINVAL;
1018
1019         return pages;
1020 }
1021
1022 int __perf_evlist__parse_mmap_pages(unsigned int *mmap_pages, const char *str)
1023 {
1024         unsigned long max = UINT_MAX;
1025         long pages;
1026
1027         if (max > SIZE_MAX / page_size)
1028                 max = SIZE_MAX / page_size;
1029
1030         pages = parse_pages_arg(str, 1, max);
1031         if (pages < 0) {
1032                 pr_err("Invalid argument for --mmap_pages/-m\n");
1033                 return -1;
1034         }
1035
1036         *mmap_pages = pages;
1037         return 0;
1038 }
1039
1040 int perf_evlist__parse_mmap_pages(const struct option *opt, const char *str,
1041                                   int unset __maybe_unused)
1042 {
1043         return __perf_evlist__parse_mmap_pages(opt->value, str);
1044 }
1045
1046 /**
1047  * perf_evlist__mmap_ex - Create mmaps to receive events.
1048  * @evlist: list of events
1049  * @pages: map length in pages
1050  * @overwrite: overwrite older events?
1051  * @auxtrace_pages - auxtrace map length in pages
1052  * @auxtrace_overwrite - overwrite older auxtrace data?
1053  *
1054  * If @overwrite is %false the user needs to signal event consumption using
1055  * perf_mmap__write_tail().  Using perf_evlist__mmap_read() does this
1056  * automatically.
1057  *
1058  * Similarly, if @auxtrace_overwrite is %false the user needs to signal data
1059  * consumption using auxtrace_mmap__write_tail().
1060  *
1061  * Return: %0 on success, negative error code otherwise.
1062  */
1063 int perf_evlist__mmap_ex(struct perf_evlist *evlist, unsigned int pages,
1064                          bool overwrite, unsigned int auxtrace_pages,
1065                          bool auxtrace_overwrite)
1066 {
1067         struct perf_evsel *evsel;
1068         const struct cpu_map *cpus = evlist->cpus;
1069         const struct thread_map *threads = evlist->threads;
1070         struct mmap_params mp = {
1071                 .prot = PROT_READ | (overwrite ? 0 : PROT_WRITE),
1072         };
1073
1074         if (evlist->mmap == NULL && perf_evlist__alloc_mmap(evlist) < 0)
1075                 return -ENOMEM;
1076
1077         if (evlist->pollfd.entries == NULL && perf_evlist__alloc_pollfd(evlist) < 0)
1078                 return -ENOMEM;
1079
1080         evlist->overwrite = overwrite;
1081         evlist->mmap_len = perf_evlist__mmap_size(pages);
1082         pr_debug("mmap size %zuB\n", evlist->mmap_len);
1083         mp.mask = evlist->mmap_len - page_size - 1;
1084
1085         auxtrace_mmap_params__init(&mp.auxtrace_mp, evlist->mmap_len,
1086                                    auxtrace_pages, auxtrace_overwrite);
1087
1088         evlist__for_each(evlist, evsel) {
1089                 if ((evsel->attr.read_format & PERF_FORMAT_ID) &&
1090                     evsel->sample_id == NULL &&
1091                     perf_evsel__alloc_id(evsel, cpu_map__nr(cpus), threads->nr) < 0)
1092                         return -ENOMEM;
1093         }
1094
1095         if (cpu_map__empty(cpus))
1096                 return perf_evlist__mmap_per_thread(evlist, &mp);
1097
1098         return perf_evlist__mmap_per_cpu(evlist, &mp);
1099 }
1100
1101 int perf_evlist__mmap(struct perf_evlist *evlist, unsigned int pages,
1102                       bool overwrite)
1103 {
1104         return perf_evlist__mmap_ex(evlist, pages, overwrite, 0, false);
1105 }
1106
1107 static int perf_evlist__propagate_maps(struct perf_evlist *evlist,
1108                                        bool has_user_cpus)
1109 {
1110         struct perf_evsel *evsel;
1111
1112         evlist__for_each(evlist, evsel) {
1113                 /*
1114                  * We already have cpus for evsel (via PMU sysfs) so
1115                  * keep it, if there's no target cpu list defined.
1116                  */
1117                 if (evsel->cpus && has_user_cpus)
1118                         cpu_map__put(evsel->cpus);
1119
1120                 if (!evsel->cpus || has_user_cpus)
1121                         evsel->cpus = cpu_map__get(evlist->cpus);
1122
1123                 evsel->threads = thread_map__get(evlist->threads);
1124
1125                 if ((evlist->cpus && !evsel->cpus) ||
1126                     (evlist->threads && !evsel->threads))
1127                         return -ENOMEM;
1128         }
1129
1130         return 0;
1131 }
1132
1133 int perf_evlist__create_maps(struct perf_evlist *evlist, struct target *target)
1134 {
1135         evlist->threads = thread_map__new_str(target->pid, target->tid,
1136                                               target->uid);
1137
1138         if (evlist->threads == NULL)
1139                 return -1;
1140
1141         if (target__uses_dummy_map(target))
1142                 evlist->cpus = cpu_map__dummy_new();
1143         else
1144                 evlist->cpus = cpu_map__new(target->cpu_list);
1145
1146         if (evlist->cpus == NULL)
1147                 goto out_delete_threads;
1148
1149         return perf_evlist__propagate_maps(evlist, !!target->cpu_list);
1150
1151 out_delete_threads:
1152         thread_map__put(evlist->threads);
1153         evlist->threads = NULL;
1154         return -1;
1155 }
1156
1157 int perf_evlist__set_maps(struct perf_evlist *evlist,
1158                           struct cpu_map *cpus,
1159                           struct thread_map *threads)
1160 {
1161         if (evlist->cpus)
1162                 cpu_map__put(evlist->cpus);
1163
1164         evlist->cpus = cpus;
1165
1166         if (evlist->threads)
1167                 thread_map__put(evlist->threads);
1168
1169         evlist->threads = threads;
1170
1171         return perf_evlist__propagate_maps(evlist, false);
1172 }
1173
1174 int perf_evlist__apply_filters(struct perf_evlist *evlist, struct perf_evsel **err_evsel)
1175 {
1176         struct perf_evsel *evsel;
1177         int err = 0;
1178         const int ncpus = cpu_map__nr(evlist->cpus),
1179                   nthreads = thread_map__nr(evlist->threads);
1180
1181         evlist__for_each(evlist, evsel) {
1182                 if (evsel->filter == NULL)
1183                         continue;
1184
1185                 /*
1186                  * filters only work for tracepoint event, which doesn't have cpu limit.
1187                  * So evlist and evsel should always be same.
1188                  */
1189                 err = perf_evsel__apply_filter(evsel, ncpus, nthreads, evsel->filter);
1190                 if (err) {
1191                         *err_evsel = evsel;
1192                         break;
1193                 }
1194         }
1195
1196         return err;
1197 }
1198
1199 int perf_evlist__set_filter(struct perf_evlist *evlist, const char *filter)
1200 {
1201         struct perf_evsel *evsel;
1202         int err = 0;
1203
1204         evlist__for_each(evlist, evsel) {
1205                 err = perf_evsel__set_filter(evsel, filter);
1206                 if (err)
1207                         break;
1208         }
1209
1210         return err;
1211 }
1212
1213 int perf_evlist__set_filter_pids(struct perf_evlist *evlist, size_t npids, pid_t *pids)
1214 {
1215         char *filter;
1216         int ret = -1;
1217         size_t i;
1218
1219         for (i = 0; i < npids; ++i) {
1220                 if (i == 0) {
1221                         if (asprintf(&filter, "common_pid != %d", pids[i]) < 0)
1222                                 return -1;
1223                 } else {
1224                         char *tmp;
1225
1226                         if (asprintf(&tmp, "%s && common_pid != %d", filter, pids[i]) < 0)
1227                                 goto out_free;
1228
1229                         free(filter);
1230                         filter = tmp;
1231                 }
1232         }
1233
1234         ret = perf_evlist__set_filter(evlist, filter);
1235 out_free:
1236         free(filter);
1237         return ret;
1238 }
1239
1240 int perf_evlist__set_filter_pid(struct perf_evlist *evlist, pid_t pid)
1241 {
1242         return perf_evlist__set_filter_pids(evlist, 1, &pid);
1243 }
1244
1245 bool perf_evlist__valid_sample_type(struct perf_evlist *evlist)
1246 {
1247         struct perf_evsel *pos;
1248
1249         if (evlist->nr_entries == 1)
1250                 return true;
1251
1252         if (evlist->id_pos < 0 || evlist->is_pos < 0)
1253                 return false;
1254
1255         evlist__for_each(evlist, pos) {
1256                 if (pos->id_pos != evlist->id_pos ||
1257                     pos->is_pos != evlist->is_pos)
1258                         return false;
1259         }
1260
1261         return true;
1262 }
1263
1264 u64 __perf_evlist__combined_sample_type(struct perf_evlist *evlist)
1265 {
1266         struct perf_evsel *evsel;
1267
1268         if (evlist->combined_sample_type)
1269                 return evlist->combined_sample_type;
1270
1271         evlist__for_each(evlist, evsel)
1272                 evlist->combined_sample_type |= evsel->attr.sample_type;
1273
1274         return evlist->combined_sample_type;
1275 }
1276
1277 u64 perf_evlist__combined_sample_type(struct perf_evlist *evlist)
1278 {
1279         evlist->combined_sample_type = 0;
1280         return __perf_evlist__combined_sample_type(evlist);
1281 }
1282
1283 u64 perf_evlist__combined_branch_type(struct perf_evlist *evlist)
1284 {
1285         struct perf_evsel *evsel;
1286         u64 branch_type = 0;
1287
1288         evlist__for_each(evlist, evsel)
1289                 branch_type |= evsel->attr.branch_sample_type;
1290         return branch_type;
1291 }
1292
1293 bool perf_evlist__valid_read_format(struct perf_evlist *evlist)
1294 {
1295         struct perf_evsel *first = perf_evlist__first(evlist), *pos = first;
1296         u64 read_format = first->attr.read_format;
1297         u64 sample_type = first->attr.sample_type;
1298
1299         evlist__for_each(evlist, pos) {
1300                 if (read_format != pos->attr.read_format)
1301                         return false;
1302         }
1303
1304         /* PERF_SAMPLE_READ imples PERF_FORMAT_ID. */
1305         if ((sample_type & PERF_SAMPLE_READ) &&
1306             !(read_format & PERF_FORMAT_ID)) {
1307                 return false;
1308         }
1309
1310         return true;
1311 }
1312
1313 u64 perf_evlist__read_format(struct perf_evlist *evlist)
1314 {
1315         struct perf_evsel *first = perf_evlist__first(evlist);
1316         return first->attr.read_format;
1317 }
1318
1319 u16 perf_evlist__id_hdr_size(struct perf_evlist *evlist)
1320 {
1321         struct perf_evsel *first = perf_evlist__first(evlist);
1322         struct perf_sample *data;
1323         u64 sample_type;
1324         u16 size = 0;
1325
1326         if (!first->attr.sample_id_all)
1327                 goto out;
1328
1329         sample_type = first->attr.sample_type;
1330
1331         if (sample_type & PERF_SAMPLE_TID)
1332                 size += sizeof(data->tid) * 2;
1333
1334        if (sample_type & PERF_SAMPLE_TIME)
1335                 size += sizeof(data->time);
1336
1337         if (sample_type & PERF_SAMPLE_ID)
1338                 size += sizeof(data->id);
1339
1340         if (sample_type & PERF_SAMPLE_STREAM_ID)
1341                 size += sizeof(data->stream_id);
1342
1343         if (sample_type & PERF_SAMPLE_CPU)
1344                 size += sizeof(data->cpu) * 2;
1345
1346         if (sample_type & PERF_SAMPLE_IDENTIFIER)
1347                 size += sizeof(data->id);
1348 out:
1349         return size;
1350 }
1351
1352 bool perf_evlist__valid_sample_id_all(struct perf_evlist *evlist)
1353 {
1354         struct perf_evsel *first = perf_evlist__first(evlist), *pos = first;
1355
1356         evlist__for_each_continue(evlist, pos) {
1357                 if (first->attr.sample_id_all != pos->attr.sample_id_all)
1358                         return false;
1359         }
1360
1361         return true;
1362 }
1363
1364 bool perf_evlist__sample_id_all(struct perf_evlist *evlist)
1365 {
1366         struct perf_evsel *first = perf_evlist__first(evlist);
1367         return first->attr.sample_id_all;
1368 }
1369
1370 void perf_evlist__set_selected(struct perf_evlist *evlist,
1371                                struct perf_evsel *evsel)
1372 {
1373         evlist->selected = evsel;
1374 }
1375
1376 void perf_evlist__close(struct perf_evlist *evlist)
1377 {
1378         struct perf_evsel *evsel;
1379         int ncpus = cpu_map__nr(evlist->cpus);
1380         int nthreads = thread_map__nr(evlist->threads);
1381         int n;
1382
1383         evlist__for_each_reverse(evlist, evsel) {
1384                 n = evsel->cpus ? evsel->cpus->nr : ncpus;
1385                 perf_evsel__close(evsel, n, nthreads);
1386         }
1387 }
1388
1389 static int perf_evlist__create_syswide_maps(struct perf_evlist *evlist)
1390 {
1391         int err = -ENOMEM;
1392
1393         /*
1394          * Try reading /sys/devices/system/cpu/online to get
1395          * an all cpus map.
1396          *
1397          * FIXME: -ENOMEM is the best we can do here, the cpu_map
1398          * code needs an overhaul to properly forward the
1399          * error, and we may not want to do that fallback to a
1400          * default cpu identity map :-\
1401          */
1402         evlist->cpus = cpu_map__new(NULL);
1403         if (evlist->cpus == NULL)
1404                 goto out;
1405
1406         evlist->threads = thread_map__new_dummy();
1407         if (evlist->threads == NULL)
1408                 goto out_free_cpus;
1409
1410         err = 0;
1411 out:
1412         return err;
1413 out_free_cpus:
1414         cpu_map__put(evlist->cpus);
1415         evlist->cpus = NULL;
1416         goto out;
1417 }
1418
1419 int perf_evlist__open(struct perf_evlist *evlist)
1420 {
1421         struct perf_evsel *evsel;
1422         int err;
1423
1424         /*
1425          * Default: one fd per CPU, all threads, aka systemwide
1426          * as sys_perf_event_open(cpu = -1, thread = -1) is EINVAL
1427          */
1428         if (evlist->threads == NULL && evlist->cpus == NULL) {
1429                 err = perf_evlist__create_syswide_maps(evlist);
1430                 if (err < 0)
1431                         goto out_err;
1432         }
1433
1434         perf_evlist__update_id_pos(evlist);
1435
1436         evlist__for_each(evlist, evsel) {
1437                 err = perf_evsel__open(evsel, evlist->cpus, evlist->threads);
1438                 if (err < 0)
1439                         goto out_err;
1440         }
1441
1442         return 0;
1443 out_err:
1444         perf_evlist__close(evlist);
1445         errno = -err;
1446         return err;
1447 }
1448
1449 int perf_evlist__prepare_workload(struct perf_evlist *evlist, struct target *target,
1450                                   const char *argv[], bool pipe_output,
1451                                   void (*exec_error)(int signo, siginfo_t *info, void *ucontext))
1452 {
1453         int child_ready_pipe[2], go_pipe[2];
1454         char bf;
1455
1456         if (pipe(child_ready_pipe) < 0) {
1457                 perror("failed to create 'ready' pipe");
1458                 return -1;
1459         }
1460
1461         if (pipe(go_pipe) < 0) {
1462                 perror("failed to create 'go' pipe");
1463                 goto out_close_ready_pipe;
1464         }
1465
1466         evlist->workload.pid = fork();
1467         if (evlist->workload.pid < 0) {
1468                 perror("failed to fork");
1469                 goto out_close_pipes;
1470         }
1471
1472         if (!evlist->workload.pid) {
1473                 int ret;
1474
1475                 if (pipe_output)
1476                         dup2(2, 1);
1477
1478                 signal(SIGTERM, SIG_DFL);
1479
1480                 close(child_ready_pipe[0]);
1481                 close(go_pipe[1]);
1482                 fcntl(go_pipe[0], F_SETFD, FD_CLOEXEC);
1483
1484                 /*
1485                  * Tell the parent we're ready to go
1486                  */
1487                 close(child_ready_pipe[1]);
1488
1489                 /*
1490                  * Wait until the parent tells us to go.
1491                  */
1492                 ret = read(go_pipe[0], &bf, 1);
1493                 /*
1494                  * The parent will ask for the execvp() to be performed by
1495                  * writing exactly one byte, in workload.cork_fd, usually via
1496                  * perf_evlist__start_workload().
1497                  *
1498                  * For cancelling the workload without actually running it,
1499                  * the parent will just close workload.cork_fd, without writing
1500                  * anything, i.e. read will return zero and we just exit()
1501                  * here.
1502                  */
1503                 if (ret != 1) {
1504                         if (ret == -1)
1505                                 perror("unable to read pipe");
1506                         exit(ret);
1507                 }
1508
1509                 execvp(argv[0], (char **)argv);
1510
1511                 if (exec_error) {
1512                         union sigval val;
1513
1514                         val.sival_int = errno;
1515                         if (sigqueue(getppid(), SIGUSR1, val))
1516                                 perror(argv[0]);
1517                 } else
1518                         perror(argv[0]);
1519                 exit(-1);
1520         }
1521
1522         if (exec_error) {
1523                 struct sigaction act = {
1524                         .sa_flags     = SA_SIGINFO,
1525                         .sa_sigaction = exec_error,
1526                 };
1527                 sigaction(SIGUSR1, &act, NULL);
1528         }
1529
1530         if (target__none(target)) {
1531                 if (evlist->threads == NULL) {
1532                         fprintf(stderr, "FATAL: evlist->threads need to be set at this point (%s:%d).\n",
1533                                 __func__, __LINE__);
1534                         goto out_close_pipes;
1535                 }
1536                 thread_map__set_pid(evlist->threads, 0, evlist->workload.pid);
1537         }
1538
1539         close(child_ready_pipe[1]);
1540         close(go_pipe[0]);
1541         /*
1542          * wait for child to settle
1543          */
1544         if (read(child_ready_pipe[0], &bf, 1) == -1) {
1545                 perror("unable to read pipe");
1546                 goto out_close_pipes;
1547         }
1548
1549         fcntl(go_pipe[1], F_SETFD, FD_CLOEXEC);
1550         evlist->workload.cork_fd = go_pipe[1];
1551         close(child_ready_pipe[0]);
1552         return 0;
1553
1554 out_close_pipes:
1555         close(go_pipe[0]);
1556         close(go_pipe[1]);
1557 out_close_ready_pipe:
1558         close(child_ready_pipe[0]);
1559         close(child_ready_pipe[1]);
1560         return -1;
1561 }
1562
1563 int perf_evlist__start_workload(struct perf_evlist *evlist)
1564 {
1565         if (evlist->workload.cork_fd > 0) {
1566                 char bf = 0;
1567                 int ret;
1568                 /*
1569                  * Remove the cork, let it rip!
1570                  */
1571                 ret = write(evlist->workload.cork_fd, &bf, 1);
1572                 if (ret < 0)
1573                         perror("enable to write to pipe");
1574
1575                 close(evlist->workload.cork_fd);
1576                 return ret;
1577         }
1578
1579         return 0;
1580 }
1581
1582 int perf_evlist__parse_sample(struct perf_evlist *evlist, union perf_event *event,
1583                               struct perf_sample *sample)
1584 {
1585         struct perf_evsel *evsel = perf_evlist__event2evsel(evlist, event);
1586
1587         if (!evsel)
1588                 return -EFAULT;
1589         return perf_evsel__parse_sample(evsel, event, sample);
1590 }
1591
1592 size_t perf_evlist__fprintf(struct perf_evlist *evlist, FILE *fp)
1593 {
1594         struct perf_evsel *evsel;
1595         size_t printed = 0;
1596
1597         evlist__for_each(evlist, evsel) {
1598                 printed += fprintf(fp, "%s%s", evsel->idx ? ", " : "",
1599                                    perf_evsel__name(evsel));
1600         }
1601
1602         return printed + fprintf(fp, "\n");
1603 }
1604
1605 int perf_evlist__strerror_open(struct perf_evlist *evlist __maybe_unused,
1606                                int err, char *buf, size_t size)
1607 {
1608         int printed, value;
1609         char sbuf[STRERR_BUFSIZE], *emsg = strerror_r(err, sbuf, sizeof(sbuf));
1610
1611         switch (err) {
1612         case EACCES:
1613         case EPERM:
1614                 printed = scnprintf(buf, size,
1615                                     "Error:\t%s.\n"
1616                                     "Hint:\tCheck /proc/sys/kernel/perf_event_paranoid setting.", emsg);
1617
1618                 value = perf_event_paranoid();
1619
1620                 printed += scnprintf(buf + printed, size - printed, "\nHint:\t");
1621
1622                 if (value >= 2) {
1623                         printed += scnprintf(buf + printed, size - printed,
1624                                              "For your workloads it needs to be <= 1\nHint:\t");
1625                 }
1626                 printed += scnprintf(buf + printed, size - printed,
1627                                      "For system wide tracing it needs to be set to -1.\n");
1628
1629                 printed += scnprintf(buf + printed, size - printed,
1630                                     "Hint:\tTry: 'sudo sh -c \"echo -1 > /proc/sys/kernel/perf_event_paranoid\"'\n"
1631                                     "Hint:\tThe current value is %d.", value);
1632                 break;
1633         default:
1634                 scnprintf(buf, size, "%s", emsg);
1635                 break;
1636         }
1637
1638         return 0;
1639 }
1640
1641 int perf_evlist__strerror_mmap(struct perf_evlist *evlist, int err, char *buf, size_t size)
1642 {
1643         char sbuf[STRERR_BUFSIZE], *emsg = strerror_r(err, sbuf, sizeof(sbuf));
1644         int pages_attempted = evlist->mmap_len / 1024, pages_max_per_user, printed = 0;
1645
1646         switch (err) {
1647         case EPERM:
1648                 sysctl__read_int("kernel/perf_event_mlock_kb", &pages_max_per_user);
1649                 printed += scnprintf(buf + printed, size - printed,
1650                                      "Error:\t%s.\n"
1651                                      "Hint:\tCheck /proc/sys/kernel/perf_event_mlock_kb (%d kB) setting.\n"
1652                                      "Hint:\tTried using %zd kB.\n",
1653                                      emsg, pages_max_per_user, pages_attempted);
1654
1655                 if (pages_attempted >= pages_max_per_user) {
1656                         printed += scnprintf(buf + printed, size - printed,
1657                                              "Hint:\tTry 'sudo sh -c \"echo %d > /proc/sys/kernel/perf_event_mlock_kb\"', or\n",
1658                                              pages_max_per_user + pages_attempted);
1659                 }
1660
1661                 printed += scnprintf(buf + printed, size - printed,
1662                                      "Hint:\tTry using a smaller -m/--mmap-pages value.");
1663                 break;
1664         default:
1665                 scnprintf(buf, size, "%s", emsg);
1666                 break;
1667         }
1668
1669         return 0;
1670 }
1671
1672 void perf_evlist__to_front(struct perf_evlist *evlist,
1673                            struct perf_evsel *move_evsel)
1674 {
1675         struct perf_evsel *evsel, *n;
1676         LIST_HEAD(move);
1677
1678         if (move_evsel == perf_evlist__first(evlist))
1679                 return;
1680
1681         evlist__for_each_safe(evlist, n, evsel) {
1682                 if (evsel->leader == move_evsel->leader)
1683                         list_move_tail(&evsel->node, &move);
1684         }
1685
1686         list_splice(&move, &evlist->entries);
1687 }
1688
1689 void perf_evlist__set_tracking_event(struct perf_evlist *evlist,
1690                                      struct perf_evsel *tracking_evsel)
1691 {
1692         struct perf_evsel *evsel;
1693
1694         if (tracking_evsel->tracking)
1695                 return;
1696
1697         evlist__for_each(evlist, evsel) {
1698                 if (evsel != tracking_evsel)
1699                         evsel->tracking = false;
1700         }
1701
1702         tracking_evsel->tracking = true;
1703 }