arch/x86/events/intel/bts.c

   1 /*
   2  * BTS PMU driver for perf
   3  * Copyright (c) 2013-2014, Intel Corporation.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  */
  14
  15 #undef DEBUG
  16
  17 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  18
  19 #include <linux/bitops.h>
  20 #include <linux/types.h>
  21 #include <linux/slab.h>
  22 #include <linux/debugfs.h>
  23 #include <linux/device.h>
  24 #include <linux/coredump.h>
  25
  26 #include <asm-generic/sizes.h>
  27 #include <asm/perf_event.h>
  28
  29 #include "../perf_event.h"
  30
  31 struct bts_ctx {
  32         struct perf_output_handle       handle;
  33         struct debug_store              ds_back;
  34         int                             state;
  35 };
  36
  37 /* BTS context states: */
  38 enum {
  39         /* no ongoing AUX transactions */
  40         BTS_STATE_STOPPED = 0,
  41         /* AUX transaction is on, BTS tracing is disabled */
  42         BTS_STATE_INACTIVE,
  43         /* AUX transaction is on, BTS tracing is running */
  44         BTS_STATE_ACTIVE,
  45 };
  46
  47 static DEFINE_PER_CPU(struct bts_ctx, bts_ctx);
  48
  49 #define BTS_RECORD_SIZE         24
  50 #define BTS_SAFETY_MARGIN       4080
  51
  52 struct bts_phys {
  53         struct page     *page;
  54         unsigned long   size;
  55         unsigned long   offset;
  56         unsigned long   displacement;
  57 };
  58
  59 struct bts_buffer {
  60         size_t          real_size;      /* multiple of BTS_RECORD_SIZE */
  61         unsigned int    nr_pages;
  62         unsigned int    nr_bufs;
  63         unsigned int    cur_buf;
  64         bool            snapshot;
  65         local_t         data_size;
  66         local_t         head;
  67         unsigned long   end;
  68         void            **data_pages;
  69         struct bts_phys buf[0];
  70 };
  71
  72 static struct pmu bts_pmu;
  73
  74 static size_t buf_size(struct page *page)
  75 {
  76         return 1 << (PAGE_SHIFT + page_private(page));
  77 }
  78
  79 static void *
  80 bts_buffer_setup_aux(struct perf_event *event, void **pages,
  81                      int nr_pages, bool overwrite)
  82 {
  83         struct bts_buffer *buf;
  84         struct page *page;
  85         int cpu = event->cpu;
  86         int node = (cpu == -1) ? cpu : cpu_to_node(cpu);
  87         unsigned long offset;
  88         size_t size = nr_pages << PAGE_SHIFT;
  89         int pg, nbuf, pad;
  90
  91         /* count all the high order buffers */
  92         for (pg = 0, nbuf = 0; pg < nr_pages;) {
  93                 page = virt_to_page(pages[pg]);
  94                 if (WARN_ON_ONCE(!PagePrivate(page) && nr_pages > 1))
  95                         return NULL;
  96                 pg += 1 << page_private(page);
  97                 nbuf++;
  98         }
  99
 100         /*
 101          * to avoid interrupts in overwrite mode, only allow one physical
 102          */
 103         if (overwrite && nbuf > 1)
 104                 return NULL;
 105
 106         buf = kzalloc_node(offsetof(struct bts_buffer, buf[nbuf]), GFP_KERNEL, node);
 107         if (!buf)
 108                 return NULL;
 109
 110         buf->nr_pages = nr_pages;
 111         buf->nr_bufs = nbuf;
 112         buf->snapshot = overwrite;
 113         buf->data_pages = pages;
 114         buf->real_size = size - size % BTS_RECORD_SIZE;
 115
 116         for (pg = 0, nbuf = 0, offset = 0, pad = 0; nbuf < buf->nr_bufs; nbuf++) {
 117                 unsigned int __nr_pages;
 118
 119                 page = virt_to_page(pages[pg]);
 120                 __nr_pages = PagePrivate(page) ? 1 << page_private(page) : 1;
 121                 buf->buf[nbuf].page = page;
 122                 buf->buf[nbuf].offset = offset;
 123                 buf->buf[nbuf].displacement = (pad ? BTS_RECORD_SIZE - pad : 0);
 124                 buf->buf[nbuf].size = buf_size(page) - buf->buf[nbuf].displacement;
 125                 pad = buf->buf[nbuf].size % BTS_RECORD_SIZE;
 126                 buf->buf[nbuf].size -= pad;
 127
 128                 pg += __nr_pages;
 129                 offset += __nr_pages << PAGE_SHIFT;
 130         }
 131
 132         return buf;
 133 }
 134
 135 static void bts_buffer_free_aux(void *data)
 136 {
 137         kfree(data);
 138 }
 139
 140 static unsigned long bts_buffer_offset(struct bts_buffer *buf, unsigned int idx)
 141 {
 142         return buf->buf[idx].offset + buf->buf[idx].displacement;
 143 }
 144
 145 static void
 146 bts_config_buffer(struct bts_buffer *buf)
 147 {
 148         int cpu = raw_smp_processor_id();
 149         struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
 150         struct bts_phys *phys = &buf->buf[buf->cur_buf];
 151         unsigned long index, thresh = 0, end = phys->size;
 152         struct page *page = phys->page;
 153
 154         index = local_read(&buf->head);
 155
 156         if (!buf->snapshot) {
 157                 if (buf->end < phys->offset + buf_size(page))
 158                         end = buf->end - phys->offset - phys->displacement;
 159
 160                 index -= phys->offset + phys->displacement;
 161
 162                 if (end - index > BTS_SAFETY_MARGIN)
 163                         thresh = end - BTS_SAFETY_MARGIN;
 164                 else if (end - index > BTS_RECORD_SIZE)
 165                         thresh = end - BTS_RECORD_SIZE;
 166                 else
 167                         thresh = end;
 168         }
 169
 170         ds->bts_buffer_base = (u64)(long)page_address(page) + phys->displacement;
 171         ds->bts_index = ds->bts_buffer_base + index;
 172         ds->bts_absolute_maximum = ds->bts_buffer_base + end;
 173         ds->bts_interrupt_threshold = !buf->snapshot
 174                 ? ds->bts_buffer_base + thresh
 175                 : ds->bts_absolute_maximum + BTS_RECORD_SIZE;
 176 }
 177
 178 static void bts_buffer_pad_out(struct bts_phys *phys, unsigned long head)
 179 {
 180         unsigned long index = head - phys->offset;
 181
 182         memset(page_address(phys->page) + index, 0, phys->size - index);
 183 }
 184
 185 static void bts_update(struct bts_ctx *bts)
 186 {
 187         int cpu = raw_smp_processor_id();
 188         struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
 189         struct bts_buffer *buf = perf_get_aux(&bts->handle);
 190         unsigned long index = ds->bts_index - ds->bts_buffer_base, old, head;
 191
 192         if (!buf)
 193                 return;
 194
 195         head = index + bts_buffer_offset(buf, buf->cur_buf);
 196         old = local_xchg(&buf->head, head);
 197
 198         if (!buf->snapshot) {
 199                 if (old == head)
 200                         return;
 201
 202                 if (ds->bts_index >= ds->bts_absolute_maximum)
 203                         perf_aux_output_flag(&bts->handle,
 204                                              PERF_AUX_FLAG_TRUNCATED);
 205
 206                 /*
 207                  * old and head are always in the same physical buffer, so we
 208                  * can subtract them to get the data size.
 209                  */
 210                 local_add(head - old, &buf->data_size);
 211         } else {
 212                 local_set(&buf->data_size, head);
 213         }
 214 }
 215
 216 static int
 217 bts_buffer_reset(struct bts_buffer *buf, struct perf_output_handle *handle);
 218
 219 /*
 220  * Ordering PMU callbacks wrt themselves and the PMI is done by means
 221  * of bts::state, which:
 222  *  - is set when bts::handle::event is valid, that is, between
 223  *    perf_aux_output_begin() and perf_aux_output_end();
 224  *  - is zero otherwise;
 225  *  - is ordered against bts::handle::event with a compiler barrier.
 226  */
 227
 228 static void __bts_event_start(struct perf_event *event)
 229 {
 230         struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
 231         struct bts_buffer *buf = perf_get_aux(&bts->handle);
 232         u64 config = 0;
 233
 234         if (!buf->snapshot)
 235                 config |= ARCH_PERFMON_EVENTSEL_INT;
 236         if (!event->attr.exclude_kernel)
 237                 config |= ARCH_PERFMON_EVENTSEL_OS;
 238         if (!event->attr.exclude_user)
 239                 config |= ARCH_PERFMON_EVENTSEL_USR;
 240
 241         bts_config_buffer(buf);
 242
 243         /*
 244          * local barrier to make sure that ds configuration made it
 245          * before we enable BTS and bts::state goes ACTIVE
 246          */
 247         wmb();
 248
 249         /* INACTIVE/STOPPED -> ACTIVE */
 250         WRITE_ONCE(bts->state, BTS_STATE_ACTIVE);
 251
 252         intel_pmu_enable_bts(config);
 253
 254 }
 255
 256 static void bts_event_start(struct perf_event *event, int flags)
 257 {
 258         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 259         struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
 260         struct bts_buffer *buf;
 261
 262         buf = perf_aux_output_begin(&bts->handle, event);
 263         if (!buf)
 264                 goto fail_stop;
 265
 266         if (bts_buffer_reset(buf, &bts->handle))
 267                 goto fail_end_stop;
 268
 269         bts->ds_back.bts_buffer_base = cpuc->ds->bts_buffer_base;
 270         bts->ds_back.bts_absolute_maximum = cpuc->ds->bts_absolute_maximum;
 271         bts->ds_back.bts_interrupt_threshold = cpuc->ds->bts_interrupt_threshold;
 272
 273         perf_event_itrace_started(event);
 274         event->hw.state = 0;
 275
 276         __bts_event_start(event);
 277
 278         return;
 279
 280 fail_end_stop:
 281         perf_aux_output_end(&bts->handle, 0);
 282
 283 fail_stop:
 284         event->hw.state = PERF_HES_STOPPED;
 285 }
 286
 287 static void __bts_event_stop(struct perf_event *event, int state)
 288 {
 289         struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
 290
 291         /* ACTIVE -> INACTIVE(PMI)/STOPPED(->stop()) */
 292         WRITE_ONCE(bts->state, state);
 293
 294         /*
 295          * No extra synchronization is mandated by the documentation to have
 296          * BTS data stores globally visible.
 297          */
 298         intel_pmu_disable_bts();
 299 }
 300
 301 static void bts_event_stop(struct perf_event *event, int flags)
 302 {
 303         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 304         struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
 305         struct bts_buffer *buf = NULL;
 306         int state = READ_ONCE(bts->state);
 307
 308         if (state == BTS_STATE_ACTIVE)
 309                 __bts_event_stop(event, BTS_STATE_STOPPED);
 310
 311         if (state != BTS_STATE_STOPPED)
 312                 buf = perf_get_aux(&bts->handle);
 313
 314         event->hw.state |= PERF_HES_STOPPED;
 315
 316         if (flags & PERF_EF_UPDATE) {
 317                 bts_update(bts);
 318
 319                 if (buf) {
 320                         if (buf->snapshot)
 321                                 bts->handle.head =
 322                                         local_xchg(&buf->data_size,
 323                                                    buf->nr_pages << PAGE_SHIFT);
 324                         perf_aux_output_end(&bts->handle,
 325                                             local_xchg(&buf->data_size, 0));
 326                 }
 327
 328                 cpuc->ds->bts_index = bts->ds_back.bts_buffer_base;
 329                 cpuc->ds->bts_buffer_base = bts->ds_back.bts_buffer_base;
 330                 cpuc->ds->bts_absolute_maximum = bts->ds_back.bts_absolute_maximum;
 331                 cpuc->ds->bts_interrupt_threshold = bts->ds_back.bts_interrupt_threshold;
 332         }
 333 }
 334
 335 void intel_bts_enable_local(void)
 336 {
 337         struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
 338         int state = READ_ONCE(bts->state);
 339
 340         /*
 341          * Here we transition from INACTIVE to ACTIVE;
 342          * if we instead are STOPPED from the interrupt handler,
 343          * stay that way. Can't be ACTIVE here though.
 344          */
 345         if (WARN_ON_ONCE(state == BTS_STATE_ACTIVE))
 346                 return;
 347
 348         if (state == BTS_STATE_STOPPED)
 349                 return;
 350
 351         if (bts->handle.event)
 352                 __bts_event_start(bts->handle.event);
 353 }
 354
 355 void intel_bts_disable_local(void)
 356 {
 357         struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
 358
 359         /*
 360          * Here we transition from ACTIVE to INACTIVE;
 361          * do nothing for STOPPED or INACTIVE.
 362          */
 363         if (READ_ONCE(bts->state) != BTS_STATE_ACTIVE)
 364                 return;
 365
 366         if (bts->handle.event)
 367                 __bts_event_stop(bts->handle.event, BTS_STATE_INACTIVE);
 368 }
 369
 370 static int
 371 bts_buffer_reset(struct bts_buffer *buf, struct perf_output_handle *handle)
 372 {
 373         unsigned long head, space, next_space, pad, gap, skip, wakeup;
 374         unsigned int next_buf;
 375         struct bts_phys *phys, *next_phys;
 376         int ret;
 377
 378         if (buf->snapshot)
 379                 return 0;
 380
 381         head = handle->head & ((buf->nr_pages << PAGE_SHIFT) - 1);
 382
 383         phys = &buf->buf[buf->cur_buf];
 384         space = phys->offset + phys->displacement + phys->size - head;
 385         pad = space;
 386         if (space > handle->size) {
 387                 space = handle->size;
 388                 space -= space % BTS_RECORD_SIZE;
 389         }
 390         if (space <= BTS_SAFETY_MARGIN) {
 391                 /* See if next phys buffer has more space */
 392                 next_buf = buf->cur_buf + 1;
 393                 if (next_buf >= buf->nr_bufs)
 394                         next_buf = 0;
 395                 next_phys = &buf->buf[next_buf];
 396                 gap = buf_size(phys->page) - phys->displacement - phys->size +
 397                       next_phys->displacement;
 398                 skip = pad + gap;
 399                 if (handle->size >= skip) {
 400                         next_space = next_phys->size;
 401                         if (next_space + skip > handle->size) {
 402                                 next_space = handle->size - skip;
 403                                 next_space -= next_space % BTS_RECORD_SIZE;
 404                         }
 405                         if (next_space > space || !space) {
 406                                 if (pad)
 407                                         bts_buffer_pad_out(phys, head);
 408                                 ret = perf_aux_output_skip(handle, skip);
 409                                 if (ret)
 410                                         return ret;
 411                                 /* Advance to next phys buffer */
 412                                 phys = next_phys;
 413                                 space = next_space;
 414                                 head = phys->offset + phys->displacement;
 415                                 /*
 416                                  * After this, cur_buf and head won't match ds
 417                                  * anymore, so we must not be racing with
 418                                  * bts_update().
 419                                  */
 420                                 buf->cur_buf = next_buf;
 421                                 local_set(&buf->head, head);
 422                         }
 423                 }
 424         }
 425
 426         /* Don't go far beyond wakeup watermark */
 427         wakeup = BTS_SAFETY_MARGIN + BTS_RECORD_SIZE + handle->wakeup -
 428                  handle->head;
 429         if (space > wakeup) {
 430                 space = wakeup;
 431                 space -= space % BTS_RECORD_SIZE;
 432         }
 433
 434         buf->end = head + space;
 435
 436         /*
 437          * If we have no space, the lost notification would have been sent when
 438          * we hit absolute_maximum - see bts_update()
 439          */
 440         if (!space)
 441                 return -ENOSPC;
 442
 443         return 0;
 444 }
 445
 446 int intel_bts_interrupt(void)
 447 {
 448         struct debug_store *ds = this_cpu_ptr(&cpu_hw_events)->ds;
 449         struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
 450         struct perf_event *event = bts->handle.event;
 451         struct bts_buffer *buf;
 452         s64 old_head;
 453         int err = -ENOSPC, handled = 0;
 454
 455         /*
 456          * The only surefire way of knowing if this NMI is ours is by checking
 457          * the write ptr against the PMI threshold.
 458          */
 459         if (ds && (ds->bts_index >= ds->bts_interrupt_threshold))
 460                 handled = 1;
 461
 462         /*
 463          * this is wrapped in intel_bts_enable_local/intel_bts_disable_local,
 464          * so we can only be INACTIVE or STOPPED
 465          */
 466         if (READ_ONCE(bts->state) == BTS_STATE_STOPPED)
 467                 return handled;
 468
 469         buf = perf_get_aux(&bts->handle);
 470         if (!buf)
 471                 return handled;
 472
 473         /*
 474          * Skip snapshot counters: they don't use the interrupt, but
 475          * there's no other way of telling, because the pointer will
 476          * keep moving
 477          */
 478         if (buf->snapshot)
 479                 return 0;
 480
 481         old_head = local_read(&buf->head);
 482         bts_update(bts);
 483
 484         /* no new data */
 485         if (old_head == local_read(&buf->head))
 486                 return handled;
 487
 488         perf_aux_output_end(&bts->handle, local_xchg(&buf->data_size, 0));
 489
 490         buf = perf_aux_output_begin(&bts->handle, event);
 491         if (buf)
 492                 err = bts_buffer_reset(buf, &bts->handle);
 493
 494         if (err) {
 495                 WRITE_ONCE(bts->state, BTS_STATE_STOPPED);
 496
 497                 if (buf) {
 498                         /*
 499                          * BTS_STATE_STOPPED should be visible before
 500                          * cleared handle::event
 501                          */
 502                         barrier();
 503                         perf_aux_output_end(&bts->handle, 0);
 504                 }
 505         }
 506
 507         return 1;
 508 }
 509
 510 static void bts_event_del(struct perf_event *event, int mode)
 511 {
 512         bts_event_stop(event, PERF_EF_UPDATE);
 513 }
 514
 515 static int bts_event_add(struct perf_event *event, int mode)
 516 {
 517         struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
 518         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 519         struct hw_perf_event *hwc = &event->hw;
 520
 521         event->hw.state = PERF_HES_STOPPED;
 522
 523         if (test_bit(INTEL_PMC_IDX_FIXED_BTS, cpuc->active_mask))
 524                 return -EBUSY;
 525
 526         if (bts->handle.event)
 527                 return -EBUSY;
 528
 529         if (mode & PERF_EF_START) {
 530                 bts_event_start(event, 0);
 531                 if (hwc->state & PERF_HES_STOPPED)
 532                         return -EINVAL;
 533         }
 534
 535         return 0;
 536 }
 537
 538 static void bts_event_destroy(struct perf_event *event)
 539 {
 540         x86_release_hardware();
 541         x86_del_exclusive(x86_lbr_exclusive_bts);
 542 }
 543
 544 static int bts_event_init(struct perf_event *event)
 545 {
 546         int ret;
 547
 548         if (event->attr.type != bts_pmu.type)
 549                 return -ENOENT;
 550
 551         /*
 552          * BTS leaks kernel addresses even when CPL0 tracing is
 553          * disabled, so disallow intel_bts driver for unprivileged
 554          * users on paranoid systems since it provides trace data
 555          * to the user in a zero-copy fashion.
 556          *
 557          * Note that the default paranoia setting permits unprivileged
 558          * users to profile the kernel.
 559          */
 560         if (event->attr.exclude_kernel && perf_paranoid_kernel() &&
 561             !capable(CAP_SYS_ADMIN))
 562                 return -EACCES;
 563
 564         if (x86_add_exclusive(x86_lbr_exclusive_bts))
 565                 return -EBUSY;
 566
 567         ret = x86_reserve_hardware();
 568         if (ret) {
 569                 x86_del_exclusive(x86_lbr_exclusive_bts);
 570                 return ret;
 571         }
 572
 573         event->destroy = bts_event_destroy;
 574
 575         return 0;
 576 }
 577
 578 static void bts_event_read(struct perf_event *event)
 579 {
 580 }
 581
 582 static __init int bts_init(void)
 583 {
 584         if (!boot_cpu_has(X86_FEATURE_DTES64) || !x86_pmu.bts)
 585                 return -ENODEV;
 586
 587         if (boot_cpu_has(X86_FEATURE_PTI)) {
 588                 /*
 589                  * BTS hardware writes through a virtual memory map we must
 590                  * either use the kernel physical map, or the user mapping of
 591                  * the AUX buffer.
 592                  *
 593                  * However, since this driver supports per-CPU and per-task inherit
 594                  * we cannot use the user mapping since it will not be available
 595                  * if we're not running the owning process.
 596                  *
 597                  * With PTI we can't use the kernal map either, because its not
 598                  * there when we run userspace.
 599                  *
 600                  * For now, disable this driver when using PTI.
 601                  */
 602                 return -ENODEV;
 603         }
 604
 605         bts_pmu.capabilities    = PERF_PMU_CAP_AUX_NO_SG | PERF_PMU_CAP_ITRACE |
 606                                   PERF_PMU_CAP_EXCLUSIVE;
 607         bts_pmu.task_ctx_nr     = perf_sw_context;
 608         bts_pmu.event_init      = bts_event_init;
 609         bts_pmu.add             = bts_event_add;
 610         bts_pmu.del             = bts_event_del;
 611         bts_pmu.start           = bts_event_start;
 612         bts_pmu.stop            = bts_event_stop;
 613         bts_pmu.read            = bts_event_read;
 614         bts_pmu.setup_aux       = bts_buffer_setup_aux;
 615         bts_pmu.free_aux        = bts_buffer_free_aux;
 616
 617         return perf_pmu_register(&bts_pmu, "intel_bts", -1);
 618 }
 619 arch_initcall(bts_init);