arch/x86/events/intel/bts.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * BTS PMU driver for perf
   4  * Copyright (c) 2013-2014, Intel Corporation.
   5  */
   6
   7 #undef DEBUG
   8
   9 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  10
  11 #include <linux/bitops.h>
  12 #include <linux/types.h>
  13 #include <linux/slab.h>
  14 #include <linux/debugfs.h>
  15 #include <linux/device.h>
  16 #include <linux/coredump.h>
  17
  18 #include <linux/sizes.h>
  19 #include <asm/perf_event.h>
  20
  21 #include "../perf_event.h"
  22
  23 struct bts_ctx {
  24         struct perf_output_handle       handle;
  25         struct debug_store              ds_back;
  26         int                             state;
  27 };
  28
  29 /* BTS context states: */
  30 enum {
  31         /* no ongoing AUX transactions */
  32         BTS_STATE_STOPPED = 0,
  33         /* AUX transaction is on, BTS tracing is disabled */
  34         BTS_STATE_INACTIVE,
  35         /* AUX transaction is on, BTS tracing is running */
  36         BTS_STATE_ACTIVE,
  37 };
  38
  39 static DEFINE_PER_CPU(struct bts_ctx, bts_ctx);
  40
  41 #define BTS_RECORD_SIZE         24
  42 #define BTS_SAFETY_MARGIN       4080
  43
  44 struct bts_phys {
  45         struct page     *page;
  46         unsigned long   size;
  47         unsigned long   offset;
  48         unsigned long   displacement;
  49 };
  50
  51 struct bts_buffer {
  52         size_t          real_size;      /* multiple of BTS_RECORD_SIZE */
  53         unsigned int    nr_pages;
  54         unsigned int    nr_bufs;
  55         unsigned int    cur_buf;
  56         bool            snapshot;
  57         local_t         data_size;
  58         local_t         head;
  59         unsigned long   end;
  60         void            **data_pages;
  61         struct bts_phys buf[0];
  62 };
  63
  64 static struct pmu bts_pmu;
  65
  66 static size_t buf_size(struct page *page)
  67 {
  68         return 1 << (PAGE_SHIFT + page_private(page));
  69 }
  70
  71 static void *
  72 bts_buffer_setup_aux(struct perf_event *event, void **pages,
  73                      int nr_pages, bool overwrite)
  74 {
  75         struct bts_buffer *buf;
  76         struct page *page;
  77         int cpu = event->cpu;
  78         int node = (cpu == -1) ? cpu : cpu_to_node(cpu);
  79         unsigned long offset;
  80         size_t size = nr_pages << PAGE_SHIFT;
  81         int pg, nbuf, pad;
  82
  83         /* count all the high order buffers */
  84         for (pg = 0, nbuf = 0; pg < nr_pages;) {
  85                 page = virt_to_page(pages[pg]);
  86                 if (WARN_ON_ONCE(!PagePrivate(page) && nr_pages > 1))
  87                         return NULL;
  88                 pg += 1 << page_private(page);
  89                 nbuf++;
  90         }
  91
  92         /*
  93          * to avoid interrupts in overwrite mode, only allow one physical
  94          */
  95         if (overwrite && nbuf > 1)
  96                 return NULL;
  97
  98         buf = kzalloc_node(offsetof(struct bts_buffer, buf[nbuf]), GFP_KERNEL, node);
  99         if (!buf)
 100                 return NULL;
 101
 102         buf->nr_pages = nr_pages;
 103         buf->nr_bufs = nbuf;
 104         buf->snapshot = overwrite;
 105         buf->data_pages = pages;
 106         buf->real_size = size - size % BTS_RECORD_SIZE;
 107
 108         for (pg = 0, nbuf = 0, offset = 0, pad = 0; nbuf < buf->nr_bufs; nbuf++) {
 109                 unsigned int __nr_pages;
 110
 111                 page = virt_to_page(pages[pg]);
 112                 __nr_pages = PagePrivate(page) ? 1 << page_private(page) : 1;
 113                 buf->buf[nbuf].page = page;
 114                 buf->buf[nbuf].offset = offset;
 115                 buf->buf[nbuf].displacement = (pad ? BTS_RECORD_SIZE - pad : 0);
 116                 buf->buf[nbuf].size = buf_size(page) - buf->buf[nbuf].displacement;
 117                 pad = buf->buf[nbuf].size % BTS_RECORD_SIZE;
 118                 buf->buf[nbuf].size -= pad;
 119
 120                 pg += __nr_pages;
 121                 offset += __nr_pages << PAGE_SHIFT;
 122         }
 123
 124         return buf;
 125 }
 126
 127 static void bts_buffer_free_aux(void *data)
 128 {
 129         kfree(data);
 130 }
 131
 132 static unsigned long bts_buffer_offset(struct bts_buffer *buf, unsigned int idx)
 133 {
 134         return buf->buf[idx].offset + buf->buf[idx].displacement;
 135 }
 136
 137 static void
 138 bts_config_buffer(struct bts_buffer *buf)
 139 {
 140         int cpu = raw_smp_processor_id();
 141         struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
 142         struct bts_phys *phys = &buf->buf[buf->cur_buf];
 143         unsigned long index, thresh = 0, end = phys->size;
 144         struct page *page = phys->page;
 145
 146         index = local_read(&buf->head);
 147
 148         if (!buf->snapshot) {
 149                 if (buf->end < phys->offset + buf_size(page))
 150                         end = buf->end - phys->offset - phys->displacement;
 151
 152                 index -= phys->offset + phys->displacement;
 153
 154                 if (end - index > BTS_SAFETY_MARGIN)
 155                         thresh = end - BTS_SAFETY_MARGIN;
 156                 else if (end - index > BTS_RECORD_SIZE)
 157                         thresh = end - BTS_RECORD_SIZE;
 158                 else
 159                         thresh = end;
 160         }
 161
 162         ds->bts_buffer_base = (u64)(long)page_address(page) + phys->displacement;
 163         ds->bts_index = ds->bts_buffer_base + index;
 164         ds->bts_absolute_maximum = ds->bts_buffer_base + end;
 165         ds->bts_interrupt_threshold = !buf->snapshot
 166                 ? ds->bts_buffer_base + thresh
 167                 : ds->bts_absolute_maximum + BTS_RECORD_SIZE;
 168 }
 169
 170 static void bts_buffer_pad_out(struct bts_phys *phys, unsigned long head)
 171 {
 172         unsigned long index = head - phys->offset;
 173
 174         memset(page_address(phys->page) + index, 0, phys->size - index);
 175 }
 176
 177 static void bts_update(struct bts_ctx *bts)
 178 {
 179         int cpu = raw_smp_processor_id();
 180         struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
 181         struct bts_buffer *buf = perf_get_aux(&bts->handle);
 182         unsigned long index = ds->bts_index - ds->bts_buffer_base, old, head;
 183
 184         if (!buf)
 185                 return;
 186
 187         head = index + bts_buffer_offset(buf, buf->cur_buf);
 188         old = local_xchg(&buf->head, head);
 189
 190         if (!buf->snapshot) {
 191                 if (old == head)
 192                         return;
 193
 194                 if (ds->bts_index >= ds->bts_absolute_maximum)
 195                         perf_aux_output_flag(&bts->handle,
 196                                              PERF_AUX_FLAG_TRUNCATED);
 197
 198                 /*
 199                  * old and head are always in the same physical buffer, so we
 200                  * can subtract them to get the data size.
 201                  */
 202                 local_add(head - old, &buf->data_size);
 203         } else {
 204                 local_set(&buf->data_size, head);
 205         }
 206 }
 207
 208 static int
 209 bts_buffer_reset(struct bts_buffer *buf, struct perf_output_handle *handle);
 210
 211 /*
 212  * Ordering PMU callbacks wrt themselves and the PMI is done by means
 213  * of bts::state, which:
 214  *  - is set when bts::handle::event is valid, that is, between
 215  *    perf_aux_output_begin() and perf_aux_output_end();
 216  *  - is zero otherwise;
 217  *  - is ordered against bts::handle::event with a compiler barrier.
 218  */
 219
 220 static void __bts_event_start(struct perf_event *event)
 221 {
 222         struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
 223         struct bts_buffer *buf = perf_get_aux(&bts->handle);
 224         u64 config = 0;
 225
 226         if (!buf->snapshot)
 227                 config |= ARCH_PERFMON_EVENTSEL_INT;
 228         if (!event->attr.exclude_kernel)
 229                 config |= ARCH_PERFMON_EVENTSEL_OS;
 230         if (!event->attr.exclude_user)
 231                 config |= ARCH_PERFMON_EVENTSEL_USR;
 232
 233         bts_config_buffer(buf);
 234
 235         /*
 236          * local barrier to make sure that ds configuration made it
 237          * before we enable BTS and bts::state goes ACTIVE
 238          */
 239         wmb();
 240
 241         /* INACTIVE/STOPPED -> ACTIVE */
 242         WRITE_ONCE(bts->state, BTS_STATE_ACTIVE);
 243
 244         intel_pmu_enable_bts(config);
 245
 246 }
 247
 248 static void bts_event_start(struct perf_event *event, int flags)
 249 {
 250         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 251         struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
 252         struct bts_buffer *buf;
 253
 254         buf = perf_aux_output_begin(&bts->handle, event);
 255         if (!buf)
 256                 goto fail_stop;
 257
 258         if (bts_buffer_reset(buf, &bts->handle))
 259                 goto fail_end_stop;
 260
 261         bts->ds_back.bts_buffer_base = cpuc->ds->bts_buffer_base;
 262         bts->ds_back.bts_absolute_maximum = cpuc->ds->bts_absolute_maximum;
 263         bts->ds_back.bts_interrupt_threshold = cpuc->ds->bts_interrupt_threshold;
 264
 265         perf_event_itrace_started(event);
 266         event->hw.state = 0;
 267
 268         __bts_event_start(event);
 269
 270         return;
 271
 272 fail_end_stop:
 273         perf_aux_output_end(&bts->handle, 0);
 274
 275 fail_stop:
 276         event->hw.state = PERF_HES_STOPPED;
 277 }
 278
 279 static void __bts_event_stop(struct perf_event *event, int state)
 280 {
 281         struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
 282
 283         /* ACTIVE -> INACTIVE(PMI)/STOPPED(->stop()) */
 284         WRITE_ONCE(bts->state, state);
 285
 286         /*
 287          * No extra synchronization is mandated by the documentation to have
 288          * BTS data stores globally visible.
 289          */
 290         intel_pmu_disable_bts();
 291 }
 292
 293 static void bts_event_stop(struct perf_event *event, int flags)
 294 {
 295         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 296         struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
 297         struct bts_buffer *buf = NULL;
 298         int state = READ_ONCE(bts->state);
 299
 300         if (state == BTS_STATE_ACTIVE)
 301                 __bts_event_stop(event, BTS_STATE_STOPPED);
 302
 303         if (state != BTS_STATE_STOPPED)
 304                 buf = perf_get_aux(&bts->handle);
 305
 306         event->hw.state |= PERF_HES_STOPPED;
 307
 308         if (flags & PERF_EF_UPDATE) {
 309                 bts_update(bts);
 310
 311                 if (buf) {
 312                         if (buf->snapshot)
 313                                 bts->handle.head =
 314                                         local_xchg(&buf->data_size,
 315                                                    buf->nr_pages << PAGE_SHIFT);
 316                         perf_aux_output_end(&bts->handle,
 317                                             local_xchg(&buf->data_size, 0));
 318                 }
 319
 320                 cpuc->ds->bts_index = bts->ds_back.bts_buffer_base;
 321                 cpuc->ds->bts_buffer_base = bts->ds_back.bts_buffer_base;
 322                 cpuc->ds->bts_absolute_maximum = bts->ds_back.bts_absolute_maximum;
 323                 cpuc->ds->bts_interrupt_threshold = bts->ds_back.bts_interrupt_threshold;
 324         }
 325 }
 326
 327 void intel_bts_enable_local(void)
 328 {
 329         struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
 330         int state = READ_ONCE(bts->state);
 331
 332         /*
 333          * Here we transition from INACTIVE to ACTIVE;
 334          * if we instead are STOPPED from the interrupt handler,
 335          * stay that way. Can't be ACTIVE here though.
 336          */
 337         if (WARN_ON_ONCE(state == BTS_STATE_ACTIVE))
 338                 return;
 339
 340         if (state == BTS_STATE_STOPPED)
 341                 return;
 342
 343         if (bts->handle.event)
 344                 __bts_event_start(bts->handle.event);
 345 }
 346
 347 void intel_bts_disable_local(void)
 348 {
 349         struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
 350
 351         /*
 352          * Here we transition from ACTIVE to INACTIVE;
 353          * do nothing for STOPPED or INACTIVE.
 354          */
 355         if (READ_ONCE(bts->state) != BTS_STATE_ACTIVE)
 356                 return;
 357
 358         if (bts->handle.event)
 359                 __bts_event_stop(bts->handle.event, BTS_STATE_INACTIVE);
 360 }
 361
 362 static int
 363 bts_buffer_reset(struct bts_buffer *buf, struct perf_output_handle *handle)
 364 {
 365         unsigned long head, space, next_space, pad, gap, skip, wakeup;
 366         unsigned int next_buf;
 367         struct bts_phys *phys, *next_phys;
 368         int ret;
 369
 370         if (buf->snapshot)
 371                 return 0;
 372
 373         head = handle->head & ((buf->nr_pages << PAGE_SHIFT) - 1);
 374
 375         phys = &buf->buf[buf->cur_buf];
 376         space = phys->offset + phys->displacement + phys->size - head;
 377         pad = space;
 378         if (space > handle->size) {
 379                 space = handle->size;
 380                 space -= space % BTS_RECORD_SIZE;
 381         }
 382         if (space <= BTS_SAFETY_MARGIN) {
 383                 /* See if next phys buffer has more space */
 384                 next_buf = buf->cur_buf + 1;
 385                 if (next_buf >= buf->nr_bufs)
 386                         next_buf = 0;
 387                 next_phys = &buf->buf[next_buf];
 388                 gap = buf_size(phys->page) - phys->displacement - phys->size +
 389                       next_phys->displacement;
 390                 skip = pad + gap;
 391                 if (handle->size >= skip) {
 392                         next_space = next_phys->size;
 393                         if (next_space + skip > handle->size) {
 394                                 next_space = handle->size - skip;
 395                                 next_space -= next_space % BTS_RECORD_SIZE;
 396                         }
 397                         if (next_space > space || !space) {
 398                                 if (pad)
 399                                         bts_buffer_pad_out(phys, head);
 400                                 ret = perf_aux_output_skip(handle, skip);
 401                                 if (ret)
 402                                         return ret;
 403                                 /* Advance to next phys buffer */
 404                                 phys = next_phys;
 405                                 space = next_space;
 406                                 head = phys->offset + phys->displacement;
 407                                 /*
 408                                  * After this, cur_buf and head won't match ds
 409                                  * anymore, so we must not be racing with
 410                                  * bts_update().
 411                                  */
 412                                 buf->cur_buf = next_buf;
 413                                 local_set(&buf->head, head);
 414                         }
 415                 }
 416         }
 417
 418         /* Don't go far beyond wakeup watermark */
 419         wakeup = BTS_SAFETY_MARGIN + BTS_RECORD_SIZE + handle->wakeup -
 420                  handle->head;
 421         if (space > wakeup) {
 422                 space = wakeup;
 423                 space -= space % BTS_RECORD_SIZE;
 424         }
 425
 426         buf->end = head + space;
 427
 428         /*
 429          * If we have no space, the lost notification would have been sent when
 430          * we hit absolute_maximum - see bts_update()
 431          */
 432         if (!space)
 433                 return -ENOSPC;
 434
 435         return 0;
 436 }
 437
 438 int intel_bts_interrupt(void)
 439 {
 440         struct debug_store *ds = this_cpu_ptr(&cpu_hw_events)->ds;
 441         struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
 442         struct perf_event *event = bts->handle.event;
 443         struct bts_buffer *buf;
 444         s64 old_head;
 445         int err = -ENOSPC, handled = 0;
 446
 447         /*
 448          * The only surefire way of knowing if this NMI is ours is by checking
 449          * the write ptr against the PMI threshold.
 450          */
 451         if (ds && (ds->bts_index >= ds->bts_interrupt_threshold))
 452                 handled = 1;
 453
 454         /*
 455          * this is wrapped in intel_bts_enable_local/intel_bts_disable_local,
 456          * so we can only be INACTIVE or STOPPED
 457          */
 458         if (READ_ONCE(bts->state) == BTS_STATE_STOPPED)
 459                 return handled;
 460
 461         buf = perf_get_aux(&bts->handle);
 462         if (!buf)
 463                 return handled;
 464
 465         /*
 466          * Skip snapshot counters: they don't use the interrupt, but
 467          * there's no other way of telling, because the pointer will
 468          * keep moving
 469          */
 470         if (buf->snapshot)
 471                 return 0;
 472
 473         old_head = local_read(&buf->head);
 474         bts_update(bts);
 475
 476         /* no new data */
 477         if (old_head == local_read(&buf->head))
 478                 return handled;
 479
 480         perf_aux_output_end(&bts->handle, local_xchg(&buf->data_size, 0));
 481
 482         buf = perf_aux_output_begin(&bts->handle, event);
 483         if (buf)
 484                 err = bts_buffer_reset(buf, &bts->handle);
 485
 486         if (err) {
 487                 WRITE_ONCE(bts->state, BTS_STATE_STOPPED);
 488
 489                 if (buf) {
 490                         /*
 491                          * BTS_STATE_STOPPED should be visible before
 492                          * cleared handle::event
 493                          */
 494                         barrier();
 495                         perf_aux_output_end(&bts->handle, 0);
 496                 }
 497         }
 498
 499         return 1;
 500 }
 501
 502 static void bts_event_del(struct perf_event *event, int mode)
 503 {
 504         bts_event_stop(event, PERF_EF_UPDATE);
 505 }
 506
 507 static int bts_event_add(struct perf_event *event, int mode)
 508 {
 509         struct bts_ctx *bts = this_cpu_ptr(&bts_ctx);
 510         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 511         struct hw_perf_event *hwc = &event->hw;
 512
 513         event->hw.state = PERF_HES_STOPPED;
 514
 515         if (test_bit(INTEL_PMC_IDX_FIXED_BTS, cpuc->active_mask))
 516                 return -EBUSY;
 517
 518         if (bts->handle.event)
 519                 return -EBUSY;
 520
 521         if (mode & PERF_EF_START) {
 522                 bts_event_start(event, 0);
 523                 if (hwc->state & PERF_HES_STOPPED)
 524                         return -EINVAL;
 525         }
 526
 527         return 0;
 528 }
 529
 530 static void bts_event_destroy(struct perf_event *event)
 531 {
 532         x86_release_hardware();
 533         x86_del_exclusive(x86_lbr_exclusive_bts);
 534 }
 535
 536 static int bts_event_init(struct perf_event *event)
 537 {
 538         int ret;
 539
 540         if (event->attr.type != bts_pmu.type)
 541                 return -ENOENT;
 542
 543         /*
 544          * BTS leaks kernel addresses even when CPL0 tracing is
 545          * disabled, so disallow intel_bts driver for unprivileged
 546          * users on paranoid systems since it provides trace data
 547          * to the user in a zero-copy fashion.
 548          *
 549          * Note that the default paranoia setting permits unprivileged
 550          * users to profile the kernel.
 551          */
 552         if (event->attr.exclude_kernel && perf_paranoid_kernel() &&
 553             !capable(CAP_SYS_ADMIN))
 554                 return -EACCES;
 555
 556         if (x86_add_exclusive(x86_lbr_exclusive_bts))
 557                 return -EBUSY;
 558
 559         ret = x86_reserve_hardware();
 560         if (ret) {
 561                 x86_del_exclusive(x86_lbr_exclusive_bts);
 562                 return ret;
 563         }
 564
 565         event->destroy = bts_event_destroy;
 566
 567         return 0;
 568 }
 569
 570 static void bts_event_read(struct perf_event *event)
 571 {
 572 }
 573
 574 static __init int bts_init(void)
 575 {
 576         if (!boot_cpu_has(X86_FEATURE_DTES64) || !x86_pmu.bts)
 577                 return -ENODEV;
 578
 579         if (boot_cpu_has(X86_FEATURE_PTI)) {
 580                 /*
 581                  * BTS hardware writes through a virtual memory map we must
 582                  * either use the kernel physical map, or the user mapping of
 583                  * the AUX buffer.
 584                  *
 585                  * However, since this driver supports per-CPU and per-task inherit
 586                  * we cannot use the user mapping since it will not be available
 587                  * if we're not running the owning process.
 588                  *
 589                  * With PTI we can't use the kernal map either, because its not
 590                  * there when we run userspace.
 591                  *
 592                  * For now, disable this driver when using PTI.
 593                  */
 594                 return -ENODEV;
 595         }
 596
 597         bts_pmu.capabilities    = PERF_PMU_CAP_AUX_NO_SG | PERF_PMU_CAP_ITRACE |
 598                                   PERF_PMU_CAP_EXCLUSIVE;
 599         bts_pmu.task_ctx_nr     = perf_sw_context;
 600         bts_pmu.event_init      = bts_event_init;
 601         bts_pmu.add             = bts_event_add;
 602         bts_pmu.del             = bts_event_del;
 603         bts_pmu.start           = bts_event_start;
 604         bts_pmu.stop            = bts_event_stop;
 605         bts_pmu.read            = bts_event_read;
 606         bts_pmu.setup_aux       = bts_buffer_setup_aux;
 607         bts_pmu.free_aux        = bts_buffer_free_aux;
 608
 609         return perf_pmu_register(&bts_pmu, "intel_bts", -1);
 610 }
 611 arch_initcall(bts_init);