drivers/gpu/drm/i915/i915_pmu.c

   1 /*
   2  * SPDX-License-Identifier: MIT
   3  *
   4  * Copyright © 2017-2018 Intel Corporation
   5  */
   6
   7 #include <linux/irq.h>
   8 #include <linux/pm_runtime.h>
   9 #include "i915_pmu.h"
  10 #include "intel_ringbuffer.h"
  11 #include "i915_drv.h"
  12
  13 /* Frequency for the sampling timer for events which need it. */
  14 #define FREQUENCY 200
  15 #define PERIOD max_t(u64, 10000, NSEC_PER_SEC / FREQUENCY)
  16
  17 #define ENGINE_SAMPLE_MASK \
  18         (BIT(I915_SAMPLE_BUSY) | \
  19          BIT(I915_SAMPLE_WAIT) | \
  20          BIT(I915_SAMPLE_SEMA))
  21
  22 #define ENGINE_SAMPLE_BITS (1 << I915_PMU_SAMPLE_BITS)
  23
  24 static cpumask_t i915_pmu_cpumask;
  25
  26 static u8 engine_config_sample(u64 config)
  27 {
  28         return config & I915_PMU_SAMPLE_MASK;
  29 }
  30
  31 static u8 engine_event_sample(struct perf_event *event)
  32 {
  33         return engine_config_sample(event->attr.config);
  34 }
  35
  36 static u8 engine_event_class(struct perf_event *event)
  37 {
  38         return (event->attr.config >> I915_PMU_CLASS_SHIFT) & 0xff;
  39 }
  40
  41 static u8 engine_event_instance(struct perf_event *event)
  42 {
  43         return (event->attr.config >> I915_PMU_SAMPLE_BITS) & 0xff;
  44 }
  45
  46 static bool is_engine_config(u64 config)
  47 {
  48         return config < __I915_PMU_OTHER(0);
  49 }
  50
  51 static unsigned int config_enabled_bit(u64 config)
  52 {
  53         if (is_engine_config(config))
  54                 return engine_config_sample(config);
  55         else
  56                 return ENGINE_SAMPLE_BITS + (config - __I915_PMU_OTHER(0));
  57 }
  58
  59 static u64 config_enabled_mask(u64 config)
  60 {
  61         return BIT_ULL(config_enabled_bit(config));
  62 }
  63
  64 static bool is_engine_event(struct perf_event *event)
  65 {
  66         return is_engine_config(event->attr.config);
  67 }
  68
  69 static unsigned int event_enabled_bit(struct perf_event *event)
  70 {
  71         return config_enabled_bit(event->attr.config);
  72 }
  73
  74 static bool pmu_needs_timer(struct drm_i915_private *i915, bool gpu_active)
  75 {
  76         u64 enable;
  77
  78         /*
  79          * Only some counters need the sampling timer.
  80          *
  81          * We start with a bitmask of all currently enabled events.
  82          */
  83         enable = i915->pmu.enable;
  84
  85         /*
  86          * Mask out all the ones which do not need the timer, or in
  87          * other words keep all the ones that could need the timer.
  88          */
  89         enable &= config_enabled_mask(I915_PMU_ACTUAL_FREQUENCY) |
  90                   config_enabled_mask(I915_PMU_REQUESTED_FREQUENCY) |
  91                   ENGINE_SAMPLE_MASK;
  92
  93         /*
  94          * When the GPU is idle per-engine counters do not need to be
  95          * running so clear those bits out.
  96          */
  97         if (!gpu_active)
  98                 enable &= ~ENGINE_SAMPLE_MASK;
  99         /*
 100          * Also there is software busyness tracking available we do not
 101          * need the timer for I915_SAMPLE_BUSY counter.
 102          *
 103          * Use RCS as proxy for all engines.
 104          */
 105         else if (intel_engine_supports_stats(i915->engine[RCS]))
 106                 enable &= ~BIT(I915_SAMPLE_BUSY);
 107
 108         /*
 109          * If some bits remain it means we need the sampling timer running.
 110          */
 111         return enable;
 112 }
 113
 114 void i915_pmu_gt_parked(struct drm_i915_private *i915)
 115 {
 116         if (!i915->pmu.base.event_init)
 117                 return;
 118
 119         spin_lock_irq(&i915->pmu.lock);
 120         /*
 121          * Signal sampling timer to stop if only engine events are enabled and
 122          * GPU went idle.
 123          */
 124         i915->pmu.timer_enabled = pmu_needs_timer(i915, false);
 125         spin_unlock_irq(&i915->pmu.lock);
 126 }
 127
 128 static void __i915_pmu_maybe_start_timer(struct drm_i915_private *i915)
 129 {
 130         if (!i915->pmu.timer_enabled && pmu_needs_timer(i915, true)) {
 131                 i915->pmu.timer_enabled = true;
 132                 i915->pmu.timer_last = ktime_get();
 133                 hrtimer_start_range_ns(&i915->pmu.timer,
 134                                        ns_to_ktime(PERIOD), 0,
 135                                        HRTIMER_MODE_REL_PINNED);
 136         }
 137 }
 138
 139 void i915_pmu_gt_unparked(struct drm_i915_private *i915)
 140 {
 141         if (!i915->pmu.base.event_init)
 142                 return;
 143
 144         spin_lock_irq(&i915->pmu.lock);
 145         /*
 146          * Re-enable sampling timer when GPU goes active.
 147          */
 148         __i915_pmu_maybe_start_timer(i915);
 149         spin_unlock_irq(&i915->pmu.lock);
 150 }
 151
 152 static bool grab_forcewake(struct drm_i915_private *i915, bool fw)
 153 {
 154         if (!fw)
 155                 intel_uncore_forcewake_get(i915, FORCEWAKE_ALL);
 156
 157         return true;
 158 }
 159
 160 static void
 161 add_sample(struct i915_pmu_sample *sample, u32 val)
 162 {
 163         sample->cur += val;
 164 }
 165
 166 static void
 167 engines_sample(struct drm_i915_private *dev_priv, unsigned int period_ns)
 168 {
 169         struct intel_engine_cs *engine;
 170         enum intel_engine_id id;
 171         intel_wakeref_t wakeref;
 172         bool fw = false;
 173
 174         if ((dev_priv->pmu.enable & ENGINE_SAMPLE_MASK) == 0)
 175                 return;
 176
 177         if (!dev_priv->gt.awake)
 178                 return;
 179
 180         wakeref = intel_runtime_pm_get_if_in_use(dev_priv);
 181         if (!wakeref)
 182                 return;
 183
 184         for_each_engine(engine, dev_priv, id) {
 185                 u32 current_seqno = intel_engine_get_seqno(engine);
 186                 u32 last_seqno = intel_engine_last_submit(engine);
 187                 u32 val;
 188
 189                 val = !i915_seqno_passed(current_seqno, last_seqno);
 190
 191                 if (val)
 192                         add_sample(&engine->pmu.sample[I915_SAMPLE_BUSY],
 193                                    period_ns);
 194
 195                 if (val && (engine->pmu.enable &
 196                     (BIT(I915_SAMPLE_WAIT) | BIT(I915_SAMPLE_SEMA)))) {
 197                         fw = grab_forcewake(dev_priv, fw);
 198
 199                         val = I915_READ_FW(RING_CTL(engine->mmio_base));
 200                 } else {
 201                         val = 0;
 202                 }
 203
 204                 if (val & RING_WAIT)
 205                         add_sample(&engine->pmu.sample[I915_SAMPLE_WAIT],
 206                                    period_ns);
 207
 208                 if (val & RING_WAIT_SEMAPHORE)
 209                         add_sample(&engine->pmu.sample[I915_SAMPLE_SEMA],
 210                                    period_ns);
 211         }
 212
 213         if (fw)
 214                 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
 215
 216         intel_runtime_pm_put(dev_priv, wakeref);
 217 }
 218
 219 static void
 220 add_sample_mult(struct i915_pmu_sample *sample, u32 val, u32 mul)
 221 {
 222         sample->cur += mul_u32_u32(val, mul);
 223 }
 224
 225 static void
 226 frequency_sample(struct drm_i915_private *dev_priv, unsigned int period_ns)
 227 {
 228         if (dev_priv->pmu.enable &
 229             config_enabled_mask(I915_PMU_ACTUAL_FREQUENCY)) {
 230                 u32 val;
 231
 232                 val = dev_priv->gt_pm.rps.cur_freq;
 233                 if (dev_priv->gt.awake) {
 234                         intel_wakeref_t wakeref;
 235
 236                         with_intel_runtime_pm_if_in_use(dev_priv, wakeref)
 237                                 val = intel_get_cagf(dev_priv,
 238                                                      I915_READ_NOTRACE(GEN6_RPSTAT1));
 239                 }
 240
 241                 add_sample_mult(&dev_priv->pmu.sample[__I915_SAMPLE_FREQ_ACT],
 242                                 intel_gpu_freq(dev_priv, val),
 243                                 period_ns / 1000);
 244         }
 245
 246         if (dev_priv->pmu.enable &
 247             config_enabled_mask(I915_PMU_REQUESTED_FREQUENCY)) {
 248                 add_sample_mult(&dev_priv->pmu.sample[__I915_SAMPLE_FREQ_REQ],
 249                                 intel_gpu_freq(dev_priv,
 250                                                dev_priv->gt_pm.rps.cur_freq),
 251                                 period_ns / 1000);
 252         }
 253 }
 254
 255 static enum hrtimer_restart i915_sample(struct hrtimer *hrtimer)
 256 {
 257         struct drm_i915_private *i915 =
 258                 container_of(hrtimer, struct drm_i915_private, pmu.timer);
 259         unsigned int period_ns;
 260         ktime_t now;
 261
 262         if (!READ_ONCE(i915->pmu.timer_enabled))
 263                 return HRTIMER_NORESTART;
 264
 265         now = ktime_get();
 266         period_ns = ktime_to_ns(ktime_sub(now, i915->pmu.timer_last));
 267         i915->pmu.timer_last = now;
 268
 269         /*
 270          * Strictly speaking the passed in period may not be 100% accurate for
 271          * all internal calculation, since some amount of time can be spent on
 272          * grabbing the forcewake. However the potential error from timer call-
 273          * back delay greatly dominates this so we keep it simple.
 274          */
 275         engines_sample(i915, period_ns);
 276         frequency_sample(i915, period_ns);
 277
 278         hrtimer_forward(hrtimer, now, ns_to_ktime(PERIOD));
 279
 280         return HRTIMER_RESTART;
 281 }
 282
 283 static u64 count_interrupts(struct drm_i915_private *i915)
 284 {
 285         /* open-coded kstat_irqs() */
 286         struct irq_desc *desc = irq_to_desc(i915->drm.pdev->irq);
 287         u64 sum = 0;
 288         int cpu;
 289
 290         if (!desc || !desc->kstat_irqs)
 291                 return 0;
 292
 293         for_each_possible_cpu(cpu)
 294                 sum += *per_cpu_ptr(desc->kstat_irqs, cpu);
 295
 296         return sum;
 297 }
 298
 299 static void engine_event_destroy(struct perf_event *event)
 300 {
 301         struct drm_i915_private *i915 =
 302                 container_of(event->pmu, typeof(*i915), pmu.base);
 303         struct intel_engine_cs *engine;
 304
 305         engine = intel_engine_lookup_user(i915,
 306                                           engine_event_class(event),
 307                                           engine_event_instance(event));
 308         if (WARN_ON_ONCE(!engine))
 309                 return;
 310
 311         if (engine_event_sample(event) == I915_SAMPLE_BUSY &&
 312             intel_engine_supports_stats(engine))
 313                 intel_disable_engine_stats(engine);
 314 }
 315
 316 static void i915_pmu_event_destroy(struct perf_event *event)
 317 {
 318         WARN_ON(event->parent);
 319
 320         if (is_engine_event(event))
 321                 engine_event_destroy(event);
 322 }
 323
 324 static int
 325 engine_event_status(struct intel_engine_cs *engine,
 326                     enum drm_i915_pmu_engine_sample sample)
 327 {
 328         switch (sample) {
 329         case I915_SAMPLE_BUSY:
 330         case I915_SAMPLE_WAIT:
 331                 break;
 332         case I915_SAMPLE_SEMA:
 333                 if (INTEL_GEN(engine->i915) < 6)
 334                         return -ENODEV;
 335                 break;
 336         default:
 337                 return -ENOENT;
 338         }
 339
 340         return 0;
 341 }
 342
 343 static int
 344 config_status(struct drm_i915_private *i915, u64 config)
 345 {
 346         switch (config) {
 347         case I915_PMU_ACTUAL_FREQUENCY:
 348                 if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915))
 349                         /* Requires a mutex for sampling! */
 350                         return -ENODEV;
 351                 /* Fall-through. */
 352         case I915_PMU_REQUESTED_FREQUENCY:
 353                 if (INTEL_GEN(i915) < 6)
 354                         return -ENODEV;
 355                 break;
 356         case I915_PMU_INTERRUPTS:
 357                 break;
 358         case I915_PMU_RC6_RESIDENCY:
 359                 if (!HAS_RC6(i915))
 360                         return -ENODEV;
 361                 break;
 362         default:
 363                 return -ENOENT;
 364         }
 365
 366         return 0;
 367 }
 368
 369 static int engine_event_init(struct perf_event *event)
 370 {
 371         struct drm_i915_private *i915 =
 372                 container_of(event->pmu, typeof(*i915), pmu.base);
 373         struct intel_engine_cs *engine;
 374         u8 sample;
 375         int ret;
 376
 377         engine = intel_engine_lookup_user(i915, engine_event_class(event),
 378                                           engine_event_instance(event));
 379         if (!engine)
 380                 return -ENODEV;
 381
 382         sample = engine_event_sample(event);
 383         ret = engine_event_status(engine, sample);
 384         if (ret)
 385                 return ret;
 386
 387         if (sample == I915_SAMPLE_BUSY && intel_engine_supports_stats(engine))
 388                 ret = intel_enable_engine_stats(engine);
 389
 390         return ret;
 391 }
 392
 393 static int i915_pmu_event_init(struct perf_event *event)
 394 {
 395         struct drm_i915_private *i915 =
 396                 container_of(event->pmu, typeof(*i915), pmu.base);
 397         int ret;
 398
 399         if (event->attr.type != event->pmu->type)
 400                 return -ENOENT;
 401
 402         /* unsupported modes and filters */
 403         if (event->attr.sample_period) /* no sampling */
 404                 return -EINVAL;
 405
 406         if (has_branch_stack(event))
 407                 return -EOPNOTSUPP;
 408
 409         if (event->cpu < 0)
 410                 return -EINVAL;
 411
 412         /* only allow running on one cpu at a time */
 413         if (!cpumask_test_cpu(event->cpu, &i915_pmu_cpumask))
 414                 return -EINVAL;
 415
 416         if (is_engine_event(event))
 417                 ret = engine_event_init(event);
 418         else
 419                 ret = config_status(i915, event->attr.config);
 420         if (ret)
 421                 return ret;
 422
 423         if (!event->parent)
 424                 event->destroy = i915_pmu_event_destroy;
 425
 426         return 0;
 427 }
 428
 429 static u64 __get_rc6(struct drm_i915_private *i915)
 430 {
 431         u64 val;
 432
 433         val = intel_rc6_residency_ns(i915,
 434                                      IS_VALLEYVIEW(i915) ?
 435                                      VLV_GT_RENDER_RC6 :
 436                                      GEN6_GT_GFX_RC6);
 437
 438         if (HAS_RC6p(i915))
 439                 val += intel_rc6_residency_ns(i915, GEN6_GT_GFX_RC6p);
 440
 441         if (HAS_RC6pp(i915))
 442                 val += intel_rc6_residency_ns(i915, GEN6_GT_GFX_RC6pp);
 443
 444         return val;
 445 }
 446
 447 static u64 get_rc6(struct drm_i915_private *i915)
 448 {
 449 #if IS_ENABLED(CONFIG_PM)
 450         intel_wakeref_t wakeref;
 451         unsigned long flags;
 452         u64 val;
 453
 454         wakeref = intel_runtime_pm_get_if_in_use(i915);
 455         if (wakeref) {
 456                 val = __get_rc6(i915);
 457                 intel_runtime_pm_put(i915, wakeref);
 458
 459                 /*
 460                  * If we are coming back from being runtime suspended we must
 461                  * be careful not to report a larger value than returned
 462                  * previously.
 463                  */
 464
 465                 spin_lock_irqsave(&i915->pmu.lock, flags);
 466
 467                 if (val >= i915->pmu.sample[__I915_SAMPLE_RC6_ESTIMATED].cur) {
 468                         i915->pmu.sample[__I915_SAMPLE_RC6_ESTIMATED].cur = 0;
 469                         i915->pmu.sample[__I915_SAMPLE_RC6].cur = val;
 470                 } else {
 471                         val = i915->pmu.sample[__I915_SAMPLE_RC6_ESTIMATED].cur;
 472                 }
 473
 474                 spin_unlock_irqrestore(&i915->pmu.lock, flags);
 475         } else {
 476                 struct pci_dev *pdev = i915->drm.pdev;
 477                 struct device *kdev = &pdev->dev;
 478
 479                 /*
 480                  * We are runtime suspended.
 481                  *
 482                  * Report the delta from when the device was suspended to now,
 483                  * on top of the last known real value, as the approximated RC6
 484                  * counter value.
 485                  */
 486                 spin_lock_irqsave(&i915->pmu.lock, flags);
 487
 488                 /*
 489                  * After the above branch intel_runtime_pm_get_if_in_use failed
 490                  * to get the runtime PM reference we cannot assume we are in
 491                  * runtime suspend since we can either: a) race with coming out
 492                  * of it before we took the power.lock, or b) there are other
 493                  * states than suspended which can bring us here.
 494                  *
 495                  * We need to double-check that we are indeed currently runtime
 496                  * suspended and if not we cannot do better than report the last
 497                  * known RC6 value.
 498                  */
 499                 if (pm_runtime_status_suspended(kdev)) {
 500                         val = pm_runtime_suspended_time(kdev);
 501
 502                         if (!i915->pmu.sample[__I915_SAMPLE_RC6_ESTIMATED].cur)
 503                                 i915->pmu.suspended_time_last = val;
 504
 505                         val -= i915->pmu.suspended_time_last;
 506                         val += i915->pmu.sample[__I915_SAMPLE_RC6].cur;
 507
 508                         i915->pmu.sample[__I915_SAMPLE_RC6_ESTIMATED].cur = val;
 509                 } else if (i915->pmu.sample[__I915_SAMPLE_RC6_ESTIMATED].cur) {
 510                         val = i915->pmu.sample[__I915_SAMPLE_RC6_ESTIMATED].cur;
 511                 } else {
 512                         val = i915->pmu.sample[__I915_SAMPLE_RC6].cur;
 513                 }
 514
 515                 spin_unlock_irqrestore(&i915->pmu.lock, flags);
 516         }
 517
 518         return val;
 519 #else
 520         return __get_rc6(i915);
 521 #endif
 522 }
 523
 524 static u64 __i915_pmu_event_read(struct perf_event *event)
 525 {
 526         struct drm_i915_private *i915 =
 527                 container_of(event->pmu, typeof(*i915), pmu.base);
 528         u64 val = 0;
 529
 530         if (is_engine_event(event)) {
 531                 u8 sample = engine_event_sample(event);
 532                 struct intel_engine_cs *engine;
 533
 534                 engine = intel_engine_lookup_user(i915,
 535                                                   engine_event_class(event),
 536                                                   engine_event_instance(event));
 537
 538                 if (WARN_ON_ONCE(!engine)) {
 539                         /* Do nothing */
 540                 } else if (sample == I915_SAMPLE_BUSY &&
 541                            intel_engine_supports_stats(engine)) {
 542                         val = ktime_to_ns(intel_engine_get_busy_time(engine));
 543                 } else {
 544                         val = engine->pmu.sample[sample].cur;
 545                 }
 546         } else {
 547                 switch (event->attr.config) {
 548                 case I915_PMU_ACTUAL_FREQUENCY:
 549                         val =
 550                            div_u64(i915->pmu.sample[__I915_SAMPLE_FREQ_ACT].cur,
 551                                    USEC_PER_SEC /* to MHz */);
 552                         break;
 553                 case I915_PMU_REQUESTED_FREQUENCY:
 554                         val =
 555                            div_u64(i915->pmu.sample[__I915_SAMPLE_FREQ_REQ].cur,
 556                                    USEC_PER_SEC /* to MHz */);
 557                         break;
 558                 case I915_PMU_INTERRUPTS:
 559                         val = count_interrupts(i915);
 560                         break;
 561                 case I915_PMU_RC6_RESIDENCY:
 562                         val = get_rc6(i915);
 563                         break;
 564                 }
 565         }
 566
 567         return val;
 568 }
 569
 570 static void i915_pmu_event_read(struct perf_event *event)
 571 {
 572         struct hw_perf_event *hwc = &event->hw;
 573         u64 prev, new;
 574
 575 again:
 576         prev = local64_read(&hwc->prev_count);
 577         new = __i915_pmu_event_read(event);
 578
 579         if (local64_cmpxchg(&hwc->prev_count, prev, new) != prev)
 580                 goto again;
 581
 582         local64_add(new - prev, &event->count);
 583 }
 584
 585 static void i915_pmu_enable(struct perf_event *event)
 586 {
 587         struct drm_i915_private *i915 =
 588                 container_of(event->pmu, typeof(*i915), pmu.base);
 589         unsigned int bit = event_enabled_bit(event);
 590         unsigned long flags;
 591
 592         spin_lock_irqsave(&i915->pmu.lock, flags);
 593
 594         /*
 595          * Update the bitmask of enabled events and increment
 596          * the event reference counter.
 597          */
 598         BUILD_BUG_ON(ARRAY_SIZE(i915->pmu.enable_count) != I915_PMU_MASK_BITS);
 599         GEM_BUG_ON(bit >= ARRAY_SIZE(i915->pmu.enable_count));
 600         GEM_BUG_ON(i915->pmu.enable_count[bit] == ~0);
 601         i915->pmu.enable |= BIT_ULL(bit);
 602         i915->pmu.enable_count[bit]++;
 603
 604         /*
 605          * Start the sampling timer if needed and not already enabled.
 606          */
 607         __i915_pmu_maybe_start_timer(i915);
 608
 609         /*
 610          * For per-engine events the bitmask and reference counting
 611          * is stored per engine.
 612          */
 613         if (is_engine_event(event)) {
 614                 u8 sample = engine_event_sample(event);
 615                 struct intel_engine_cs *engine;
 616
 617                 engine = intel_engine_lookup_user(i915,
 618                                                   engine_event_class(event),
 619                                                   engine_event_instance(event));
 620
 621                 BUILD_BUG_ON(ARRAY_SIZE(engine->pmu.enable_count) !=
 622                              I915_ENGINE_SAMPLE_COUNT);
 623                 BUILD_BUG_ON(ARRAY_SIZE(engine->pmu.sample) !=
 624                              I915_ENGINE_SAMPLE_COUNT);
 625                 GEM_BUG_ON(sample >= ARRAY_SIZE(engine->pmu.enable_count));
 626                 GEM_BUG_ON(sample >= ARRAY_SIZE(engine->pmu.sample));
 627                 GEM_BUG_ON(engine->pmu.enable_count[sample] == ~0);
 628
 629                 engine->pmu.enable |= BIT(sample);
 630                 engine->pmu.enable_count[sample]++;
 631         }
 632
 633         spin_unlock_irqrestore(&i915->pmu.lock, flags);
 634
 635         /*
 636          * Store the current counter value so we can report the correct delta
 637          * for all listeners. Even when the event was already enabled and has
 638          * an existing non-zero value.
 639          */
 640         local64_set(&event->hw.prev_count, __i915_pmu_event_read(event));
 641 }
 642
 643 static void i915_pmu_disable(struct perf_event *event)
 644 {
 645         struct drm_i915_private *i915 =
 646                 container_of(event->pmu, typeof(*i915), pmu.base);
 647         unsigned int bit = event_enabled_bit(event);
 648         unsigned long flags;
 649
 650         spin_lock_irqsave(&i915->pmu.lock, flags);
 651
 652         if (is_engine_event(event)) {
 653                 u8 sample = engine_event_sample(event);
 654                 struct intel_engine_cs *engine;
 655
 656                 engine = intel_engine_lookup_user(i915,
 657                                                   engine_event_class(event),
 658                                                   engine_event_instance(event));
 659
 660                 GEM_BUG_ON(sample >= ARRAY_SIZE(engine->pmu.enable_count));
 661                 GEM_BUG_ON(sample >= ARRAY_SIZE(engine->pmu.sample));
 662                 GEM_BUG_ON(engine->pmu.enable_count[sample] == 0);
 663
 664                 /*
 665                  * Decrement the reference count and clear the enabled
 666                  * bitmask when the last listener on an event goes away.
 667                  */
 668                 if (--engine->pmu.enable_count[sample] == 0)
 669                         engine->pmu.enable &= ~BIT(sample);
 670         }
 671
 672         GEM_BUG_ON(bit >= ARRAY_SIZE(i915->pmu.enable_count));
 673         GEM_BUG_ON(i915->pmu.enable_count[bit] == 0);
 674         /*
 675          * Decrement the reference count and clear the enabled
 676          * bitmask when the last listener on an event goes away.
 677          */
 678         if (--i915->pmu.enable_count[bit] == 0) {
 679                 i915->pmu.enable &= ~BIT_ULL(bit);
 680                 i915->pmu.timer_enabled &= pmu_needs_timer(i915, true);
 681         }
 682
 683         spin_unlock_irqrestore(&i915->pmu.lock, flags);
 684 }
 685
 686 static void i915_pmu_event_start(struct perf_event *event, int flags)
 687 {
 688         i915_pmu_enable(event);
 689         event->hw.state = 0;
 690 }
 691
 692 static void i915_pmu_event_stop(struct perf_event *event, int flags)
 693 {
 694         if (flags & PERF_EF_UPDATE)
 695                 i915_pmu_event_read(event);
 696         i915_pmu_disable(event);
 697         event->hw.state = PERF_HES_STOPPED;
 698 }
 699
 700 static int i915_pmu_event_add(struct perf_event *event, int flags)
 701 {
 702         if (flags & PERF_EF_START)
 703                 i915_pmu_event_start(event, flags);
 704
 705         return 0;
 706 }
 707
 708 static void i915_pmu_event_del(struct perf_event *event, int flags)
 709 {
 710         i915_pmu_event_stop(event, PERF_EF_UPDATE);
 711 }
 712
 713 static int i915_pmu_event_event_idx(struct perf_event *event)
 714 {
 715         return 0;
 716 }
 717
 718 struct i915_str_attribute {
 719         struct device_attribute attr;
 720         const char *str;
 721 };
 722
 723 static ssize_t i915_pmu_format_show(struct device *dev,
 724                                     struct device_attribute *attr, char *buf)
 725 {
 726         struct i915_str_attribute *eattr;
 727
 728         eattr = container_of(attr, struct i915_str_attribute, attr);
 729         return sprintf(buf, "%s\n", eattr->str);
 730 }
 731
 732 #define I915_PMU_FORMAT_ATTR(_name, _config) \
 733         (&((struct i915_str_attribute[]) { \
 734                 { .attr = __ATTR(_name, 0444, i915_pmu_format_show, NULL), \
 735                   .str = _config, } \
 736         })[0].attr.attr)
 737
 738 static struct attribute *i915_pmu_format_attrs[] = {
 739         I915_PMU_FORMAT_ATTR(i915_eventid, "config:0-20"),
 740         NULL,
 741 };
 742
 743 static const struct attribute_group i915_pmu_format_attr_group = {
 744         .name = "format",
 745         .attrs = i915_pmu_format_attrs,
 746 };
 747
 748 struct i915_ext_attribute {
 749         struct device_attribute attr;
 750         unsigned long val;
 751 };
 752
 753 static ssize_t i915_pmu_event_show(struct device *dev,
 754                                    struct device_attribute *attr, char *buf)
 755 {
 756         struct i915_ext_attribute *eattr;
 757
 758         eattr = container_of(attr, struct i915_ext_attribute, attr);
 759         return sprintf(buf, "config=0x%lx\n", eattr->val);
 760 }
 761
 762 static struct attribute_group i915_pmu_events_attr_group = {
 763         .name = "events",
 764         /* Patch in attrs at runtime. */
 765 };
 766
 767 static ssize_t
 768 i915_pmu_get_attr_cpumask(struct device *dev,
 769                           struct device_attribute *attr,
 770                           char *buf)
 771 {
 772         return cpumap_print_to_pagebuf(true, buf, &i915_pmu_cpumask);
 773 }
 774
 775 static DEVICE_ATTR(cpumask, 0444, i915_pmu_get_attr_cpumask, NULL);
 776
 777 static struct attribute *i915_cpumask_attrs[] = {
 778         &dev_attr_cpumask.attr,
 779         NULL,
 780 };
 781
 782 static const struct attribute_group i915_pmu_cpumask_attr_group = {
 783         .attrs = i915_cpumask_attrs,
 784 };
 785
 786 static const struct attribute_group *i915_pmu_attr_groups[] = {
 787         &i915_pmu_format_attr_group,
 788         &i915_pmu_events_attr_group,
 789         &i915_pmu_cpumask_attr_group,
 790         NULL
 791 };
 792
 793 #define __event(__config, __name, __unit) \
 794 { \
 795         .config = (__config), \
 796         .name = (__name), \
 797         .unit = (__unit), \
 798 }
 799
 800 #define __engine_event(__sample, __name) \
 801 { \
 802         .sample = (__sample), \
 803         .name = (__name), \
 804 }
 805
 806 static struct i915_ext_attribute *
 807 add_i915_attr(struct i915_ext_attribute *attr, const char *name, u64 config)
 808 {
 809         sysfs_attr_init(&attr->attr.attr);
 810         attr->attr.attr.name = name;
 811         attr->attr.attr.mode = 0444;
 812         attr->attr.show = i915_pmu_event_show;
 813         attr->val = config;
 814
 815         return ++attr;
 816 }
 817
 818 static struct perf_pmu_events_attr *
 819 add_pmu_attr(struct perf_pmu_events_attr *attr, const char *name,
 820              const char *str)
 821 {
 822         sysfs_attr_init(&attr->attr.attr);
 823         attr->attr.attr.name = name;
 824         attr->attr.attr.mode = 0444;
 825         attr->attr.show = perf_event_sysfs_show;
 826         attr->event_str = str;
 827
 828         return ++attr;
 829 }
 830
 831 static struct attribute **
 832 create_event_attributes(struct drm_i915_private *i915)
 833 {
 834         static const struct {
 835                 u64 config;
 836                 const char *name;
 837                 const char *unit;
 838         } events[] = {
 839                 __event(I915_PMU_ACTUAL_FREQUENCY, "actual-frequency", "MHz"),
 840                 __event(I915_PMU_REQUESTED_FREQUENCY, "requested-frequency", "MHz"),
 841                 __event(I915_PMU_INTERRUPTS, "interrupts", NULL),
 842                 __event(I915_PMU_RC6_RESIDENCY, "rc6-residency", "ns"),
 843         };
 844         static const struct {
 845                 enum drm_i915_pmu_engine_sample sample;
 846                 char *name;
 847         } engine_events[] = {
 848                 __engine_event(I915_SAMPLE_BUSY, "busy"),
 849                 __engine_event(I915_SAMPLE_SEMA, "sema"),
 850                 __engine_event(I915_SAMPLE_WAIT, "wait"),
 851         };
 852         unsigned int count = 0;
 853         struct perf_pmu_events_attr *pmu_attr = NULL, *pmu_iter;
 854         struct i915_ext_attribute *i915_attr = NULL, *i915_iter;
 855         struct attribute **attr = NULL, **attr_iter;
 856         struct intel_engine_cs *engine;
 857         enum intel_engine_id id;
 858         unsigned int i;
 859
 860         /* Count how many counters we will be exposing. */
 861         for (i = 0; i < ARRAY_SIZE(events); i++) {
 862                 if (!config_status(i915, events[i].config))
 863                         count++;
 864         }
 865
 866         for_each_engine(engine, i915, id) {
 867                 for (i = 0; i < ARRAY_SIZE(engine_events); i++) {
 868                         if (!engine_event_status(engine,
 869                                                  engine_events[i].sample))
 870                                 count++;
 871                 }
 872         }
 873
 874         /* Allocate attribute objects and table. */
 875         i915_attr = kcalloc(count, sizeof(*i915_attr), GFP_KERNEL);
 876         if (!i915_attr)
 877                 goto err_alloc;
 878
 879         pmu_attr = kcalloc(count, sizeof(*pmu_attr), GFP_KERNEL);
 880         if (!pmu_attr)
 881                 goto err_alloc;
 882
 883         /* Max one pointer of each attribute type plus a termination entry. */
 884         attr = kcalloc(count * 2 + 1, sizeof(*attr), GFP_KERNEL);
 885         if (!attr)
 886                 goto err_alloc;
 887
 888         i915_iter = i915_attr;
 889         pmu_iter = pmu_attr;
 890         attr_iter = attr;
 891
 892         /* Initialize supported non-engine counters. */
 893         for (i = 0; i < ARRAY_SIZE(events); i++) {
 894                 char *str;
 895
 896                 if (config_status(i915, events[i].config))
 897                         continue;
 898
 899                 str = kstrdup(events[i].name, GFP_KERNEL);
 900                 if (!str)
 901                         goto err;
 902
 903                 *attr_iter++ = &i915_iter->attr.attr;
 904                 i915_iter = add_i915_attr(i915_iter, str, events[i].config);
 905
 906                 if (events[i].unit) {
 907                         str = kasprintf(GFP_KERNEL, "%s.unit", events[i].name);
 908                         if (!str)
 909                                 goto err;
 910
 911                         *attr_iter++ = &pmu_iter->attr.attr;
 912                         pmu_iter = add_pmu_attr(pmu_iter, str, events[i].unit);
 913                 }
 914         }
 915
 916         /* Initialize supported engine counters. */
 917         for_each_engine(engine, i915, id) {
 918                 for (i = 0; i < ARRAY_SIZE(engine_events); i++) {
 919                         char *str;
 920
 921                         if (engine_event_status(engine,
 922                                                 engine_events[i].sample))
 923                                 continue;
 924
 925                         str = kasprintf(GFP_KERNEL, "%s-%s",
 926                                         engine->name, engine_events[i].name);
 927                         if (!str)
 928                                 goto err;
 929
 930                         *attr_iter++ = &i915_iter->attr.attr;
 931                         i915_iter =
 932                                 add_i915_attr(i915_iter, str,
 933                                               __I915_PMU_ENGINE(engine->uabi_class,
 934                                                                 engine->instance,
 935                                                                 engine_events[i].sample));
 936
 937                         str = kasprintf(GFP_KERNEL, "%s-%s.unit",
 938                                         engine->name, engine_events[i].name);
 939                         if (!str)
 940                                 goto err;
 941
 942                         *attr_iter++ = &pmu_iter->attr.attr;
 943                         pmu_iter = add_pmu_attr(pmu_iter, str, "ns");
 944                 }
 945         }
 946
 947         i915->pmu.i915_attr = i915_attr;
 948         i915->pmu.pmu_attr = pmu_attr;
 949
 950         return attr;
 951
 952 err:;
 953         for (attr_iter = attr; *attr_iter; attr_iter++)
 954                 kfree((*attr_iter)->name);
 955
 956 err_alloc:
 957         kfree(attr);
 958         kfree(i915_attr);
 959         kfree(pmu_attr);
 960
 961         return NULL;
 962 }
 963
 964 static void free_event_attributes(struct drm_i915_private *i915)
 965 {
 966         struct attribute **attr_iter = i915_pmu_events_attr_group.attrs;
 967
 968         for (; *attr_iter; attr_iter++)
 969                 kfree((*attr_iter)->name);
 970
 971         kfree(i915_pmu_events_attr_group.attrs);
 972         kfree(i915->pmu.i915_attr);
 973         kfree(i915->pmu.pmu_attr);
 974
 975         i915_pmu_events_attr_group.attrs = NULL;
 976         i915->pmu.i915_attr = NULL;
 977         i915->pmu.pmu_attr = NULL;
 978 }
 979
 980 static int i915_pmu_cpu_online(unsigned int cpu, struct hlist_node *node)
 981 {
 982         struct i915_pmu *pmu = hlist_entry_safe(node, typeof(*pmu), node);
 983
 984         GEM_BUG_ON(!pmu->base.event_init);
 985
 986         /* Select the first online CPU as a designated reader. */
 987         if (!cpumask_weight(&i915_pmu_cpumask))
 988                 cpumask_set_cpu(cpu, &i915_pmu_cpumask);
 989
 990         return 0;
 991 }
 992
 993 static int i915_pmu_cpu_offline(unsigned int cpu, struct hlist_node *node)
 994 {
 995         struct i915_pmu *pmu = hlist_entry_safe(node, typeof(*pmu), node);
 996         unsigned int target;
 997
 998         GEM_BUG_ON(!pmu->base.event_init);
 999
1000         if (cpumask_test_and_clear_cpu(cpu, &i915_pmu_cpumask)) {
1001                 target = cpumask_any_but(topology_sibling_cpumask(cpu), cpu);
1002                 /* Migrate events if there is a valid target */
1003                 if (target < nr_cpu_ids) {
1004                         cpumask_set_cpu(target, &i915_pmu_cpumask);
1005                         perf_pmu_migrate_context(&pmu->base, cpu, target);
1006                 }
1007         }
1008
1009         return 0;
1010 }
1011
1012 static enum cpuhp_state cpuhp_slot = CPUHP_INVALID;
1013
1014 static int i915_pmu_register_cpuhp_state(struct drm_i915_private *i915)
1015 {
1016         enum cpuhp_state slot;
1017         int ret;
1018
1019         ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
1020                                       "perf/x86/intel/i915:online",
1021                                       i915_pmu_cpu_online,
1022                                       i915_pmu_cpu_offline);
1023         if (ret < 0)
1024                 return ret;
1025
1026         slot = ret;
1027         ret = cpuhp_state_add_instance(slot, &i915->pmu.node);
1028         if (ret) {
1029                 cpuhp_remove_multi_state(slot);
1030                 return ret;
1031         }
1032
1033         cpuhp_slot = slot;
1034         return 0;
1035 }
1036
1037 static void i915_pmu_unregister_cpuhp_state(struct drm_i915_private *i915)
1038 {
1039         WARN_ON(cpuhp_slot == CPUHP_INVALID);
1040         WARN_ON(cpuhp_state_remove_instance(cpuhp_slot, &i915->pmu.node));
1041         cpuhp_remove_multi_state(cpuhp_slot);
1042 }
1043
1044 void i915_pmu_register(struct drm_i915_private *i915)
1045 {
1046         int ret;
1047
1048         if (INTEL_GEN(i915) <= 2) {
1049                 DRM_INFO("PMU not supported for this GPU.");
1050                 return;
1051         }
1052
1053         i915_pmu_events_attr_group.attrs = create_event_attributes(i915);
1054         if (!i915_pmu_events_attr_group.attrs) {
1055                 ret = -ENOMEM;
1056                 goto err;
1057         }
1058
1059         i915->pmu.base.attr_groups      = i915_pmu_attr_groups;
1060         i915->pmu.base.task_ctx_nr      = perf_invalid_context;
1061         i915->pmu.base.event_init       = i915_pmu_event_init;
1062         i915->pmu.base.add              = i915_pmu_event_add;
1063         i915->pmu.base.del              = i915_pmu_event_del;
1064         i915->pmu.base.start            = i915_pmu_event_start;
1065         i915->pmu.base.stop             = i915_pmu_event_stop;
1066         i915->pmu.base.read             = i915_pmu_event_read;
1067         i915->pmu.base.event_idx        = i915_pmu_event_event_idx;
1068
1069         spin_lock_init(&i915->pmu.lock);
1070         hrtimer_init(&i915->pmu.timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1071         i915->pmu.timer.function = i915_sample;
1072
1073         ret = perf_pmu_register(&i915->pmu.base, "i915", -1);
1074         if (ret)
1075                 goto err;
1076
1077         ret = i915_pmu_register_cpuhp_state(i915);
1078         if (ret)
1079                 goto err_unreg;
1080
1081         return;
1082
1083 err_unreg:
1084         perf_pmu_unregister(&i915->pmu.base);
1085 err:
1086         i915->pmu.base.event_init = NULL;
1087         free_event_attributes(i915);
1088         DRM_NOTE("Failed to register PMU! (err=%d)\n", ret);
1089 }
1090
1091 void i915_pmu_unregister(struct drm_i915_private *i915)
1092 {
1093         if (!i915->pmu.base.event_init)
1094                 return;
1095
1096         WARN_ON(i915->pmu.enable);
1097
1098         hrtimer_cancel(&i915->pmu.timer);
1099
1100         i915_pmu_unregister_cpuhp_state(i915);
1101
1102         perf_pmu_unregister(&i915->pmu.base);
1103         i915->pmu.base.event_init = NULL;
1104         free_event_attributes(i915);
1105 }