arch/arm/mach-imx/mmdc.c

   1 /*
   2  * Copyright 2017 NXP
   3  * Copyright 2011,2016 Freescale Semiconductor, Inc.
   4  * Copyright 2011 Linaro Ltd.
   5  *
   6  * The code contained herein is licensed under the GNU General Public
   7  * License. You may obtain a copy of the GNU General Public License
   8  * Version 2 or later at the following locations:
   9  *
  10  * http://www.opensource.org/licenses/gpl-license.html
  11  * http://www.gnu.org/copyleft/gpl.html
  12  */
  13
  14 #include <linux/hrtimer.h>
  15 #include <linux/init.h>
  16 #include <linux/interrupt.h>
  17 #include <linux/io.h>
  18 #include <linux/module.h>
  19 #include <linux/of.h>
  20 #include <linux/of_address.h>
  21 #include <linux/of_device.h>
  22 #include <linux/perf_event.h>
  23 #include <linux/slab.h>
  24
  25 #include "common.h"
  26
  27 #define MMDC_MAPSR              0x404
  28 #define BP_MMDC_MAPSR_PSD       0
  29 #define BP_MMDC_MAPSR_PSS       4
  30
  31 #define MMDC_MDMISC             0x18
  32 #define BM_MMDC_MDMISC_DDR_TYPE 0x18
  33 #define BP_MMDC_MDMISC_DDR_TYPE 0x3
  34
  35 #define TOTAL_CYCLES            0x0
  36 #define BUSY_CYCLES             0x1
  37 #define READ_ACCESSES           0x2
  38 #define WRITE_ACCESSES          0x3
  39 #define READ_BYTES              0x4
  40 #define WRITE_BYTES             0x5
  41
  42 /* Enables, resets, freezes, overflow profiling*/
  43 #define DBG_DIS                 0x0
  44 #define DBG_EN                  0x1
  45 #define DBG_RST                 0x2
  46 #define PRF_FRZ                 0x4
  47 #define CYC_OVF                 0x8
  48 #define PROFILE_SEL             0x10
  49
  50 #define MMDC_MADPCR0    0x410
  51 #define MMDC_MADPCR1    0x414
  52 #define MMDC_MADPSR0    0x418
  53 #define MMDC_MADPSR1    0x41C
  54 #define MMDC_MADPSR2    0x420
  55 #define MMDC_MADPSR3    0x424
  56 #define MMDC_MADPSR4    0x428
  57 #define MMDC_MADPSR5    0x42C
  58
  59 #define MMDC_NUM_COUNTERS       6
  60
  61 #define MMDC_FLAG_PROFILE_SEL   0x1
  62 #define MMDC_PRF_AXI_ID_CLEAR   0x0
  63
  64 #define to_mmdc_pmu(p) container_of(p, struct mmdc_pmu, pmu)
  65
  66 static int ddr_type;
  67
  68 struct fsl_mmdc_devtype_data {
  69         unsigned int flags;
  70 };
  71
  72 static const struct fsl_mmdc_devtype_data imx6q_data = {
  73 };
  74
  75 static const struct fsl_mmdc_devtype_data imx6qp_data = {
  76         .flags = MMDC_FLAG_PROFILE_SEL,
  77 };
  78
  79 static const struct of_device_id imx_mmdc_dt_ids[] = {
  80         { .compatible = "fsl,imx6q-mmdc", .data = (void *)&imx6q_data},
  81         { .compatible = "fsl,imx6qp-mmdc", .data = (void *)&imx6qp_data},
  82         { /* sentinel */ }
  83 };
  84
  85 #ifdef CONFIG_PERF_EVENTS
  86
  87 static enum cpuhp_state cpuhp_mmdc_state;
  88 static DEFINE_IDA(mmdc_ida);
  89
  90 PMU_EVENT_ATTR_STRING(total-cycles, mmdc_pmu_total_cycles, "event=0x00")
  91 PMU_EVENT_ATTR_STRING(busy-cycles, mmdc_pmu_busy_cycles, "event=0x01")
  92 PMU_EVENT_ATTR_STRING(read-accesses, mmdc_pmu_read_accesses, "event=0x02")
  93 PMU_EVENT_ATTR_STRING(write-accesses, mmdc_pmu_write_accesses, "event=0x03")
  94 PMU_EVENT_ATTR_STRING(read-bytes, mmdc_pmu_read_bytes, "event=0x04")
  95 PMU_EVENT_ATTR_STRING(read-bytes.unit, mmdc_pmu_read_bytes_unit, "MB");
  96 PMU_EVENT_ATTR_STRING(read-bytes.scale, mmdc_pmu_read_bytes_scale, "0.000001");
  97 PMU_EVENT_ATTR_STRING(write-bytes, mmdc_pmu_write_bytes, "event=0x05")
  98 PMU_EVENT_ATTR_STRING(write-bytes.unit, mmdc_pmu_write_bytes_unit, "MB");
  99 PMU_EVENT_ATTR_STRING(write-bytes.scale, mmdc_pmu_write_bytes_scale, "0.000001");
 100
 101 struct mmdc_pmu {
 102         struct pmu pmu;
 103         void __iomem *mmdc_base;
 104         cpumask_t cpu;
 105         struct hrtimer hrtimer;
 106         unsigned int active_events;
 107         struct device *dev;
 108         struct perf_event *mmdc_events[MMDC_NUM_COUNTERS];
 109         struct hlist_node node;
 110         struct fsl_mmdc_devtype_data *devtype_data;
 111 };
 112
 113 /*
 114  * Polling period is set to one second, overflow of total-cycles (the fastest
 115  * increasing counter) takes ten seconds so one second is safe
 116  */
 117 static unsigned int mmdc_pmu_poll_period_us = 1000000;
 118
 119 module_param_named(pmu_pmu_poll_period_us, mmdc_pmu_poll_period_us, uint,
 120                 S_IRUGO | S_IWUSR);
 121
 122 static ktime_t mmdc_pmu_timer_period(void)
 123 {
 124         return ns_to_ktime((u64)mmdc_pmu_poll_period_us * 1000);
 125 }
 126
 127 static ssize_t mmdc_pmu_cpumask_show(struct device *dev,
 128                 struct device_attribute *attr, char *buf)
 129 {
 130         struct mmdc_pmu *pmu_mmdc = dev_get_drvdata(dev);
 131
 132         return cpumap_print_to_pagebuf(true, buf, &pmu_mmdc->cpu);
 133 }
 134
 135 static struct device_attribute mmdc_pmu_cpumask_attr =
 136         __ATTR(cpumask, S_IRUGO, mmdc_pmu_cpumask_show, NULL);
 137
 138 static struct attribute *mmdc_pmu_cpumask_attrs[] = {
 139         &mmdc_pmu_cpumask_attr.attr,
 140         NULL,
 141 };
 142
 143 static struct attribute_group mmdc_pmu_cpumask_attr_group = {
 144         .attrs = mmdc_pmu_cpumask_attrs,
 145 };
 146
 147 static struct attribute *mmdc_pmu_events_attrs[] = {
 148         &mmdc_pmu_total_cycles.attr.attr,
 149         &mmdc_pmu_busy_cycles.attr.attr,
 150         &mmdc_pmu_read_accesses.attr.attr,
 151         &mmdc_pmu_write_accesses.attr.attr,
 152         &mmdc_pmu_read_bytes.attr.attr,
 153         &mmdc_pmu_read_bytes_unit.attr.attr,
 154         &mmdc_pmu_read_bytes_scale.attr.attr,
 155         &mmdc_pmu_write_bytes.attr.attr,
 156         &mmdc_pmu_write_bytes_unit.attr.attr,
 157         &mmdc_pmu_write_bytes_scale.attr.attr,
 158         NULL,
 159 };
 160
 161 static struct attribute_group mmdc_pmu_events_attr_group = {
 162         .name = "events",
 163         .attrs = mmdc_pmu_events_attrs,
 164 };
 165
 166 PMU_FORMAT_ATTR(event, "config:0-63");
 167 PMU_FORMAT_ATTR(axi_id, "config1:0-63");
 168
 169 static struct attribute *mmdc_pmu_format_attrs[] = {
 170         &format_attr_event.attr,
 171         &format_attr_axi_id.attr,
 172         NULL,
 173 };
 174
 175 static struct attribute_group mmdc_pmu_format_attr_group = {
 176         .name = "format",
 177         .attrs = mmdc_pmu_format_attrs,
 178 };
 179
 180 static const struct attribute_group *attr_groups[] = {
 181         &mmdc_pmu_events_attr_group,
 182         &mmdc_pmu_format_attr_group,
 183         &mmdc_pmu_cpumask_attr_group,
 184         NULL,
 185 };
 186
 187 static u32 mmdc_pmu_read_counter(struct mmdc_pmu *pmu_mmdc, int cfg)
 188 {
 189         void __iomem *mmdc_base, *reg;
 190
 191         mmdc_base = pmu_mmdc->mmdc_base;
 192
 193         switch (cfg) {
 194         case TOTAL_CYCLES:
 195                 reg = mmdc_base + MMDC_MADPSR0;
 196                 break;
 197         case BUSY_CYCLES:
 198                 reg = mmdc_base + MMDC_MADPSR1;
 199                 break;
 200         case READ_ACCESSES:
 201                 reg = mmdc_base + MMDC_MADPSR2;
 202                 break;
 203         case WRITE_ACCESSES:
 204                 reg = mmdc_base + MMDC_MADPSR3;
 205                 break;
 206         case READ_BYTES:
 207                 reg = mmdc_base + MMDC_MADPSR4;
 208                 break;
 209         case WRITE_BYTES:
 210                 reg = mmdc_base + MMDC_MADPSR5;
 211                 break;
 212         default:
 213                 return WARN_ONCE(1,
 214                         "invalid configuration %d for mmdc counter", cfg);
 215         }
 216         return readl(reg);
 217 }
 218
 219 static int mmdc_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)
 220 {
 221         struct mmdc_pmu *pmu_mmdc = hlist_entry_safe(node, struct mmdc_pmu, node);
 222         int target;
 223
 224         if (!cpumask_test_and_clear_cpu(cpu, &pmu_mmdc->cpu))
 225                 return 0;
 226
 227         target = cpumask_any_but(cpu_online_mask, cpu);
 228         if (target >= nr_cpu_ids)
 229                 return 0;
 230
 231         perf_pmu_migrate_context(&pmu_mmdc->pmu, cpu, target);
 232         cpumask_set_cpu(target, &pmu_mmdc->cpu);
 233
 234         return 0;
 235 }
 236
 237 static bool mmdc_pmu_group_event_is_valid(struct perf_event *event,
 238                                           struct pmu *pmu,
 239                                           unsigned long *used_counters)
 240 {
 241         int cfg = event->attr.config;
 242
 243         if (is_software_event(event))
 244                 return true;
 245
 246         if (event->pmu != pmu)
 247                 return false;
 248
 249         return !test_and_set_bit(cfg, used_counters);
 250 }
 251
 252 /*
 253  * Each event has a single fixed-purpose counter, so we can only have a
 254  * single active event for each at any point in time. Here we just check
 255  * for duplicates, and rely on mmdc_pmu_event_init to verify that the HW
 256  * event numbers are valid.
 257  */
 258 static bool mmdc_pmu_group_is_valid(struct perf_event *event)
 259 {
 260         struct pmu *pmu = event->pmu;
 261         struct perf_event *leader = event->group_leader;
 262         struct perf_event *sibling;
 263         unsigned long counter_mask = 0;
 264
 265         set_bit(leader->attr.config, &counter_mask);
 266
 267         if (event != leader) {
 268                 if (!mmdc_pmu_group_event_is_valid(event, pmu, &counter_mask))
 269                         return false;
 270         }
 271
 272         list_for_each_entry(sibling, &leader->sibling_list, group_entry) {
 273                 if (!mmdc_pmu_group_event_is_valid(sibling, pmu, &counter_mask))
 274                         return false;
 275         }
 276
 277         return true;
 278 }
 279
 280 static int mmdc_pmu_event_init(struct perf_event *event)
 281 {
 282         struct mmdc_pmu *pmu_mmdc = to_mmdc_pmu(event->pmu);
 283         int cfg = event->attr.config;
 284
 285         if (event->attr.type != event->pmu->type)
 286                 return -ENOENT;
 287
 288         if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK)
 289                 return -EOPNOTSUPP;
 290
 291         if (event->cpu < 0) {
 292                 dev_warn(pmu_mmdc->dev, "Can't provide per-task data!\n");
 293                 return -EOPNOTSUPP;
 294         }
 295
 296         if (event->attr.exclude_user            ||
 297                         event->attr.exclude_kernel      ||
 298                         event->attr.exclude_hv          ||
 299                         event->attr.exclude_idle        ||
 300                         event->attr.exclude_host        ||
 301                         event->attr.exclude_guest       ||
 302                         event->attr.sample_period)
 303                 return -EINVAL;
 304
 305         if (cfg < 0 || cfg >= MMDC_NUM_COUNTERS)
 306                 return -EINVAL;
 307
 308         if (!mmdc_pmu_group_is_valid(event))
 309                 return -EINVAL;
 310
 311         event->cpu = cpumask_first(&pmu_mmdc->cpu);
 312         return 0;
 313 }
 314
 315 static void mmdc_pmu_event_update(struct perf_event *event)
 316 {
 317         struct mmdc_pmu *pmu_mmdc = to_mmdc_pmu(event->pmu);
 318         struct hw_perf_event *hwc = &event->hw;
 319         u64 delta, prev_raw_count, new_raw_count;
 320
 321         do {
 322                 prev_raw_count = local64_read(&hwc->prev_count);
 323                 new_raw_count = mmdc_pmu_read_counter(pmu_mmdc,
 324                                                       event->attr.config);
 325         } while (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
 326                 new_raw_count) != prev_raw_count);
 327
 328         delta = (new_raw_count - prev_raw_count) & 0xFFFFFFFF;
 329
 330         local64_add(delta, &event->count);
 331 }
 332
 333 static void mmdc_pmu_event_start(struct perf_event *event, int flags)
 334 {
 335         struct mmdc_pmu *pmu_mmdc = to_mmdc_pmu(event->pmu);
 336         struct hw_perf_event *hwc = &event->hw;
 337         void __iomem *mmdc_base, *reg;
 338         u32 val;
 339
 340         mmdc_base = pmu_mmdc->mmdc_base;
 341         reg = mmdc_base + MMDC_MADPCR0;
 342
 343         /*
 344          * hrtimer is required because mmdc does not provide an interrupt so
 345          * polling is necessary
 346          */
 347         hrtimer_start(&pmu_mmdc->hrtimer, mmdc_pmu_timer_period(),
 348                         HRTIMER_MODE_REL_PINNED);
 349
 350         local64_set(&hwc->prev_count, 0);
 351
 352         writel(DBG_RST, reg);
 353
 354         /*
 355          * Write the AXI id parameter to MADPCR1.
 356          */
 357         val = event->attr.config1;
 358         reg = mmdc_base + MMDC_MADPCR1;
 359         writel(val, reg);
 360
 361         reg = mmdc_base + MMDC_MADPCR0;
 362         val = DBG_EN;
 363         if (pmu_mmdc->devtype_data->flags & MMDC_FLAG_PROFILE_SEL)
 364                 val |= PROFILE_SEL;
 365
 366         writel(val, reg);
 367 }
 368
 369 static int mmdc_pmu_event_add(struct perf_event *event, int flags)
 370 {
 371         struct mmdc_pmu *pmu_mmdc = to_mmdc_pmu(event->pmu);
 372         struct hw_perf_event *hwc = &event->hw;
 373
 374         int cfg = event->attr.config;
 375
 376         if (flags & PERF_EF_START)
 377                 mmdc_pmu_event_start(event, flags);
 378
 379         if (pmu_mmdc->mmdc_events[cfg] != NULL)
 380                 return -EAGAIN;
 381
 382         pmu_mmdc->mmdc_events[cfg] = event;
 383         pmu_mmdc->active_events++;
 384
 385         local64_set(&hwc->prev_count, mmdc_pmu_read_counter(pmu_mmdc, cfg));
 386
 387         return 0;
 388 }
 389
 390 static void mmdc_pmu_event_stop(struct perf_event *event, int flags)
 391 {
 392         struct mmdc_pmu *pmu_mmdc = to_mmdc_pmu(event->pmu);
 393         void __iomem *mmdc_base, *reg;
 394
 395         mmdc_base = pmu_mmdc->mmdc_base;
 396         reg = mmdc_base + MMDC_MADPCR0;
 397
 398         writel(PRF_FRZ, reg);
 399
 400         reg = mmdc_base + MMDC_MADPCR1;
 401         writel(MMDC_PRF_AXI_ID_CLEAR, reg);
 402
 403         mmdc_pmu_event_update(event);
 404 }
 405
 406 static void mmdc_pmu_event_del(struct perf_event *event, int flags)
 407 {
 408         struct mmdc_pmu *pmu_mmdc = to_mmdc_pmu(event->pmu);
 409         int cfg = event->attr.config;
 410
 411         pmu_mmdc->mmdc_events[cfg] = NULL;
 412         pmu_mmdc->active_events--;
 413
 414         if (pmu_mmdc->active_events == 0)
 415                 hrtimer_cancel(&pmu_mmdc->hrtimer);
 416
 417         mmdc_pmu_event_stop(event, PERF_EF_UPDATE);
 418 }
 419
 420 static void mmdc_pmu_overflow_handler(struct mmdc_pmu *pmu_mmdc)
 421 {
 422         int i;
 423
 424         for (i = 0; i < MMDC_NUM_COUNTERS; i++) {
 425                 struct perf_event *event = pmu_mmdc->mmdc_events[i];
 426
 427                 if (event)
 428                         mmdc_pmu_event_update(event);
 429         }
 430 }
 431
 432 static enum hrtimer_restart mmdc_pmu_timer_handler(struct hrtimer *hrtimer)
 433 {
 434         struct mmdc_pmu *pmu_mmdc = container_of(hrtimer, struct mmdc_pmu,
 435                         hrtimer);
 436
 437         mmdc_pmu_overflow_handler(pmu_mmdc);
 438         hrtimer_forward_now(hrtimer, mmdc_pmu_timer_period());
 439
 440         return HRTIMER_RESTART;
 441 }
 442
 443 static int mmdc_pmu_init(struct mmdc_pmu *pmu_mmdc,
 444                 void __iomem *mmdc_base, struct device *dev)
 445 {
 446         int mmdc_num;
 447
 448         *pmu_mmdc = (struct mmdc_pmu) {
 449                 .pmu = (struct pmu) {
 450                         .task_ctx_nr    = perf_invalid_context,
 451                         .attr_groups    = attr_groups,
 452                         .event_init     = mmdc_pmu_event_init,
 453                         .add            = mmdc_pmu_event_add,
 454                         .del            = mmdc_pmu_event_del,
 455                         .start          = mmdc_pmu_event_start,
 456                         .stop           = mmdc_pmu_event_stop,
 457                         .read           = mmdc_pmu_event_update,
 458                 },
 459                 .mmdc_base = mmdc_base,
 460                 .dev = dev,
 461                 .active_events = 0,
 462         };
 463
 464         mmdc_num = ida_simple_get(&mmdc_ida, 0, 0, GFP_KERNEL);
 465
 466         return mmdc_num;
 467 }
 468
 469 static int imx_mmdc_remove(struct platform_device *pdev)
 470 {
 471         struct mmdc_pmu *pmu_mmdc = platform_get_drvdata(pdev);
 472
 473         cpuhp_state_remove_instance_nocalls(cpuhp_mmdc_state, &pmu_mmdc->node);
 474         perf_pmu_unregister(&pmu_mmdc->pmu);
 475         kfree(pmu_mmdc);
 476         return 0;
 477 }
 478
 479 static int imx_mmdc_perf_init(struct platform_device *pdev, void __iomem *mmdc_base)
 480 {
 481         struct mmdc_pmu *pmu_mmdc;
 482         char *name;
 483         int mmdc_num;
 484         int ret;
 485         const struct of_device_id *of_id =
 486                 of_match_device(imx_mmdc_dt_ids, &pdev->dev);
 487
 488         pmu_mmdc = kzalloc(sizeof(*pmu_mmdc), GFP_KERNEL);
 489         if (!pmu_mmdc) {
 490                 pr_err("failed to allocate PMU device!\n");
 491                 return -ENOMEM;
 492         }
 493
 494         /* The first instance registers the hotplug state */
 495         if (!cpuhp_mmdc_state) {
 496                 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
 497                                               "perf/arm/mmdc:online", NULL,
 498                                               mmdc_pmu_offline_cpu);
 499                 if (ret < 0) {
 500                         pr_err("cpuhp_setup_state_multi failed\n");
 501                         goto pmu_free;
 502                 }
 503                 cpuhp_mmdc_state = ret;
 504         }
 505
 506         mmdc_num = mmdc_pmu_init(pmu_mmdc, mmdc_base, &pdev->dev);
 507         if (mmdc_num == 0)
 508                 name = "mmdc";
 509         else
 510                 name = devm_kasprintf(&pdev->dev,
 511                                 GFP_KERNEL, "mmdc%d", mmdc_num);
 512
 513         pmu_mmdc->devtype_data = (struct fsl_mmdc_devtype_data *)of_id->data;
 514
 515         hrtimer_init(&pmu_mmdc->hrtimer, CLOCK_MONOTONIC,
 516                         HRTIMER_MODE_REL);
 517         pmu_mmdc->hrtimer.function = mmdc_pmu_timer_handler;
 518
 519         cpumask_set_cpu(raw_smp_processor_id(), &pmu_mmdc->cpu);
 520
 521         /* Register the pmu instance for cpu hotplug */
 522         cpuhp_state_add_instance_nocalls(cpuhp_mmdc_state, &pmu_mmdc->node);
 523
 524         ret = perf_pmu_register(&(pmu_mmdc->pmu), name, -1);
 525         if (ret)
 526                 goto pmu_register_err;
 527
 528         platform_set_drvdata(pdev, pmu_mmdc);
 529         return 0;
 530
 531 pmu_register_err:
 532         pr_warn("MMDC Perf PMU failed (%d), disabled\n", ret);
 533         cpuhp_state_remove_instance_nocalls(cpuhp_mmdc_state, &pmu_mmdc->node);
 534         hrtimer_cancel(&pmu_mmdc->hrtimer);
 535 pmu_free:
 536         kfree(pmu_mmdc);
 537         return ret;
 538 }
 539
 540 #else
 541 #define imx_mmdc_remove NULL
 542 #define imx_mmdc_perf_init(pdev, mmdc_base) 0
 543 #endif
 544
 545 static int imx_mmdc_probe(struct platform_device *pdev)
 546 {
 547         struct device_node *np = pdev->dev.of_node;
 548         void __iomem *mmdc_base, *reg;
 549         u32 val;
 550         int timeout = 0x400;
 551
 552         mmdc_base = of_iomap(np, 0);
 553         WARN_ON(!mmdc_base);
 554
 555         reg = mmdc_base + MMDC_MDMISC;
 556         /* Get ddr type */
 557         val = readl_relaxed(reg);
 558         ddr_type = (val & BM_MMDC_MDMISC_DDR_TYPE) >>
 559                  BP_MMDC_MDMISC_DDR_TYPE;
 560
 561         reg = mmdc_base + MMDC_MAPSR;
 562
 563         /* Enable automatic power saving */
 564         val = readl_relaxed(reg);
 565         val &= ~(1 << BP_MMDC_MAPSR_PSD);
 566         writel_relaxed(val, reg);
 567
 568         /* Ensure it's successfully enabled */
 569         while (!(readl_relaxed(reg) & 1 << BP_MMDC_MAPSR_PSS) && --timeout)
 570                 cpu_relax();
 571
 572         if (unlikely(!timeout)) {
 573                 pr_warn("%s: failed to enable automatic power saving\n",
 574                         __func__);
 575                 return -EBUSY;
 576         }
 577
 578         return imx_mmdc_perf_init(pdev, mmdc_base);
 579 }
 580
 581 int imx_mmdc_get_ddr_type(void)
 582 {
 583         return ddr_type;
 584 }
 585
 586 static struct platform_driver imx_mmdc_driver = {
 587         .driver         = {
 588                 .name   = "imx-mmdc",
 589                 .of_match_table = imx_mmdc_dt_ids,
 590         },
 591         .probe          = imx_mmdc_probe,
 592         .remove         = imx_mmdc_remove,
 593 };
 594
 595 static int __init imx_mmdc_init(void)
 596 {
 597         return platform_driver_register(&imx_mmdc_driver);
 598 }
 599 postcore_initcall(imx_mmdc_init);