drivers/perf/alibaba_uncore_drw_pmu.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Alibaba DDR Sub-System Driveway PMU driver
   4  *
   5  * Copyright (C) 2022 Alibaba Inc
   6  */
   7
   8 #define ALI_DRW_PMUNAME         "ali_drw"
   9 #define ALI_DRW_DRVNAME         ALI_DRW_PMUNAME "_pmu"
  10 #define pr_fmt(fmt)             ALI_DRW_DRVNAME ": " fmt
  11
  12 #include <linux/acpi.h>
  13 #include <linux/bitfield.h>
  14 #include <linux/bitmap.h>
  15 #include <linux/bitops.h>
  16 #include <linux/cpuhotplug.h>
  17 #include <linux/cpumask.h>
  18 #include <linux/device.h>
  19 #include <linux/errno.h>
  20 #include <linux/interrupt.h>
  21 #include <linux/irq.h>
  22 #include <linux/kernel.h>
  23 #include <linux/list.h>
  24 #include <linux/module.h>
  25 #include <linux/mutex.h>
  26 #include <linux/perf_event.h>
  27 #include <linux/platform_device.h>
  28 #include <linux/printk.h>
  29 #include <linux/rculist.h>
  30 #include <linux/refcount.h>
  31
  32
  33 #define ALI_DRW_PMU_COMMON_MAX_COUNTERS                 16
  34 #define ALI_DRW_PMU_TEST_SEL_COMMON_COUNTER_BASE        19
  35
  36 #define ALI_DRW_PMU_PA_SHIFT                    12
  37 #define ALI_DRW_PMU_CNT_INIT                    0x00000000
  38 #define ALI_DRW_CNT_MAX_PERIOD                  0xffffffff
  39 #define ALI_DRW_PMU_CYCLE_EVT_ID                0x80
  40
  41 #define ALI_DRW_PMU_CNT_CTRL                    0xC00
  42 #define ALI_DRW_PMU_CNT_RST                     BIT(2)
  43 #define ALI_DRW_PMU_CNT_STOP                    BIT(1)
  44 #define ALI_DRW_PMU_CNT_START                   BIT(0)
  45
  46 #define ALI_DRW_PMU_CNT_STATE                   0xC04
  47 #define ALI_DRW_PMU_TEST_CTRL                   0xC08
  48 #define ALI_DRW_PMU_CNT_PRELOAD                 0xC0C
  49
  50 #define ALI_DRW_PMU_CYCLE_CNT_HIGH_MASK         GENMASK(23, 0)
  51 #define ALI_DRW_PMU_CYCLE_CNT_LOW_MASK          GENMASK(31, 0)
  52 #define ALI_DRW_PMU_CYCLE_CNT_HIGH              0xC10
  53 #define ALI_DRW_PMU_CYCLE_CNT_LOW               0xC14
  54
  55 /* PMU EVENT SEL 0-3 are paired in 32-bit registers on a 4-byte stride */
  56 #define ALI_DRW_PMU_EVENT_SEL0                  0xC68
  57 /* counter 0-3 use sel0, counter 4-7 use sel1...*/
  58 #define ALI_DRW_PMU_EVENT_SELn(n) \
  59         (ALI_DRW_PMU_EVENT_SEL0 + (n / 4) * 0x4)
  60 #define ALI_DRW_PMCOM_CNT_EN                    BIT(7)
  61 #define ALI_DRW_PMCOM_CNT_EVENT_MASK            GENMASK(5, 0)
  62 #define ALI_DRW_PMCOM_CNT_EVENT_OFFSET(n) \
  63         (8 * (n % 4))
  64
  65 /* PMU COMMON COUNTER 0-15, are paired in 32-bit registers on a 4-byte stride */
  66 #define ALI_DRW_PMU_COMMON_COUNTER0             0xC78
  67 #define ALI_DRW_PMU_COMMON_COUNTERn(n) \
  68         (ALI_DRW_PMU_COMMON_COUNTER0 + 0x4 * (n))
  69
  70 #define ALI_DRW_PMU_OV_INTR_ENABLE_CTL          0xCB8
  71 #define ALI_DRW_PMU_OV_INTR_DISABLE_CTL         0xCBC
  72 #define ALI_DRW_PMU_OV_INTR_ENABLE_STATUS       0xCC0
  73 #define ALI_DRW_PMU_OV_INTR_CLR                 0xCC4
  74 #define ALI_DRW_PMU_OV_INTR_STATUS              0xCC8
  75 #define ALI_DRW_PMCOM_CNT_OV_INTR_MASK          GENMASK(23, 8)
  76 #define ALI_DRW_PMBW_CNT_OV_INTR_MASK           GENMASK(7, 0)
  77 #define ALI_DRW_PMU_OV_INTR_MASK                GENMASK_ULL(63, 0)
  78
  79 static int ali_drw_cpuhp_state_num;
  80
  81 static LIST_HEAD(ali_drw_pmu_irqs);
  82 static DEFINE_MUTEX(ali_drw_pmu_irqs_lock);
  83
  84 struct ali_drw_pmu_irq {
  85         struct hlist_node node;
  86         struct list_head irqs_node;
  87         struct list_head pmus_node;
  88         int irq_num;
  89         int cpu;
  90         refcount_t refcount;
  91 };
  92
  93 struct ali_drw_pmu {
  94         void __iomem *cfg_base;
  95         struct device *dev;
  96
  97         struct list_head pmus_node;
  98         struct ali_drw_pmu_irq *irq;
  99         int irq_num;
 100         int cpu;
 101         DECLARE_BITMAP(used_mask, ALI_DRW_PMU_COMMON_MAX_COUNTERS);
 102         struct perf_event *events[ALI_DRW_PMU_COMMON_MAX_COUNTERS];
 103         int evtids[ALI_DRW_PMU_COMMON_MAX_COUNTERS];
 104
 105         struct pmu pmu;
 106 };
 107
 108 #define to_ali_drw_pmu(p) (container_of(p, struct ali_drw_pmu, pmu))
 109
 110 #define DRW_CONFIG_EVENTID              GENMASK(7, 0)
 111 #define GET_DRW_EVENTID(event)  FIELD_GET(DRW_CONFIG_EVENTID, (event)->attr.config)
 112
 113 static ssize_t ali_drw_pmu_format_show(struct device *dev,
 114                                 struct device_attribute *attr, char *buf)
 115 {
 116         struct dev_ext_attribute *eattr;
 117
 118         eattr = container_of(attr, struct dev_ext_attribute, attr);
 119
 120         return sprintf(buf, "%s\n", (char *)eattr->var);
 121 }
 122
 123 /*
 124  * PMU event attributes
 125  */
 126 static ssize_t ali_drw_pmu_event_show(struct device *dev,
 127                                struct device_attribute *attr, char *page)
 128 {
 129         struct dev_ext_attribute *eattr;
 130
 131         eattr = container_of(attr, struct dev_ext_attribute, attr);
 132
 133         return sprintf(page, "config=0x%lx\n", (unsigned long)eattr->var);
 134 }
 135
 136 #define ALI_DRW_PMU_ATTR(_name, _func, _config)                            \
 137                 (&((struct dev_ext_attribute[]) {                               \
 138                                 { __ATTR(_name, 0444, _func, NULL), (void *)_config }   \
 139                 })[0].attr.attr)
 140
 141 #define ALI_DRW_PMU_FORMAT_ATTR(_name, _config)            \
 142         ALI_DRW_PMU_ATTR(_name, ali_drw_pmu_format_show, (void *)_config)
 143 #define ALI_DRW_PMU_EVENT_ATTR(_name, _config)             \
 144         ALI_DRW_PMU_ATTR(_name, ali_drw_pmu_event_show, (unsigned long)_config)
 145
 146 static struct attribute *ali_drw_pmu_events_attrs[] = {
 147         ALI_DRW_PMU_EVENT_ATTR(hif_rd_or_wr,                    0x0),
 148         ALI_DRW_PMU_EVENT_ATTR(hif_wr,                          0x1),
 149         ALI_DRW_PMU_EVENT_ATTR(hif_rd,                          0x2),
 150         ALI_DRW_PMU_EVENT_ATTR(hif_rmw,                         0x3),
 151         ALI_DRW_PMU_EVENT_ATTR(hif_hi_pri_rd,                   0x4),
 152         ALI_DRW_PMU_EVENT_ATTR(dfi_wr_data_cycles,              0x7),
 153         ALI_DRW_PMU_EVENT_ATTR(dfi_rd_data_cycles,              0x8),
 154         ALI_DRW_PMU_EVENT_ATTR(hpr_xact_when_critical,          0x9),
 155         ALI_DRW_PMU_EVENT_ATTR(lpr_xact_when_critical,          0xA),
 156         ALI_DRW_PMU_EVENT_ATTR(wr_xact_when_critical,           0xB),
 157         ALI_DRW_PMU_EVENT_ATTR(op_is_activate,                  0xC),
 158         ALI_DRW_PMU_EVENT_ATTR(op_is_rd_or_wr,                  0xD),
 159         ALI_DRW_PMU_EVENT_ATTR(op_is_rd_activate,               0xE),
 160         ALI_DRW_PMU_EVENT_ATTR(op_is_rd,                        0xF),
 161         ALI_DRW_PMU_EVENT_ATTR(op_is_wr,                        0x10),
 162         ALI_DRW_PMU_EVENT_ATTR(op_is_mwr,                       0x11),
 163         ALI_DRW_PMU_EVENT_ATTR(op_is_precharge,                 0x12),
 164         ALI_DRW_PMU_EVENT_ATTR(precharge_for_rdwr,              0x13),
 165         ALI_DRW_PMU_EVENT_ATTR(precharge_for_other,             0x14),
 166         ALI_DRW_PMU_EVENT_ATTR(rdwr_transitions,                0x15),
 167         ALI_DRW_PMU_EVENT_ATTR(write_combine,                   0x16),
 168         ALI_DRW_PMU_EVENT_ATTR(war_hazard,                      0x17),
 169         ALI_DRW_PMU_EVENT_ATTR(raw_hazard,                      0x18),
 170         ALI_DRW_PMU_EVENT_ATTR(waw_hazard,                      0x19),
 171         ALI_DRW_PMU_EVENT_ATTR(op_is_enter_selfref_rk0,         0x1A),
 172         ALI_DRW_PMU_EVENT_ATTR(op_is_enter_selfref_rk1,         0x1B),
 173         ALI_DRW_PMU_EVENT_ATTR(op_is_enter_selfref_rk2,         0x1C),
 174         ALI_DRW_PMU_EVENT_ATTR(op_is_enter_selfref_rk3,         0x1D),
 175         ALI_DRW_PMU_EVENT_ATTR(op_is_enter_powerdown_rk0,       0x1E),
 176         ALI_DRW_PMU_EVENT_ATTR(op_is_enter_powerdown_rk1,       0x1F),
 177         ALI_DRW_PMU_EVENT_ATTR(op_is_enter_powerdown_rk2,       0x20),
 178         ALI_DRW_PMU_EVENT_ATTR(op_is_enter_powerdown_rk3,       0x21),
 179         ALI_DRW_PMU_EVENT_ATTR(selfref_mode_rk0,                0x26),
 180         ALI_DRW_PMU_EVENT_ATTR(selfref_mode_rk1,                0x27),
 181         ALI_DRW_PMU_EVENT_ATTR(selfref_mode_rk2,                0x28),
 182         ALI_DRW_PMU_EVENT_ATTR(selfref_mode_rk3,                0x29),
 183         ALI_DRW_PMU_EVENT_ATTR(op_is_refresh,                   0x2A),
 184         ALI_DRW_PMU_EVENT_ATTR(op_is_crit_ref,                  0x2B),
 185         ALI_DRW_PMU_EVENT_ATTR(op_is_load_mode,                 0x2D),
 186         ALI_DRW_PMU_EVENT_ATTR(op_is_zqcl,                      0x2E),
 187         ALI_DRW_PMU_EVENT_ATTR(visible_window_limit_reached_rd, 0x30),
 188         ALI_DRW_PMU_EVENT_ATTR(visible_window_limit_reached_wr, 0x31),
 189         ALI_DRW_PMU_EVENT_ATTR(op_is_dqsosc_mpc,                0x34),
 190         ALI_DRW_PMU_EVENT_ATTR(op_is_dqsosc_mrr,                0x35),
 191         ALI_DRW_PMU_EVENT_ATTR(op_is_tcr_mrr,                   0x36),
 192         ALI_DRW_PMU_EVENT_ATTR(op_is_zqstart,                   0x37),
 193         ALI_DRW_PMU_EVENT_ATTR(op_is_zqlatch,                   0x38),
 194         ALI_DRW_PMU_EVENT_ATTR(chi_txreq,                       0x39),
 195         ALI_DRW_PMU_EVENT_ATTR(chi_txdat,                       0x3A),
 196         ALI_DRW_PMU_EVENT_ATTR(chi_rxdat,                       0x3B),
 197         ALI_DRW_PMU_EVENT_ATTR(chi_rxrsp,                       0x3C),
 198         ALI_DRW_PMU_EVENT_ATTR(tsz_vio,                         0x3D),
 199         ALI_DRW_PMU_EVENT_ATTR(cycle,                           0x80),
 200         NULL,
 201 };
 202
 203 static struct attribute_group ali_drw_pmu_events_attr_group = {
 204         .name = "events",
 205         .attrs = ali_drw_pmu_events_attrs,
 206 };
 207
 208 static struct attribute *ali_drw_pmu_format_attr[] = {
 209         ALI_DRW_PMU_FORMAT_ATTR(event, "config:0-7"),
 210         NULL,
 211 };
 212
 213 static const struct attribute_group ali_drw_pmu_format_group = {
 214         .name = "format",
 215         .attrs = ali_drw_pmu_format_attr,
 216 };
 217
 218 static ssize_t ali_drw_pmu_cpumask_show(struct device *dev,
 219                                         struct device_attribute *attr,
 220                                         char *buf)
 221 {
 222         struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(dev_get_drvdata(dev));
 223
 224         return cpumap_print_to_pagebuf(true, buf, cpumask_of(drw_pmu->cpu));
 225 }
 226
 227 static struct device_attribute ali_drw_pmu_cpumask_attr =
 228                 __ATTR(cpumask, 0444, ali_drw_pmu_cpumask_show, NULL);
 229
 230 static struct attribute *ali_drw_pmu_cpumask_attrs[] = {
 231         &ali_drw_pmu_cpumask_attr.attr,
 232         NULL,
 233 };
 234
 235 static const struct attribute_group ali_drw_pmu_cpumask_attr_group = {
 236         .attrs = ali_drw_pmu_cpumask_attrs,
 237 };
 238
 239 static ssize_t ali_drw_pmu_identifier_show(struct device *dev,
 240                                         struct device_attribute *attr,
 241                                         char *page)
 242 {
 243         return sysfs_emit(page, "%s\n", "ali_drw_pmu");
 244 }
 245
 246 static umode_t ali_drw_pmu_identifier_attr_visible(struct kobject *kobj,
 247                                                 struct attribute *attr, int n)
 248 {
 249         return attr->mode;
 250 }
 251
 252 static struct device_attribute ali_drw_pmu_identifier_attr =
 253         __ATTR(identifier, 0444, ali_drw_pmu_identifier_show, NULL);
 254
 255 static struct attribute *ali_drw_pmu_identifier_attrs[] = {
 256         &ali_drw_pmu_identifier_attr.attr,
 257         NULL
 258 };
 259
 260 static const struct attribute_group ali_drw_pmu_identifier_attr_group = {
 261         .attrs = ali_drw_pmu_identifier_attrs,
 262         .is_visible = ali_drw_pmu_identifier_attr_visible
 263 };
 264
 265 static const struct attribute_group *ali_drw_pmu_attr_groups[] = {
 266         &ali_drw_pmu_events_attr_group,
 267         &ali_drw_pmu_cpumask_attr_group,
 268         &ali_drw_pmu_format_group,
 269         &ali_drw_pmu_identifier_attr_group,
 270         NULL,
 271 };
 272
 273 /* find a counter for event, then in add func, hw.idx will equal to counter */
 274 static int ali_drw_get_counter_idx(struct perf_event *event)
 275 {
 276         struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
 277         int idx;
 278
 279         for (idx = 0; idx < ALI_DRW_PMU_COMMON_MAX_COUNTERS; ++idx) {
 280                 if (!test_and_set_bit(idx, drw_pmu->used_mask))
 281                         return idx;
 282         }
 283
 284         /* The counters are all in use. */
 285         return -EBUSY;
 286 }
 287
 288 static u64 ali_drw_pmu_read_counter(struct perf_event *event)
 289 {
 290         struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
 291         u64 cycle_high, cycle_low;
 292
 293         if (GET_DRW_EVENTID(event) == ALI_DRW_PMU_CYCLE_EVT_ID) {
 294                 cycle_high = readl(drw_pmu->cfg_base + ALI_DRW_PMU_CYCLE_CNT_HIGH);
 295                 cycle_high &= ALI_DRW_PMU_CYCLE_CNT_HIGH_MASK;
 296                 cycle_low = readl(drw_pmu->cfg_base + ALI_DRW_PMU_CYCLE_CNT_LOW);
 297                 cycle_low &= ALI_DRW_PMU_CYCLE_CNT_LOW_MASK;
 298                 return (cycle_high << 32 | cycle_low);
 299         }
 300
 301         return readl(drw_pmu->cfg_base +
 302                      ALI_DRW_PMU_COMMON_COUNTERn(event->hw.idx));
 303 }
 304
 305 static void ali_drw_pmu_event_update(struct perf_event *event)
 306 {
 307         struct hw_perf_event *hwc = &event->hw;
 308         u64 delta, prev, now;
 309
 310         do {
 311                 prev = local64_read(&hwc->prev_count);
 312                 now = ali_drw_pmu_read_counter(event);
 313         } while (local64_cmpxchg(&hwc->prev_count, prev, now) != prev);
 314
 315         /* handle overflow. */
 316         delta = now - prev;
 317         if (GET_DRW_EVENTID(event) == ALI_DRW_PMU_CYCLE_EVT_ID)
 318                 delta &= ALI_DRW_PMU_OV_INTR_MASK;
 319         else
 320                 delta &= ALI_DRW_CNT_MAX_PERIOD;
 321         local64_add(delta, &event->count);
 322 }
 323
 324 static void ali_drw_pmu_event_set_period(struct perf_event *event)
 325 {
 326         u64 pre_val;
 327         struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
 328
 329         /* set a preload counter for test purpose */
 330         writel(ALI_DRW_PMU_TEST_SEL_COMMON_COUNTER_BASE + event->hw.idx,
 331                drw_pmu->cfg_base + ALI_DRW_PMU_TEST_CTRL);
 332
 333         /* set conunter initial value */
 334         pre_val = ALI_DRW_PMU_CNT_INIT;
 335         writel(pre_val, drw_pmu->cfg_base + ALI_DRW_PMU_CNT_PRELOAD);
 336         local64_set(&event->hw.prev_count, pre_val);
 337
 338         /* set sel mode to zero to start test */
 339         writel(0x0, drw_pmu->cfg_base + ALI_DRW_PMU_TEST_CTRL);
 340 }
 341
 342 static void ali_drw_pmu_enable_counter(struct perf_event *event)
 343 {
 344         u32 val, subval, reg, shift;
 345         int counter = event->hw.idx;
 346         struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
 347
 348         reg = ALI_DRW_PMU_EVENT_SELn(counter);
 349         val = readl(drw_pmu->cfg_base + reg);
 350         subval = FIELD_PREP(ALI_DRW_PMCOM_CNT_EN, 1) |
 351                  FIELD_PREP(ALI_DRW_PMCOM_CNT_EVENT_MASK, drw_pmu->evtids[counter]);
 352
 353         shift = ALI_DRW_PMCOM_CNT_EVENT_OFFSET(counter);
 354         val &= ~(GENMASK(7, 0) << shift);
 355         val |= subval << shift;
 356
 357         writel(val, drw_pmu->cfg_base + reg);
 358 }
 359
 360 static void ali_drw_pmu_disable_counter(struct perf_event *event)
 361 {
 362         u32 val, reg, subval, shift;
 363         struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
 364         int counter = event->hw.idx;
 365
 366         reg = ALI_DRW_PMU_EVENT_SELn(counter);
 367         val = readl(drw_pmu->cfg_base + reg);
 368         subval = FIELD_PREP(ALI_DRW_PMCOM_CNT_EN, 0) |
 369                  FIELD_PREP(ALI_DRW_PMCOM_CNT_EVENT_MASK, 0);
 370
 371         shift = ALI_DRW_PMCOM_CNT_EVENT_OFFSET(counter);
 372         val &= ~(GENMASK(7, 0) << shift);
 373         val |= subval << shift;
 374
 375         writel(val, drw_pmu->cfg_base + reg);
 376 }
 377
 378 static irqreturn_t ali_drw_pmu_isr(int irq_num, void *data)
 379 {
 380         struct ali_drw_pmu_irq *irq = data;
 381         struct ali_drw_pmu *drw_pmu;
 382         irqreturn_t ret = IRQ_NONE;
 383
 384         rcu_read_lock();
 385         list_for_each_entry_rcu(drw_pmu, &irq->pmus_node, pmus_node) {
 386                 unsigned long status, clr_status;
 387                 struct perf_event *event;
 388                 unsigned int idx;
 389
 390                 for (idx = 0; idx < ALI_DRW_PMU_COMMON_MAX_COUNTERS; idx++) {
 391                         event = drw_pmu->events[idx];
 392                         if (!event)
 393                                 continue;
 394                         ali_drw_pmu_disable_counter(event);
 395                 }
 396
 397                 /* common counter intr status */
 398                 status = readl(drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_STATUS);
 399                 status = FIELD_GET(ALI_DRW_PMCOM_CNT_OV_INTR_MASK, status);
 400                 if (status) {
 401                         for_each_set_bit(idx, &status,
 402                                          ALI_DRW_PMU_COMMON_MAX_COUNTERS) {
 403                                 event = drw_pmu->events[idx];
 404                                 if (WARN_ON_ONCE(!event))
 405                                         continue;
 406                                 ali_drw_pmu_event_update(event);
 407                                 ali_drw_pmu_event_set_period(event);
 408                         }
 409
 410                         /* clear common counter intr status */
 411                         clr_status = FIELD_PREP(ALI_DRW_PMCOM_CNT_OV_INTR_MASK, 1);
 412                         writel(clr_status,
 413                                drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_CLR);
 414                 }
 415
 416                 for (idx = 0; idx < ALI_DRW_PMU_COMMON_MAX_COUNTERS; idx++) {
 417                         event = drw_pmu->events[idx];
 418                         if (!event)
 419                                 continue;
 420                         if (!(event->hw.state & PERF_HES_STOPPED))
 421                                 ali_drw_pmu_enable_counter(event);
 422                 }
 423                 if (status)
 424                         ret = IRQ_HANDLED;
 425         }
 426         rcu_read_unlock();
 427         return ret;
 428 }
 429
 430 static struct ali_drw_pmu_irq *__ali_drw_pmu_init_irq(struct platform_device
 431                                                       *pdev, int irq_num)
 432 {
 433         int ret;
 434         struct ali_drw_pmu_irq *irq;
 435
 436         list_for_each_entry(irq, &ali_drw_pmu_irqs, irqs_node) {
 437                 if (irq->irq_num == irq_num
 438                     && refcount_inc_not_zero(&irq->refcount))
 439                         return irq;
 440         }
 441
 442         irq = kzalloc(sizeof(*irq), GFP_KERNEL);
 443         if (!irq)
 444                 return ERR_PTR(-ENOMEM);
 445
 446         INIT_LIST_HEAD(&irq->pmus_node);
 447
 448         /* Pick one CPU to be the preferred one to use */
 449         irq->cpu = smp_processor_id();
 450         refcount_set(&irq->refcount, 1);
 451
 452         /*
 453          * FIXME: one of DDRSS Driveway PMU overflow interrupt shares the same
 454          * irq number with MPAM ERR_IRQ. To register DDRSS PMU and MPAM drivers
 455          * successfully, add IRQF_SHARED flag. Howerer, PMU interrupt should not
 456          * share with other component.
 457          */
 458         ret = devm_request_irq(&pdev->dev, irq_num, ali_drw_pmu_isr,
 459                                IRQF_SHARED, dev_name(&pdev->dev), irq);
 460         if (ret < 0) {
 461                 dev_err(&pdev->dev,
 462                         "Fail to request IRQ:%d ret:%d\n", irq_num, ret);
 463                 goto out_free;
 464         }
 465
 466         ret = irq_set_affinity_hint(irq_num, cpumask_of(irq->cpu));
 467         if (ret)
 468                 goto out_free;
 469
 470         ret = cpuhp_state_add_instance_nocalls(ali_drw_cpuhp_state_num,
 471                                              &irq->node);
 472         if (ret)
 473                 goto out_free;
 474
 475         irq->irq_num = irq_num;
 476         list_add(&irq->irqs_node, &ali_drw_pmu_irqs);
 477
 478         return irq;
 479
 480 out_free:
 481         kfree(irq);
 482         return ERR_PTR(ret);
 483 }
 484
 485 static int ali_drw_pmu_init_irq(struct ali_drw_pmu *drw_pmu,
 486                                 struct platform_device *pdev)
 487 {
 488         int irq_num;
 489         struct ali_drw_pmu_irq *irq;
 490
 491         /* Read and init IRQ */
 492         irq_num = platform_get_irq(pdev, 0);
 493         if (irq_num < 0)
 494                 return irq_num;
 495
 496         mutex_lock(&ali_drw_pmu_irqs_lock);
 497         irq = __ali_drw_pmu_init_irq(pdev, irq_num);
 498         mutex_unlock(&ali_drw_pmu_irqs_lock);
 499
 500         if (IS_ERR(irq))
 501                 return PTR_ERR(irq);
 502
 503         drw_pmu->irq = irq;
 504
 505         mutex_lock(&ali_drw_pmu_irqs_lock);
 506         list_add_rcu(&drw_pmu->pmus_node, &irq->pmus_node);
 507         mutex_unlock(&ali_drw_pmu_irqs_lock);
 508
 509         return 0;
 510 }
 511
 512 static void ali_drw_pmu_uninit_irq(struct ali_drw_pmu *drw_pmu)
 513 {
 514         struct ali_drw_pmu_irq *irq = drw_pmu->irq;
 515
 516         mutex_lock(&ali_drw_pmu_irqs_lock);
 517         list_del_rcu(&drw_pmu->pmus_node);
 518
 519         if (!refcount_dec_and_test(&irq->refcount)) {
 520                 mutex_unlock(&ali_drw_pmu_irqs_lock);
 521                 return;
 522         }
 523
 524         list_del(&irq->irqs_node);
 525         mutex_unlock(&ali_drw_pmu_irqs_lock);
 526
 527         WARN_ON(irq_set_affinity_hint(irq->irq_num, NULL));
 528         cpuhp_state_remove_instance_nocalls(ali_drw_cpuhp_state_num,
 529                                             &irq->node);
 530         kfree(irq);
 531 }
 532
 533 static int ali_drw_pmu_event_init(struct perf_event *event)
 534 {
 535         struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
 536         struct hw_perf_event *hwc = &event->hw;
 537         struct perf_event *sibling;
 538         struct device *dev = drw_pmu->pmu.dev;
 539
 540         if (event->attr.type != event->pmu->type)
 541                 return -ENOENT;
 542
 543         if (is_sampling_event(event)) {
 544                 dev_err(dev, "Sampling not supported!\n");
 545                 return -EOPNOTSUPP;
 546         }
 547
 548         if (event->attach_state & PERF_ATTACH_TASK) {
 549                 dev_err(dev, "Per-task counter cannot allocate!\n");
 550                 return -EOPNOTSUPP;
 551         }
 552
 553         event->cpu = drw_pmu->cpu;
 554         if (event->cpu < 0) {
 555                 dev_err(dev, "Per-task mode not supported!\n");
 556                 return -EOPNOTSUPP;
 557         }
 558
 559         if (event->group_leader != event &&
 560             !is_software_event(event->group_leader)) {
 561                 dev_err(dev, "driveway only allow one event!\n");
 562                 return -EINVAL;
 563         }
 564
 565         for_each_sibling_event(sibling, event->group_leader) {
 566                 if (sibling != event && !is_software_event(sibling)) {
 567                         dev_err(dev, "driveway event not allowed!\n");
 568                         return -EINVAL;
 569                 }
 570         }
 571
 572         /* reset all the pmu counters */
 573         writel(ALI_DRW_PMU_CNT_RST, drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL);
 574
 575         hwc->idx = -1;
 576
 577         return 0;
 578 }
 579
 580 static void ali_drw_pmu_start(struct perf_event *event, int flags)
 581 {
 582         struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
 583
 584         event->hw.state = 0;
 585
 586         if (GET_DRW_EVENTID(event) == ALI_DRW_PMU_CYCLE_EVT_ID) {
 587                 writel(ALI_DRW_PMU_CNT_START,
 588                        drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL);
 589                 return;
 590         }
 591
 592         ali_drw_pmu_event_set_period(event);
 593         if (flags & PERF_EF_RELOAD) {
 594                 unsigned long prev_raw_count =
 595                     local64_read(&event->hw.prev_count);
 596                 writel(prev_raw_count,
 597                        drw_pmu->cfg_base + ALI_DRW_PMU_CNT_PRELOAD);
 598         }
 599
 600         ali_drw_pmu_enable_counter(event);
 601
 602         writel(ALI_DRW_PMU_CNT_START, drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL);
 603 }
 604
 605 static void ali_drw_pmu_stop(struct perf_event *event, int flags)
 606 {
 607         struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
 608
 609         if (event->hw.state & PERF_HES_STOPPED)
 610                 return;
 611
 612         if (GET_DRW_EVENTID(event) != ALI_DRW_PMU_CYCLE_EVT_ID)
 613                 ali_drw_pmu_disable_counter(event);
 614
 615         writel(ALI_DRW_PMU_CNT_STOP, drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL);
 616
 617         ali_drw_pmu_event_update(event);
 618         event->hw.state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
 619 }
 620
 621 static int ali_drw_pmu_add(struct perf_event *event, int flags)
 622 {
 623         struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
 624         struct hw_perf_event *hwc = &event->hw;
 625         int idx = -1;
 626         int evtid;
 627
 628         evtid = GET_DRW_EVENTID(event);
 629
 630         if (evtid != ALI_DRW_PMU_CYCLE_EVT_ID) {
 631                 idx = ali_drw_get_counter_idx(event);
 632                 if (idx < 0)
 633                         return idx;
 634                 drw_pmu->events[idx] = event;
 635                 drw_pmu->evtids[idx] = evtid;
 636         }
 637         hwc->idx = idx;
 638
 639         hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
 640
 641         if (flags & PERF_EF_START)
 642                 ali_drw_pmu_start(event, PERF_EF_RELOAD);
 643
 644         /* Propagate our changes to the userspace mapping. */
 645         perf_event_update_userpage(event);
 646
 647         return 0;
 648 }
 649
 650 static void ali_drw_pmu_del(struct perf_event *event, int flags)
 651 {
 652         struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
 653         struct hw_perf_event *hwc = &event->hw;
 654         int idx = hwc->idx;
 655
 656         ali_drw_pmu_stop(event, PERF_EF_UPDATE);
 657
 658         if (idx >= 0 && idx < ALI_DRW_PMU_COMMON_MAX_COUNTERS) {
 659                 drw_pmu->events[idx] = NULL;
 660                 drw_pmu->evtids[idx] = 0;
 661                 clear_bit(idx, drw_pmu->used_mask);
 662         }
 663
 664         perf_event_update_userpage(event);
 665 }
 666
 667 static void ali_drw_pmu_read(struct perf_event *event)
 668 {
 669         ali_drw_pmu_event_update(event);
 670 }
 671
 672 static int ali_drw_pmu_probe(struct platform_device *pdev)
 673 {
 674         struct ali_drw_pmu *drw_pmu;
 675         struct resource *res;
 676         char *name;
 677         int ret;
 678
 679         drw_pmu = devm_kzalloc(&pdev->dev, sizeof(*drw_pmu), GFP_KERNEL);
 680         if (!drw_pmu)
 681                 return -ENOMEM;
 682
 683         drw_pmu->dev = &pdev->dev;
 684         platform_set_drvdata(pdev, drw_pmu);
 685
 686         drw_pmu->cfg_base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
 687         if (IS_ERR(drw_pmu->cfg_base))
 688                 return PTR_ERR(drw_pmu->cfg_base);
 689
 690         name = devm_kasprintf(drw_pmu->dev, GFP_KERNEL, "ali_drw_%llx",
 691                               (u64) (res->start >> ALI_DRW_PMU_PA_SHIFT));
 692         if (!name)
 693                 return -ENOMEM;
 694
 695         writel(ALI_DRW_PMU_CNT_RST, drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL);
 696
 697         /* enable the generation of interrupt by all common counters */
 698         writel(ALI_DRW_PMCOM_CNT_OV_INTR_MASK,
 699                drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_ENABLE_CTL);
 700
 701         /* clearing interrupt status */
 702         writel(0xffffff, drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_CLR);
 703
 704         drw_pmu->cpu = smp_processor_id();
 705
 706         ret = ali_drw_pmu_init_irq(drw_pmu, pdev);
 707         if (ret)
 708                 return ret;
 709
 710         drw_pmu->pmu = (struct pmu) {
 711                 .module         = THIS_MODULE,
 712                 .task_ctx_nr    = perf_invalid_context,
 713                 .event_init     = ali_drw_pmu_event_init,
 714                 .add            = ali_drw_pmu_add,
 715                 .del            = ali_drw_pmu_del,
 716                 .start          = ali_drw_pmu_start,
 717                 .stop           = ali_drw_pmu_stop,
 718                 .read           = ali_drw_pmu_read,
 719                 .attr_groups    = ali_drw_pmu_attr_groups,
 720                 .capabilities   = PERF_PMU_CAP_NO_EXCLUDE,
 721         };
 722
 723         ret = perf_pmu_register(&drw_pmu->pmu, name, -1);
 724         if (ret) {
 725                 dev_err(drw_pmu->dev, "DRW Driveway PMU PMU register failed!\n");
 726                 ali_drw_pmu_uninit_irq(drw_pmu);
 727         }
 728
 729         return ret;
 730 }
 731
 732 static void ali_drw_pmu_remove(struct platform_device *pdev)
 733 {
 734         struct ali_drw_pmu *drw_pmu = platform_get_drvdata(pdev);
 735
 736         /* disable the generation of interrupt by all common counters */
 737         writel(ALI_DRW_PMCOM_CNT_OV_INTR_MASK,
 738                drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_DISABLE_CTL);
 739
 740         ali_drw_pmu_uninit_irq(drw_pmu);
 741         perf_pmu_unregister(&drw_pmu->pmu);
 742 }
 743
 744 static int ali_drw_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)
 745 {
 746         struct ali_drw_pmu_irq *irq;
 747         struct ali_drw_pmu *drw_pmu;
 748         unsigned int target;
 749         int ret;
 750         cpumask_t node_online_cpus;
 751
 752         irq = hlist_entry_safe(node, struct ali_drw_pmu_irq, node);
 753         if (cpu != irq->cpu)
 754                 return 0;
 755
 756         ret = cpumask_and(&node_online_cpus,
 757                           cpumask_of_node(cpu_to_node(cpu)), cpu_online_mask);
 758         if (ret)
 759                 target = cpumask_any_but(&node_online_cpus, cpu);
 760         else
 761                 target = cpumask_any_but(cpu_online_mask, cpu);
 762
 763         if (target >= nr_cpu_ids)
 764                 return 0;
 765
 766         /* We're only reading, but this isn't the place to be involving RCU */
 767         mutex_lock(&ali_drw_pmu_irqs_lock);
 768         list_for_each_entry(drw_pmu, &irq->pmus_node, pmus_node)
 769                 perf_pmu_migrate_context(&drw_pmu->pmu, irq->cpu, target);
 770         mutex_unlock(&ali_drw_pmu_irqs_lock);
 771
 772         WARN_ON(irq_set_affinity_hint(irq->irq_num, cpumask_of(target)));
 773         irq->cpu = target;
 774
 775         return 0;
 776 }
 777
 778 /*
 779  * Due to historical reasons, the HID used in the production environment is
 780  * ARMHD700, so we leave ARMHD700 as Compatible ID.
 781  */
 782 static const struct acpi_device_id ali_drw_acpi_match[] = {
 783         {"BABA5000", 0},
 784         {"ARMHD700", 0},
 785         {}
 786 };
 787
 788 MODULE_DEVICE_TABLE(acpi, ali_drw_acpi_match);
 789
 790 static struct platform_driver ali_drw_pmu_driver = {
 791         .driver = {
 792                    .name = "ali_drw_pmu",
 793                    .acpi_match_table = ali_drw_acpi_match,
 794                    },
 795         .probe = ali_drw_pmu_probe,
 796         .remove_new = ali_drw_pmu_remove,
 797 };
 798
 799 static int __init ali_drw_pmu_init(void)
 800 {
 801         int ret;
 802
 803         ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
 804                                       "ali_drw_pmu:online",
 805                                       NULL, ali_drw_pmu_offline_cpu);
 806
 807         if (ret < 0) {
 808                 pr_err("DRW Driveway PMU: setup hotplug failed, ret = %d\n",
 809                        ret);
 810                 return ret;
 811         }
 812         ali_drw_cpuhp_state_num = ret;
 813
 814         ret = platform_driver_register(&ali_drw_pmu_driver);
 815         if (ret)
 816                 cpuhp_remove_multi_state(ali_drw_cpuhp_state_num);
 817
 818         return ret;
 819 }
 820
 821 static void __exit ali_drw_pmu_exit(void)
 822 {
 823         platform_driver_unregister(&ali_drw_pmu_driver);
 824         cpuhp_remove_multi_state(ali_drw_cpuhp_state_num);
 825 }
 826
 827 module_init(ali_drw_pmu_init);
 828 module_exit(ali_drw_pmu_exit);
 829
 830 MODULE_AUTHOR("Hongbo Yao <yaohongbo@linux.alibaba.com>");
 831 MODULE_AUTHOR("Neng Chen <nengchen@linux.alibaba.com>");
 832 MODULE_AUTHOR("Shuai Xue <xueshuai@linux.alibaba.com>");
 833 MODULE_DESCRIPTION("Alibaba DDR Sub-System Driveway PMU driver");
 834 MODULE_LICENSE("GPL v2");