arch/arm/kernel/perf_event_v7.c

   1 /*
   2  * ARMv7 Cortex-A8 and Cortex-A9 Performance Events handling code.
   3  *
   4  * ARMv7 support: Jean Pihet <jpihet@mvista.com>
   5  * 2010 (c) MontaVista Software, LLC.
   6  *
   7  * Copied from ARMv6 code, with the low level code inspired
   8  *  by the ARMv7 Oprofile code.
   9  *
  10  * Cortex-A8 has up to 4 configurable performance counters and
  11  *  a single cycle counter.
  12  * Cortex-A9 has up to 31 configurable performance counters and
  13  *  a single cycle counter.
  14  *
  15  * All counters can be enabled/disabled and IRQ masked separately. The cycle
  16  *  counter and all 4 performance counters together can be reset separately.
  17  */
  18
  19 #ifdef CONFIG_CPU_V7
  20
  21 static struct arm_pmu armv7pmu;
  22
  23 /*
  24  * Common ARMv7 event types
  25  *
  26  * Note: An implementation may not be able to count all of these events
  27  * but the encodings are considered to be `reserved' in the case that
  28  * they are not available.
  29  */
  30 enum armv7_perf_types {
  31         ARMV7_PERFCTR_PMNC_SW_INCR                      = 0x00,
  32         ARMV7_PERFCTR_L1_ICACHE_REFILL                  = 0x01,
  33         ARMV7_PERFCTR_ITLB_REFILL                       = 0x02,
  34         ARMV7_PERFCTR_L1_DCACHE_REFILL                  = 0x03,
  35         ARMV7_PERFCTR_L1_DCACHE_ACCESS                  = 0x04,
  36         ARMV7_PERFCTR_DTLB_REFILL                       = 0x05,
  37         ARMV7_PERFCTR_MEM_READ                          = 0x06,
  38         ARMV7_PERFCTR_MEM_WRITE                         = 0x07,
  39         ARMV7_PERFCTR_INSTR_EXECUTED                    = 0x08,
  40         ARMV7_PERFCTR_EXC_TAKEN                         = 0x09,
  41         ARMV7_PERFCTR_EXC_EXECUTED                      = 0x0A,
  42         ARMV7_PERFCTR_CID_WRITE                         = 0x0B,
  43
  44         /*
  45          * ARMV7_PERFCTR_PC_WRITE is equivalent to HW_BRANCH_INSTRUCTIONS.
  46          * It counts:
  47          *  - all (taken) branch instructions,
  48          *  - instructions that explicitly write the PC,
  49          *  - exception generating instructions.
  50          */
  51         ARMV7_PERFCTR_PC_WRITE                          = 0x0C,
  52         ARMV7_PERFCTR_PC_IMM_BRANCH                     = 0x0D,
  53         ARMV7_PERFCTR_PC_PROC_RETURN                    = 0x0E,
  54         ARMV7_PERFCTR_MEM_UNALIGNED_ACCESS              = 0x0F,
  55         ARMV7_PERFCTR_PC_BRANCH_MIS_PRED                = 0x10,
  56         ARMV7_PERFCTR_CLOCK_CYCLES                      = 0x11,
  57         ARMV7_PERFCTR_PC_BRANCH_PRED                    = 0x12,
  58
  59         /* These events are defined by the PMUv2 supplement (ARM DDI 0457A). */
  60         ARMV7_PERFCTR_MEM_ACCESS                        = 0x13,
  61         ARMV7_PERFCTR_L1_ICACHE_ACCESS                  = 0x14,
  62         ARMV7_PERFCTR_L1_DCACHE_WB                      = 0x15,
  63         ARMV7_PERFCTR_L2_CACHE_ACCESS                   = 0x16,
  64         ARMV7_PERFCTR_L2_CACHE_REFILL                   = 0x17,
  65         ARMV7_PERFCTR_L2_CACHE_WB                       = 0x18,
  66         ARMV7_PERFCTR_BUS_ACCESS                        = 0x19,
  67         ARMV7_PERFCTR_MEM_ERROR                         = 0x1A,
  68         ARMV7_PERFCTR_INSTR_SPEC                        = 0x1B,
  69         ARMV7_PERFCTR_TTBR_WRITE                        = 0x1C,
  70         ARMV7_PERFCTR_BUS_CYCLES                        = 0x1D,
  71
  72         ARMV7_PERFCTR_CPU_CYCLES                        = 0xFF
  73 };
  74
  75 /* ARMv7 Cortex-A8 specific event types */
  76 enum armv7_a8_perf_types {
  77         ARMV7_A8_PERFCTR_L2_CACHE_ACCESS                = 0x43,
  78         ARMV7_A8_PERFCTR_L2_CACHE_REFILL                = 0x44,
  79         ARMV7_A8_PERFCTR_L1_ICACHE_ACCESS               = 0x50,
  80         ARMV7_A8_PERFCTR_STALL_ISIDE                    = 0x56,
  81 };
  82
  83 /* ARMv7 Cortex-A9 specific event types */
  84 enum armv7_a9_perf_types {
  85         ARMV7_A9_PERFCTR_INSTR_CORE_RENAME              = 0x68,
  86         ARMV7_A9_PERFCTR_STALL_ICACHE                   = 0x60,
  87         ARMV7_A9_PERFCTR_STALL_DISPATCH                 = 0x66,
  88 };
  89
  90 /* ARMv7 Cortex-A5 specific event types */
  91 enum armv7_a5_perf_types {
  92         ARMV7_A5_PERFCTR_PREFETCH_LINEFILL              = 0xc2,
  93         ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP         = 0xc3,
  94 };
  95
  96 /* ARMv7 Cortex-A15 specific event types */
  97 enum armv7_a15_perf_types {
  98         ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_READ         = 0x40,
  99         ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_WRITE        = 0x41,
 100         ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_READ         = 0x42,
 101         ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_WRITE        = 0x43,
 102
 103         ARMV7_A15_PERFCTR_DTLB_REFILL_L1_READ           = 0x4C,
 104         ARMV7_A15_PERFCTR_DTLB_REFILL_L1_WRITE          = 0x4D,
 105
 106         ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_READ          = 0x50,
 107         ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_WRITE         = 0x51,
 108         ARMV7_A15_PERFCTR_L2_CACHE_REFILL_READ          = 0x52,
 109         ARMV7_A15_PERFCTR_L2_CACHE_REFILL_WRITE         = 0x53,
 110
 111         ARMV7_A15_PERFCTR_PC_WRITE_SPEC                 = 0x76,
 112 };
 113
 114 /*
 115  * Cortex-A8 HW events mapping
 116  *
 117  * The hardware events that we support. We do support cache operations but
 118  * we have harvard caches and no way to combine instruction and data
 119  * accesses/misses in hardware.
 120  */
 121 static const unsigned armv7_a8_perf_map[PERF_COUNT_HW_MAX] = {
 122         [PERF_COUNT_HW_CPU_CYCLES]              = ARMV7_PERFCTR_CPU_CYCLES,
 123         [PERF_COUNT_HW_INSTRUCTIONS]            = ARMV7_PERFCTR_INSTR_EXECUTED,
 124         [PERF_COUNT_HW_CACHE_REFERENCES]        = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 125         [PERF_COUNT_HW_CACHE_MISSES]            = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 126         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]     = ARMV7_PERFCTR_PC_WRITE,
 127         [PERF_COUNT_HW_BRANCH_MISSES]           = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 128         [PERF_COUNT_HW_BUS_CYCLES]              = HW_OP_UNSUPPORTED,
 129         [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = ARMV7_A8_PERFCTR_STALL_ISIDE,
 130         [PERF_COUNT_HW_STALLED_CYCLES_BACKEND]  = HW_OP_UNSUPPORTED,
 131 };
 132
 133 static const unsigned armv7_a8_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 134                                           [PERF_COUNT_HW_CACHE_OP_MAX]
 135                                           [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 136         [C(L1D)] = {
 137                 /*
 138                  * The performance counters don't differentiate between read
 139                  * and write accesses/misses so this isn't strictly correct,
 140                  * but it's the best we can do. Writes and reads get
 141                  * combined.
 142                  */
 143                 [C(OP_READ)] = {
 144                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 145                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 146                 },
 147                 [C(OP_WRITE)] = {
 148                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 149                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 150                 },
 151                 [C(OP_PREFETCH)] = {
 152                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 153                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 154                 },
 155         },
 156         [C(L1I)] = {
 157                 [C(OP_READ)] = {
 158                         [C(RESULT_ACCESS)]      = ARMV7_A8_PERFCTR_L1_ICACHE_ACCESS,
 159                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_L1_ICACHE_REFILL,
 160                 },
 161                 [C(OP_WRITE)] = {
 162                         [C(RESULT_ACCESS)]      = ARMV7_A8_PERFCTR_L1_ICACHE_ACCESS,
 163                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_L1_ICACHE_REFILL,
 164                 },
 165                 [C(OP_PREFETCH)] = {
 166                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 167                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 168                 },
 169         },
 170         [C(LL)] = {
 171                 [C(OP_READ)] = {
 172                         [C(RESULT_ACCESS)]      = ARMV7_A8_PERFCTR_L2_CACHE_ACCESS,
 173                         [C(RESULT_MISS)]        = ARMV7_A8_PERFCTR_L2_CACHE_REFILL,
 174                 },
 175                 [C(OP_WRITE)] = {
 176                         [C(RESULT_ACCESS)]      = ARMV7_A8_PERFCTR_L2_CACHE_ACCESS,
 177                         [C(RESULT_MISS)]        = ARMV7_A8_PERFCTR_L2_CACHE_REFILL,
 178                 },
 179                 [C(OP_PREFETCH)] = {
 180                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 181                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 182                 },
 183         },
 184         [C(DTLB)] = {
 185                 [C(OP_READ)] = {
 186                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 187                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_DTLB_REFILL,
 188                 },
 189                 [C(OP_WRITE)] = {
 190                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 191                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_DTLB_REFILL,
 192                 },
 193                 [C(OP_PREFETCH)] = {
 194                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 195                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 196                 },
 197         },
 198         [C(ITLB)] = {
 199                 [C(OP_READ)] = {
 200                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 201                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_ITLB_REFILL,
 202                 },
 203                 [C(OP_WRITE)] = {
 204                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 205                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_ITLB_REFILL,
 206                 },
 207                 [C(OP_PREFETCH)] = {
 208                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 209                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 210                 },
 211         },
 212         [C(BPU)] = {
 213                 [C(OP_READ)] = {
 214                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_PC_BRANCH_PRED,
 215                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 216                 },
 217                 [C(OP_WRITE)] = {
 218                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_PC_BRANCH_PRED,
 219                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 220                 },
 221                 [C(OP_PREFETCH)] = {
 222                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 223                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 224                 },
 225         },
 226         [C(NODE)] = {
 227                 [C(OP_READ)] = {
 228                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 229                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 230                 },
 231                 [C(OP_WRITE)] = {
 232                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 233                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 234                 },
 235                 [C(OP_PREFETCH)] = {
 236                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 237                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 238                 },
 239         },
 240 };
 241
 242 /*
 243  * Cortex-A9 HW events mapping
 244  */
 245 static const unsigned armv7_a9_perf_map[PERF_COUNT_HW_MAX] = {
 246         [PERF_COUNT_HW_CPU_CYCLES]              = ARMV7_PERFCTR_CPU_CYCLES,
 247         [PERF_COUNT_HW_INSTRUCTIONS]            = ARMV7_A9_PERFCTR_INSTR_CORE_RENAME,
 248         [PERF_COUNT_HW_CACHE_REFERENCES]        = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 249         [PERF_COUNT_HW_CACHE_MISSES]            = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 250         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]     = ARMV7_PERFCTR_PC_WRITE,
 251         [PERF_COUNT_HW_BRANCH_MISSES]           = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 252         [PERF_COUNT_HW_BUS_CYCLES]              = HW_OP_UNSUPPORTED,
 253         [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = ARMV7_A9_PERFCTR_STALL_ICACHE,
 254         [PERF_COUNT_HW_STALLED_CYCLES_BACKEND]  = ARMV7_A9_PERFCTR_STALL_DISPATCH,
 255 };
 256
 257 static const unsigned armv7_a9_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 258                                           [PERF_COUNT_HW_CACHE_OP_MAX]
 259                                           [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 260         [C(L1D)] = {
 261                 /*
 262                  * The performance counters don't differentiate between read
 263                  * and write accesses/misses so this isn't strictly correct,
 264                  * but it's the best we can do. Writes and reads get
 265                  * combined.
 266                  */
 267                 [C(OP_READ)] = {
 268                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 269                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 270                 },
 271                 [C(OP_WRITE)] = {
 272                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 273                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 274                 },
 275                 [C(OP_PREFETCH)] = {
 276                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 277                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 278                 },
 279         },
 280         [C(L1I)] = {
 281                 [C(OP_READ)] = {
 282                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 283                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_L1_ICACHE_REFILL,
 284                 },
 285                 [C(OP_WRITE)] = {
 286                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 287                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_L1_ICACHE_REFILL,
 288                 },
 289                 [C(OP_PREFETCH)] = {
 290                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 291                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 292                 },
 293         },
 294         [C(LL)] = {
 295                 [C(OP_READ)] = {
 296                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 297                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 298                 },
 299                 [C(OP_WRITE)] = {
 300                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 301                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 302                 },
 303                 [C(OP_PREFETCH)] = {
 304                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 305                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 306                 },
 307         },
 308         [C(DTLB)] = {
 309                 [C(OP_READ)] = {
 310                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 311                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_DTLB_REFILL,
 312                 },
 313                 [C(OP_WRITE)] = {
 314                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 315                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_DTLB_REFILL,
 316                 },
 317                 [C(OP_PREFETCH)] = {
 318                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 319                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 320                 },
 321         },
 322         [C(ITLB)] = {
 323                 [C(OP_READ)] = {
 324                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 325                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_ITLB_REFILL,
 326                 },
 327                 [C(OP_WRITE)] = {
 328                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 329                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_ITLB_REFILL,
 330                 },
 331                 [C(OP_PREFETCH)] = {
 332                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 333                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 334                 },
 335         },
 336         [C(BPU)] = {
 337                 [C(OP_READ)] = {
 338                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_PC_BRANCH_PRED,
 339                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 340                 },
 341                 [C(OP_WRITE)] = {
 342                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_PC_BRANCH_PRED,
 343                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 344                 },
 345                 [C(OP_PREFETCH)] = {
 346                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 347                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 348                 },
 349         },
 350         [C(NODE)] = {
 351                 [C(OP_READ)] = {
 352                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 353                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 354                 },
 355                 [C(OP_WRITE)] = {
 356                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 357                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 358                 },
 359                 [C(OP_PREFETCH)] = {
 360                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 361                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 362                 },
 363         },
 364 };
 365
 366 /*
 367  * Cortex-A5 HW events mapping
 368  */
 369 static const unsigned armv7_a5_perf_map[PERF_COUNT_HW_MAX] = {
 370         [PERF_COUNT_HW_CPU_CYCLES]              = ARMV7_PERFCTR_CPU_CYCLES,
 371         [PERF_COUNT_HW_INSTRUCTIONS]            = ARMV7_PERFCTR_INSTR_EXECUTED,
 372         [PERF_COUNT_HW_CACHE_REFERENCES]        = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 373         [PERF_COUNT_HW_CACHE_MISSES]            = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 374         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]     = ARMV7_PERFCTR_PC_WRITE,
 375         [PERF_COUNT_HW_BRANCH_MISSES]           = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 376         [PERF_COUNT_HW_BUS_CYCLES]              = HW_OP_UNSUPPORTED,
 377         [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = HW_OP_UNSUPPORTED,
 378         [PERF_COUNT_HW_STALLED_CYCLES_BACKEND]  = HW_OP_UNSUPPORTED,
 379 };
 380
 381 static const unsigned armv7_a5_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 382                                         [PERF_COUNT_HW_CACHE_OP_MAX]
 383                                         [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 384         [C(L1D)] = {
 385                 [C(OP_READ)] = {
 386                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 387                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 388                 },
 389                 [C(OP_WRITE)] = {
 390                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 391                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 392                 },
 393                 [C(OP_PREFETCH)] = {
 394                         [C(RESULT_ACCESS)]      = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL,
 395                         [C(RESULT_MISS)]        = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP,
 396                 },
 397         },
 398         [C(L1I)] = {
 399                 [C(OP_READ)] = {
 400                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_L1_ICACHE_ACCESS,
 401                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_L1_ICACHE_REFILL,
 402                 },
 403                 [C(OP_WRITE)] = {
 404                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_L1_ICACHE_ACCESS,
 405                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_L1_ICACHE_REFILL,
 406                 },
 407                 /*
 408                  * The prefetch counters don't differentiate between the I
 409                  * side and the D side.
 410                  */
 411                 [C(OP_PREFETCH)] = {
 412                         [C(RESULT_ACCESS)]      = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL,
 413                         [C(RESULT_MISS)]        = ARMV7_A5_PERFCTR_PREFETCH_LINEFILL_DROP,
 414                 },
 415         },
 416         [C(LL)] = {
 417                 [C(OP_READ)] = {
 418                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 419                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 420                 },
 421                 [C(OP_WRITE)] = {
 422                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 423                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 424                 },
 425                 [C(OP_PREFETCH)] = {
 426                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 427                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 428                 },
 429         },
 430         [C(DTLB)] = {
 431                 [C(OP_READ)] = {
 432                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 433                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_DTLB_REFILL,
 434                 },
 435                 [C(OP_WRITE)] = {
 436                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 437                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_DTLB_REFILL,
 438                 },
 439                 [C(OP_PREFETCH)] = {
 440                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 441                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 442                 },
 443         },
 444         [C(ITLB)] = {
 445                 [C(OP_READ)] = {
 446                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 447                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_ITLB_REFILL,
 448                 },
 449                 [C(OP_WRITE)] = {
 450                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 451                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_ITLB_REFILL,
 452                 },
 453                 [C(OP_PREFETCH)] = {
 454                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 455                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 456                 },
 457         },
 458         [C(BPU)] = {
 459                 [C(OP_READ)] = {
 460                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_PC_BRANCH_PRED,
 461                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 462                 },
 463                 [C(OP_WRITE)] = {
 464                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_PC_BRANCH_PRED,
 465                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 466                 },
 467                 [C(OP_PREFETCH)] = {
 468                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 469                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 470                 },
 471         },
 472 };
 473
 474 /*
 475  * Cortex-A15 HW events mapping
 476  */
 477 static const unsigned armv7_a15_perf_map[PERF_COUNT_HW_MAX] = {
 478         [PERF_COUNT_HW_CPU_CYCLES]              = ARMV7_PERFCTR_CPU_CYCLES,
 479         [PERF_COUNT_HW_INSTRUCTIONS]            = ARMV7_PERFCTR_INSTR_EXECUTED,
 480         [PERF_COUNT_HW_CACHE_REFERENCES]        = ARMV7_PERFCTR_L1_DCACHE_ACCESS,
 481         [PERF_COUNT_HW_CACHE_MISSES]            = ARMV7_PERFCTR_L1_DCACHE_REFILL,
 482         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]     = ARMV7_A15_PERFCTR_PC_WRITE_SPEC,
 483         [PERF_COUNT_HW_BRANCH_MISSES]           = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 484         [PERF_COUNT_HW_BUS_CYCLES]              = ARMV7_PERFCTR_BUS_CYCLES,
 485         [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = HW_OP_UNSUPPORTED,
 486         [PERF_COUNT_HW_STALLED_CYCLES_BACKEND]  = HW_OP_UNSUPPORTED,
 487 };
 488
 489 static const unsigned armv7_a15_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
 490                                         [PERF_COUNT_HW_CACHE_OP_MAX]
 491                                         [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 492         [C(L1D)] = {
 493                 [C(OP_READ)] = {
 494                         [C(RESULT_ACCESS)]      = ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_READ,
 495                         [C(RESULT_MISS)]        = ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_READ,
 496                 },
 497                 [C(OP_WRITE)] = {
 498                         [C(RESULT_ACCESS)]      = ARMV7_A15_PERFCTR_L1_DCACHE_ACCESS_WRITE,
 499                         [C(RESULT_MISS)]        = ARMV7_A15_PERFCTR_L1_DCACHE_REFILL_WRITE,
 500                 },
 501                 [C(OP_PREFETCH)] = {
 502                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 503                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 504                 },
 505         },
 506         [C(L1I)] = {
 507                 /*
 508                  * Not all performance counters differentiate between read
 509                  * and write accesses/misses so we're not always strictly
 510                  * correct, but it's the best we can do. Writes and reads get
 511                  * combined in these cases.
 512                  */
 513                 [C(OP_READ)] = {
 514                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_L1_ICACHE_ACCESS,
 515                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_L1_ICACHE_REFILL,
 516                 },
 517                 [C(OP_WRITE)] = {
 518                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_L1_ICACHE_ACCESS,
 519                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_L1_ICACHE_REFILL,
 520                 },
 521                 [C(OP_PREFETCH)] = {
 522                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 523                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 524                 },
 525         },
 526         [C(LL)] = {
 527                 [C(OP_READ)] = {
 528                         [C(RESULT_ACCESS)]      = ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_READ,
 529                         [C(RESULT_MISS)]        = ARMV7_A15_PERFCTR_L2_CACHE_REFILL_READ,
 530                 },
 531                 [C(OP_WRITE)] = {
 532                         [C(RESULT_ACCESS)]      = ARMV7_A15_PERFCTR_L2_CACHE_ACCESS_WRITE,
 533                         [C(RESULT_MISS)]        = ARMV7_A15_PERFCTR_L2_CACHE_REFILL_WRITE,
 534                 },
 535                 [C(OP_PREFETCH)] = {
 536                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 537                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 538                 },
 539         },
 540         [C(DTLB)] = {
 541                 [C(OP_READ)] = {
 542                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 543                         [C(RESULT_MISS)]        = ARMV7_A15_PERFCTR_DTLB_REFILL_L1_READ,
 544                 },
 545                 [C(OP_WRITE)] = {
 546                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 547                         [C(RESULT_MISS)]        = ARMV7_A15_PERFCTR_DTLB_REFILL_L1_WRITE,
 548                 },
 549                 [C(OP_PREFETCH)] = {
 550                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 551                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 552                 },
 553         },
 554         [C(ITLB)] = {
 555                 [C(OP_READ)] = {
 556                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 557                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_ITLB_REFILL,
 558                 },
 559                 [C(OP_WRITE)] = {
 560                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 561                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_ITLB_REFILL,
 562                 },
 563                 [C(OP_PREFETCH)] = {
 564                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 565                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 566                 },
 567         },
 568         [C(BPU)] = {
 569                 [C(OP_READ)] = {
 570                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_PC_BRANCH_PRED,
 571                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 572                 },
 573                 [C(OP_WRITE)] = {
 574                         [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_PC_BRANCH_PRED,
 575                         [C(RESULT_MISS)]        = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
 576                 },
 577                 [C(OP_PREFETCH)] = {
 578                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 579                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 580                 },
 581         },
 582 };
 583
 584 /*
 585  * Perf Events' indices
 586  */
 587 #define ARMV7_IDX_CYCLE_COUNTER 0
 588 #define ARMV7_IDX_COUNTER0      1
 589 #define ARMV7_IDX_COUNTER_LAST  (ARMV7_IDX_CYCLE_COUNTER + cpu_pmu->num_events - 1)
 590
 591 #define ARMV7_MAX_COUNTERS      32
 592 #define ARMV7_COUNTER_MASK      (ARMV7_MAX_COUNTERS - 1)
 593
 594 /*
 595  * ARMv7 low level PMNC access
 596  */
 597
 598 /*
 599  * Perf Event to low level counters mapping
 600  */
 601 #define ARMV7_IDX_TO_COUNTER(x) \
 602         (((x) - ARMV7_IDX_COUNTER0) & ARMV7_COUNTER_MASK)
 603
 604 /*
 605  * Per-CPU PMNC: config reg
 606  */
 607 #define ARMV7_PMNC_E            (1 << 0) /* Enable all counters */
 608 #define ARMV7_PMNC_P            (1 << 1) /* Reset all counters */
 609 #define ARMV7_PMNC_C            (1 << 2) /* Cycle counter reset */
 610 #define ARMV7_PMNC_D            (1 << 3) /* CCNT counts every 64th cpu cycle */
 611 #define ARMV7_PMNC_X            (1 << 4) /* Export to ETM */
 612 #define ARMV7_PMNC_DP           (1 << 5) /* Disable CCNT if non-invasive debug*/
 613 #define ARMV7_PMNC_N_SHIFT      11       /* Number of counters supported */
 614 #define ARMV7_PMNC_N_MASK       0x1f
 615 #define ARMV7_PMNC_MASK         0x3f     /* Mask for writable bits */
 616
 617 /*
 618  * FLAG: counters overflow flag status reg
 619  */
 620 #define ARMV7_FLAG_MASK         0xffffffff      /* Mask for writable bits */
 621 #define ARMV7_OVERFLOWED_MASK   ARMV7_FLAG_MASK
 622
 623 /*
 624  * PMXEVTYPER: Event selection reg
 625  */
 626 #define ARMV7_EVTYPE_MASK       0xc00000ff      /* Mask for writable bits */
 627 #define ARMV7_EVTYPE_EVENT      0xff            /* Mask for EVENT bits */
 628
 629 /*
 630  * Event filters for PMUv2
 631  */
 632 #define ARMV7_EXCLUDE_PL1       (1 << 31)
 633 #define ARMV7_EXCLUDE_USER      (1 << 30)
 634 #define ARMV7_INCLUDE_HYP       (1 << 27)
 635
 636 static inline u32 armv7_pmnc_read(void)
 637 {
 638         u32 val;
 639         asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r"(val));
 640         return val;
 641 }
 642
 643 static inline void armv7_pmnc_write(u32 val)
 644 {
 645         val &= ARMV7_PMNC_MASK;
 646         isb();
 647         asm volatile("mcr p15, 0, %0, c9, c12, 0" : : "r"(val));
 648 }
 649
 650 static inline int armv7_pmnc_has_overflowed(u32 pmnc)
 651 {
 652         return pmnc & ARMV7_OVERFLOWED_MASK;
 653 }
 654
 655 static inline int armv7_pmnc_counter_valid(int idx)
 656 {
 657         return idx >= ARMV7_IDX_CYCLE_COUNTER && idx <= ARMV7_IDX_COUNTER_LAST;
 658 }
 659
 660 static inline int armv7_pmnc_counter_has_overflowed(u32 pmnc, int idx)
 661 {
 662         int ret = 0;
 663         u32 counter;
 664
 665         if (!armv7_pmnc_counter_valid(idx)) {
 666                 pr_err("CPU%u checking wrong counter %d overflow status\n",
 667                         smp_processor_id(), idx);
 668         } else {
 669                 counter = ARMV7_IDX_TO_COUNTER(idx);
 670                 ret = pmnc & BIT(counter);
 671         }
 672
 673         return ret;
 674 }
 675
 676 static inline int armv7_pmnc_select_counter(int idx)
 677 {
 678         u32 counter;
 679
 680         if (!armv7_pmnc_counter_valid(idx)) {
 681                 pr_err("CPU%u selecting wrong PMNC counter %d\n",
 682                         smp_processor_id(), idx);
 683                 return -EINVAL;
 684         }
 685
 686         counter = ARMV7_IDX_TO_COUNTER(idx);
 687         asm volatile("mcr p15, 0, %0, c9, c12, 5" : : "r" (counter));
 688         isb();
 689
 690         return idx;
 691 }
 692
 693 static inline u32 armv7pmu_read_counter(int idx)
 694 {
 695         u32 value = 0;
 696
 697         if (!armv7_pmnc_counter_valid(idx))
 698                 pr_err("CPU%u reading wrong counter %d\n",
 699                         smp_processor_id(), idx);
 700         else if (idx == ARMV7_IDX_CYCLE_COUNTER)
 701                 asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (value));
 702         else if (armv7_pmnc_select_counter(idx) == idx)
 703                 asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (value));
 704
 705         return value;
 706 }
 707
 708 static inline void armv7pmu_write_counter(int idx, u32 value)
 709 {
 710         if (!armv7_pmnc_counter_valid(idx))
 711                 pr_err("CPU%u writing wrong counter %d\n",
 712                         smp_processor_id(), idx);
 713         else if (idx == ARMV7_IDX_CYCLE_COUNTER)
 714                 asm volatile("mcr p15, 0, %0, c9, c13, 0" : : "r" (value));
 715         else if (armv7_pmnc_select_counter(idx) == idx)
 716                 asm volatile("mcr p15, 0, %0, c9, c13, 2" : : "r" (value));
 717 }
 718
 719 static inline void armv7_pmnc_write_evtsel(int idx, u32 val)
 720 {
 721         if (armv7_pmnc_select_counter(idx) == idx) {
 722                 val &= ARMV7_EVTYPE_MASK;
 723                 asm volatile("mcr p15, 0, %0, c9, c13, 1" : : "r" (val));
 724         }
 725 }
 726
 727 static inline int armv7_pmnc_enable_counter(int idx)
 728 {
 729         u32 counter;
 730
 731         if (!armv7_pmnc_counter_valid(idx)) {
 732                 pr_err("CPU%u enabling wrong PMNC counter %d\n",
 733                         smp_processor_id(), idx);
 734                 return -EINVAL;
 735         }
 736
 737         counter = ARMV7_IDX_TO_COUNTER(idx);
 738         asm volatile("mcr p15, 0, %0, c9, c12, 1" : : "r" (BIT(counter)));
 739         return idx;
 740 }
 741
 742 static inline int armv7_pmnc_disable_counter(int idx)
 743 {
 744         u32 counter;
 745
 746         if (!armv7_pmnc_counter_valid(idx)) {
 747                 pr_err("CPU%u disabling wrong PMNC counter %d\n",
 748                         smp_processor_id(), idx);
 749                 return -EINVAL;
 750         }
 751
 752         counter = ARMV7_IDX_TO_COUNTER(idx);
 753         asm volatile("mcr p15, 0, %0, c9, c12, 2" : : "r" (BIT(counter)));
 754         return idx;
 755 }
 756
 757 static inline int armv7_pmnc_enable_intens(int idx)
 758 {
 759         u32 counter;
 760
 761         if (!armv7_pmnc_counter_valid(idx)) {
 762                 pr_err("CPU%u enabling wrong PMNC counter IRQ enable %d\n",
 763                         smp_processor_id(), idx);
 764                 return -EINVAL;
 765         }
 766
 767         counter = ARMV7_IDX_TO_COUNTER(idx);
 768         asm volatile("mcr p15, 0, %0, c9, c14, 1" : : "r" (BIT(counter)));
 769         return idx;
 770 }
 771
 772 static inline int armv7_pmnc_disable_intens(int idx)
 773 {
 774         u32 counter;
 775
 776         if (!armv7_pmnc_counter_valid(idx)) {
 777                 pr_err("CPU%u disabling wrong PMNC counter IRQ enable %d\n",
 778                         smp_processor_id(), idx);
 779                 return -EINVAL;
 780         }
 781
 782         counter = ARMV7_IDX_TO_COUNTER(idx);
 783         asm volatile("mcr p15, 0, %0, c9, c14, 2" : : "r" (BIT(counter)));
 784         return idx;
 785 }
 786
 787 static inline u32 armv7_pmnc_getreset_flags(void)
 788 {
 789         u32 val;
 790
 791         /* Read */
 792         asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val));
 793
 794         /* Write to clear flags */
 795         val &= ARMV7_FLAG_MASK;
 796         asm volatile("mcr p15, 0, %0, c9, c12, 3" : : "r" (val));
 797
 798         return val;
 799 }
 800
 801 #ifdef DEBUG
 802 static void armv7_pmnc_dump_regs(void)
 803 {
 804         u32 val;
 805         unsigned int cnt;
 806
 807         printk(KERN_INFO "PMNC registers dump:\n");
 808
 809         asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r" (val));
 810         printk(KERN_INFO "PMNC  =0x%08x\n", val);
 811
 812         asm volatile("mrc p15, 0, %0, c9, c12, 1" : "=r" (val));
 813         printk(KERN_INFO "CNTENS=0x%08x\n", val);
 814
 815         asm volatile("mrc p15, 0, %0, c9, c14, 1" : "=r" (val));
 816         printk(KERN_INFO "INTENS=0x%08x\n", val);
 817
 818         asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val));
 819         printk(KERN_INFO "FLAGS =0x%08x\n", val);
 820
 821         asm volatile("mrc p15, 0, %0, c9, c12, 5" : "=r" (val));
 822         printk(KERN_INFO "SELECT=0x%08x\n", val);
 823
 824         asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (val));
 825         printk(KERN_INFO "CCNT  =0x%08x\n", val);
 826
 827         for (cnt = ARMV7_IDX_COUNTER0; cnt <= ARMV7_IDX_COUNTER_LAST; cnt++) {
 828                 armv7_pmnc_select_counter(cnt);
 829                 asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (val));
 830                 printk(KERN_INFO "CNT[%d] count =0x%08x\n",
 831                         ARMV7_IDX_TO_COUNTER(cnt), val);
 832                 asm volatile("mrc p15, 0, %0, c9, c13, 1" : "=r" (val));
 833                 printk(KERN_INFO "CNT[%d] evtsel=0x%08x\n",
 834                         ARMV7_IDX_TO_COUNTER(cnt), val);
 835         }
 836 }
 837 #endif
 838
 839 static void armv7pmu_enable_event(struct hw_perf_event *hwc, int idx)
 840 {
 841         unsigned long flags;
 842         struct pmu_hw_events *events = cpu_pmu->get_hw_events();
 843
 844         /*
 845          * Enable counter and interrupt, and set the counter to count
 846          * the event that we're interested in.
 847          */
 848         raw_spin_lock_irqsave(&events->pmu_lock, flags);
 849
 850         /*
 851          * Disable counter
 852          */
 853         armv7_pmnc_disable_counter(idx);
 854
 855         /*
 856          * Set event (if destined for PMNx counters)
 857          * We only need to set the event for the cycle counter if we
 858          * have the ability to perform event filtering.
 859          */
 860         if (armv7pmu.set_event_filter || idx != ARMV7_IDX_CYCLE_COUNTER)
 861                 armv7_pmnc_write_evtsel(idx, hwc->config_base);
 862
 863         /*
 864          * Enable interrupt for this counter
 865          */
 866         armv7_pmnc_enable_intens(idx);
 867
 868         /*
 869          * Enable counter
 870          */
 871         armv7_pmnc_enable_counter(idx);
 872
 873         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
 874 }
 875
 876 static void armv7pmu_disable_event(struct hw_perf_event *hwc, int idx)
 877 {
 878         unsigned long flags;
 879         struct pmu_hw_events *events = cpu_pmu->get_hw_events();
 880
 881         /*
 882          * Disable counter and interrupt
 883          */
 884         raw_spin_lock_irqsave(&events->pmu_lock, flags);
 885
 886         /*
 887          * Disable counter
 888          */
 889         armv7_pmnc_disable_counter(idx);
 890
 891         /*
 892          * Disable interrupt for this counter
 893          */
 894         armv7_pmnc_disable_intens(idx);
 895
 896         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
 897 }
 898
 899 static irqreturn_t armv7pmu_handle_irq(int irq_num, void *dev)
 900 {
 901         u32 pmnc;
 902         struct perf_sample_data data;
 903         struct pmu_hw_events *cpuc;
 904         struct pt_regs *regs;
 905         int idx;
 906
 907         /*
 908          * Get and reset the IRQ flags
 909          */
 910         pmnc = armv7_pmnc_getreset_flags();
 911
 912         /*
 913          * Did an overflow occur?
 914          */
 915         if (!armv7_pmnc_has_overflowed(pmnc))
 916                 return IRQ_NONE;
 917
 918         /*
 919          * Handle the counter(s) overflow(s)
 920          */
 921         regs = get_irq_regs();
 922
 923         perf_sample_data_init(&data, 0);
 924
 925         cpuc = &__get_cpu_var(cpu_hw_events);
 926         for (idx = 0; idx < cpu_pmu->num_events; ++idx) {
 927                 struct perf_event *event = cpuc->events[idx];
 928                 struct hw_perf_event *hwc;
 929
 930                 /*
 931                  * We have a single interrupt for all counters. Check that
 932                  * each counter has overflowed before we process it.
 933                  */
 934                 if (!armv7_pmnc_counter_has_overflowed(pmnc, idx))
 935                         continue;
 936
 937                 hwc = &event->hw;
 938                 armpmu_event_update(event, hwc, idx, 1);
 939                 data.period = event->hw.last_period;
 940                 if (!armpmu_event_set_period(event, hwc, idx))
 941                         continue;
 942
 943                 if (perf_event_overflow(event, &data, regs))
 944                         cpu_pmu->disable(hwc, idx);
 945         }
 946
 947         /*
 948          * Handle the pending perf events.
 949          *
 950          * Note: this call *must* be run with interrupts disabled. For
 951          * platforms that can have the PMU interrupts raised as an NMI, this
 952          * will not work.
 953          */
 954         irq_work_run();
 955
 956         return IRQ_HANDLED;
 957 }
 958
 959 static void armv7pmu_start(void)
 960 {
 961         unsigned long flags;
 962         struct pmu_hw_events *events = cpu_pmu->get_hw_events();
 963
 964         raw_spin_lock_irqsave(&events->pmu_lock, flags);
 965         /* Enable all counters */
 966         armv7_pmnc_write(armv7_pmnc_read() | ARMV7_PMNC_E);
 967         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
 968 }
 969
 970 static void armv7pmu_stop(void)
 971 {
 972         unsigned long flags;
 973         struct pmu_hw_events *events = cpu_pmu->get_hw_events();
 974
 975         raw_spin_lock_irqsave(&events->pmu_lock, flags);
 976         /* Disable all counters */
 977         armv7_pmnc_write(armv7_pmnc_read() & ~ARMV7_PMNC_E);
 978         raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
 979 }
 980
 981 static int armv7pmu_get_event_idx(struct pmu_hw_events *cpuc,
 982                                   struct hw_perf_event *event)
 983 {
 984         int idx;
 985         unsigned long evtype = event->config_base & ARMV7_EVTYPE_EVENT;
 986
 987         /* Always place a cycle counter into the cycle counter. */
 988         if (evtype == ARMV7_PERFCTR_CPU_CYCLES) {
 989                 if (test_and_set_bit(ARMV7_IDX_CYCLE_COUNTER, cpuc->used_mask))
 990                         return -EAGAIN;
 991
 992                 return ARMV7_IDX_CYCLE_COUNTER;
 993         }
 994
 995         /*
 996          * For anything other than a cycle counter, try and use
 997          * the events counters
 998          */
 999         for (idx = ARMV7_IDX_COUNTER0; idx < cpu_pmu->num_events; ++idx) {
1000                 if (!test_and_set_bit(idx, cpuc->used_mask))
1001                         return idx;
1002         }
1003
1004         /* The counters are all in use. */
1005         return -EAGAIN;
1006 }
1007
1008 /*
1009  * Add an event filter to a given event. This will only work for PMUv2 PMUs.
1010  */
1011 static int armv7pmu_set_event_filter(struct hw_perf_event *event,
1012                                      struct perf_event_attr *attr)
1013 {
1014         unsigned long config_base = 0;
1015
1016         if (attr->exclude_idle)
1017                 return -EPERM;
1018         if (attr->exclude_user)
1019                 config_base |= ARMV7_EXCLUDE_USER;
1020         if (attr->exclude_kernel)
1021                 config_base |= ARMV7_EXCLUDE_PL1;
1022         if (!attr->exclude_hv)
1023                 config_base |= ARMV7_INCLUDE_HYP;
1024
1025         /*
1026          * Install the filter into config_base as this is used to
1027          * construct the event type.
1028          */
1029         event->config_base = config_base;
1030
1031         return 0;
1032 }
1033
1034 static void armv7pmu_reset(void *info)
1035 {
1036         u32 idx, nb_cnt = cpu_pmu->num_events;
1037
1038         /* The counter and interrupt enable registers are unknown at reset. */
1039         for (idx = ARMV7_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx)
1040                 armv7pmu_disable_event(NULL, idx);
1041
1042         /* Initialize & Reset PMNC: C and P bits */
1043         armv7_pmnc_write(ARMV7_PMNC_P | ARMV7_PMNC_C);
1044 }
1045
1046 static int armv7_a8_map_event(struct perf_event *event)
1047 {
1048         return map_cpu_event(event, &armv7_a8_perf_map,
1049                                 &armv7_a8_perf_cache_map, 0xFF);
1050 }
1051
1052 static int armv7_a9_map_event(struct perf_event *event)
1053 {
1054         return map_cpu_event(event, &armv7_a9_perf_map,
1055                                 &armv7_a9_perf_cache_map, 0xFF);
1056 }
1057
1058 static int armv7_a5_map_event(struct perf_event *event)
1059 {
1060         return map_cpu_event(event, &armv7_a5_perf_map,
1061                                 &armv7_a5_perf_cache_map, 0xFF);
1062 }
1063
1064 static int armv7_a15_map_event(struct perf_event *event)
1065 {
1066         return map_cpu_event(event, &armv7_a15_perf_map,
1067                                 &armv7_a15_perf_cache_map, 0xFF);
1068 }
1069
1070 static struct arm_pmu armv7pmu = {
1071         .handle_irq             = armv7pmu_handle_irq,
1072         .enable                 = armv7pmu_enable_event,
1073         .disable                = armv7pmu_disable_event,
1074         .read_counter           = armv7pmu_read_counter,
1075         .write_counter          = armv7pmu_write_counter,
1076         .get_event_idx          = armv7pmu_get_event_idx,
1077         .start                  = armv7pmu_start,
1078         .stop                   = armv7pmu_stop,
1079         .reset                  = armv7pmu_reset,
1080         .max_period             = (1LLU << 32) - 1,
1081 };
1082
1083 static u32 __init armv7_read_num_pmnc_events(void)
1084 {
1085         u32 nb_cnt;
1086
1087         /* Read the nb of CNTx counters supported from PMNC */
1088         nb_cnt = (armv7_pmnc_read() >> ARMV7_PMNC_N_SHIFT) & ARMV7_PMNC_N_MASK;
1089
1090         /* Add the CPU cycles counter and return */
1091         return nb_cnt + 1;
1092 }
1093
1094 static struct arm_pmu *__init armv7_a8_pmu_init(void)
1095 {
1096         armv7pmu.id             = ARM_PERF_PMU_ID_CA8;
1097         armv7pmu.name           = "ARMv7 Cortex-A8";
1098         armv7pmu.map_event      = armv7_a8_map_event;
1099         armv7pmu.num_events     = armv7_read_num_pmnc_events();
1100         return &armv7pmu;
1101 }
1102
1103 static struct arm_pmu *__init armv7_a9_pmu_init(void)
1104 {
1105         armv7pmu.id             = ARM_PERF_PMU_ID_CA9;
1106         armv7pmu.name           = "ARMv7 Cortex-A9";
1107         armv7pmu.map_event      = armv7_a9_map_event;
1108         armv7pmu.num_events     = armv7_read_num_pmnc_events();
1109         return &armv7pmu;
1110 }
1111
1112 static struct arm_pmu *__init armv7_a5_pmu_init(void)
1113 {
1114         armv7pmu.id             = ARM_PERF_PMU_ID_CA5;
1115         armv7pmu.name           = "ARMv7 Cortex-A5";
1116         armv7pmu.map_event      = armv7_a5_map_event;
1117         armv7pmu.num_events     = armv7_read_num_pmnc_events();
1118         return &armv7pmu;
1119 }
1120
1121 static struct arm_pmu *__init armv7_a15_pmu_init(void)
1122 {
1123         armv7pmu.id             = ARM_PERF_PMU_ID_CA15;
1124         armv7pmu.name           = "ARMv7 Cortex-A15";
1125         armv7pmu.map_event      = armv7_a15_map_event;
1126         armv7pmu.num_events     = armv7_read_num_pmnc_events();
1127         armv7pmu.set_event_filter = armv7pmu_set_event_filter;
1128         return &armv7pmu;
1129 }
1130 #else
1131 static struct arm_pmu *__init armv7_a8_pmu_init(void)
1132 {
1133         return NULL;
1134 }
1135
1136 static struct arm_pmu *__init armv7_a9_pmu_init(void)
1137 {
1138         return NULL;
1139 }
1140
1141 static struct arm_pmu *__init armv7_a5_pmu_init(void)
1142 {
1143         return NULL;
1144 }
1145
1146 static struct arm_pmu *__init armv7_a15_pmu_init(void)
1147 {
1148         return NULL;
1149 }
1150 #endif  /* CONFIG_CPU_V7 */