arch/x86/kernel/cpu/perf_event.c

   1 /*
   2  * Performance events x86 architecture code
   3  *
   4  *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
   5  *  Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
   6  *  Copyright (C) 2009 Jaswinder Singh Rajput
   7  *  Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
   8  *  Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
   9  *  Copyright (C) 2009 Intel Corporation, <markus.t.metzger@intel.com>
  10  *
  11  *  For licencing details see kernel-base/COPYING
  12  */
  13
  14 #include <linux/perf_event.h>
  15 #include <linux/capability.h>
  16 #include <linux/notifier.h>
  17 #include <linux/hardirq.h>
  18 #include <linux/kprobes.h>
  19 #include <linux/module.h>
  20 #include <linux/kdebug.h>
  21 #include <linux/sched.h>
  22 #include <linux/uaccess.h>
  23 #include <linux/highmem.h>
  24 #include <linux/cpu.h>
  25
  26 #include <asm/apic.h>
  27 #include <asm/stacktrace.h>
  28 #include <asm/nmi.h>
  29
  30 static u64 perf_event_mask __read_mostly;
  31
  32 /* The maximal number of PEBS events: */
  33 #define MAX_PEBS_EVENTS 4
  34
  35 /* The size of a BTS record in bytes: */
  36 #define BTS_RECORD_SIZE         24
  37
  38 /* The size of a per-cpu BTS buffer in bytes: */
  39 #define BTS_BUFFER_SIZE         (BTS_RECORD_SIZE * 2048)
  40
  41 /* The BTS overflow threshold in bytes from the end of the buffer: */
  42 #define BTS_OVFL_TH             (BTS_RECORD_SIZE * 128)
  43
  44
  45 /*
  46  * Bits in the debugctlmsr controlling branch tracing.
  47  */
  48 #define X86_DEBUGCTL_TR                 (1 << 6)
  49 #define X86_DEBUGCTL_BTS                (1 << 7)
  50 #define X86_DEBUGCTL_BTINT              (1 << 8)
  51 #define X86_DEBUGCTL_BTS_OFF_OS         (1 << 9)
  52 #define X86_DEBUGCTL_BTS_OFF_USR        (1 << 10)
  53
  54 /*
  55  * A debug store configuration.
  56  *
  57  * We only support architectures that use 64bit fields.
  58  */
  59 struct debug_store {
  60         u64     bts_buffer_base;
  61         u64     bts_index;
  62         u64     bts_absolute_maximum;
  63         u64     bts_interrupt_threshold;
  64         u64     pebs_buffer_base;
  65         u64     pebs_index;
  66         u64     pebs_absolute_maximum;
  67         u64     pebs_interrupt_threshold;
  68         u64     pebs_event_reset[MAX_PEBS_EVENTS];
  69 };
  70
  71 struct cpu_hw_events {
  72         struct perf_event       *events[X86_PMC_IDX_MAX];
  73         unsigned long           used_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
  74         unsigned long           active_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
  75         unsigned long           interrupts;
  76         int                     enabled;
  77         struct debug_store      *ds;
  78 };
  79
  80 struct event_constraint {
  81         unsigned long   idxmsk[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
  82         int             code;
  83 };
  84
  85 #define EVENT_CONSTRAINT(c, m) { .code = (c), .idxmsk[0] = (m) }
  86 #define EVENT_CONSTRAINT_END  { .code = 0, .idxmsk[0] = 0 }
  87
  88 #define for_each_event_constraint(e, c) \
  89         for ((e) = (c); (e)->idxmsk[0]; (e)++)
  90
  91
  92 /*
  93  * struct x86_pmu - generic x86 pmu
  94  */
  95 struct x86_pmu {
  96         const char      *name;
  97         int             version;
  98         int             (*handle_irq)(struct pt_regs *);
  99         void            (*disable_all)(void);
 100         void            (*enable_all)(void);
 101         void            (*enable)(struct hw_perf_event *, int);
 102         void            (*disable)(struct hw_perf_event *, int);
 103         unsigned        eventsel;
 104         unsigned        perfctr;
 105         u64             (*event_map)(int);
 106         u64             (*raw_event)(u64);
 107         int             max_events;
 108         int             num_events;
 109         int             num_events_fixed;
 110         int             event_bits;
 111         u64             event_mask;
 112         int             apic;
 113         u64             max_period;
 114         u64             intel_ctrl;
 115         void            (*enable_bts)(u64 config);
 116         void            (*disable_bts)(void);
 117         int             (*get_event_idx)(struct cpu_hw_events *cpuc,
 118                                          struct hw_perf_event *hwc);
 119 };
 120
 121 static struct x86_pmu x86_pmu __read_mostly;
 122
 123 static DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = {
 124         .enabled = 1,
 125 };
 126
 127 static const struct event_constraint *event_constraints;
 128
 129 /*
 130  * Not sure about some of these
 131  */
 132 static const u64 p6_perfmon_event_map[] =
 133 {
 134   [PERF_COUNT_HW_CPU_CYCLES]            = 0x0079,
 135   [PERF_COUNT_HW_INSTRUCTIONS]          = 0x00c0,
 136   [PERF_COUNT_HW_CACHE_REFERENCES]      = 0x0f2e,
 137   [PERF_COUNT_HW_CACHE_MISSES]          = 0x012e,
 138   [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]   = 0x00c4,
 139   [PERF_COUNT_HW_BRANCH_MISSES]         = 0x00c5,
 140   [PERF_COUNT_HW_BUS_CYCLES]            = 0x0062,
 141 };
 142
 143 static u64 p6_pmu_event_map(int hw_event)
 144 {
 145         return p6_perfmon_event_map[hw_event];
 146 }
 147
 148 /*
 149  * Event setting that is specified not to count anything.
 150  * We use this to effectively disable a counter.
 151  *
 152  * L2_RQSTS with 0 MESI unit mask.
 153  */
 154 #define P6_NOP_EVENT                    0x0000002EULL
 155
 156 static u64 p6_pmu_raw_event(u64 hw_event)
 157 {
 158 #define P6_EVNTSEL_EVENT_MASK           0x000000FFULL
 159 #define P6_EVNTSEL_UNIT_MASK            0x0000FF00ULL
 160 #define P6_EVNTSEL_EDGE_MASK            0x00040000ULL
 161 #define P6_EVNTSEL_INV_MASK             0x00800000ULL
 162 #define P6_EVNTSEL_REG_MASK             0xFF000000ULL
 163
 164 #define P6_EVNTSEL_MASK                 \
 165         (P6_EVNTSEL_EVENT_MASK |        \
 166          P6_EVNTSEL_UNIT_MASK  |        \
 167          P6_EVNTSEL_EDGE_MASK  |        \
 168          P6_EVNTSEL_INV_MASK   |        \
 169          P6_EVNTSEL_REG_MASK)
 170
 171         return hw_event & P6_EVNTSEL_MASK;
 172 }
 173
 174 static const struct event_constraint intel_p6_event_constraints[] =
 175 {
 176         EVENT_CONSTRAINT(0xc1, 0x1),    /* FLOPS */
 177         EVENT_CONSTRAINT(0x10, 0x1),    /* FP_COMP_OPS_EXE */
 178         EVENT_CONSTRAINT(0x11, 0x1),    /* FP_ASSIST */
 179         EVENT_CONSTRAINT(0x12, 0x2),    /* MUL */
 180         EVENT_CONSTRAINT(0x13, 0x2),    /* DIV */
 181         EVENT_CONSTRAINT(0x14, 0x1),    /* CYCLES_DIV_BUSY */
 182         EVENT_CONSTRAINT_END
 183 };
 184
 185 /*
 186  * Intel PerfMon v3. Used on Core2 and later.
 187  */
 188 static const u64 intel_perfmon_event_map[] =
 189 {
 190   [PERF_COUNT_HW_CPU_CYCLES]            = 0x003c,
 191   [PERF_COUNT_HW_INSTRUCTIONS]          = 0x00c0,
 192   [PERF_COUNT_HW_CACHE_REFERENCES]      = 0x4f2e,
 193   [PERF_COUNT_HW_CACHE_MISSES]          = 0x412e,
 194   [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]   = 0x00c4,
 195   [PERF_COUNT_HW_BRANCH_MISSES]         = 0x00c5,
 196   [PERF_COUNT_HW_BUS_CYCLES]            = 0x013c,
 197 };
 198
 199 static const struct event_constraint intel_core_event_constraints[] =
 200 {
 201         EVENT_CONSTRAINT(0x10, 0x1),    /* FP_COMP_OPS_EXE */
 202         EVENT_CONSTRAINT(0x11, 0x2),    /* FP_ASSIST */
 203         EVENT_CONSTRAINT(0x12, 0x2),    /* MUL */
 204         EVENT_CONSTRAINT(0x13, 0x2),    /* DIV */
 205         EVENT_CONSTRAINT(0x14, 0x1),    /* CYCLES_DIV_BUSY */
 206         EVENT_CONSTRAINT(0x18, 0x1),    /* IDLE_DURING_DIV */
 207         EVENT_CONSTRAINT(0x19, 0x2),    /* DELAYED_BYPASS */
 208         EVENT_CONSTRAINT(0xa1, 0x1),    /* RS_UOPS_DISPATCH_CYCLES */
 209         EVENT_CONSTRAINT(0xcb, 0x1),    /* MEM_LOAD_RETIRED */
 210         EVENT_CONSTRAINT_END
 211 };
 212
 213 static const struct event_constraint intel_nehalem_event_constraints[] =
 214 {
 215         EVENT_CONSTRAINT(0x40, 0x3),    /* L1D_CACHE_LD */
 216         EVENT_CONSTRAINT(0x41, 0x3),    /* L1D_CACHE_ST */
 217         EVENT_CONSTRAINT(0x42, 0x3),    /* L1D_CACHE_LOCK */
 218         EVENT_CONSTRAINT(0x43, 0x3),    /* L1D_ALL_REF */
 219         EVENT_CONSTRAINT(0x4e, 0x3),    /* L1D_PREFETCH */
 220         EVENT_CONSTRAINT(0x4c, 0x3),    /* LOAD_HIT_PRE */
 221         EVENT_CONSTRAINT(0x51, 0x3),    /* L1D */
 222         EVENT_CONSTRAINT(0x52, 0x3),    /* L1D_CACHE_PREFETCH_LOCK_FB_HIT */
 223         EVENT_CONSTRAINT(0x53, 0x3),    /* L1D_CACHE_LOCK_FB_HIT */
 224         EVENT_CONSTRAINT(0xc5, 0x3),    /* CACHE_LOCK_CYCLES */
 225         EVENT_CONSTRAINT_END
 226 };
 227
 228 static u64 intel_pmu_event_map(int hw_event)
 229 {
 230         return intel_perfmon_event_map[hw_event];
 231 }
 232
 233 /*
 234  * Generalized hw caching related hw_event table, filled
 235  * in on a per model basis. A value of 0 means
 236  * 'not supported', -1 means 'hw_event makes no sense on
 237  * this CPU', any other value means the raw hw_event
 238  * ID.
 239  */
 240
 241 #define C(x) PERF_COUNT_HW_CACHE_##x
 242
 243 static u64 __read_mostly hw_cache_event_ids
 244                                 [PERF_COUNT_HW_CACHE_MAX]
 245                                 [PERF_COUNT_HW_CACHE_OP_MAX]
 246                                 [PERF_COUNT_HW_CACHE_RESULT_MAX];
 247
 248 static __initconst u64 nehalem_hw_cache_event_ids
 249                                 [PERF_COUNT_HW_CACHE_MAX]
 250                                 [PERF_COUNT_HW_CACHE_OP_MAX]
 251                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
 252 {
 253  [ C(L1D) ] = {
 254         [ C(OP_READ) ] = {
 255                 [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI            */
 256                 [ C(RESULT_MISS)   ] = 0x0140, /* L1D_CACHE_LD.I_STATE         */
 257         },
 258         [ C(OP_WRITE) ] = {
 259                 [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI            */
 260                 [ C(RESULT_MISS)   ] = 0x0141, /* L1D_CACHE_ST.I_STATE         */
 261         },
 262         [ C(OP_PREFETCH) ] = {
 263                 [ C(RESULT_ACCESS) ] = 0x014e, /* L1D_PREFETCH.REQUESTS        */
 264                 [ C(RESULT_MISS)   ] = 0x024e, /* L1D_PREFETCH.MISS            */
 265         },
 266  },
 267  [ C(L1I ) ] = {
 268         [ C(OP_READ) ] = {
 269                 [ C(RESULT_ACCESS) ] = 0x0380, /* L1I.READS                    */
 270                 [ C(RESULT_MISS)   ] = 0x0280, /* L1I.MISSES                   */
 271         },
 272         [ C(OP_WRITE) ] = {
 273                 [ C(RESULT_ACCESS) ] = -1,
 274                 [ C(RESULT_MISS)   ] = -1,
 275         },
 276         [ C(OP_PREFETCH) ] = {
 277                 [ C(RESULT_ACCESS) ] = 0x0,
 278                 [ C(RESULT_MISS)   ] = 0x0,
 279         },
 280  },
 281  [ C(LL  ) ] = {
 282         [ C(OP_READ) ] = {
 283                 [ C(RESULT_ACCESS) ] = 0x0324, /* L2_RQSTS.LOADS               */
 284                 [ C(RESULT_MISS)   ] = 0x0224, /* L2_RQSTS.LD_MISS             */
 285         },
 286         [ C(OP_WRITE) ] = {
 287                 [ C(RESULT_ACCESS) ] = 0x0c24, /* L2_RQSTS.RFOS                */
 288                 [ C(RESULT_MISS)   ] = 0x0824, /* L2_RQSTS.RFO_MISS            */
 289         },
 290         [ C(OP_PREFETCH) ] = {
 291                 [ C(RESULT_ACCESS) ] = 0x4f2e, /* LLC Reference                */
 292                 [ C(RESULT_MISS)   ] = 0x412e, /* LLC Misses                   */
 293         },
 294  },
 295  [ C(DTLB) ] = {
 296         [ C(OP_READ) ] = {
 297                 [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI   (alias)  */
 298                 [ C(RESULT_MISS)   ] = 0x0108, /* DTLB_LOAD_MISSES.ANY         */
 299         },
 300         [ C(OP_WRITE) ] = {
 301                 [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI   (alias)  */
 302                 [ C(RESULT_MISS)   ] = 0x010c, /* MEM_STORE_RETIRED.DTLB_MISS  */
 303         },
 304         [ C(OP_PREFETCH) ] = {
 305                 [ C(RESULT_ACCESS) ] = 0x0,
 306                 [ C(RESULT_MISS)   ] = 0x0,
 307         },
 308  },
 309  [ C(ITLB) ] = {
 310         [ C(OP_READ) ] = {
 311                 [ C(RESULT_ACCESS) ] = 0x01c0, /* INST_RETIRED.ANY_P           */
 312                 [ C(RESULT_MISS)   ] = 0x20c8, /* ITLB_MISS_RETIRED            */
 313         },
 314         [ C(OP_WRITE) ] = {
 315                 [ C(RESULT_ACCESS) ] = -1,
 316                 [ C(RESULT_MISS)   ] = -1,
 317         },
 318         [ C(OP_PREFETCH) ] = {
 319                 [ C(RESULT_ACCESS) ] = -1,
 320                 [ C(RESULT_MISS)   ] = -1,
 321         },
 322  },
 323  [ C(BPU ) ] = {
 324         [ C(OP_READ) ] = {
 325                 [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ALL_BRANCHES */
 326                 [ C(RESULT_MISS)   ] = 0x03e8, /* BPU_CLEARS.ANY               */
 327         },
 328         [ C(OP_WRITE) ] = {
 329                 [ C(RESULT_ACCESS) ] = -1,
 330                 [ C(RESULT_MISS)   ] = -1,
 331         },
 332         [ C(OP_PREFETCH) ] = {
 333                 [ C(RESULT_ACCESS) ] = -1,
 334                 [ C(RESULT_MISS)   ] = -1,
 335         },
 336  },
 337 };
 338
 339 static __initconst u64 core2_hw_cache_event_ids
 340                                 [PERF_COUNT_HW_CACHE_MAX]
 341                                 [PERF_COUNT_HW_CACHE_OP_MAX]
 342                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
 343 {
 344  [ C(L1D) ] = {
 345         [ C(OP_READ) ] = {
 346                 [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI          */
 347                 [ C(RESULT_MISS)   ] = 0x0140, /* L1D_CACHE_LD.I_STATE       */
 348         },
 349         [ C(OP_WRITE) ] = {
 350                 [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI          */
 351                 [ C(RESULT_MISS)   ] = 0x0141, /* L1D_CACHE_ST.I_STATE       */
 352         },
 353         [ C(OP_PREFETCH) ] = {
 354                 [ C(RESULT_ACCESS) ] = 0x104e, /* L1D_PREFETCH.REQUESTS      */
 355                 [ C(RESULT_MISS)   ] = 0,
 356         },
 357  },
 358  [ C(L1I ) ] = {
 359         [ C(OP_READ) ] = {
 360                 [ C(RESULT_ACCESS) ] = 0x0080, /* L1I.READS                  */
 361                 [ C(RESULT_MISS)   ] = 0x0081, /* L1I.MISSES                 */
 362         },
 363         [ C(OP_WRITE) ] = {
 364                 [ C(RESULT_ACCESS) ] = -1,
 365                 [ C(RESULT_MISS)   ] = -1,
 366         },
 367         [ C(OP_PREFETCH) ] = {
 368                 [ C(RESULT_ACCESS) ] = 0,
 369                 [ C(RESULT_MISS)   ] = 0,
 370         },
 371  },
 372  [ C(LL  ) ] = {
 373         [ C(OP_READ) ] = {
 374                 [ C(RESULT_ACCESS) ] = 0x4f29, /* L2_LD.MESI                 */
 375                 [ C(RESULT_MISS)   ] = 0x4129, /* L2_LD.ISTATE               */
 376         },
 377         [ C(OP_WRITE) ] = {
 378                 [ C(RESULT_ACCESS) ] = 0x4f2A, /* L2_ST.MESI                 */
 379                 [ C(RESULT_MISS)   ] = 0x412A, /* L2_ST.ISTATE               */
 380         },
 381         [ C(OP_PREFETCH) ] = {
 382                 [ C(RESULT_ACCESS) ] = 0,
 383                 [ C(RESULT_MISS)   ] = 0,
 384         },
 385  },
 386  [ C(DTLB) ] = {
 387         [ C(OP_READ) ] = {
 388                 [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI  (alias) */
 389                 [ C(RESULT_MISS)   ] = 0x0208, /* DTLB_MISSES.MISS_LD        */
 390         },
 391         [ C(OP_WRITE) ] = {
 392                 [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI  (alias) */
 393                 [ C(RESULT_MISS)   ] = 0x0808, /* DTLB_MISSES.MISS_ST        */
 394         },
 395         [ C(OP_PREFETCH) ] = {
 396                 [ C(RESULT_ACCESS) ] = 0,
 397                 [ C(RESULT_MISS)   ] = 0,
 398         },
 399  },
 400  [ C(ITLB) ] = {
 401         [ C(OP_READ) ] = {
 402                 [ C(RESULT_ACCESS) ] = 0x00c0, /* INST_RETIRED.ANY_P         */
 403                 [ C(RESULT_MISS)   ] = 0x1282, /* ITLBMISSES                 */
 404         },
 405         [ C(OP_WRITE) ] = {
 406                 [ C(RESULT_ACCESS) ] = -1,
 407                 [ C(RESULT_MISS)   ] = -1,
 408         },
 409         [ C(OP_PREFETCH) ] = {
 410                 [ C(RESULT_ACCESS) ] = -1,
 411                 [ C(RESULT_MISS)   ] = -1,
 412         },
 413  },
 414  [ C(BPU ) ] = {
 415         [ C(OP_READ) ] = {
 416                 [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ANY        */
 417                 [ C(RESULT_MISS)   ] = 0x00c5, /* BP_INST_RETIRED.MISPRED    */
 418         },
 419         [ C(OP_WRITE) ] = {
 420                 [ C(RESULT_ACCESS) ] = -1,
 421                 [ C(RESULT_MISS)   ] = -1,
 422         },
 423         [ C(OP_PREFETCH) ] = {
 424                 [ C(RESULT_ACCESS) ] = -1,
 425                 [ C(RESULT_MISS)   ] = -1,
 426         },
 427  },
 428 };
 429
 430 static __initconst u64 atom_hw_cache_event_ids
 431                                 [PERF_COUNT_HW_CACHE_MAX]
 432                                 [PERF_COUNT_HW_CACHE_OP_MAX]
 433                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
 434 {
 435  [ C(L1D) ] = {
 436         [ C(OP_READ) ] = {
 437                 [ C(RESULT_ACCESS) ] = 0x2140, /* L1D_CACHE.LD               */
 438                 [ C(RESULT_MISS)   ] = 0,
 439         },
 440         [ C(OP_WRITE) ] = {
 441                 [ C(RESULT_ACCESS) ] = 0x2240, /* L1D_CACHE.ST               */
 442                 [ C(RESULT_MISS)   ] = 0,
 443         },
 444         [ C(OP_PREFETCH) ] = {
 445                 [ C(RESULT_ACCESS) ] = 0x0,
 446                 [ C(RESULT_MISS)   ] = 0,
 447         },
 448  },
 449  [ C(L1I ) ] = {
 450         [ C(OP_READ) ] = {
 451                 [ C(RESULT_ACCESS) ] = 0x0380, /* L1I.READS                  */
 452                 [ C(RESULT_MISS)   ] = 0x0280, /* L1I.MISSES                 */
 453         },
 454         [ C(OP_WRITE) ] = {
 455                 [ C(RESULT_ACCESS) ] = -1,
 456                 [ C(RESULT_MISS)   ] = -1,
 457         },
 458         [ C(OP_PREFETCH) ] = {
 459                 [ C(RESULT_ACCESS) ] = 0,
 460                 [ C(RESULT_MISS)   ] = 0,
 461         },
 462  },
 463  [ C(LL  ) ] = {
 464         [ C(OP_READ) ] = {
 465                 [ C(RESULT_ACCESS) ] = 0x4f29, /* L2_LD.MESI                 */
 466                 [ C(RESULT_MISS)   ] = 0x4129, /* L2_LD.ISTATE               */
 467         },
 468         [ C(OP_WRITE) ] = {
 469                 [ C(RESULT_ACCESS) ] = 0x4f2A, /* L2_ST.MESI                 */
 470                 [ C(RESULT_MISS)   ] = 0x412A, /* L2_ST.ISTATE               */
 471         },
 472         [ C(OP_PREFETCH) ] = {
 473                 [ C(RESULT_ACCESS) ] = 0,
 474                 [ C(RESULT_MISS)   ] = 0,
 475         },
 476  },
 477  [ C(DTLB) ] = {
 478         [ C(OP_READ) ] = {
 479                 [ C(RESULT_ACCESS) ] = 0x2140, /* L1D_CACHE_LD.MESI  (alias) */
 480                 [ C(RESULT_MISS)   ] = 0x0508, /* DTLB_MISSES.MISS_LD        */
 481         },
 482         [ C(OP_WRITE) ] = {
 483                 [ C(RESULT_ACCESS) ] = 0x2240, /* L1D_CACHE_ST.MESI  (alias) */
 484                 [ C(RESULT_MISS)   ] = 0x0608, /* DTLB_MISSES.MISS_ST        */
 485         },
 486         [ C(OP_PREFETCH) ] = {
 487                 [ C(RESULT_ACCESS) ] = 0,
 488                 [ C(RESULT_MISS)   ] = 0,
 489         },
 490  },
 491  [ C(ITLB) ] = {
 492         [ C(OP_READ) ] = {
 493                 [ C(RESULT_ACCESS) ] = 0x00c0, /* INST_RETIRED.ANY_P         */
 494                 [ C(RESULT_MISS)   ] = 0x0282, /* ITLB.MISSES                */
 495         },
 496         [ C(OP_WRITE) ] = {
 497                 [ C(RESULT_ACCESS) ] = -1,
 498                 [ C(RESULT_MISS)   ] = -1,
 499         },
 500         [ C(OP_PREFETCH) ] = {
 501                 [ C(RESULT_ACCESS) ] = -1,
 502                 [ C(RESULT_MISS)   ] = -1,
 503         },
 504  },
 505  [ C(BPU ) ] = {
 506         [ C(OP_READ) ] = {
 507                 [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ANY        */
 508                 [ C(RESULT_MISS)   ] = 0x00c5, /* BP_INST_RETIRED.MISPRED    */
 509         },
 510         [ C(OP_WRITE) ] = {
 511                 [ C(RESULT_ACCESS) ] = -1,
 512                 [ C(RESULT_MISS)   ] = -1,
 513         },
 514         [ C(OP_PREFETCH) ] = {
 515                 [ C(RESULT_ACCESS) ] = -1,
 516                 [ C(RESULT_MISS)   ] = -1,
 517         },
 518  },
 519 };
 520
 521 static u64 intel_pmu_raw_event(u64 hw_event)
 522 {
 523 #define CORE_EVNTSEL_EVENT_MASK         0x000000FFULL
 524 #define CORE_EVNTSEL_UNIT_MASK          0x0000FF00ULL
 525 #define CORE_EVNTSEL_EDGE_MASK          0x00040000ULL
 526 #define CORE_EVNTSEL_INV_MASK           0x00800000ULL
 527 #define CORE_EVNTSEL_REG_MASK           0xFF000000ULL
 528
 529 #define CORE_EVNTSEL_MASK               \
 530         (CORE_EVNTSEL_EVENT_MASK |      \
 531          CORE_EVNTSEL_UNIT_MASK  |      \
 532          CORE_EVNTSEL_EDGE_MASK  |      \
 533          CORE_EVNTSEL_INV_MASK  |       \
 534          CORE_EVNTSEL_REG_MASK)
 535
 536         return hw_event & CORE_EVNTSEL_MASK;
 537 }
 538
 539 static __initconst u64 amd_hw_cache_event_ids
 540                                 [PERF_COUNT_HW_CACHE_MAX]
 541                                 [PERF_COUNT_HW_CACHE_OP_MAX]
 542                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
 543 {
 544  [ C(L1D) ] = {
 545         [ C(OP_READ) ] = {
 546                 [ C(RESULT_ACCESS) ] = 0x0040, /* Data Cache Accesses        */
 547                 [ C(RESULT_MISS)   ] = 0x0041, /* Data Cache Misses          */
 548         },
 549         [ C(OP_WRITE) ] = {
 550                 [ C(RESULT_ACCESS) ] = 0x0142, /* Data Cache Refills :system */
 551                 [ C(RESULT_MISS)   ] = 0,
 552         },
 553         [ C(OP_PREFETCH) ] = {
 554                 [ C(RESULT_ACCESS) ] = 0x0267, /* Data Prefetcher :attempts  */
 555                 [ C(RESULT_MISS)   ] = 0x0167, /* Data Prefetcher :cancelled */
 556         },
 557  },
 558  [ C(L1I ) ] = {
 559         [ C(OP_READ) ] = {
 560                 [ C(RESULT_ACCESS) ] = 0x0080, /* Instruction cache fetches  */
 561                 [ C(RESULT_MISS)   ] = 0x0081, /* Instruction cache misses   */
 562         },
 563         [ C(OP_WRITE) ] = {
 564                 [ C(RESULT_ACCESS) ] = -1,
 565                 [ C(RESULT_MISS)   ] = -1,
 566         },
 567         [ C(OP_PREFETCH) ] = {
 568                 [ C(RESULT_ACCESS) ] = 0x014B, /* Prefetch Instructions :Load */
 569                 [ C(RESULT_MISS)   ] = 0,
 570         },
 571  },
 572  [ C(LL  ) ] = {
 573         [ C(OP_READ) ] = {
 574                 [ C(RESULT_ACCESS) ] = 0x037D, /* Requests to L2 Cache :IC+DC */
 575                 [ C(RESULT_MISS)   ] = 0x037E, /* L2 Cache Misses : IC+DC     */
 576         },
 577         [ C(OP_WRITE) ] = {
 578                 [ C(RESULT_ACCESS) ] = 0x017F, /* L2 Fill/Writeback           */
 579                 [ C(RESULT_MISS)   ] = 0,
 580         },
 581         [ C(OP_PREFETCH) ] = {
 582                 [ C(RESULT_ACCESS) ] = 0,
 583                 [ C(RESULT_MISS)   ] = 0,
 584         },
 585  },
 586  [ C(DTLB) ] = {
 587         [ C(OP_READ) ] = {
 588                 [ C(RESULT_ACCESS) ] = 0x0040, /* Data Cache Accesses        */
 589                 [ C(RESULT_MISS)   ] = 0x0046, /* L1 DTLB and L2 DLTB Miss   */
 590         },
 591         [ C(OP_WRITE) ] = {
 592                 [ C(RESULT_ACCESS) ] = 0,
 593                 [ C(RESULT_MISS)   ] = 0,
 594         },
 595         [ C(OP_PREFETCH) ] = {
 596                 [ C(RESULT_ACCESS) ] = 0,
 597                 [ C(RESULT_MISS)   ] = 0,
 598         },
 599  },
 600  [ C(ITLB) ] = {
 601         [ C(OP_READ) ] = {
 602                 [ C(RESULT_ACCESS) ] = 0x0080, /* Instruction fecthes        */
 603                 [ C(RESULT_MISS)   ] = 0x0085, /* Instr. fetch ITLB misses   */
 604         },
 605         [ C(OP_WRITE) ] = {
 606                 [ C(RESULT_ACCESS) ] = -1,
 607                 [ C(RESULT_MISS)   ] = -1,
 608         },
 609         [ C(OP_PREFETCH) ] = {
 610                 [ C(RESULT_ACCESS) ] = -1,
 611                 [ C(RESULT_MISS)   ] = -1,
 612         },
 613  },
 614  [ C(BPU ) ] = {
 615         [ C(OP_READ) ] = {
 616                 [ C(RESULT_ACCESS) ] = 0x00c2, /* Retired Branch Instr.      */
 617                 [ C(RESULT_MISS)   ] = 0x00c3, /* Retired Mispredicted BI    */
 618         },
 619         [ C(OP_WRITE) ] = {
 620                 [ C(RESULT_ACCESS) ] = -1,
 621                 [ C(RESULT_MISS)   ] = -1,
 622         },
 623         [ C(OP_PREFETCH) ] = {
 624                 [ C(RESULT_ACCESS) ] = -1,
 625                 [ C(RESULT_MISS)   ] = -1,
 626         },
 627  },
 628 };
 629
 630 /*
 631  * AMD Performance Monitor K7 and later.
 632  */
 633 static const u64 amd_perfmon_event_map[] =
 634 {
 635   [PERF_COUNT_HW_CPU_CYCLES]            = 0x0076,
 636   [PERF_COUNT_HW_INSTRUCTIONS]          = 0x00c0,
 637   [PERF_COUNT_HW_CACHE_REFERENCES]      = 0x0080,
 638   [PERF_COUNT_HW_CACHE_MISSES]          = 0x0081,
 639   [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]   = 0x00c4,
 640   [PERF_COUNT_HW_BRANCH_MISSES]         = 0x00c5,
 641 };
 642
 643 static u64 amd_pmu_event_map(int hw_event)
 644 {
 645         return amd_perfmon_event_map[hw_event];
 646 }
 647
 648 static u64 amd_pmu_raw_event(u64 hw_event)
 649 {
 650 #define K7_EVNTSEL_EVENT_MASK   0x7000000FFULL
 651 #define K7_EVNTSEL_UNIT_MASK    0x00000FF00ULL
 652 #define K7_EVNTSEL_EDGE_MASK    0x000040000ULL
 653 #define K7_EVNTSEL_INV_MASK     0x000800000ULL
 654 #define K7_EVNTSEL_REG_MASK     0x0FF000000ULL
 655
 656 #define K7_EVNTSEL_MASK                 \
 657         (K7_EVNTSEL_EVENT_MASK |        \
 658          K7_EVNTSEL_UNIT_MASK  |        \
 659          K7_EVNTSEL_EDGE_MASK  |        \
 660          K7_EVNTSEL_INV_MASK   |        \
 661          K7_EVNTSEL_REG_MASK)
 662
 663         return hw_event & K7_EVNTSEL_MASK;
 664 }
 665
 666 /*
 667  * Propagate event elapsed time into the generic event.
 668  * Can only be executed on the CPU where the event is active.
 669  * Returns the delta events processed.
 670  */
 671 static u64
 672 x86_perf_event_update(struct perf_event *event,
 673                         struct hw_perf_event *hwc, int idx)
 674 {
 675         int shift = 64 - x86_pmu.event_bits;
 676         u64 prev_raw_count, new_raw_count;
 677         s64 delta;
 678
 679         if (idx == X86_PMC_IDX_FIXED_BTS)
 680                 return 0;
 681
 682         /*
 683          * Careful: an NMI might modify the previous event value.
 684          *
 685          * Our tactic to handle this is to first atomically read and
 686          * exchange a new raw count - then add that new-prev delta
 687          * count to the generic event atomically:
 688          */
 689 again:
 690         prev_raw_count = atomic64_read(&hwc->prev_count);
 691         rdmsrl(hwc->event_base + idx, new_raw_count);
 692
 693         if (atomic64_cmpxchg(&hwc->prev_count, prev_raw_count,
 694                                         new_raw_count) != prev_raw_count)
 695                 goto again;
 696
 697         /*
 698          * Now we have the new raw value and have updated the prev
 699          * timestamp already. We can now calculate the elapsed delta
 700          * (event-)time and add that to the generic event.
 701          *
 702          * Careful, not all hw sign-extends above the physical width
 703          * of the count.
 704          */
 705         delta = (new_raw_count << shift) - (prev_raw_count << shift);
 706         delta >>= shift;
 707
 708         atomic64_add(delta, &event->count);
 709         atomic64_sub(delta, &hwc->period_left);
 710
 711         return new_raw_count;
 712 }
 713
 714 static atomic_t active_events;
 715 static DEFINE_MUTEX(pmc_reserve_mutex);
 716
 717 static bool reserve_pmc_hardware(void)
 718 {
 719 #ifdef CONFIG_X86_LOCAL_APIC
 720         int i;
 721
 722         if (nmi_watchdog == NMI_LOCAL_APIC)
 723                 disable_lapic_nmi_watchdog();
 724
 725         for (i = 0; i < x86_pmu.num_events; i++) {
 726                 if (!reserve_perfctr_nmi(x86_pmu.perfctr + i))
 727                         goto perfctr_fail;
 728         }
 729
 730         for (i = 0; i < x86_pmu.num_events; i++) {
 731                 if (!reserve_evntsel_nmi(x86_pmu.eventsel + i))
 732                         goto eventsel_fail;
 733         }
 734 #endif
 735
 736         return true;
 737
 738 #ifdef CONFIG_X86_LOCAL_APIC
 739 eventsel_fail:
 740         for (i--; i >= 0; i--)
 741                 release_evntsel_nmi(x86_pmu.eventsel + i);
 742
 743         i = x86_pmu.num_events;
 744
 745 perfctr_fail:
 746         for (i--; i >= 0; i--)
 747                 release_perfctr_nmi(x86_pmu.perfctr + i);
 748
 749         if (nmi_watchdog == NMI_LOCAL_APIC)
 750                 enable_lapic_nmi_watchdog();
 751
 752         return false;
 753 #endif
 754 }
 755
 756 static void release_pmc_hardware(void)
 757 {
 758 #ifdef CONFIG_X86_LOCAL_APIC
 759         int i;
 760
 761         for (i = 0; i < x86_pmu.num_events; i++) {
 762                 release_perfctr_nmi(x86_pmu.perfctr + i);
 763                 release_evntsel_nmi(x86_pmu.eventsel + i);
 764         }
 765
 766         if (nmi_watchdog == NMI_LOCAL_APIC)
 767                 enable_lapic_nmi_watchdog();
 768 #endif
 769 }
 770
 771 static inline bool bts_available(void)
 772 {
 773         return x86_pmu.enable_bts != NULL;
 774 }
 775
 776 static inline void init_debug_store_on_cpu(int cpu)
 777 {
 778         struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
 779
 780         if (!ds)
 781                 return;
 782
 783         wrmsr_on_cpu(cpu, MSR_IA32_DS_AREA,
 784                      (u32)((u64)(unsigned long)ds),
 785                      (u32)((u64)(unsigned long)ds >> 32));
 786 }
 787
 788 static inline void fini_debug_store_on_cpu(int cpu)
 789 {
 790         if (!per_cpu(cpu_hw_events, cpu).ds)
 791                 return;
 792
 793         wrmsr_on_cpu(cpu, MSR_IA32_DS_AREA, 0, 0);
 794 }
 795
 796 static void release_bts_hardware(void)
 797 {
 798         int cpu;
 799
 800         if (!bts_available())
 801                 return;
 802
 803         get_online_cpus();
 804
 805         for_each_online_cpu(cpu)
 806                 fini_debug_store_on_cpu(cpu);
 807
 808         for_each_possible_cpu(cpu) {
 809                 struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
 810
 811                 if (!ds)
 812                         continue;
 813
 814                 per_cpu(cpu_hw_events, cpu).ds = NULL;
 815
 816                 kfree((void *)(unsigned long)ds->bts_buffer_base);
 817                 kfree(ds);
 818         }
 819
 820         put_online_cpus();
 821 }
 822
 823 static int reserve_bts_hardware(void)
 824 {
 825         int cpu, err = 0;
 826
 827         if (!bts_available())
 828                 return 0;
 829
 830         get_online_cpus();
 831
 832         for_each_possible_cpu(cpu) {
 833                 struct debug_store *ds;
 834                 void *buffer;
 835
 836                 err = -ENOMEM;
 837                 buffer = kzalloc(BTS_BUFFER_SIZE, GFP_KERNEL);
 838                 if (unlikely(!buffer))
 839                         break;
 840
 841                 ds = kzalloc(sizeof(*ds), GFP_KERNEL);
 842                 if (unlikely(!ds)) {
 843                         kfree(buffer);
 844                         break;
 845                 }
 846
 847                 ds->bts_buffer_base = (u64)(unsigned long)buffer;
 848                 ds->bts_index = ds->bts_buffer_base;
 849                 ds->bts_absolute_maximum =
 850                         ds->bts_buffer_base + BTS_BUFFER_SIZE;
 851                 ds->bts_interrupt_threshold =
 852                         ds->bts_absolute_maximum - BTS_OVFL_TH;
 853
 854                 per_cpu(cpu_hw_events, cpu).ds = ds;
 855                 err = 0;
 856         }
 857
 858         if (err)
 859                 release_bts_hardware();
 860         else {
 861                 for_each_online_cpu(cpu)
 862                         init_debug_store_on_cpu(cpu);
 863         }
 864
 865         put_online_cpus();
 866
 867         return err;
 868 }
 869
 870 static void hw_perf_event_destroy(struct perf_event *event)
 871 {
 872         if (atomic_dec_and_mutex_lock(&active_events, &pmc_reserve_mutex)) {
 873                 release_pmc_hardware();
 874                 release_bts_hardware();
 875                 mutex_unlock(&pmc_reserve_mutex);
 876         }
 877 }
 878
 879 static inline int x86_pmu_initialized(void)
 880 {
 881         return x86_pmu.handle_irq != NULL;
 882 }
 883
 884 static inline int
 885 set_ext_hw_attr(struct hw_perf_event *hwc, struct perf_event_attr *attr)
 886 {
 887         unsigned int cache_type, cache_op, cache_result;
 888         u64 config, val;
 889
 890         config = attr->config;
 891
 892         cache_type = (config >>  0) & 0xff;
 893         if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
 894                 return -EINVAL;
 895
 896         cache_op = (config >>  8) & 0xff;
 897         if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
 898                 return -EINVAL;
 899
 900         cache_result = (config >> 16) & 0xff;
 901         if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
 902                 return -EINVAL;
 903
 904         val = hw_cache_event_ids[cache_type][cache_op][cache_result];
 905
 906         if (val == 0)
 907                 return -ENOENT;
 908
 909         if (val == -1)
 910                 return -EINVAL;
 911
 912         hwc->config |= val;
 913
 914         return 0;
 915 }
 916
 917 static void intel_pmu_enable_bts(u64 config)
 918 {
 919         unsigned long debugctlmsr;
 920
 921         debugctlmsr = get_debugctlmsr();
 922
 923         debugctlmsr |= X86_DEBUGCTL_TR;
 924         debugctlmsr |= X86_DEBUGCTL_BTS;
 925         debugctlmsr |= X86_DEBUGCTL_BTINT;
 926
 927         if (!(config & ARCH_PERFMON_EVENTSEL_OS))
 928                 debugctlmsr |= X86_DEBUGCTL_BTS_OFF_OS;
 929
 930         if (!(config & ARCH_PERFMON_EVENTSEL_USR))
 931                 debugctlmsr |= X86_DEBUGCTL_BTS_OFF_USR;
 932
 933         update_debugctlmsr(debugctlmsr);
 934 }
 935
 936 static void intel_pmu_disable_bts(void)
 937 {
 938         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
 939         unsigned long debugctlmsr;
 940
 941         if (!cpuc->ds)
 942                 return;
 943
 944         debugctlmsr = get_debugctlmsr();
 945
 946         debugctlmsr &=
 947                 ~(X86_DEBUGCTL_TR | X86_DEBUGCTL_BTS | X86_DEBUGCTL_BTINT |
 948                   X86_DEBUGCTL_BTS_OFF_OS | X86_DEBUGCTL_BTS_OFF_USR);
 949
 950         update_debugctlmsr(debugctlmsr);
 951 }
 952
 953 /*
 954  * Setup the hardware configuration for a given attr_type
 955  */
 956 static int __hw_perf_event_init(struct perf_event *event)
 957 {
 958         struct perf_event_attr *attr = &event->attr;
 959         struct hw_perf_event *hwc = &event->hw;
 960         u64 config;
 961         int err;
 962
 963         if (!x86_pmu_initialized())
 964                 return -ENODEV;
 965
 966         err = 0;
 967         if (!atomic_inc_not_zero(&active_events)) {
 968                 mutex_lock(&pmc_reserve_mutex);
 969                 if (atomic_read(&active_events) == 0) {
 970                         if (!reserve_pmc_hardware())
 971                                 err = -EBUSY;
 972                         else
 973                                 err = reserve_bts_hardware();
 974                 }
 975                 if (!err)
 976                         atomic_inc(&active_events);
 977                 mutex_unlock(&pmc_reserve_mutex);
 978         }
 979         if (err)
 980                 return err;
 981
 982         event->destroy = hw_perf_event_destroy;
 983
 984         /*
 985          * Generate PMC IRQs:
 986          * (keep 'enabled' bit clear for now)
 987          */
 988         hwc->config = ARCH_PERFMON_EVENTSEL_INT;
 989
 990         hwc->idx = -1;
 991
 992         /*
 993          * Count user and OS events unless requested not to.
 994          */
 995         if (!attr->exclude_user)
 996                 hwc->config |= ARCH_PERFMON_EVENTSEL_USR;
 997         if (!attr->exclude_kernel)
 998                 hwc->config |= ARCH_PERFMON_EVENTSEL_OS;
 999
1000         if (!hwc->sample_period) {
1001                 hwc->sample_period = x86_pmu.max_period;
1002                 hwc->last_period = hwc->sample_period;
1003                 atomic64_set(&hwc->period_left, hwc->sample_period);
1004         } else {
1005                 /*
1006                  * If we have a PMU initialized but no APIC
1007                  * interrupts, we cannot sample hardware
1008                  * events (user-space has to fall back and
1009                  * sample via a hrtimer based software event):
1010                  */
1011                 if (!x86_pmu.apic)
1012                         return -EOPNOTSUPP;
1013         }
1014
1015         /*
1016          * Raw hw_event type provide the config in the hw_event structure
1017          */
1018         if (attr->type == PERF_TYPE_RAW) {
1019                 hwc->config |= x86_pmu.raw_event(attr->config);
1020                 return 0;
1021         }
1022
1023         if (attr->type == PERF_TYPE_HW_CACHE)
1024                 return set_ext_hw_attr(hwc, attr);
1025
1026         if (attr->config >= x86_pmu.max_events)
1027                 return -EINVAL;
1028
1029         /*
1030          * The generic map:
1031          */
1032         config = x86_pmu.event_map(attr->config);
1033
1034         if (config == 0)
1035                 return -ENOENT;
1036
1037         if (config == -1LL)
1038                 return -EINVAL;
1039
1040         /*
1041          * Branch tracing:
1042          */
1043         if ((attr->config == PERF_COUNT_HW_BRANCH_INSTRUCTIONS) &&
1044             (hwc->sample_period == 1)) {
1045                 /* BTS is not supported by this architecture. */
1046                 if (!bts_available())
1047                         return -EOPNOTSUPP;
1048
1049                 /* BTS is currently only allowed for user-mode. */
1050                 if (hwc->config & ARCH_PERFMON_EVENTSEL_OS)
1051                         return -EOPNOTSUPP;
1052         }
1053
1054         hwc->config |= config;
1055
1056         return 0;
1057 }
1058
1059 static void p6_pmu_disable_all(void)
1060 {
1061         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1062         u64 val;
1063
1064         if (!cpuc->enabled)
1065                 return;
1066
1067         cpuc->enabled = 0;
1068         barrier();
1069
1070         /* p6 only has one enable register */
1071         rdmsrl(MSR_P6_EVNTSEL0, val);
1072         val &= ~ARCH_PERFMON_EVENTSEL0_ENABLE;
1073         wrmsrl(MSR_P6_EVNTSEL0, val);
1074 }
1075
1076 static void intel_pmu_disable_all(void)
1077 {
1078         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1079
1080         if (!cpuc->enabled)
1081                 return;
1082
1083         cpuc->enabled = 0;
1084         barrier();
1085
1086         wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0);
1087
1088         if (test_bit(X86_PMC_IDX_FIXED_BTS, cpuc->active_mask))
1089                 intel_pmu_disable_bts();
1090 }
1091
1092 static void amd_pmu_disable_all(void)
1093 {
1094         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1095         int idx;
1096
1097         if (!cpuc->enabled)
1098                 return;
1099
1100         cpuc->enabled = 0;
1101         /*
1102          * ensure we write the disable before we start disabling the
1103          * events proper, so that amd_pmu_enable_event() does the
1104          * right thing.
1105          */
1106         barrier();
1107
1108         for (idx = 0; idx < x86_pmu.num_events; idx++) {
1109                 u64 val;
1110
1111                 if (!test_bit(idx, cpuc->active_mask))
1112                         continue;
1113                 rdmsrl(MSR_K7_EVNTSEL0 + idx, val);
1114                 if (!(val & ARCH_PERFMON_EVENTSEL0_ENABLE))
1115                         continue;
1116                 val &= ~ARCH_PERFMON_EVENTSEL0_ENABLE;
1117                 wrmsrl(MSR_K7_EVNTSEL0 + idx, val);
1118         }
1119 }
1120
1121 void hw_perf_disable(void)
1122 {
1123         if (!x86_pmu_initialized())
1124                 return;
1125         return x86_pmu.disable_all();
1126 }
1127
1128 static void p6_pmu_enable_all(void)
1129 {
1130         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1131         unsigned long val;
1132
1133         if (cpuc->enabled)
1134                 return;
1135
1136         cpuc->enabled = 1;
1137         barrier();
1138
1139         /* p6 only has one enable register */
1140         rdmsrl(MSR_P6_EVNTSEL0, val);
1141         val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
1142         wrmsrl(MSR_P6_EVNTSEL0, val);
1143 }
1144
1145 static void intel_pmu_enable_all(void)
1146 {
1147         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1148
1149         if (cpuc->enabled)
1150                 return;
1151
1152         cpuc->enabled = 1;
1153         barrier();
1154
1155         wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, x86_pmu.intel_ctrl);
1156
1157         if (test_bit(X86_PMC_IDX_FIXED_BTS, cpuc->active_mask)) {
1158                 struct perf_event *event =
1159                         cpuc->events[X86_PMC_IDX_FIXED_BTS];
1160
1161                 if (WARN_ON_ONCE(!event))
1162                         return;
1163
1164                 intel_pmu_enable_bts(event->hw.config);
1165         }
1166 }
1167
1168 static void amd_pmu_enable_all(void)
1169 {
1170         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1171         int idx;
1172
1173         if (cpuc->enabled)
1174                 return;
1175
1176         cpuc->enabled = 1;
1177         barrier();
1178
1179         for (idx = 0; idx < x86_pmu.num_events; idx++) {
1180                 struct perf_event *event = cpuc->events[idx];
1181                 u64 val;
1182
1183                 if (!test_bit(idx, cpuc->active_mask))
1184                         continue;
1185
1186                 val = event->hw.config;
1187                 val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
1188                 wrmsrl(MSR_K7_EVNTSEL0 + idx, val);
1189         }
1190 }
1191
1192 void hw_perf_enable(void)
1193 {
1194         if (!x86_pmu_initialized())
1195                 return;
1196         x86_pmu.enable_all();
1197 }
1198
1199 static inline u64 intel_pmu_get_status(void)
1200 {
1201         u64 status;
1202
1203         rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
1204
1205         return status;
1206 }
1207
1208 static inline void intel_pmu_ack_status(u64 ack)
1209 {
1210         wrmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, ack);
1211 }
1212
1213 static inline void x86_pmu_enable_event(struct hw_perf_event *hwc, int idx)
1214 {
1215         (void)checking_wrmsrl(hwc->config_base + idx,
1216                               hwc->config | ARCH_PERFMON_EVENTSEL0_ENABLE);
1217 }
1218
1219 static inline void x86_pmu_disable_event(struct hw_perf_event *hwc, int idx)
1220 {
1221         (void)checking_wrmsrl(hwc->config_base + idx, hwc->config);
1222 }
1223
1224 static inline void
1225 intel_pmu_disable_fixed(struct hw_perf_event *hwc, int __idx)
1226 {
1227         int idx = __idx - X86_PMC_IDX_FIXED;
1228         u64 ctrl_val, mask;
1229
1230         mask = 0xfULL << (idx * 4);
1231
1232         rdmsrl(hwc->config_base, ctrl_val);
1233         ctrl_val &= ~mask;
1234         (void)checking_wrmsrl(hwc->config_base, ctrl_val);
1235 }
1236
1237 static inline void
1238 p6_pmu_disable_event(struct hw_perf_event *hwc, int idx)
1239 {
1240         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1241         u64 val = P6_NOP_EVENT;
1242
1243         if (cpuc->enabled)
1244                 val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
1245
1246         (void)checking_wrmsrl(hwc->config_base + idx, val);
1247 }
1248
1249 static inline void
1250 intel_pmu_disable_event(struct hw_perf_event *hwc, int idx)
1251 {
1252         if (unlikely(idx == X86_PMC_IDX_FIXED_BTS)) {
1253                 intel_pmu_disable_bts();
1254                 return;
1255         }
1256
1257         if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
1258                 intel_pmu_disable_fixed(hwc, idx);
1259                 return;
1260         }
1261
1262         x86_pmu_disable_event(hwc, idx);
1263 }
1264
1265 static inline void
1266 amd_pmu_disable_event(struct hw_perf_event *hwc, int idx)
1267 {
1268         x86_pmu_disable_event(hwc, idx);
1269 }
1270
1271 static DEFINE_PER_CPU(u64 [X86_PMC_IDX_MAX], pmc_prev_left);
1272
1273 /*
1274  * Set the next IRQ period, based on the hwc->period_left value.
1275  * To be called with the event disabled in hw:
1276  */
1277 static int
1278 x86_perf_event_set_period(struct perf_event *event,
1279                              struct hw_perf_event *hwc, int idx)
1280 {
1281         s64 left = atomic64_read(&hwc->period_left);
1282         s64 period = hwc->sample_period;
1283         int err, ret = 0;
1284
1285         if (idx == X86_PMC_IDX_FIXED_BTS)
1286                 return 0;
1287
1288         /*
1289          * If we are way outside a reasoable range then just skip forward:
1290          */
1291         if (unlikely(left <= -period)) {
1292                 left = period;
1293                 atomic64_set(&hwc->period_left, left);
1294                 hwc->last_period = period;
1295                 ret = 1;
1296         }
1297
1298         if (unlikely(left <= 0)) {
1299                 left += period;
1300                 atomic64_set(&hwc->period_left, left);
1301                 hwc->last_period = period;
1302                 ret = 1;
1303         }
1304         /*
1305          * Quirk: certain CPUs dont like it if just 1 hw_event is left:
1306          */
1307         if (unlikely(left < 2))
1308                 left = 2;
1309
1310         if (left > x86_pmu.max_period)
1311                 left = x86_pmu.max_period;
1312
1313         per_cpu(pmc_prev_left[idx], smp_processor_id()) = left;
1314
1315         /*
1316          * The hw event starts counting from this event offset,
1317          * mark it to be able to extra future deltas:
1318          */
1319         atomic64_set(&hwc->prev_count, (u64)-left);
1320
1321         err = checking_wrmsrl(hwc->event_base + idx,
1322                              (u64)(-left) & x86_pmu.event_mask);
1323
1324         perf_event_update_userpage(event);
1325
1326         return ret;
1327 }
1328
1329 static inline void
1330 intel_pmu_enable_fixed(struct hw_perf_event *hwc, int __idx)
1331 {
1332         int idx = __idx - X86_PMC_IDX_FIXED;
1333         u64 ctrl_val, bits, mask;
1334         int err;
1335
1336         /*
1337          * Enable IRQ generation (0x8),
1338          * and enable ring-3 counting (0x2) and ring-0 counting (0x1)
1339          * if requested:
1340          */
1341         bits = 0x8ULL;
1342         if (hwc->config & ARCH_PERFMON_EVENTSEL_USR)
1343                 bits |= 0x2;
1344         if (hwc->config & ARCH_PERFMON_EVENTSEL_OS)
1345                 bits |= 0x1;
1346         bits <<= (idx * 4);
1347         mask = 0xfULL << (idx * 4);
1348
1349         rdmsrl(hwc->config_base, ctrl_val);
1350         ctrl_val &= ~mask;
1351         ctrl_val |= bits;
1352         err = checking_wrmsrl(hwc->config_base, ctrl_val);
1353 }
1354
1355 static void p6_pmu_enable_event(struct hw_perf_event *hwc, int idx)
1356 {
1357         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1358         u64 val;
1359
1360         val = hwc->config;
1361         if (cpuc->enabled)
1362                 val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
1363
1364         (void)checking_wrmsrl(hwc->config_base + idx, val);
1365 }
1366
1367
1368 static void intel_pmu_enable_event(struct hw_perf_event *hwc, int idx)
1369 {
1370         if (unlikely(idx == X86_PMC_IDX_FIXED_BTS)) {
1371                 if (!__get_cpu_var(cpu_hw_events).enabled)
1372                         return;
1373
1374                 intel_pmu_enable_bts(hwc->config);
1375                 return;
1376         }
1377
1378         if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
1379                 intel_pmu_enable_fixed(hwc, idx);
1380                 return;
1381         }
1382
1383         x86_pmu_enable_event(hwc, idx);
1384 }
1385
1386 static void amd_pmu_enable_event(struct hw_perf_event *hwc, int idx)
1387 {
1388         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1389
1390         if (cpuc->enabled)
1391                 x86_pmu_enable_event(hwc, idx);
1392 }
1393
1394 static int fixed_mode_idx(struct hw_perf_event *hwc)
1395 {
1396         unsigned int hw_event;
1397
1398         hw_event = hwc->config & ARCH_PERFMON_EVENT_MASK;
1399
1400         if (unlikely((hw_event ==
1401                       x86_pmu.event_map(PERF_COUNT_HW_BRANCH_INSTRUCTIONS)) &&
1402                      (hwc->sample_period == 1)))
1403                 return X86_PMC_IDX_FIXED_BTS;
1404
1405         if (!x86_pmu.num_events_fixed)
1406                 return -1;
1407
1408         /*
1409          * fixed counters do not take all possible filters
1410          */
1411         if (hwc->config & ARCH_PERFMON_EVENT_FILTER_MASK)
1412                 return -1;
1413
1414         if (unlikely(hw_event == x86_pmu.event_map(PERF_COUNT_HW_INSTRUCTIONS)))
1415                 return X86_PMC_IDX_FIXED_INSTRUCTIONS;
1416         if (unlikely(hw_event == x86_pmu.event_map(PERF_COUNT_HW_CPU_CYCLES)))
1417                 return X86_PMC_IDX_FIXED_CPU_CYCLES;
1418         if (unlikely(hw_event == x86_pmu.event_map(PERF_COUNT_HW_BUS_CYCLES)))
1419                 return X86_PMC_IDX_FIXED_BUS_CYCLES;
1420
1421         return -1;
1422 }
1423
1424 /*
1425  * generic counter allocator: get next free counter
1426  */
1427 static int
1428 gen_get_event_idx(struct cpu_hw_events *cpuc, struct hw_perf_event *hwc)
1429 {
1430         int idx;
1431
1432         idx = find_first_zero_bit(cpuc->used_mask, x86_pmu.num_events);
1433         return idx == x86_pmu.num_events ? -1 : idx;
1434 }
1435
1436 /*
1437  * intel-specific counter allocator: check event constraints
1438  */
1439 static int
1440 intel_get_event_idx(struct cpu_hw_events *cpuc, struct hw_perf_event *hwc)
1441 {
1442         const struct event_constraint *event_constraint;
1443         int i, code;
1444
1445         if (!event_constraints)
1446                 goto skip;
1447
1448         code = hwc->config & CORE_EVNTSEL_EVENT_MASK;
1449
1450         for_each_event_constraint(event_constraint, event_constraints) {
1451                 if (code == event_constraint->code) {
1452                         for_each_bit(i, event_constraint->idxmsk, X86_PMC_IDX_MAX) {
1453                                 if (!test_and_set_bit(i, cpuc->used_mask))
1454                                         return i;
1455                         }
1456                         return -1;
1457                 }
1458         }
1459 skip:
1460         return gen_get_event_idx(cpuc, hwc);
1461 }
1462
1463 static int
1464 x86_schedule_event(struct cpu_hw_events *cpuc, struct hw_perf_event *hwc)
1465 {
1466         int idx;
1467
1468         idx = fixed_mode_idx(hwc);
1469         if (idx == X86_PMC_IDX_FIXED_BTS) {
1470                 /* BTS is already occupied. */
1471                 if (test_and_set_bit(idx, cpuc->used_mask))
1472                         return -EAGAIN;
1473
1474                 hwc->config_base        = 0;
1475                 hwc->event_base         = 0;
1476                 hwc->idx                = idx;
1477         } else if (idx >= 0) {
1478                 /*
1479                  * Try to get the fixed event, if that is already taken
1480                  * then try to get a generic event:
1481                  */
1482                 if (test_and_set_bit(idx, cpuc->used_mask))
1483                         goto try_generic;
1484
1485                 hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
1486                 /*
1487                  * We set it so that event_base + idx in wrmsr/rdmsr maps to
1488                  * MSR_ARCH_PERFMON_FIXED_CTR0 ... CTR2:
1489                  */
1490                 hwc->event_base =
1491                         MSR_ARCH_PERFMON_FIXED_CTR0 - X86_PMC_IDX_FIXED;
1492                 hwc->idx = idx;
1493         } else {
1494                 idx = hwc->idx;
1495                 /* Try to get the previous generic event again */
1496                 if (idx == -1 || test_and_set_bit(idx, cpuc->used_mask)) {
1497 try_generic:
1498                         idx = x86_pmu.get_event_idx(cpuc, hwc);
1499                         if (idx == -1)
1500                                 return -EAGAIN;
1501
1502                         set_bit(idx, cpuc->used_mask);
1503                         hwc->idx = idx;
1504                 }
1505                 hwc->config_base = x86_pmu.eventsel;
1506                 hwc->event_base  = x86_pmu.perfctr;
1507         }
1508
1509         return idx;
1510 }
1511
1512 /*
1513  * Find a PMC slot for the freshly enabled / scheduled in event:
1514  */
1515 static int x86_pmu_enable(struct perf_event *event)
1516 {
1517         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1518         struct hw_perf_event *hwc = &event->hw;
1519         int idx;
1520
1521         idx = x86_schedule_event(cpuc, hwc);
1522         if (idx < 0)
1523                 return idx;
1524
1525         perf_events_lapic_init();
1526
1527         x86_pmu.disable(hwc, idx);
1528
1529         cpuc->events[idx] = event;
1530         set_bit(idx, cpuc->active_mask);
1531
1532         x86_perf_event_set_period(event, hwc, idx);
1533         x86_pmu.enable(hwc, idx);
1534
1535         perf_event_update_userpage(event);
1536
1537         return 0;
1538 }
1539
1540 static void x86_pmu_unthrottle(struct perf_event *event)
1541 {
1542         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1543         struct hw_perf_event *hwc = &event->hw;
1544
1545         if (WARN_ON_ONCE(hwc->idx >= X86_PMC_IDX_MAX ||
1546                                 cpuc->events[hwc->idx] != event))
1547                 return;
1548
1549         x86_pmu.enable(hwc, hwc->idx);
1550 }
1551
1552 void perf_event_print_debug(void)
1553 {
1554         u64 ctrl, status, overflow, pmc_ctrl, pmc_count, prev_left, fixed;
1555         struct cpu_hw_events *cpuc;
1556         unsigned long flags;
1557         int cpu, idx;
1558
1559         if (!x86_pmu.num_events)
1560                 return;
1561
1562         local_irq_save(flags);
1563
1564         cpu = smp_processor_id();
1565         cpuc = &per_cpu(cpu_hw_events, cpu);
1566
1567         if (x86_pmu.version >= 2) {
1568                 rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
1569                 rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
1570                 rdmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, overflow);
1571                 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, fixed);
1572
1573                 pr_info("\n");
1574                 pr_info("CPU#%d: ctrl:       %016llx\n", cpu, ctrl);
1575                 pr_info("CPU#%d: status:     %016llx\n", cpu, status);
1576                 pr_info("CPU#%d: overflow:   %016llx\n", cpu, overflow);
1577                 pr_info("CPU#%d: fixed:      %016llx\n", cpu, fixed);
1578         }
1579         pr_info("CPU#%d: used:       %016llx\n", cpu, *(u64 *)cpuc->used_mask);
1580
1581         for (idx = 0; idx < x86_pmu.num_events; idx++) {
1582                 rdmsrl(x86_pmu.eventsel + idx, pmc_ctrl);
1583                 rdmsrl(x86_pmu.perfctr  + idx, pmc_count);
1584
1585                 prev_left = per_cpu(pmc_prev_left[idx], cpu);
1586
1587                 pr_info("CPU#%d:   gen-PMC%d ctrl:  %016llx\n",
1588                         cpu, idx, pmc_ctrl);
1589                 pr_info("CPU#%d:   gen-PMC%d count: %016llx\n",
1590                         cpu, idx, pmc_count);
1591                 pr_info("CPU#%d:   gen-PMC%d left:  %016llx\n",
1592                         cpu, idx, prev_left);
1593         }
1594         for (idx = 0; idx < x86_pmu.num_events_fixed; idx++) {
1595                 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, pmc_count);
1596
1597                 pr_info("CPU#%d: fixed-PMC%d count: %016llx\n",
1598                         cpu, idx, pmc_count);
1599         }
1600         local_irq_restore(flags);
1601 }
1602
1603 static void intel_pmu_drain_bts_buffer(struct cpu_hw_events *cpuc)
1604 {
1605         struct debug_store *ds = cpuc->ds;
1606         struct bts_record {
1607                 u64     from;
1608                 u64     to;
1609                 u64     flags;
1610         };
1611         struct perf_event *event = cpuc->events[X86_PMC_IDX_FIXED_BTS];
1612         struct bts_record *at, *top;
1613         struct perf_output_handle handle;
1614         struct perf_event_header header;
1615         struct perf_sample_data data;
1616         struct pt_regs regs;
1617
1618         if (!event)
1619                 return;
1620
1621         if (!ds)
1622                 return;
1623
1624         at  = (struct bts_record *)(unsigned long)ds->bts_buffer_base;
1625         top = (struct bts_record *)(unsigned long)ds->bts_index;
1626
1627         if (top <= at)
1628                 return;
1629
1630         ds->bts_index = ds->bts_buffer_base;
1631
1632
1633         data.period     = event->hw.last_period;
1634         data.addr       = 0;
1635         regs.ip         = 0;
1636
1637         /*
1638          * Prepare a generic sample, i.e. fill in the invariant fields.
1639          * We will overwrite the from and to address before we output
1640          * the sample.
1641          */
1642         perf_prepare_sample(&header, &data, event, &regs);
1643
1644         if (perf_output_begin(&handle, event,
1645                               header.size * (top - at), 1, 1))
1646                 return;
1647
1648         for (; at < top; at++) {
1649                 data.ip         = at->from;
1650                 data.addr       = at->to;
1651
1652                 perf_output_sample(&handle, &header, &data, event);
1653         }
1654
1655         perf_output_end(&handle);
1656
1657         /* There's new data available. */
1658         event->hw.interrupts++;
1659         event->pending_kill = POLL_IN;
1660 }
1661
1662 static void x86_pmu_disable(struct perf_event *event)
1663 {
1664         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1665         struct hw_perf_event *hwc = &event->hw;
1666         int idx = hwc->idx;
1667
1668         /*
1669          * Must be done before we disable, otherwise the nmi handler
1670          * could reenable again:
1671          */
1672         clear_bit(idx, cpuc->active_mask);
1673         x86_pmu.disable(hwc, idx);
1674
1675         /*
1676          * Make sure the cleared pointer becomes visible before we
1677          * (potentially) free the event:
1678          */
1679         barrier();
1680
1681         /*
1682          * Drain the remaining delta count out of a event
1683          * that we are disabling:
1684          */
1685         x86_perf_event_update(event, hwc, idx);
1686
1687         /* Drain the remaining BTS records. */
1688         if (unlikely(idx == X86_PMC_IDX_FIXED_BTS))
1689                 intel_pmu_drain_bts_buffer(cpuc);
1690
1691         cpuc->events[idx] = NULL;
1692         clear_bit(idx, cpuc->used_mask);
1693
1694         perf_event_update_userpage(event);
1695 }
1696
1697 /*
1698  * Save and restart an expired event. Called by NMI contexts,
1699  * so it has to be careful about preempting normal event ops:
1700  */
1701 static int intel_pmu_save_and_restart(struct perf_event *event)
1702 {
1703         struct hw_perf_event *hwc = &event->hw;
1704         int idx = hwc->idx;
1705         int ret;
1706
1707         x86_perf_event_update(event, hwc, idx);
1708         ret = x86_perf_event_set_period(event, hwc, idx);
1709
1710         if (event->state == PERF_EVENT_STATE_ACTIVE)
1711                 intel_pmu_enable_event(hwc, idx);
1712
1713         return ret;
1714 }
1715
1716 static void intel_pmu_reset(void)
1717 {
1718         struct debug_store *ds = __get_cpu_var(cpu_hw_events).ds;
1719         unsigned long flags;
1720         int idx;
1721
1722         if (!x86_pmu.num_events)
1723                 return;
1724
1725         local_irq_save(flags);
1726
1727         printk("clearing PMU state on CPU#%d\n", smp_processor_id());
1728
1729         for (idx = 0; idx < x86_pmu.num_events; idx++) {
1730                 checking_wrmsrl(x86_pmu.eventsel + idx, 0ull);
1731                 checking_wrmsrl(x86_pmu.perfctr  + idx, 0ull);
1732         }
1733         for (idx = 0; idx < x86_pmu.num_events_fixed; idx++) {
1734                 checking_wrmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, 0ull);
1735         }
1736         if (ds)
1737                 ds->bts_index = ds->bts_buffer_base;
1738
1739         local_irq_restore(flags);
1740 }
1741
1742 static int p6_pmu_handle_irq(struct pt_regs *regs)
1743 {
1744         struct perf_sample_data data;
1745         struct cpu_hw_events *cpuc;
1746         struct perf_event *event;
1747         struct hw_perf_event *hwc;
1748         int idx, handled = 0;
1749         u64 val;
1750
1751         data.addr = 0;
1752
1753         cpuc = &__get_cpu_var(cpu_hw_events);
1754
1755         for (idx = 0; idx < x86_pmu.num_events; idx++) {
1756                 if (!test_bit(idx, cpuc->active_mask))
1757                         continue;
1758
1759                 event = cpuc->events[idx];
1760                 hwc = &event->hw;
1761
1762                 val = x86_perf_event_update(event, hwc, idx);
1763                 if (val & (1ULL << (x86_pmu.event_bits - 1)))
1764                         continue;
1765
1766                 /*
1767                  * event overflow
1768                  */
1769                 handled         = 1;
1770                 data.period     = event->hw.last_period;
1771
1772                 if (!x86_perf_event_set_period(event, hwc, idx))
1773                         continue;
1774
1775                 if (perf_event_overflow(event, 1, &data, regs))
1776                         p6_pmu_disable_event(hwc, idx);
1777         }
1778
1779         if (handled)
1780                 inc_irq_stat(apic_perf_irqs);
1781
1782         return handled;
1783 }
1784
1785 /*
1786  * This handler is triggered by the local APIC, so the APIC IRQ handling
1787  * rules apply:
1788  */
1789 static int intel_pmu_handle_irq(struct pt_regs *regs)
1790 {
1791         struct perf_sample_data data;
1792         struct cpu_hw_events *cpuc;
1793         int bit, loops;
1794         u64 ack, status;
1795
1796         data.addr = 0;
1797
1798         cpuc = &__get_cpu_var(cpu_hw_events);
1799
1800         perf_disable();
1801         intel_pmu_drain_bts_buffer(cpuc);
1802         status = intel_pmu_get_status();
1803         if (!status) {
1804                 perf_enable();
1805                 return 0;
1806         }
1807
1808         loops = 0;
1809 again:
1810         if (++loops > 100) {
1811                 WARN_ONCE(1, "perfevents: irq loop stuck!\n");
1812                 perf_event_print_debug();
1813                 intel_pmu_reset();
1814                 perf_enable();
1815                 return 1;
1816         }
1817
1818         inc_irq_stat(apic_perf_irqs);
1819         ack = status;
1820         for_each_bit(bit, (unsigned long *)&status, X86_PMC_IDX_MAX) {
1821                 struct perf_event *event = cpuc->events[bit];
1822
1823                 clear_bit(bit, (unsigned long *) &status);
1824                 if (!test_bit(bit, cpuc->active_mask))
1825                         continue;
1826
1827                 if (!intel_pmu_save_and_restart(event))
1828                         continue;
1829
1830                 data.period = event->hw.last_period;
1831
1832                 if (perf_event_overflow(event, 1, &data, regs))
1833                         intel_pmu_disable_event(&event->hw, bit);
1834         }
1835
1836         intel_pmu_ack_status(ack);
1837
1838         /*
1839          * Repeat if there is more work to be done:
1840          */
1841         status = intel_pmu_get_status();
1842         if (status)
1843                 goto again;
1844
1845         perf_enable();
1846
1847         return 1;
1848 }
1849
1850 static int amd_pmu_handle_irq(struct pt_regs *regs)
1851 {
1852         struct perf_sample_data data;
1853         struct cpu_hw_events *cpuc;
1854         struct perf_event *event;
1855         struct hw_perf_event *hwc;
1856         int idx, handled = 0;
1857         u64 val;
1858
1859         data.addr = 0;
1860
1861         cpuc = &__get_cpu_var(cpu_hw_events);
1862
1863         for (idx = 0; idx < x86_pmu.num_events; idx++) {
1864                 if (!test_bit(idx, cpuc->active_mask))
1865                         continue;
1866
1867                 event = cpuc->events[idx];
1868                 hwc = &event->hw;
1869
1870                 val = x86_perf_event_update(event, hwc, idx);
1871                 if (val & (1ULL << (x86_pmu.event_bits - 1)))
1872                         continue;
1873
1874                 /*
1875                  * event overflow
1876                  */
1877                 handled         = 1;
1878                 data.period     = event->hw.last_period;
1879
1880                 if (!x86_perf_event_set_period(event, hwc, idx))
1881                         continue;
1882
1883                 if (perf_event_overflow(event, 1, &data, regs))
1884                         amd_pmu_disable_event(hwc, idx);
1885         }
1886
1887         if (handled)
1888                 inc_irq_stat(apic_perf_irqs);
1889
1890         return handled;
1891 }
1892
1893 void smp_perf_pending_interrupt(struct pt_regs *regs)
1894 {
1895         irq_enter();
1896         ack_APIC_irq();
1897         inc_irq_stat(apic_pending_irqs);
1898         perf_event_do_pending();
1899         irq_exit();
1900 }
1901
1902 void set_perf_event_pending(void)
1903 {
1904 #ifdef CONFIG_X86_LOCAL_APIC
1905         if (!x86_pmu.apic || !x86_pmu_initialized())
1906                 return;
1907
1908         apic->send_IPI_self(LOCAL_PENDING_VECTOR);
1909 #endif
1910 }
1911
1912 void perf_events_lapic_init(void)
1913 {
1914 #ifdef CONFIG_X86_LOCAL_APIC
1915         if (!x86_pmu.apic || !x86_pmu_initialized())
1916                 return;
1917
1918         /*
1919          * Always use NMI for PMU
1920          */
1921         apic_write(APIC_LVTPC, APIC_DM_NMI);
1922 #endif
1923 }
1924
1925 static int __kprobes
1926 perf_event_nmi_handler(struct notifier_block *self,
1927                          unsigned long cmd, void *__args)
1928 {
1929         struct die_args *args = __args;
1930         struct pt_regs *regs;
1931
1932         if (!atomic_read(&active_events))
1933                 return NOTIFY_DONE;
1934
1935         switch (cmd) {
1936         case DIE_NMI:
1937         case DIE_NMI_IPI:
1938                 break;
1939
1940         default:
1941                 return NOTIFY_DONE;
1942         }
1943
1944         regs = args->regs;
1945
1946 #ifdef CONFIG_X86_LOCAL_APIC
1947         apic_write(APIC_LVTPC, APIC_DM_NMI);
1948 #endif
1949         /*
1950          * Can't rely on the handled return value to say it was our NMI, two
1951          * events could trigger 'simultaneously' raising two back-to-back NMIs.
1952          *
1953          * If the first NMI handles both, the latter will be empty and daze
1954          * the CPU.
1955          */
1956         x86_pmu.handle_irq(regs);
1957
1958         return NOTIFY_STOP;
1959 }
1960
1961 static __read_mostly struct notifier_block perf_event_nmi_notifier = {
1962         .notifier_call          = perf_event_nmi_handler,
1963         .next                   = NULL,
1964         .priority               = 1
1965 };
1966
1967 static __initconst struct x86_pmu p6_pmu = {
1968         .name                   = "p6",
1969         .handle_irq             = p6_pmu_handle_irq,
1970         .disable_all            = p6_pmu_disable_all,
1971         .enable_all             = p6_pmu_enable_all,
1972         .enable                 = p6_pmu_enable_event,
1973         .disable                = p6_pmu_disable_event,
1974         .eventsel               = MSR_P6_EVNTSEL0,
1975         .perfctr                = MSR_P6_PERFCTR0,
1976         .event_map              = p6_pmu_event_map,
1977         .raw_event              = p6_pmu_raw_event,
1978         .max_events             = ARRAY_SIZE(p6_perfmon_event_map),
1979         .apic                   = 1,
1980         .max_period             = (1ULL << 31) - 1,
1981         .version                = 0,
1982         .num_events             = 2,
1983         /*
1984          * Events have 40 bits implemented. However they are designed such
1985          * that bits [32-39] are sign extensions of bit 31. As such the
1986          * effective width of a event for P6-like PMU is 32 bits only.
1987          *
1988          * See IA-32 Intel Architecture Software developer manual Vol 3B
1989          */
1990         .event_bits             = 32,
1991         .event_mask             = (1ULL << 32) - 1,
1992         .get_event_idx          = intel_get_event_idx,
1993 };
1994
1995 static __initconst struct x86_pmu intel_pmu = {
1996         .name                   = "Intel",
1997         .handle_irq             = intel_pmu_handle_irq,
1998         .disable_all            = intel_pmu_disable_all,
1999         .enable_all             = intel_pmu_enable_all,
2000         .enable                 = intel_pmu_enable_event,
2001         .disable                = intel_pmu_disable_event,
2002         .eventsel               = MSR_ARCH_PERFMON_EVENTSEL0,
2003         .perfctr                = MSR_ARCH_PERFMON_PERFCTR0,
2004         .event_map              = intel_pmu_event_map,
2005         .raw_event              = intel_pmu_raw_event,
2006         .max_events             = ARRAY_SIZE(intel_perfmon_event_map),
2007         .apic                   = 1,
2008         /*
2009          * Intel PMCs cannot be accessed sanely above 32 bit width,
2010          * so we install an artificial 1<<31 period regardless of
2011          * the generic event period:
2012          */
2013         .max_period             = (1ULL << 31) - 1,
2014         .enable_bts             = intel_pmu_enable_bts,
2015         .disable_bts            = intel_pmu_disable_bts,
2016         .get_event_idx          = intel_get_event_idx,
2017 };
2018
2019 static __initconst struct x86_pmu amd_pmu = {
2020         .name                   = "AMD",
2021         .handle_irq             = amd_pmu_handle_irq,
2022         .disable_all            = amd_pmu_disable_all,
2023         .enable_all             = amd_pmu_enable_all,
2024         .enable                 = amd_pmu_enable_event,
2025         .disable                = amd_pmu_disable_event,
2026         .eventsel               = MSR_K7_EVNTSEL0,
2027         .perfctr                = MSR_K7_PERFCTR0,
2028         .event_map              = amd_pmu_event_map,
2029         .raw_event              = amd_pmu_raw_event,
2030         .max_events             = ARRAY_SIZE(amd_perfmon_event_map),
2031         .num_events             = 4,
2032         .event_bits             = 48,
2033         .event_mask             = (1ULL << 48) - 1,
2034         .apic                   = 1,
2035         /* use highest bit to detect overflow */
2036         .max_period             = (1ULL << 47) - 1,
2037         .get_event_idx          = gen_get_event_idx,
2038 };
2039
2040 static __init int p6_pmu_init(void)
2041 {
2042         switch (boot_cpu_data.x86_model) {
2043         case 1:
2044         case 3:  /* Pentium Pro */
2045         case 5:
2046         case 6:  /* Pentium II */
2047         case 7:
2048         case 8:
2049         case 11: /* Pentium III */
2050                 event_constraints = intel_p6_event_constraints;
2051                 break;
2052         case 9:
2053         case 13:
2054                 /* Pentium M */
2055                 event_constraints = intel_p6_event_constraints;
2056                 break;
2057         default:
2058                 pr_cont("unsupported p6 CPU model %d ",
2059                         boot_cpu_data.x86_model);
2060                 return -ENODEV;
2061         }
2062
2063         x86_pmu = p6_pmu;
2064
2065         if (!cpu_has_apic) {
2066                 pr_info("no APIC, boot with the \"lapic\" boot parameter to force-enable it.\n");
2067                 pr_info("no hardware sampling interrupt available.\n");
2068                 x86_pmu.apic = 0;
2069         }
2070
2071         return 0;
2072 }
2073
2074 static __init int intel_pmu_init(void)
2075 {
2076         union cpuid10_edx edx;
2077         union cpuid10_eax eax;
2078         unsigned int unused;
2079         unsigned int ebx;
2080         int version;
2081
2082         if (!cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON)) {
2083                 /* check for P6 processor family */
2084            if (boot_cpu_data.x86 == 6) {
2085                 return p6_pmu_init();
2086            } else {
2087                 return -ENODEV;
2088            }
2089         }
2090
2091         /*
2092          * Check whether the Architectural PerfMon supports
2093          * Branch Misses Retired hw_event or not.
2094          */
2095         cpuid(10, &eax.full, &ebx, &unused, &edx.full);
2096         if (eax.split.mask_length <= ARCH_PERFMON_BRANCH_MISSES_RETIRED)
2097                 return -ENODEV;
2098
2099         version = eax.split.version_id;
2100         if (version < 2)
2101                 return -ENODEV;
2102
2103         x86_pmu                         = intel_pmu;
2104         x86_pmu.version                 = version;
2105         x86_pmu.num_events              = eax.split.num_events;
2106         x86_pmu.event_bits              = eax.split.bit_width;
2107         x86_pmu.event_mask              = (1ULL << eax.split.bit_width) - 1;
2108
2109         /*
2110          * Quirk: v2 perfmon does not report fixed-purpose events, so
2111          * assume at least 3 events:
2112          */
2113         x86_pmu.num_events_fixed        = max((int)edx.split.num_events_fixed, 3);
2114
2115         /*
2116          * Install the hw-cache-events table:
2117          */
2118         switch (boot_cpu_data.x86_model) {
2119         case 15: /* original 65 nm celeron/pentium/core2/xeon, "Merom"/"Conroe" */
2120         case 22: /* single-core 65 nm celeron/core2solo "Merom-L"/"Conroe-L" */
2121         case 23: /* current 45 nm celeron/core2/xeon "Penryn"/"Wolfdale" */
2122         case 29: /* six-core 45 nm xeon "Dunnington" */
2123                 memcpy(hw_cache_event_ids, core2_hw_cache_event_ids,
2124                        sizeof(hw_cache_event_ids));
2125
2126                 pr_cont("Core2 events, ");
2127                 event_constraints = intel_core_event_constraints;
2128                 break;
2129         default:
2130         case 26:
2131                 memcpy(hw_cache_event_ids, nehalem_hw_cache_event_ids,
2132                        sizeof(hw_cache_event_ids));
2133
2134                 event_constraints = intel_nehalem_event_constraints;
2135                 pr_cont("Nehalem/Corei7 events, ");
2136                 break;
2137         case 28:
2138                 memcpy(hw_cache_event_ids, atom_hw_cache_event_ids,
2139                        sizeof(hw_cache_event_ids));
2140
2141                 pr_cont("Atom events, ");
2142                 break;
2143         }
2144         return 0;
2145 }
2146
2147 static __init int amd_pmu_init(void)
2148 {
2149         /* Performance-monitoring supported from K7 and later: */
2150         if (boot_cpu_data.x86 < 6)
2151                 return -ENODEV;
2152
2153         x86_pmu = amd_pmu;
2154
2155         /* Events are common for all AMDs */
2156         memcpy(hw_cache_event_ids, amd_hw_cache_event_ids,
2157                sizeof(hw_cache_event_ids));
2158
2159         return 0;
2160 }
2161
2162 void __init init_hw_perf_events(void)
2163 {
2164         int err;
2165
2166         pr_info("Performance Events: ");
2167
2168         switch (boot_cpu_data.x86_vendor) {
2169         case X86_VENDOR_INTEL:
2170                 err = intel_pmu_init();
2171                 break;
2172         case X86_VENDOR_AMD:
2173                 err = amd_pmu_init();
2174                 break;
2175         default:
2176                 return;
2177         }
2178         if (err != 0) {
2179                 pr_cont("no PMU driver, software events only.\n");
2180                 return;
2181         }
2182
2183         pr_cont("%s PMU driver.\n", x86_pmu.name);
2184
2185         if (x86_pmu.num_events > X86_PMC_MAX_GENERIC) {
2186                 WARN(1, KERN_ERR "hw perf events %d > max(%d), clipping!",
2187                      x86_pmu.num_events, X86_PMC_MAX_GENERIC);
2188                 x86_pmu.num_events = X86_PMC_MAX_GENERIC;
2189         }
2190         perf_event_mask = (1 << x86_pmu.num_events) - 1;
2191         perf_max_events = x86_pmu.num_events;
2192
2193         if (x86_pmu.num_events_fixed > X86_PMC_MAX_FIXED) {
2194                 WARN(1, KERN_ERR "hw perf events fixed %d > max(%d), clipping!",
2195                      x86_pmu.num_events_fixed, X86_PMC_MAX_FIXED);
2196                 x86_pmu.num_events_fixed = X86_PMC_MAX_FIXED;
2197         }
2198
2199         perf_event_mask |=
2200                 ((1LL << x86_pmu.num_events_fixed)-1) << X86_PMC_IDX_FIXED;
2201         x86_pmu.intel_ctrl = perf_event_mask;
2202
2203         perf_events_lapic_init();
2204         register_die_notifier(&perf_event_nmi_notifier);
2205
2206         pr_info("... version:                %d\n",     x86_pmu.version);
2207         pr_info("... bit width:              %d\n",     x86_pmu.event_bits);
2208         pr_info("... generic registers:      %d\n",     x86_pmu.num_events);
2209         pr_info("... value mask:             %016Lx\n", x86_pmu.event_mask);
2210         pr_info("... max period:             %016Lx\n", x86_pmu.max_period);
2211         pr_info("... fixed-purpose events:   %d\n",     x86_pmu.num_events_fixed);
2212         pr_info("... event mask:             %016Lx\n", perf_event_mask);
2213 }
2214
2215 static inline void x86_pmu_read(struct perf_event *event)
2216 {
2217         x86_perf_event_update(event, &event->hw, event->hw.idx);
2218 }
2219
2220 static const struct pmu pmu = {
2221         .enable         = x86_pmu_enable,
2222         .disable        = x86_pmu_disable,
2223         .read           = x86_pmu_read,
2224         .unthrottle     = x86_pmu_unthrottle,
2225 };
2226
2227 static int
2228 validate_event(struct cpu_hw_events *cpuc, struct perf_event *event)
2229 {
2230         struct hw_perf_event fake_event = event->hw;
2231
2232         if (event->pmu && event->pmu != &pmu)
2233                 return 0;
2234
2235         return x86_schedule_event(cpuc, &fake_event) >= 0;
2236 }
2237
2238 static int validate_group(struct perf_event *event)
2239 {
2240         struct perf_event *sibling, *leader = event->group_leader;
2241         struct cpu_hw_events fake_pmu;
2242
2243         memset(&fake_pmu, 0, sizeof(fake_pmu));
2244
2245         if (!validate_event(&fake_pmu, leader))
2246                 return -ENOSPC;
2247
2248         list_for_each_entry(sibling, &leader->sibling_list, group_entry) {
2249                 if (!validate_event(&fake_pmu, sibling))
2250                         return -ENOSPC;
2251         }
2252
2253         if (!validate_event(&fake_pmu, event))
2254                 return -ENOSPC;
2255
2256         return 0;
2257 }
2258
2259 const struct pmu *hw_perf_event_init(struct perf_event *event)
2260 {
2261         int err;
2262
2263         err = __hw_perf_event_init(event);
2264         if (!err) {
2265                 if (event->group_leader != event)
2266                         err = validate_group(event);
2267         }
2268         if (err) {
2269                 if (event->destroy)
2270                         event->destroy(event);
2271                 return ERR_PTR(err);
2272         }
2273
2274         return &pmu;
2275 }
2276
2277 /*
2278  * callchain support
2279  */
2280
2281 static inline
2282 void callchain_store(struct perf_callchain_entry *entry, u64 ip)
2283 {
2284         if (entry->nr < PERF_MAX_STACK_DEPTH)
2285                 entry->ip[entry->nr++] = ip;
2286 }
2287
2288 static DEFINE_PER_CPU(struct perf_callchain_entry, pmc_irq_entry);
2289 static DEFINE_PER_CPU(struct perf_callchain_entry, pmc_nmi_entry);
2290 static DEFINE_PER_CPU(int, in_nmi_frame);
2291
2292
2293 static void
2294 backtrace_warning_symbol(void *data, char *msg, unsigned long symbol)
2295 {
2296         /* Ignore warnings */
2297 }
2298
2299 static void backtrace_warning(void *data, char *msg)
2300 {
2301         /* Ignore warnings */
2302 }
2303
2304 static int backtrace_stack(void *data, char *name)
2305 {
2306         per_cpu(in_nmi_frame, smp_processor_id()) =
2307                         x86_is_stack_id(NMI_STACK, name);
2308
2309         return 0;
2310 }
2311
2312 static void backtrace_address(void *data, unsigned long addr, int reliable)
2313 {
2314         struct perf_callchain_entry *entry = data;
2315
2316         if (per_cpu(in_nmi_frame, smp_processor_id()))
2317                 return;
2318
2319         if (reliable)
2320                 callchain_store(entry, addr);
2321 }
2322
2323 static const struct stacktrace_ops backtrace_ops = {
2324         .warning                = backtrace_warning,
2325         .warning_symbol         = backtrace_warning_symbol,
2326         .stack                  = backtrace_stack,
2327         .address                = backtrace_address,
2328 };
2329
2330 #include "../dumpstack.h"
2331
2332 static void
2333 perf_callchain_kernel(struct pt_regs *regs, struct perf_callchain_entry *entry)
2334 {
2335         callchain_store(entry, PERF_CONTEXT_KERNEL);
2336         callchain_store(entry, regs->ip);
2337
2338         dump_trace(NULL, regs, NULL, 0, &backtrace_ops, entry);
2339 }
2340
2341 /*
2342  * best effort, GUP based copy_from_user() that assumes IRQ or NMI context
2343  */
2344 static unsigned long
2345 copy_from_user_nmi(void *to, const void __user *from, unsigned long n)
2346 {
2347         unsigned long offset, addr = (unsigned long)from;
2348         int type = in_nmi() ? KM_NMI : KM_IRQ0;
2349         unsigned long size, len = 0;
2350         struct page *page;
2351         void *map;
2352         int ret;
2353
2354         do {
2355                 ret = __get_user_pages_fast(addr, 1, 0, &page);
2356                 if (!ret)
2357                         break;
2358
2359                 offset = addr & (PAGE_SIZE - 1);
2360                 size = min(PAGE_SIZE - offset, n - len);
2361
2362                 map = kmap_atomic(page, type);
2363                 memcpy(to, map+offset, size);
2364                 kunmap_atomic(map, type);
2365                 put_page(page);
2366
2367                 len  += size;
2368                 to   += size;
2369                 addr += size;
2370
2371         } while (len < n);
2372
2373         return len;
2374 }
2375
2376 static int copy_stack_frame(const void __user *fp, struct stack_frame *frame)
2377 {
2378         unsigned long bytes;
2379
2380         bytes = copy_from_user_nmi(frame, fp, sizeof(*frame));
2381
2382         return bytes == sizeof(*frame);
2383 }
2384
2385 static void
2386 perf_callchain_user(struct pt_regs *regs, struct perf_callchain_entry *entry)
2387 {
2388         struct stack_frame frame;
2389         const void __user *fp;
2390
2391         if (!user_mode(regs))
2392                 regs = task_pt_regs(current);
2393
2394         fp = (void __user *)regs->bp;
2395
2396         callchain_store(entry, PERF_CONTEXT_USER);
2397         callchain_store(entry, regs->ip);
2398
2399         while (entry->nr < PERF_MAX_STACK_DEPTH) {
2400                 frame.next_frame             = NULL;
2401                 frame.return_address = 0;
2402
2403                 if (!copy_stack_frame(fp, &frame))
2404                         break;
2405
2406                 if ((unsigned long)fp < regs->sp)
2407                         break;
2408
2409                 callchain_store(entry, frame.return_address);
2410                 fp = frame.next_frame;
2411         }
2412 }
2413
2414 static void
2415 perf_do_callchain(struct pt_regs *regs, struct perf_callchain_entry *entry)
2416 {
2417         int is_user;
2418
2419         if (!regs)
2420                 return;
2421
2422         is_user = user_mode(regs);
2423
2424         if (!current || current->pid == 0)
2425                 return;
2426
2427         if (is_user && current->state != TASK_RUNNING)
2428                 return;
2429
2430         if (!is_user)
2431                 perf_callchain_kernel(regs, entry);
2432
2433         if (current->mm)
2434                 perf_callchain_user(regs, entry);
2435 }
2436
2437 struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
2438 {
2439         struct perf_callchain_entry *entry;
2440
2441         if (in_nmi())
2442                 entry = &__get_cpu_var(pmc_nmi_entry);
2443         else
2444                 entry = &__get_cpu_var(pmc_irq_entry);
2445
2446         entry->nr = 0;
2447
2448         perf_do_callchain(regs, entry);
2449
2450         return entry;
2451 }
2452
2453 void hw_perf_event_setup_online(int cpu)
2454 {
2455         init_debug_store_on_cpu(cpu);
2456 }