arch/x86/kernel/cpu/perf_event.c

   1 /*
   2  * Performance events x86 architecture code
   3  *
   4  *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
   5  *  Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
   6  *  Copyright (C) 2009 Jaswinder Singh Rajput
   7  *  Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
   8  *  Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
   9  *  Copyright (C) 2009 Intel Corporation, <markus.t.metzger@intel.com>
  10  *
  11  *  For licencing details see kernel-base/COPYING
  12  */
  13
  14 #include <linux/perf_event.h>
  15 #include <linux/capability.h>
  16 #include <linux/notifier.h>
  17 #include <linux/hardirq.h>
  18 #include <linux/kprobes.h>
  19 #include <linux/module.h>
  20 #include <linux/kdebug.h>
  21 #include <linux/sched.h>
  22 #include <linux/uaccess.h>
  23 #include <linux/highmem.h>
  24 #include <linux/cpu.h>
  25
  26 #include <asm/apic.h>
  27 #include <asm/stacktrace.h>
  28 #include <asm/nmi.h>
  29
  30 static u64 perf_event_mask __read_mostly;
  31
  32 /* The maximal number of PEBS events: */
  33 #define MAX_PEBS_EVENTS 4
  34
  35 /* The size of a BTS record in bytes: */
  36 #define BTS_RECORD_SIZE         24
  37
  38 /* The size of a per-cpu BTS buffer in bytes: */
  39 #define BTS_BUFFER_SIZE         (BTS_RECORD_SIZE * 2048)
  40
  41 /* The BTS overflow threshold in bytes from the end of the buffer: */
  42 #define BTS_OVFL_TH             (BTS_RECORD_SIZE * 128)
  43
  44
  45 /*
  46  * Bits in the debugctlmsr controlling branch tracing.
  47  */
  48 #define X86_DEBUGCTL_TR                 (1 << 6)
  49 #define X86_DEBUGCTL_BTS                (1 << 7)
  50 #define X86_DEBUGCTL_BTINT              (1 << 8)
  51 #define X86_DEBUGCTL_BTS_OFF_OS         (1 << 9)
  52 #define X86_DEBUGCTL_BTS_OFF_USR        (1 << 10)
  53
  54 /*
  55  * A debug store configuration.
  56  *
  57  * We only support architectures that use 64bit fields.
  58  */
  59 struct debug_store {
  60         u64     bts_buffer_base;
  61         u64     bts_index;
  62         u64     bts_absolute_maximum;
  63         u64     bts_interrupt_threshold;
  64         u64     pebs_buffer_base;
  65         u64     pebs_index;
  66         u64     pebs_absolute_maximum;
  67         u64     pebs_interrupt_threshold;
  68         u64     pebs_event_reset[MAX_PEBS_EVENTS];
  69 };
  70
  71 struct cpu_hw_events {
  72         struct perf_event       *events[X86_PMC_IDX_MAX];
  73         unsigned long           used_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
  74         unsigned long           active_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
  75         unsigned long           interrupts;
  76         int                     enabled;
  77         struct debug_store      *ds;
  78 };
  79
  80 struct event_constraint {
  81         unsigned long   idxmsk[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
  82         int             code;
  83 };
  84
  85 #define EVENT_CONSTRAINT(c, m) { .code = (c), .idxmsk[0] = (m) }
  86 #define EVENT_CONSTRAINT_END  { .code = 0, .idxmsk[0] = 0 }
  87
  88 #define for_each_event_constraint(e, c) \
  89         for ((e) = (c); (e)->idxmsk[0]; (e)++)
  90
  91
  92 /*
  93  * struct x86_pmu - generic x86 pmu
  94  */
  95 struct x86_pmu {
  96         const char      *name;
  97         int             version;
  98         int             (*handle_irq)(struct pt_regs *);
  99         void            (*disable_all)(void);
 100         void            (*enable_all)(void);
 101         void            (*enable)(struct hw_perf_event *, int);
 102         void            (*disable)(struct hw_perf_event *, int);
 103         unsigned        eventsel;
 104         unsigned        perfctr;
 105         u64             (*event_map)(int);
 106         u64             (*raw_event)(u64);
 107         int             max_events;
 108         int             num_events;
 109         int             num_events_fixed;
 110         int             event_bits;
 111         u64             event_mask;
 112         int             apic;
 113         u64             max_period;
 114         u64             intel_ctrl;
 115         void            (*enable_bts)(u64 config);
 116         void            (*disable_bts)(void);
 117         int             (*get_event_idx)(struct cpu_hw_events *cpuc,
 118                                          struct hw_perf_event *hwc);
 119 };
 120
 121 static struct x86_pmu x86_pmu __read_mostly;
 122
 123 static DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = {
 124         .enabled = 1,
 125 };
 126
 127 static const struct event_constraint *event_constraints;
 128
 129 /*
 130  * Not sure about some of these
 131  */
 132 static const u64 p6_perfmon_event_map[] =
 133 {
 134   [PERF_COUNT_HW_CPU_CYCLES]            = 0x0079,
 135   [PERF_COUNT_HW_INSTRUCTIONS]          = 0x00c0,
 136   [PERF_COUNT_HW_CACHE_REFERENCES]      = 0x0f2e,
 137   [PERF_COUNT_HW_CACHE_MISSES]          = 0x012e,
 138   [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]   = 0x00c4,
 139   [PERF_COUNT_HW_BRANCH_MISSES]         = 0x00c5,
 140   [PERF_COUNT_HW_BUS_CYCLES]            = 0x0062,
 141 };
 142
 143 static u64 p6_pmu_event_map(int hw_event)
 144 {
 145         return p6_perfmon_event_map[hw_event];
 146 }
 147
 148 /*
 149  * Event setting that is specified not to count anything.
 150  * We use this to effectively disable a counter.
 151  *
 152  * L2_RQSTS with 0 MESI unit mask.
 153  */
 154 #define P6_NOP_EVENT                    0x0000002EULL
 155
 156 static u64 p6_pmu_raw_event(u64 hw_event)
 157 {
 158 #define P6_EVNTSEL_EVENT_MASK           0x000000FFULL
 159 #define P6_EVNTSEL_UNIT_MASK            0x0000FF00ULL
 160 #define P6_EVNTSEL_EDGE_MASK            0x00040000ULL
 161 #define P6_EVNTSEL_INV_MASK             0x00800000ULL
 162 #define P6_EVNTSEL_REG_MASK             0xFF000000ULL
 163
 164 #define P6_EVNTSEL_MASK                 \
 165         (P6_EVNTSEL_EVENT_MASK |        \
 166          P6_EVNTSEL_UNIT_MASK  |        \
 167          P6_EVNTSEL_EDGE_MASK  |        \
 168          P6_EVNTSEL_INV_MASK   |        \
 169          P6_EVNTSEL_REG_MASK)
 170
 171         return hw_event & P6_EVNTSEL_MASK;
 172 }
 173
 174 static const struct event_constraint intel_p6_event_constraints[] =
 175 {
 176         EVENT_CONSTRAINT(0xc1, 0x1),    /* FLOPS */
 177         EVENT_CONSTRAINT(0x10, 0x1),    /* FP_COMP_OPS_EXE */
 178         EVENT_CONSTRAINT(0x11, 0x1),    /* FP_ASSIST */
 179         EVENT_CONSTRAINT(0x12, 0x2),    /* MUL */
 180         EVENT_CONSTRAINT(0x13, 0x2),    /* DIV */
 181         EVENT_CONSTRAINT(0x14, 0x1),    /* CYCLES_DIV_BUSY */
 182         EVENT_CONSTRAINT_END
 183 };
 184
 185 /*
 186  * Intel PerfMon v3. Used on Core2 and later.
 187  */
 188 static const u64 intel_perfmon_event_map[] =
 189 {
 190   [PERF_COUNT_HW_CPU_CYCLES]            = 0x003c,
 191   [PERF_COUNT_HW_INSTRUCTIONS]          = 0x00c0,
 192   [PERF_COUNT_HW_CACHE_REFERENCES]      = 0x4f2e,
 193   [PERF_COUNT_HW_CACHE_MISSES]          = 0x412e,
 194   [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]   = 0x00c4,
 195   [PERF_COUNT_HW_BRANCH_MISSES]         = 0x00c5,
 196   [PERF_COUNT_HW_BUS_CYCLES]            = 0x013c,
 197 };
 198
 199 static const struct event_constraint intel_core_event_constraints[] =
 200 {
 201         EVENT_CONSTRAINT(0x10, 0x1),    /* FP_COMP_OPS_EXE */
 202         EVENT_CONSTRAINT(0x11, 0x2),    /* FP_ASSIST */
 203         EVENT_CONSTRAINT(0x12, 0x2),    /* MUL */
 204         EVENT_CONSTRAINT(0x13, 0x2),    /* DIV */
 205         EVENT_CONSTRAINT(0x14, 0x1),    /* CYCLES_DIV_BUSY */
 206         EVENT_CONSTRAINT(0x18, 0x1),    /* IDLE_DURING_DIV */
 207         EVENT_CONSTRAINT(0x19, 0x2),    /* DELAYED_BYPASS */
 208         EVENT_CONSTRAINT(0xa1, 0x1),    /* RS_UOPS_DISPATCH_CYCLES */
 209         EVENT_CONSTRAINT(0xcb, 0x1),    /* MEM_LOAD_RETIRED */
 210         EVENT_CONSTRAINT_END
 211 };
 212
 213 static const struct event_constraint intel_nehalem_event_constraints[] =
 214 {
 215         EVENT_CONSTRAINT(0x40, 0x3),    /* L1D_CACHE_LD */
 216         EVENT_CONSTRAINT(0x41, 0x3),    /* L1D_CACHE_ST */
 217         EVENT_CONSTRAINT(0x42, 0x3),    /* L1D_CACHE_LOCK */
 218         EVENT_CONSTRAINT(0x43, 0x3),    /* L1D_ALL_REF */
 219         EVENT_CONSTRAINT(0x4e, 0x3),    /* L1D_PREFETCH */
 220         EVENT_CONSTRAINT(0x4c, 0x3),    /* LOAD_HIT_PRE */
 221         EVENT_CONSTRAINT(0x51, 0x3),    /* L1D */
 222         EVENT_CONSTRAINT(0x52, 0x3),    /* L1D_CACHE_PREFETCH_LOCK_FB_HIT */
 223         EVENT_CONSTRAINT(0x53, 0x3),    /* L1D_CACHE_LOCK_FB_HIT */
 224         EVENT_CONSTRAINT(0xc5, 0x3),    /* CACHE_LOCK_CYCLES */
 225         EVENT_CONSTRAINT_END
 226 };
 227
 228 static u64 intel_pmu_event_map(int hw_event)
 229 {
 230         return intel_perfmon_event_map[hw_event];
 231 }
 232
 233 /*
 234  * Generalized hw caching related hw_event table, filled
 235  * in on a per model basis. A value of 0 means
 236  * 'not supported', -1 means 'hw_event makes no sense on
 237  * this CPU', any other value means the raw hw_event
 238  * ID.
 239  */
 240
 241 #define C(x) PERF_COUNT_HW_CACHE_##x
 242
 243 static u64 __read_mostly hw_cache_event_ids
 244                                 [PERF_COUNT_HW_CACHE_MAX]
 245                                 [PERF_COUNT_HW_CACHE_OP_MAX]
 246                                 [PERF_COUNT_HW_CACHE_RESULT_MAX];
 247
 248 static __initconst u64 nehalem_hw_cache_event_ids
 249                                 [PERF_COUNT_HW_CACHE_MAX]
 250                                 [PERF_COUNT_HW_CACHE_OP_MAX]
 251                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
 252 {
 253  [ C(L1D) ] = {
 254         [ C(OP_READ) ] = {
 255                 [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI            */
 256                 [ C(RESULT_MISS)   ] = 0x0140, /* L1D_CACHE_LD.I_STATE         */
 257         },
 258         [ C(OP_WRITE) ] = {
 259                 [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI            */
 260                 [ C(RESULT_MISS)   ] = 0x0141, /* L1D_CACHE_ST.I_STATE         */
 261         },
 262         [ C(OP_PREFETCH) ] = {
 263                 [ C(RESULT_ACCESS) ] = 0x014e, /* L1D_PREFETCH.REQUESTS        */
 264                 [ C(RESULT_MISS)   ] = 0x024e, /* L1D_PREFETCH.MISS            */
 265         },
 266  },
 267  [ C(L1I ) ] = {
 268         [ C(OP_READ) ] = {
 269                 [ C(RESULT_ACCESS) ] = 0x0380, /* L1I.READS                    */
 270                 [ C(RESULT_MISS)   ] = 0x0280, /* L1I.MISSES                   */
 271         },
 272         [ C(OP_WRITE) ] = {
 273                 [ C(RESULT_ACCESS) ] = -1,
 274                 [ C(RESULT_MISS)   ] = -1,
 275         },
 276         [ C(OP_PREFETCH) ] = {
 277                 [ C(RESULT_ACCESS) ] = 0x0,
 278                 [ C(RESULT_MISS)   ] = 0x0,
 279         },
 280  },
 281  [ C(LL  ) ] = {
 282         [ C(OP_READ) ] = {
 283                 [ C(RESULT_ACCESS) ] = 0x0324, /* L2_RQSTS.LOADS               */
 284                 [ C(RESULT_MISS)   ] = 0x0224, /* L2_RQSTS.LD_MISS             */
 285         },
 286         [ C(OP_WRITE) ] = {
 287                 [ C(RESULT_ACCESS) ] = 0x0c24, /* L2_RQSTS.RFOS                */
 288                 [ C(RESULT_MISS)   ] = 0x0824, /* L2_RQSTS.RFO_MISS            */
 289         },
 290         [ C(OP_PREFETCH) ] = {
 291                 [ C(RESULT_ACCESS) ] = 0x4f2e, /* LLC Reference                */
 292                 [ C(RESULT_MISS)   ] = 0x412e, /* LLC Misses                   */
 293         },
 294  },
 295  [ C(DTLB) ] = {
 296         [ C(OP_READ) ] = {
 297                 [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI   (alias)  */
 298                 [ C(RESULT_MISS)   ] = 0x0108, /* DTLB_LOAD_MISSES.ANY         */
 299         },
 300         [ C(OP_WRITE) ] = {
 301                 [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI   (alias)  */
 302                 [ C(RESULT_MISS)   ] = 0x010c, /* MEM_STORE_RETIRED.DTLB_MISS  */
 303         },
 304         [ C(OP_PREFETCH) ] = {
 305                 [ C(RESULT_ACCESS) ] = 0x0,
 306                 [ C(RESULT_MISS)   ] = 0x0,
 307         },
 308  },
 309  [ C(ITLB) ] = {
 310         [ C(OP_READ) ] = {
 311                 [ C(RESULT_ACCESS) ] = 0x01c0, /* INST_RETIRED.ANY_P           */
 312                 [ C(RESULT_MISS)   ] = 0x20c8, /* ITLB_MISS_RETIRED            */
 313         },
 314         [ C(OP_WRITE) ] = {
 315                 [ C(RESULT_ACCESS) ] = -1,
 316                 [ C(RESULT_MISS)   ] = -1,
 317         },
 318         [ C(OP_PREFETCH) ] = {
 319                 [ C(RESULT_ACCESS) ] = -1,
 320                 [ C(RESULT_MISS)   ] = -1,
 321         },
 322  },
 323  [ C(BPU ) ] = {
 324         [ C(OP_READ) ] = {
 325                 [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ALL_BRANCHES */
 326                 [ C(RESULT_MISS)   ] = 0x03e8, /* BPU_CLEARS.ANY               */
 327         },
 328         [ C(OP_WRITE) ] = {
 329                 [ C(RESULT_ACCESS) ] = -1,
 330                 [ C(RESULT_MISS)   ] = -1,
 331         },
 332         [ C(OP_PREFETCH) ] = {
 333                 [ C(RESULT_ACCESS) ] = -1,
 334                 [ C(RESULT_MISS)   ] = -1,
 335         },
 336  },
 337 };
 338
 339 static __initconst u64 core2_hw_cache_event_ids
 340                                 [PERF_COUNT_HW_CACHE_MAX]
 341                                 [PERF_COUNT_HW_CACHE_OP_MAX]
 342                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
 343 {
 344  [ C(L1D) ] = {
 345         [ C(OP_READ) ] = {
 346                 [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI          */
 347                 [ C(RESULT_MISS)   ] = 0x0140, /* L1D_CACHE_LD.I_STATE       */
 348         },
 349         [ C(OP_WRITE) ] = {
 350                 [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI          */
 351                 [ C(RESULT_MISS)   ] = 0x0141, /* L1D_CACHE_ST.I_STATE       */
 352         },
 353         [ C(OP_PREFETCH) ] = {
 354                 [ C(RESULT_ACCESS) ] = 0x104e, /* L1D_PREFETCH.REQUESTS      */
 355                 [ C(RESULT_MISS)   ] = 0,
 356         },
 357  },
 358  [ C(L1I ) ] = {
 359         [ C(OP_READ) ] = {
 360                 [ C(RESULT_ACCESS) ] = 0x0080, /* L1I.READS                  */
 361                 [ C(RESULT_MISS)   ] = 0x0081, /* L1I.MISSES                 */
 362         },
 363         [ C(OP_WRITE) ] = {
 364                 [ C(RESULT_ACCESS) ] = -1,
 365                 [ C(RESULT_MISS)   ] = -1,
 366         },
 367         [ C(OP_PREFETCH) ] = {
 368                 [ C(RESULT_ACCESS) ] = 0,
 369                 [ C(RESULT_MISS)   ] = 0,
 370         },
 371  },
 372  [ C(LL  ) ] = {
 373         [ C(OP_READ) ] = {
 374                 [ C(RESULT_ACCESS) ] = 0x4f29, /* L2_LD.MESI                 */
 375                 [ C(RESULT_MISS)   ] = 0x4129, /* L2_LD.ISTATE               */
 376         },
 377         [ C(OP_WRITE) ] = {
 378                 [ C(RESULT_ACCESS) ] = 0x4f2A, /* L2_ST.MESI                 */
 379                 [ C(RESULT_MISS)   ] = 0x412A, /* L2_ST.ISTATE               */
 380         },
 381         [ C(OP_PREFETCH) ] = {
 382                 [ C(RESULT_ACCESS) ] = 0,
 383                 [ C(RESULT_MISS)   ] = 0,
 384         },
 385  },
 386  [ C(DTLB) ] = {
 387         [ C(OP_READ) ] = {
 388                 [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI  (alias) */
 389                 [ C(RESULT_MISS)   ] = 0x0208, /* DTLB_MISSES.MISS_LD        */
 390         },
 391         [ C(OP_WRITE) ] = {
 392                 [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI  (alias) */
 393                 [ C(RESULT_MISS)   ] = 0x0808, /* DTLB_MISSES.MISS_ST        */
 394         },
 395         [ C(OP_PREFETCH) ] = {
 396                 [ C(RESULT_ACCESS) ] = 0,
 397                 [ C(RESULT_MISS)   ] = 0,
 398         },
 399  },
 400  [ C(ITLB) ] = {
 401         [ C(OP_READ) ] = {
 402                 [ C(RESULT_ACCESS) ] = 0x00c0, /* INST_RETIRED.ANY_P         */
 403                 [ C(RESULT_MISS)   ] = 0x1282, /* ITLBMISSES                 */
 404         },
 405         [ C(OP_WRITE) ] = {
 406                 [ C(RESULT_ACCESS) ] = -1,
 407                 [ C(RESULT_MISS)   ] = -1,
 408         },
 409         [ C(OP_PREFETCH) ] = {
 410                 [ C(RESULT_ACCESS) ] = -1,
 411                 [ C(RESULT_MISS)   ] = -1,
 412         },
 413  },
 414  [ C(BPU ) ] = {
 415         [ C(OP_READ) ] = {
 416                 [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ANY        */
 417                 [ C(RESULT_MISS)   ] = 0x00c5, /* BP_INST_RETIRED.MISPRED    */
 418         },
 419         [ C(OP_WRITE) ] = {
 420                 [ C(RESULT_ACCESS) ] = -1,
 421                 [ C(RESULT_MISS)   ] = -1,
 422         },
 423         [ C(OP_PREFETCH) ] = {
 424                 [ C(RESULT_ACCESS) ] = -1,
 425                 [ C(RESULT_MISS)   ] = -1,
 426         },
 427  },
 428 };
 429
 430 static __initconst u64 atom_hw_cache_event_ids
 431                                 [PERF_COUNT_HW_CACHE_MAX]
 432                                 [PERF_COUNT_HW_CACHE_OP_MAX]
 433                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
 434 {
 435  [ C(L1D) ] = {
 436         [ C(OP_READ) ] = {
 437                 [ C(RESULT_ACCESS) ] = 0x2140, /* L1D_CACHE.LD               */
 438                 [ C(RESULT_MISS)   ] = 0,
 439         },
 440         [ C(OP_WRITE) ] = {
 441                 [ C(RESULT_ACCESS) ] = 0x2240, /* L1D_CACHE.ST               */
 442                 [ C(RESULT_MISS)   ] = 0,
 443         },
 444         [ C(OP_PREFETCH) ] = {
 445                 [ C(RESULT_ACCESS) ] = 0x0,
 446                 [ C(RESULT_MISS)   ] = 0,
 447         },
 448  },
 449  [ C(L1I ) ] = {
 450         [ C(OP_READ) ] = {
 451                 [ C(RESULT_ACCESS) ] = 0x0380, /* L1I.READS                  */
 452                 [ C(RESULT_MISS)   ] = 0x0280, /* L1I.MISSES                 */
 453         },
 454         [ C(OP_WRITE) ] = {
 455                 [ C(RESULT_ACCESS) ] = -1,
 456                 [ C(RESULT_MISS)   ] = -1,
 457         },
 458         [ C(OP_PREFETCH) ] = {
 459                 [ C(RESULT_ACCESS) ] = 0,
 460                 [ C(RESULT_MISS)   ] = 0,
 461         },
 462  },
 463  [ C(LL  ) ] = {
 464         [ C(OP_READ) ] = {
 465                 [ C(RESULT_ACCESS) ] = 0x4f29, /* L2_LD.MESI                 */
 466                 [ C(RESULT_MISS)   ] = 0x4129, /* L2_LD.ISTATE               */
 467         },
 468         [ C(OP_WRITE) ] = {
 469                 [ C(RESULT_ACCESS) ] = 0x4f2A, /* L2_ST.MESI                 */
 470                 [ C(RESULT_MISS)   ] = 0x412A, /* L2_ST.ISTATE               */
 471         },
 472         [ C(OP_PREFETCH) ] = {
 473                 [ C(RESULT_ACCESS) ] = 0,
 474                 [ C(RESULT_MISS)   ] = 0,
 475         },
 476  },
 477  [ C(DTLB) ] = {
 478         [ C(OP_READ) ] = {
 479                 [ C(RESULT_ACCESS) ] = 0x2140, /* L1D_CACHE_LD.MESI  (alias) */
 480                 [ C(RESULT_MISS)   ] = 0x0508, /* DTLB_MISSES.MISS_LD        */
 481         },
 482         [ C(OP_WRITE) ] = {
 483                 [ C(RESULT_ACCESS) ] = 0x2240, /* L1D_CACHE_ST.MESI  (alias) */
 484                 [ C(RESULT_MISS)   ] = 0x0608, /* DTLB_MISSES.MISS_ST        */
 485         },
 486         [ C(OP_PREFETCH) ] = {
 487                 [ C(RESULT_ACCESS) ] = 0,
 488                 [ C(RESULT_MISS)   ] = 0,
 489         },
 490  },
 491  [ C(ITLB) ] = {
 492         [ C(OP_READ) ] = {
 493                 [ C(RESULT_ACCESS) ] = 0x00c0, /* INST_RETIRED.ANY_P         */
 494                 [ C(RESULT_MISS)   ] = 0x0282, /* ITLB.MISSES                */
 495         },
 496         [ C(OP_WRITE) ] = {
 497                 [ C(RESULT_ACCESS) ] = -1,
 498                 [ C(RESULT_MISS)   ] = -1,
 499         },
 500         [ C(OP_PREFETCH) ] = {
 501                 [ C(RESULT_ACCESS) ] = -1,
 502                 [ C(RESULT_MISS)   ] = -1,
 503         },
 504  },
 505  [ C(BPU ) ] = {
 506         [ C(OP_READ) ] = {
 507                 [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ANY        */
 508                 [ C(RESULT_MISS)   ] = 0x00c5, /* BP_INST_RETIRED.MISPRED    */
 509         },
 510         [ C(OP_WRITE) ] = {
 511                 [ C(RESULT_ACCESS) ] = -1,
 512                 [ C(RESULT_MISS)   ] = -1,
 513         },
 514         [ C(OP_PREFETCH) ] = {
 515                 [ C(RESULT_ACCESS) ] = -1,
 516                 [ C(RESULT_MISS)   ] = -1,
 517         },
 518  },
 519 };
 520
 521 static u64 intel_pmu_raw_event(u64 hw_event)
 522 {
 523 #define CORE_EVNTSEL_EVENT_MASK         0x000000FFULL
 524 #define CORE_EVNTSEL_UNIT_MASK          0x0000FF00ULL
 525 #define CORE_EVNTSEL_EDGE_MASK          0x00040000ULL
 526 #define CORE_EVNTSEL_INV_MASK           0x00800000ULL
 527 #define CORE_EVNTSEL_REG_MASK           0xFF000000ULL
 528
 529 #define CORE_EVNTSEL_MASK               \
 530         (CORE_EVNTSEL_EVENT_MASK |      \
 531          CORE_EVNTSEL_UNIT_MASK  |      \
 532          CORE_EVNTSEL_EDGE_MASK  |      \
 533          CORE_EVNTSEL_INV_MASK  |       \
 534          CORE_EVNTSEL_REG_MASK)
 535
 536         return hw_event & CORE_EVNTSEL_MASK;
 537 }
 538
 539 static __initconst u64 amd_hw_cache_event_ids
 540                                 [PERF_COUNT_HW_CACHE_MAX]
 541                                 [PERF_COUNT_HW_CACHE_OP_MAX]
 542                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
 543 {
 544  [ C(L1D) ] = {
 545         [ C(OP_READ) ] = {
 546                 [ C(RESULT_ACCESS) ] = 0x0040, /* Data Cache Accesses        */
 547                 [ C(RESULT_MISS)   ] = 0x0041, /* Data Cache Misses          */
 548         },
 549         [ C(OP_WRITE) ] = {
 550                 [ C(RESULT_ACCESS) ] = 0x0142, /* Data Cache Refills :system */
 551                 [ C(RESULT_MISS)   ] = 0,
 552         },
 553         [ C(OP_PREFETCH) ] = {
 554                 [ C(RESULT_ACCESS) ] = 0x0267, /* Data Prefetcher :attempts  */
 555                 [ C(RESULT_MISS)   ] = 0x0167, /* Data Prefetcher :cancelled */
 556         },
 557  },
 558  [ C(L1I ) ] = {
 559         [ C(OP_READ) ] = {
 560                 [ C(RESULT_ACCESS) ] = 0x0080, /* Instruction cache fetches  */
 561                 [ C(RESULT_MISS)   ] = 0x0081, /* Instruction cache misses   */
 562         },
 563         [ C(OP_WRITE) ] = {
 564                 [ C(RESULT_ACCESS) ] = -1,
 565                 [ C(RESULT_MISS)   ] = -1,
 566         },
 567         [ C(OP_PREFETCH) ] = {
 568                 [ C(RESULT_ACCESS) ] = 0x014B, /* Prefetch Instructions :Load */
 569                 [ C(RESULT_MISS)   ] = 0,
 570         },
 571  },
 572  [ C(LL  ) ] = {
 573         [ C(OP_READ) ] = {
 574                 [ C(RESULT_ACCESS) ] = 0x037D, /* Requests to L2 Cache :IC+DC */
 575                 [ C(RESULT_MISS)   ] = 0x037E, /* L2 Cache Misses : IC+DC     */
 576         },
 577         [ C(OP_WRITE) ] = {
 578                 [ C(RESULT_ACCESS) ] = 0x017F, /* L2 Fill/Writeback           */
 579                 [ C(RESULT_MISS)   ] = 0,
 580         },
 581         [ C(OP_PREFETCH) ] = {
 582                 [ C(RESULT_ACCESS) ] = 0,
 583                 [ C(RESULT_MISS)   ] = 0,
 584         },
 585  },
 586  [ C(DTLB) ] = {
 587         [ C(OP_READ) ] = {
 588                 [ C(RESULT_ACCESS) ] = 0x0040, /* Data Cache Accesses        */
 589                 [ C(RESULT_MISS)   ] = 0x0046, /* L1 DTLB and L2 DLTB Miss   */
 590         },
 591         [ C(OP_WRITE) ] = {
 592                 [ C(RESULT_ACCESS) ] = 0,
 593                 [ C(RESULT_MISS)   ] = 0,
 594         },
 595         [ C(OP_PREFETCH) ] = {
 596                 [ C(RESULT_ACCESS) ] = 0,
 597                 [ C(RESULT_MISS)   ] = 0,
 598         },
 599  },
 600  [ C(ITLB) ] = {
 601         [ C(OP_READ) ] = {
 602                 [ C(RESULT_ACCESS) ] = 0x0080, /* Instruction fecthes        */
 603                 [ C(RESULT_MISS)   ] = 0x0085, /* Instr. fetch ITLB misses   */
 604         },
 605         [ C(OP_WRITE) ] = {
 606                 [ C(RESULT_ACCESS) ] = -1,
 607                 [ C(RESULT_MISS)   ] = -1,
 608         },
 609         [ C(OP_PREFETCH) ] = {
 610                 [ C(RESULT_ACCESS) ] = -1,
 611                 [ C(RESULT_MISS)   ] = -1,
 612         },
 613  },
 614  [ C(BPU ) ] = {
 615         [ C(OP_READ) ] = {
 616                 [ C(RESULT_ACCESS) ] = 0x00c2, /* Retired Branch Instr.      */
 617                 [ C(RESULT_MISS)   ] = 0x00c3, /* Retired Mispredicted BI    */
 618         },
 619         [ C(OP_WRITE) ] = {
 620                 [ C(RESULT_ACCESS) ] = -1,
 621                 [ C(RESULT_MISS)   ] = -1,
 622         },
 623         [ C(OP_PREFETCH) ] = {
 624                 [ C(RESULT_ACCESS) ] = -1,
 625                 [ C(RESULT_MISS)   ] = -1,
 626         },
 627  },
 628 };
 629
 630 /*
 631  * AMD Performance Monitor K7 and later.
 632  */
 633 static const u64 amd_perfmon_event_map[] =
 634 {
 635   [PERF_COUNT_HW_CPU_CYCLES]            = 0x0076,
 636   [PERF_COUNT_HW_INSTRUCTIONS]          = 0x00c0,
 637   [PERF_COUNT_HW_CACHE_REFERENCES]      = 0x0080,
 638   [PERF_COUNT_HW_CACHE_MISSES]          = 0x0081,
 639   [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]   = 0x00c4,
 640   [PERF_COUNT_HW_BRANCH_MISSES]         = 0x00c5,
 641 };
 642
 643 static u64 amd_pmu_event_map(int hw_event)
 644 {
 645         return amd_perfmon_event_map[hw_event];
 646 }
 647
 648 static u64 amd_pmu_raw_event(u64 hw_event)
 649 {
 650 #define K7_EVNTSEL_EVENT_MASK   0x7000000FFULL
 651 #define K7_EVNTSEL_UNIT_MASK    0x00000FF00ULL
 652 #define K7_EVNTSEL_EDGE_MASK    0x000040000ULL
 653 #define K7_EVNTSEL_INV_MASK     0x000800000ULL
 654 #define K7_EVNTSEL_REG_MASK     0x0FF000000ULL
 655
 656 #define K7_EVNTSEL_MASK                 \
 657         (K7_EVNTSEL_EVENT_MASK |        \
 658          K7_EVNTSEL_UNIT_MASK  |        \
 659          K7_EVNTSEL_EDGE_MASK  |        \
 660          K7_EVNTSEL_INV_MASK   |        \
 661          K7_EVNTSEL_REG_MASK)
 662
 663         return hw_event & K7_EVNTSEL_MASK;
 664 }
 665
 666 /*
 667  * Propagate event elapsed time into the generic event.
 668  * Can only be executed on the CPU where the event is active.
 669  * Returns the delta events processed.
 670  */
 671 static u64
 672 x86_perf_event_update(struct perf_event *event,
 673                         struct hw_perf_event *hwc, int idx)
 674 {
 675         int shift = 64 - x86_pmu.event_bits;
 676         u64 prev_raw_count, new_raw_count;
 677         s64 delta;
 678
 679         if (idx == X86_PMC_IDX_FIXED_BTS)
 680                 return 0;
 681
 682         /*
 683          * Careful: an NMI might modify the previous event value.
 684          *
 685          * Our tactic to handle this is to first atomically read and
 686          * exchange a new raw count - then add that new-prev delta
 687          * count to the generic event atomically:
 688          */
 689 again:
 690         prev_raw_count = atomic64_read(&hwc->prev_count);
 691         rdmsrl(hwc->event_base + idx, new_raw_count);
 692
 693         if (atomic64_cmpxchg(&hwc->prev_count, prev_raw_count,
 694                                         new_raw_count) != prev_raw_count)
 695                 goto again;
 696
 697         /*
 698          * Now we have the new raw value and have updated the prev
 699          * timestamp already. We can now calculate the elapsed delta
 700          * (event-)time and add that to the generic event.
 701          *
 702          * Careful, not all hw sign-extends above the physical width
 703          * of the count.
 704          */
 705         delta = (new_raw_count << shift) - (prev_raw_count << shift);
 706         delta >>= shift;
 707
 708         atomic64_add(delta, &event->count);
 709         atomic64_sub(delta, &hwc->period_left);
 710
 711         return new_raw_count;
 712 }
 713
 714 static atomic_t active_events;
 715 static DEFINE_MUTEX(pmc_reserve_mutex);
 716
 717 static bool reserve_pmc_hardware(void)
 718 {
 719 #ifdef CONFIG_X86_LOCAL_APIC
 720         int i;
 721
 722         if (nmi_watchdog == NMI_LOCAL_APIC)
 723                 disable_lapic_nmi_watchdog();
 724
 725         for (i = 0; i < x86_pmu.num_events; i++) {
 726                 if (!reserve_perfctr_nmi(x86_pmu.perfctr + i))
 727                         goto perfctr_fail;
 728         }
 729
 730         for (i = 0; i < x86_pmu.num_events; i++) {
 731                 if (!reserve_evntsel_nmi(x86_pmu.eventsel + i))
 732                         goto eventsel_fail;
 733         }
 734 #endif
 735
 736         return true;
 737
 738 #ifdef CONFIG_X86_LOCAL_APIC
 739 eventsel_fail:
 740         for (i--; i >= 0; i--)
 741                 release_evntsel_nmi(x86_pmu.eventsel + i);
 742
 743         i = x86_pmu.num_events;
 744
 745 perfctr_fail:
 746         for (i--; i >= 0; i--)
 747                 release_perfctr_nmi(x86_pmu.perfctr + i);
 748
 749         if (nmi_watchdog == NMI_LOCAL_APIC)
 750                 enable_lapic_nmi_watchdog();
 751
 752         return false;
 753 #endif
 754 }
 755
 756 static void release_pmc_hardware(void)
 757 {
 758 #ifdef CONFIG_X86_LOCAL_APIC
 759         int i;
 760
 761         for (i = 0; i < x86_pmu.num_events; i++) {
 762                 release_perfctr_nmi(x86_pmu.perfctr + i);
 763                 release_evntsel_nmi(x86_pmu.eventsel + i);
 764         }
 765
 766         if (nmi_watchdog == NMI_LOCAL_APIC)
 767                 enable_lapic_nmi_watchdog();
 768 #endif
 769 }
 770
 771 static inline bool bts_available(void)
 772 {
 773         return x86_pmu.enable_bts != NULL;
 774 }
 775
 776 static inline void init_debug_store_on_cpu(int cpu)
 777 {
 778         struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
 779
 780         if (!ds)
 781                 return;
 782
 783         wrmsr_on_cpu(cpu, MSR_IA32_DS_AREA,
 784                      (u32)((u64)(unsigned long)ds),
 785                      (u32)((u64)(unsigned long)ds >> 32));
 786 }
 787
 788 static inline void fini_debug_store_on_cpu(int cpu)
 789 {
 790         if (!per_cpu(cpu_hw_events, cpu).ds)
 791                 return;
 792
 793         wrmsr_on_cpu(cpu, MSR_IA32_DS_AREA, 0, 0);
 794 }
 795
 796 static void release_bts_hardware(void)
 797 {
 798         int cpu;
 799
 800         if (!bts_available())
 801                 return;
 802
 803         get_online_cpus();
 804
 805         for_each_online_cpu(cpu)
 806                 fini_debug_store_on_cpu(cpu);
 807
 808         for_each_possible_cpu(cpu) {
 809                 struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
 810
 811                 if (!ds)
 812                         continue;
 813
 814                 per_cpu(cpu_hw_events, cpu).ds = NULL;
 815
 816                 kfree((void *)(unsigned long)ds->bts_buffer_base);
 817                 kfree(ds);
 818         }
 819
 820         put_online_cpus();
 821 }
 822
 823 static int reserve_bts_hardware(void)
 824 {
 825         int cpu, err = 0;
 826
 827         if (!bts_available())
 828                 return 0;
 829
 830         get_online_cpus();
 831
 832         for_each_possible_cpu(cpu) {
 833                 struct debug_store *ds;
 834                 void *buffer;
 835
 836                 err = -ENOMEM;
 837                 buffer = kzalloc(BTS_BUFFER_SIZE, GFP_KERNEL);
 838                 if (unlikely(!buffer))
 839                         break;
 840
 841                 ds = kzalloc(sizeof(*ds), GFP_KERNEL);
 842                 if (unlikely(!ds)) {
 843                         kfree(buffer);
 844                         break;
 845                 }
 846
 847                 ds->bts_buffer_base = (u64)(unsigned long)buffer;
 848                 ds->bts_index = ds->bts_buffer_base;
 849                 ds->bts_absolute_maximum =
 850                         ds->bts_buffer_base + BTS_BUFFER_SIZE;
 851                 ds->bts_interrupt_threshold =
 852                         ds->bts_absolute_maximum - BTS_OVFL_TH;
 853
 854                 per_cpu(cpu_hw_events, cpu).ds = ds;
 855                 err = 0;
 856         }
 857
 858         if (err)
 859                 release_bts_hardware();
 860         else {
 861                 for_each_online_cpu(cpu)
 862                         init_debug_store_on_cpu(cpu);
 863         }
 864
 865         put_online_cpus();
 866
 867         return err;
 868 }
 869
 870 static void hw_perf_event_destroy(struct perf_event *event)
 871 {
 872         if (atomic_dec_and_mutex_lock(&active_events, &pmc_reserve_mutex)) {
 873                 release_pmc_hardware();
 874                 release_bts_hardware();
 875                 mutex_unlock(&pmc_reserve_mutex);
 876         }
 877 }
 878
 879 static inline int x86_pmu_initialized(void)
 880 {
 881         return x86_pmu.handle_irq != NULL;
 882 }
 883
 884 static inline int
 885 set_ext_hw_attr(struct hw_perf_event *hwc, struct perf_event_attr *attr)
 886 {
 887         unsigned int cache_type, cache_op, cache_result;
 888         u64 config, val;
 889
 890         config = attr->config;
 891
 892         cache_type = (config >>  0) & 0xff;
 893         if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
 894                 return -EINVAL;
 895
 896         cache_op = (config >>  8) & 0xff;
 897         if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
 898                 return -EINVAL;
 899
 900         cache_result = (config >> 16) & 0xff;
 901         if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
 902                 return -EINVAL;
 903
 904         val = hw_cache_event_ids[cache_type][cache_op][cache_result];
 905
 906         if (val == 0)
 907                 return -ENOENT;
 908
 909         if (val == -1)
 910                 return -EINVAL;
 911
 912         hwc->config |= val;
 913
 914         return 0;
 915 }
 916
 917 static void intel_pmu_enable_bts(u64 config)
 918 {
 919         unsigned long debugctlmsr;
 920
 921         debugctlmsr = get_debugctlmsr();
 922
 923         debugctlmsr |= X86_DEBUGCTL_TR;
 924         debugctlmsr |= X86_DEBUGCTL_BTS;
 925         debugctlmsr |= X86_DEBUGCTL_BTINT;
 926
 927         if (!(config & ARCH_PERFMON_EVENTSEL_OS))
 928                 debugctlmsr |= X86_DEBUGCTL_BTS_OFF_OS;
 929
 930         if (!(config & ARCH_PERFMON_EVENTSEL_USR))
 931                 debugctlmsr |= X86_DEBUGCTL_BTS_OFF_USR;
 932
 933         update_debugctlmsr(debugctlmsr);
 934 }
 935
 936 static void intel_pmu_disable_bts(void)
 937 {
 938         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
 939         unsigned long debugctlmsr;
 940
 941         if (!cpuc->ds)
 942                 return;
 943
 944         debugctlmsr = get_debugctlmsr();
 945
 946         debugctlmsr &=
 947                 ~(X86_DEBUGCTL_TR | X86_DEBUGCTL_BTS | X86_DEBUGCTL_BTINT |
 948                   X86_DEBUGCTL_BTS_OFF_OS | X86_DEBUGCTL_BTS_OFF_USR);
 949
 950         update_debugctlmsr(debugctlmsr);
 951 }
 952
 953 /*
 954  * Setup the hardware configuration for a given attr_type
 955  */
 956 static int __hw_perf_event_init(struct perf_event *event)
 957 {
 958         struct perf_event_attr *attr = &event->attr;
 959         struct hw_perf_event *hwc = &event->hw;
 960         u64 config;
 961         int err;
 962
 963         if (!x86_pmu_initialized())
 964                 return -ENODEV;
 965
 966         err = 0;
 967         if (!atomic_inc_not_zero(&active_events)) {
 968                 mutex_lock(&pmc_reserve_mutex);
 969                 if (atomic_read(&active_events) == 0) {
 970                         if (!reserve_pmc_hardware())
 971                                 err = -EBUSY;
 972                         else
 973                                 err = reserve_bts_hardware();
 974                 }
 975                 if (!err)
 976                         atomic_inc(&active_events);
 977                 mutex_unlock(&pmc_reserve_mutex);
 978         }
 979         if (err)
 980                 return err;
 981
 982         event->destroy = hw_perf_event_destroy;
 983
 984         /*
 985          * Generate PMC IRQs:
 986          * (keep 'enabled' bit clear for now)
 987          */
 988         hwc->config = ARCH_PERFMON_EVENTSEL_INT;
 989
 990         hwc->idx = -1;
 991
 992         /*
 993          * Count user and OS events unless requested not to.
 994          */
 995         if (!attr->exclude_user)
 996                 hwc->config |= ARCH_PERFMON_EVENTSEL_USR;
 997         if (!attr->exclude_kernel)
 998                 hwc->config |= ARCH_PERFMON_EVENTSEL_OS;
 999
1000         if (!hwc->sample_period) {
1001                 hwc->sample_period = x86_pmu.max_period;
1002                 hwc->last_period = hwc->sample_period;
1003                 atomic64_set(&hwc->period_left, hwc->sample_period);
1004         } else {
1005                 /*
1006                  * If we have a PMU initialized but no APIC
1007                  * interrupts, we cannot sample hardware
1008                  * events (user-space has to fall back and
1009                  * sample via a hrtimer based software event):
1010                  */
1011                 if (!x86_pmu.apic)
1012                         return -EOPNOTSUPP;
1013         }
1014
1015         /*
1016          * Raw hw_event type provide the config in the hw_event structure
1017          */
1018         if (attr->type == PERF_TYPE_RAW) {
1019                 hwc->config |= x86_pmu.raw_event(attr->config);
1020                 return 0;
1021         }
1022
1023         if (attr->type == PERF_TYPE_HW_CACHE)
1024                 return set_ext_hw_attr(hwc, attr);
1025
1026         if (attr->config >= x86_pmu.max_events)
1027                 return -EINVAL;
1028
1029         /*
1030          * The generic map:
1031          */
1032         config = x86_pmu.event_map(attr->config);
1033
1034         if (config == 0)
1035                 return -ENOENT;
1036
1037         if (config == -1LL)
1038                 return -EINVAL;
1039
1040         /*
1041          * Branch tracing:
1042          */
1043         if ((attr->config == PERF_COUNT_HW_BRANCH_INSTRUCTIONS) &&
1044             (hwc->sample_period == 1)) {
1045                 /* BTS is not supported by this architecture. */
1046                 if (!bts_available())
1047                         return -EOPNOTSUPP;
1048
1049                 /* BTS is currently only allowed for user-mode. */
1050                 if (hwc->config & ARCH_PERFMON_EVENTSEL_OS)
1051                         return -EOPNOTSUPP;
1052         }
1053
1054         hwc->config |= config;
1055
1056         return 0;
1057 }
1058
1059 static void p6_pmu_disable_all(void)
1060 {
1061         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1062         u64 val;
1063
1064         if (!cpuc->enabled)
1065                 return;
1066
1067         cpuc->enabled = 0;
1068         barrier();
1069
1070         /* p6 only has one enable register */
1071         rdmsrl(MSR_P6_EVNTSEL0, val);
1072         val &= ~ARCH_PERFMON_EVENTSEL0_ENABLE;
1073         wrmsrl(MSR_P6_EVNTSEL0, val);
1074 }
1075
1076 static void intel_pmu_disable_all(void)
1077 {
1078         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1079
1080         if (!cpuc->enabled)
1081                 return;
1082
1083         cpuc->enabled = 0;
1084         barrier();
1085
1086         wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0);
1087
1088         if (test_bit(X86_PMC_IDX_FIXED_BTS, cpuc->active_mask))
1089                 intel_pmu_disable_bts();
1090 }
1091
1092 static void amd_pmu_disable_all(void)
1093 {
1094         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1095         int idx;
1096
1097         if (!cpuc->enabled)
1098                 return;
1099
1100         cpuc->enabled = 0;
1101         /*
1102          * ensure we write the disable before we start disabling the
1103          * events proper, so that amd_pmu_enable_event() does the
1104          * right thing.
1105          */
1106         barrier();
1107
1108         for (idx = 0; idx < x86_pmu.num_events; idx++) {
1109                 u64 val;
1110
1111                 if (!test_bit(idx, cpuc->active_mask))
1112                         continue;
1113                 rdmsrl(MSR_K7_EVNTSEL0 + idx, val);
1114                 if (!(val & ARCH_PERFMON_EVENTSEL0_ENABLE))
1115                         continue;
1116                 val &= ~ARCH_PERFMON_EVENTSEL0_ENABLE;
1117                 wrmsrl(MSR_K7_EVNTSEL0 + idx, val);
1118         }
1119 }
1120
1121 void hw_perf_disable(void)
1122 {
1123         if (!x86_pmu_initialized())
1124                 return;
1125         return x86_pmu.disable_all();
1126 }
1127
1128 static void p6_pmu_enable_all(void)
1129 {
1130         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1131         unsigned long val;
1132
1133         if (cpuc->enabled)
1134                 return;
1135
1136         cpuc->enabled = 1;
1137         barrier();
1138
1139         /* p6 only has one enable register */
1140         rdmsrl(MSR_P6_EVNTSEL0, val);
1141         val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
1142         wrmsrl(MSR_P6_EVNTSEL0, val);
1143 }
1144
1145 static void intel_pmu_enable_all(void)
1146 {
1147         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1148
1149         if (cpuc->enabled)
1150                 return;
1151
1152         cpuc->enabled = 1;
1153         barrier();
1154
1155         wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, x86_pmu.intel_ctrl);
1156
1157         if (test_bit(X86_PMC_IDX_FIXED_BTS, cpuc->active_mask)) {
1158                 struct perf_event *event =
1159                         cpuc->events[X86_PMC_IDX_FIXED_BTS];
1160
1161                 if (WARN_ON_ONCE(!event))
1162                         return;
1163
1164                 intel_pmu_enable_bts(event->hw.config);
1165         }
1166 }
1167
1168 static void amd_pmu_enable_all(void)
1169 {
1170         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1171         int idx;
1172
1173         if (cpuc->enabled)
1174                 return;
1175
1176         cpuc->enabled = 1;
1177         barrier();
1178
1179         for (idx = 0; idx < x86_pmu.num_events; idx++) {
1180                 struct perf_event *event = cpuc->events[idx];
1181                 u64 val;
1182
1183                 if (!test_bit(idx, cpuc->active_mask))
1184                         continue;
1185
1186                 val = event->hw.config;
1187                 val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
1188                 wrmsrl(MSR_K7_EVNTSEL0 + idx, val);
1189         }
1190 }
1191
1192 void hw_perf_enable(void)
1193 {
1194         if (!x86_pmu_initialized())
1195                 return;
1196         x86_pmu.enable_all();
1197 }
1198
1199 static inline u64 intel_pmu_get_status(void)
1200 {
1201         u64 status;
1202
1203         rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
1204
1205         return status;
1206 }
1207
1208 static inline void intel_pmu_ack_status(u64 ack)
1209 {
1210         wrmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, ack);
1211 }
1212
1213 static inline void x86_pmu_enable_event(struct hw_perf_event *hwc, int idx)
1214 {
1215         (void)checking_wrmsrl(hwc->config_base + idx,
1216                               hwc->config | ARCH_PERFMON_EVENTSEL0_ENABLE);
1217 }
1218
1219 static inline void x86_pmu_disable_event(struct hw_perf_event *hwc, int idx)
1220 {
1221         (void)checking_wrmsrl(hwc->config_base + idx, hwc->config);
1222 }
1223
1224 static inline void
1225 intel_pmu_disable_fixed(struct hw_perf_event *hwc, int __idx)
1226 {
1227         int idx = __idx - X86_PMC_IDX_FIXED;
1228         u64 ctrl_val, mask;
1229
1230         mask = 0xfULL << (idx * 4);
1231
1232         rdmsrl(hwc->config_base, ctrl_val);
1233         ctrl_val &= ~mask;
1234         (void)checking_wrmsrl(hwc->config_base, ctrl_val);
1235 }
1236
1237 static inline void
1238 p6_pmu_disable_event(struct hw_perf_event *hwc, int idx)
1239 {
1240         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1241         u64 val = P6_NOP_EVENT;
1242
1243         if (cpuc->enabled)
1244                 val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
1245
1246         (void)checking_wrmsrl(hwc->config_base + idx, val);
1247 }
1248
1249 static inline void
1250 intel_pmu_disable_event(struct hw_perf_event *hwc, int idx)
1251 {
1252         if (unlikely(idx == X86_PMC_IDX_FIXED_BTS)) {
1253                 intel_pmu_disable_bts();
1254                 return;
1255         }
1256
1257         if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
1258                 intel_pmu_disable_fixed(hwc, idx);
1259                 return;
1260         }
1261
1262         x86_pmu_disable_event(hwc, idx);
1263 }
1264
1265 static inline void
1266 amd_pmu_disable_event(struct hw_perf_event *hwc, int idx)
1267 {
1268         x86_pmu_disable_event(hwc, idx);
1269 }
1270
1271 static DEFINE_PER_CPU(u64 [X86_PMC_IDX_MAX], pmc_prev_left);
1272
1273 /*
1274  * Set the next IRQ period, based on the hwc->period_left value.
1275  * To be called with the event disabled in hw:
1276  */
1277 static int
1278 x86_perf_event_set_period(struct perf_event *event,
1279                              struct hw_perf_event *hwc, int idx)
1280 {
1281         s64 left = atomic64_read(&hwc->period_left);
1282         s64 period = hwc->sample_period;
1283         int err, ret = 0;
1284
1285         if (idx == X86_PMC_IDX_FIXED_BTS)
1286                 return 0;
1287
1288         /*
1289          * If we are way outside a reasonable range then just skip forward:
1290          */
1291         if (unlikely(left <= -period)) {
1292                 left = period;
1293                 atomic64_set(&hwc->period_left, left);
1294                 hwc->last_period = period;
1295                 ret = 1;
1296         }
1297
1298         if (unlikely(left <= 0)) {
1299                 left += period;
1300                 atomic64_set(&hwc->period_left, left);
1301                 hwc->last_period = period;
1302                 ret = 1;
1303         }
1304         /*
1305          * Quirk: certain CPUs dont like it if just 1 hw_event is left:
1306          */
1307         if (unlikely(left < 2))
1308                 left = 2;
1309
1310         if (left > x86_pmu.max_period)
1311                 left = x86_pmu.max_period;
1312
1313         per_cpu(pmc_prev_left[idx], smp_processor_id()) = left;
1314
1315         /*
1316          * The hw event starts counting from this event offset,
1317          * mark it to be able to extra future deltas:
1318          */
1319         atomic64_set(&hwc->prev_count, (u64)-left);
1320
1321         err = checking_wrmsrl(hwc->event_base + idx,
1322                              (u64)(-left) & x86_pmu.event_mask);
1323
1324         perf_event_update_userpage(event);
1325
1326         return ret;
1327 }
1328
1329 static inline void
1330 intel_pmu_enable_fixed(struct hw_perf_event *hwc, int __idx)
1331 {
1332         int idx = __idx - X86_PMC_IDX_FIXED;
1333         u64 ctrl_val, bits, mask;
1334         int err;
1335
1336         /*
1337          * Enable IRQ generation (0x8),
1338          * and enable ring-3 counting (0x2) and ring-0 counting (0x1)
1339          * if requested:
1340          */
1341         bits = 0x8ULL;
1342         if (hwc->config & ARCH_PERFMON_EVENTSEL_USR)
1343                 bits |= 0x2;
1344         if (hwc->config & ARCH_PERFMON_EVENTSEL_OS)
1345                 bits |= 0x1;
1346         bits <<= (idx * 4);
1347         mask = 0xfULL << (idx * 4);
1348
1349         rdmsrl(hwc->config_base, ctrl_val);
1350         ctrl_val &= ~mask;
1351         ctrl_val |= bits;
1352         err = checking_wrmsrl(hwc->config_base, ctrl_val);
1353 }
1354
1355 static void p6_pmu_enable_event(struct hw_perf_event *hwc, int idx)
1356 {
1357         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1358         u64 val;
1359
1360         val = hwc->config;
1361         if (cpuc->enabled)
1362                 val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
1363
1364         (void)checking_wrmsrl(hwc->config_base + idx, val);
1365 }
1366
1367
1368 static void intel_pmu_enable_event(struct hw_perf_event *hwc, int idx)
1369 {
1370         if (unlikely(idx == X86_PMC_IDX_FIXED_BTS)) {
1371                 if (!__get_cpu_var(cpu_hw_events).enabled)
1372                         return;
1373
1374                 intel_pmu_enable_bts(hwc->config);
1375                 return;
1376         }
1377
1378         if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
1379                 intel_pmu_enable_fixed(hwc, idx);
1380                 return;
1381         }
1382
1383         x86_pmu_enable_event(hwc, idx);
1384 }
1385
1386 static void amd_pmu_enable_event(struct hw_perf_event *hwc, int idx)
1387 {
1388         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1389
1390         if (cpuc->enabled)
1391                 x86_pmu_enable_event(hwc, idx);
1392 }
1393
1394 static int fixed_mode_idx(struct hw_perf_event *hwc)
1395 {
1396         unsigned int hw_event;
1397
1398         hw_event = hwc->config & ARCH_PERFMON_EVENT_MASK;
1399
1400         if (unlikely((hw_event ==
1401                       x86_pmu.event_map(PERF_COUNT_HW_BRANCH_INSTRUCTIONS)) &&
1402                      (hwc->sample_period == 1)))
1403                 return X86_PMC_IDX_FIXED_BTS;
1404
1405         if (!x86_pmu.num_events_fixed)
1406                 return -1;
1407
1408         /*
1409          * fixed counters do not take all possible filters
1410          */
1411         if (hwc->config & ARCH_PERFMON_EVENT_FILTER_MASK)
1412                 return -1;
1413
1414         if (unlikely(hw_event == x86_pmu.event_map(PERF_COUNT_HW_INSTRUCTIONS)))
1415                 return X86_PMC_IDX_FIXED_INSTRUCTIONS;
1416         if (unlikely(hw_event == x86_pmu.event_map(PERF_COUNT_HW_CPU_CYCLES)))
1417                 return X86_PMC_IDX_FIXED_CPU_CYCLES;
1418         if (unlikely(hw_event == x86_pmu.event_map(PERF_COUNT_HW_BUS_CYCLES)))
1419                 return X86_PMC_IDX_FIXED_BUS_CYCLES;
1420
1421         return -1;
1422 }
1423
1424 /*
1425  * generic counter allocator: get next free counter
1426  */
1427 static int
1428 gen_get_event_idx(struct cpu_hw_events *cpuc, struct hw_perf_event *hwc)
1429 {
1430         int idx;
1431
1432         idx = find_first_zero_bit(cpuc->used_mask, x86_pmu.num_events);
1433         return idx == x86_pmu.num_events ? -1 : idx;
1434 }
1435
1436 /*
1437  * intel-specific counter allocator: check event constraints
1438  */
1439 static int
1440 intel_get_event_idx(struct cpu_hw_events *cpuc, struct hw_perf_event *hwc)
1441 {
1442         const struct event_constraint *event_constraint;
1443         int i, code;
1444
1445         if (!event_constraints)
1446                 goto skip;
1447
1448         code = hwc->config & CORE_EVNTSEL_EVENT_MASK;
1449
1450         for_each_event_constraint(event_constraint, event_constraints) {
1451                 if (code == event_constraint->code) {
1452                         for_each_bit(i, event_constraint->idxmsk, X86_PMC_IDX_MAX) {
1453                                 if (!test_and_set_bit(i, cpuc->used_mask))
1454                                         return i;
1455                         }
1456                         return -1;
1457                 }
1458         }
1459 skip:
1460         return gen_get_event_idx(cpuc, hwc);
1461 }
1462
1463 static int
1464 x86_schedule_event(struct cpu_hw_events *cpuc, struct hw_perf_event *hwc)
1465 {
1466         int idx;
1467
1468         idx = fixed_mode_idx(hwc);
1469         if (idx == X86_PMC_IDX_FIXED_BTS) {
1470                 /* BTS is already occupied. */
1471                 if (test_and_set_bit(idx, cpuc->used_mask))
1472                         return -EAGAIN;
1473
1474                 hwc->config_base        = 0;
1475                 hwc->event_base         = 0;
1476                 hwc->idx                = idx;
1477         } else if (idx >= 0) {
1478                 /*
1479                  * Try to get the fixed event, if that is already taken
1480                  * then try to get a generic event:
1481                  */
1482                 if (test_and_set_bit(idx, cpuc->used_mask))
1483                         goto try_generic;
1484
1485                 hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
1486                 /*
1487                  * We set it so that event_base + idx in wrmsr/rdmsr maps to
1488                  * MSR_ARCH_PERFMON_FIXED_CTR0 ... CTR2:
1489                  */
1490                 hwc->event_base =
1491                         MSR_ARCH_PERFMON_FIXED_CTR0 - X86_PMC_IDX_FIXED;
1492                 hwc->idx = idx;
1493         } else {
1494                 idx = hwc->idx;
1495                 /* Try to get the previous generic event again */
1496                 if (idx == -1 || test_and_set_bit(idx, cpuc->used_mask)) {
1497 try_generic:
1498                         idx = x86_pmu.get_event_idx(cpuc, hwc);
1499                         if (idx == -1)
1500                                 return -EAGAIN;
1501
1502                         set_bit(idx, cpuc->used_mask);
1503                         hwc->idx = idx;
1504                 }
1505                 hwc->config_base = x86_pmu.eventsel;
1506                 hwc->event_base  = x86_pmu.perfctr;
1507         }
1508
1509         return idx;
1510 }
1511
1512 /*
1513  * Find a PMC slot for the freshly enabled / scheduled in event:
1514  */
1515 static int x86_pmu_enable(struct perf_event *event)
1516 {
1517         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1518         struct hw_perf_event *hwc = &event->hw;
1519         int idx;
1520
1521         idx = x86_schedule_event(cpuc, hwc);
1522         if (idx < 0)
1523                 return idx;
1524
1525         perf_events_lapic_init();
1526
1527         x86_pmu.disable(hwc, idx);
1528
1529         cpuc->events[idx] = event;
1530         set_bit(idx, cpuc->active_mask);
1531
1532         x86_perf_event_set_period(event, hwc, idx);
1533         x86_pmu.enable(hwc, idx);
1534
1535         perf_event_update_userpage(event);
1536
1537         return 0;
1538 }
1539
1540 static void x86_pmu_unthrottle(struct perf_event *event)
1541 {
1542         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1543         struct hw_perf_event *hwc = &event->hw;
1544
1545         if (WARN_ON_ONCE(hwc->idx >= X86_PMC_IDX_MAX ||
1546                                 cpuc->events[hwc->idx] != event))
1547                 return;
1548
1549         x86_pmu.enable(hwc, hwc->idx);
1550 }
1551
1552 void perf_event_print_debug(void)
1553 {
1554         u64 ctrl, status, overflow, pmc_ctrl, pmc_count, prev_left, fixed;
1555         struct cpu_hw_events *cpuc;
1556         unsigned long flags;
1557         int cpu, idx;
1558
1559         if (!x86_pmu.num_events)
1560                 return;
1561
1562         local_irq_save(flags);
1563
1564         cpu = smp_processor_id();
1565         cpuc = &per_cpu(cpu_hw_events, cpu);
1566
1567         if (x86_pmu.version >= 2) {
1568                 rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
1569                 rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
1570                 rdmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, overflow);
1571                 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, fixed);
1572
1573                 pr_info("\n");
1574                 pr_info("CPU#%d: ctrl:       %016llx\n", cpu, ctrl);
1575                 pr_info("CPU#%d: status:     %016llx\n", cpu, status);
1576                 pr_info("CPU#%d: overflow:   %016llx\n", cpu, overflow);
1577                 pr_info("CPU#%d: fixed:      %016llx\n", cpu, fixed);
1578         }
1579         pr_info("CPU#%d: used:       %016llx\n", cpu, *(u64 *)cpuc->used_mask);
1580
1581         for (idx = 0; idx < x86_pmu.num_events; idx++) {
1582                 rdmsrl(x86_pmu.eventsel + idx, pmc_ctrl);
1583                 rdmsrl(x86_pmu.perfctr  + idx, pmc_count);
1584
1585                 prev_left = per_cpu(pmc_prev_left[idx], cpu);
1586
1587                 pr_info("CPU#%d:   gen-PMC%d ctrl:  %016llx\n",
1588                         cpu, idx, pmc_ctrl);
1589                 pr_info("CPU#%d:   gen-PMC%d count: %016llx\n",
1590                         cpu, idx, pmc_count);
1591                 pr_info("CPU#%d:   gen-PMC%d left:  %016llx\n",
1592                         cpu, idx, prev_left);
1593         }
1594         for (idx = 0; idx < x86_pmu.num_events_fixed; idx++) {
1595                 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, pmc_count);
1596
1597                 pr_info("CPU#%d: fixed-PMC%d count: %016llx\n",
1598                         cpu, idx, pmc_count);
1599         }
1600         local_irq_restore(flags);
1601 }
1602
1603 static void intel_pmu_drain_bts_buffer(struct cpu_hw_events *cpuc)
1604 {
1605         struct debug_store *ds = cpuc->ds;
1606         struct bts_record {
1607                 u64     from;
1608                 u64     to;
1609                 u64     flags;
1610         };
1611         struct perf_event *event = cpuc->events[X86_PMC_IDX_FIXED_BTS];
1612         struct bts_record *at, *top;
1613         struct perf_output_handle handle;
1614         struct perf_event_header header;
1615         struct perf_sample_data data;
1616         struct pt_regs regs;
1617
1618         if (!event)
1619                 return;
1620
1621         if (!ds)
1622                 return;
1623
1624         at  = (struct bts_record *)(unsigned long)ds->bts_buffer_base;
1625         top = (struct bts_record *)(unsigned long)ds->bts_index;
1626
1627         if (top <= at)
1628                 return;
1629
1630         ds->bts_index = ds->bts_buffer_base;
1631
1632
1633         data.period     = event->hw.last_period;
1634         data.addr       = 0;
1635         data.raw        = NULL;
1636         regs.ip         = 0;
1637
1638         /*
1639          * Prepare a generic sample, i.e. fill in the invariant fields.
1640          * We will overwrite the from and to address before we output
1641          * the sample.
1642          */
1643         perf_prepare_sample(&header, &data, event, &regs);
1644
1645         if (perf_output_begin(&handle, event,
1646                               header.size * (top - at), 1, 1))
1647                 return;
1648
1649         for (; at < top; at++) {
1650                 data.ip         = at->from;
1651                 data.addr       = at->to;
1652
1653                 perf_output_sample(&handle, &header, &data, event);
1654         }
1655
1656         perf_output_end(&handle);
1657
1658         /* There's new data available. */
1659         event->hw.interrupts++;
1660         event->pending_kill = POLL_IN;
1661 }
1662
1663 static void x86_pmu_disable(struct perf_event *event)
1664 {
1665         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1666         struct hw_perf_event *hwc = &event->hw;
1667         int idx = hwc->idx;
1668
1669         /*
1670          * Must be done before we disable, otherwise the nmi handler
1671          * could reenable again:
1672          */
1673         clear_bit(idx, cpuc->active_mask);
1674         x86_pmu.disable(hwc, idx);
1675
1676         /*
1677          * Make sure the cleared pointer becomes visible before we
1678          * (potentially) free the event:
1679          */
1680         barrier();
1681
1682         /*
1683          * Drain the remaining delta count out of a event
1684          * that we are disabling:
1685          */
1686         x86_perf_event_update(event, hwc, idx);
1687
1688         /* Drain the remaining BTS records. */
1689         if (unlikely(idx == X86_PMC_IDX_FIXED_BTS))
1690                 intel_pmu_drain_bts_buffer(cpuc);
1691
1692         cpuc->events[idx] = NULL;
1693         clear_bit(idx, cpuc->used_mask);
1694
1695         perf_event_update_userpage(event);
1696 }
1697
1698 /*
1699  * Save and restart an expired event. Called by NMI contexts,
1700  * so it has to be careful about preempting normal event ops:
1701  */
1702 static int intel_pmu_save_and_restart(struct perf_event *event)
1703 {
1704         struct hw_perf_event *hwc = &event->hw;
1705         int idx = hwc->idx;
1706         int ret;
1707
1708         x86_perf_event_update(event, hwc, idx);
1709         ret = x86_perf_event_set_period(event, hwc, idx);
1710
1711         if (event->state == PERF_EVENT_STATE_ACTIVE)
1712                 intel_pmu_enable_event(hwc, idx);
1713
1714         return ret;
1715 }
1716
1717 static void intel_pmu_reset(void)
1718 {
1719         struct debug_store *ds = __get_cpu_var(cpu_hw_events).ds;
1720         unsigned long flags;
1721         int idx;
1722
1723         if (!x86_pmu.num_events)
1724                 return;
1725
1726         local_irq_save(flags);
1727
1728         printk("clearing PMU state on CPU#%d\n", smp_processor_id());
1729
1730         for (idx = 0; idx < x86_pmu.num_events; idx++) {
1731                 checking_wrmsrl(x86_pmu.eventsel + idx, 0ull);
1732                 checking_wrmsrl(x86_pmu.perfctr  + idx, 0ull);
1733         }
1734         for (idx = 0; idx < x86_pmu.num_events_fixed; idx++) {
1735                 checking_wrmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, 0ull);
1736         }
1737         if (ds)
1738                 ds->bts_index = ds->bts_buffer_base;
1739
1740         local_irq_restore(flags);
1741 }
1742
1743 static int p6_pmu_handle_irq(struct pt_regs *regs)
1744 {
1745         struct perf_sample_data data;
1746         struct cpu_hw_events *cpuc;
1747         struct perf_event *event;
1748         struct hw_perf_event *hwc;
1749         int idx, handled = 0;
1750         u64 val;
1751
1752         data.addr = 0;
1753         data.raw = NULL;
1754
1755         cpuc = &__get_cpu_var(cpu_hw_events);
1756
1757         for (idx = 0; idx < x86_pmu.num_events; idx++) {
1758                 if (!test_bit(idx, cpuc->active_mask))
1759                         continue;
1760
1761                 event = cpuc->events[idx];
1762                 hwc = &event->hw;
1763
1764                 val = x86_perf_event_update(event, hwc, idx);
1765                 if (val & (1ULL << (x86_pmu.event_bits - 1)))
1766                         continue;
1767
1768                 /*
1769                  * event overflow
1770                  */
1771                 handled         = 1;
1772                 data.period     = event->hw.last_period;
1773
1774                 if (!x86_perf_event_set_period(event, hwc, idx))
1775                         continue;
1776
1777                 if (perf_event_overflow(event, 1, &data, regs))
1778                         p6_pmu_disable_event(hwc, idx);
1779         }
1780
1781         if (handled)
1782                 inc_irq_stat(apic_perf_irqs);
1783
1784         return handled;
1785 }
1786
1787 /*
1788  * This handler is triggered by the local APIC, so the APIC IRQ handling
1789  * rules apply:
1790  */
1791 static int intel_pmu_handle_irq(struct pt_regs *regs)
1792 {
1793         struct perf_sample_data data;
1794         struct cpu_hw_events *cpuc;
1795         int bit, loops;
1796         u64 ack, status;
1797
1798         data.addr = 0;
1799         data.raw = NULL;
1800
1801         cpuc = &__get_cpu_var(cpu_hw_events);
1802
1803         perf_disable();
1804         intel_pmu_drain_bts_buffer(cpuc);
1805         status = intel_pmu_get_status();
1806         if (!status) {
1807                 perf_enable();
1808                 return 0;
1809         }
1810
1811         loops = 0;
1812 again:
1813         if (++loops > 100) {
1814                 WARN_ONCE(1, "perfevents: irq loop stuck!\n");
1815                 perf_event_print_debug();
1816                 intel_pmu_reset();
1817                 perf_enable();
1818                 return 1;
1819         }
1820
1821         inc_irq_stat(apic_perf_irqs);
1822         ack = status;
1823         for_each_bit(bit, (unsigned long *)&status, X86_PMC_IDX_MAX) {
1824                 struct perf_event *event = cpuc->events[bit];
1825
1826                 clear_bit(bit, (unsigned long *) &status);
1827                 if (!test_bit(bit, cpuc->active_mask))
1828                         continue;
1829
1830                 if (!intel_pmu_save_and_restart(event))
1831                         continue;
1832
1833                 data.period = event->hw.last_period;
1834
1835                 if (perf_event_overflow(event, 1, &data, regs))
1836                         intel_pmu_disable_event(&event->hw, bit);
1837         }
1838
1839         intel_pmu_ack_status(ack);
1840
1841         /*
1842          * Repeat if there is more work to be done:
1843          */
1844         status = intel_pmu_get_status();
1845         if (status)
1846                 goto again;
1847
1848         perf_enable();
1849
1850         return 1;
1851 }
1852
1853 static int amd_pmu_handle_irq(struct pt_regs *regs)
1854 {
1855         struct perf_sample_data data;
1856         struct cpu_hw_events *cpuc;
1857         struct perf_event *event;
1858         struct hw_perf_event *hwc;
1859         int idx, handled = 0;
1860         u64 val;
1861
1862         data.addr = 0;
1863         data.raw = NULL;
1864
1865         cpuc = &__get_cpu_var(cpu_hw_events);
1866
1867         for (idx = 0; idx < x86_pmu.num_events; idx++) {
1868                 if (!test_bit(idx, cpuc->active_mask))
1869                         continue;
1870
1871                 event = cpuc->events[idx];
1872                 hwc = &event->hw;
1873
1874                 val = x86_perf_event_update(event, hwc, idx);
1875                 if (val & (1ULL << (x86_pmu.event_bits - 1)))
1876                         continue;
1877
1878                 /*
1879                  * event overflow
1880                  */
1881                 handled         = 1;
1882                 data.period     = event->hw.last_period;
1883
1884                 if (!x86_perf_event_set_period(event, hwc, idx))
1885                         continue;
1886
1887                 if (perf_event_overflow(event, 1, &data, regs))
1888                         amd_pmu_disable_event(hwc, idx);
1889         }
1890
1891         if (handled)
1892                 inc_irq_stat(apic_perf_irqs);
1893
1894         return handled;
1895 }
1896
1897 void smp_perf_pending_interrupt(struct pt_regs *regs)
1898 {
1899         irq_enter();
1900         ack_APIC_irq();
1901         inc_irq_stat(apic_pending_irqs);
1902         perf_event_do_pending();
1903         irq_exit();
1904 }
1905
1906 void set_perf_event_pending(void)
1907 {
1908 #ifdef CONFIG_X86_LOCAL_APIC
1909         if (!x86_pmu.apic || !x86_pmu_initialized())
1910                 return;
1911
1912         apic->send_IPI_self(LOCAL_PENDING_VECTOR);
1913 #endif
1914 }
1915
1916 void perf_events_lapic_init(void)
1917 {
1918 #ifdef CONFIG_X86_LOCAL_APIC
1919         if (!x86_pmu.apic || !x86_pmu_initialized())
1920                 return;
1921
1922         /*
1923          * Always use NMI for PMU
1924          */
1925         apic_write(APIC_LVTPC, APIC_DM_NMI);
1926 #endif
1927 }
1928
1929 static int __kprobes
1930 perf_event_nmi_handler(struct notifier_block *self,
1931                          unsigned long cmd, void *__args)
1932 {
1933         struct die_args *args = __args;
1934         struct pt_regs *regs;
1935
1936         if (!atomic_read(&active_events))
1937                 return NOTIFY_DONE;
1938
1939         switch (cmd) {
1940         case DIE_NMI:
1941         case DIE_NMI_IPI:
1942                 break;
1943
1944         default:
1945                 return NOTIFY_DONE;
1946         }
1947
1948         regs = args->regs;
1949
1950 #ifdef CONFIG_X86_LOCAL_APIC
1951         apic_write(APIC_LVTPC, APIC_DM_NMI);
1952 #endif
1953         /*
1954          * Can't rely on the handled return value to say it was our NMI, two
1955          * events could trigger 'simultaneously' raising two back-to-back NMIs.
1956          *
1957          * If the first NMI handles both, the latter will be empty and daze
1958          * the CPU.
1959          */
1960         x86_pmu.handle_irq(regs);
1961
1962         return NOTIFY_STOP;
1963 }
1964
1965 static __read_mostly struct notifier_block perf_event_nmi_notifier = {
1966         .notifier_call          = perf_event_nmi_handler,
1967         .next                   = NULL,
1968         .priority               = 1
1969 };
1970
1971 static __initconst struct x86_pmu p6_pmu = {
1972         .name                   = "p6",
1973         .handle_irq             = p6_pmu_handle_irq,
1974         .disable_all            = p6_pmu_disable_all,
1975         .enable_all             = p6_pmu_enable_all,
1976         .enable                 = p6_pmu_enable_event,
1977         .disable                = p6_pmu_disable_event,
1978         .eventsel               = MSR_P6_EVNTSEL0,
1979         .perfctr                = MSR_P6_PERFCTR0,
1980         .event_map              = p6_pmu_event_map,
1981         .raw_event              = p6_pmu_raw_event,
1982         .max_events             = ARRAY_SIZE(p6_perfmon_event_map),
1983         .apic                   = 1,
1984         .max_period             = (1ULL << 31) - 1,
1985         .version                = 0,
1986         .num_events             = 2,
1987         /*
1988          * Events have 40 bits implemented. However they are designed such
1989          * that bits [32-39] are sign extensions of bit 31. As such the
1990          * effective width of a event for P6-like PMU is 32 bits only.
1991          *
1992          * See IA-32 Intel Architecture Software developer manual Vol 3B
1993          */
1994         .event_bits             = 32,
1995         .event_mask             = (1ULL << 32) - 1,
1996         .get_event_idx          = intel_get_event_idx,
1997 };
1998
1999 static __initconst struct x86_pmu intel_pmu = {
2000         .name                   = "Intel",
2001         .handle_irq             = intel_pmu_handle_irq,
2002         .disable_all            = intel_pmu_disable_all,
2003         .enable_all             = intel_pmu_enable_all,
2004         .enable                 = intel_pmu_enable_event,
2005         .disable                = intel_pmu_disable_event,
2006         .eventsel               = MSR_ARCH_PERFMON_EVENTSEL0,
2007         .perfctr                = MSR_ARCH_PERFMON_PERFCTR0,
2008         .event_map              = intel_pmu_event_map,
2009         .raw_event              = intel_pmu_raw_event,
2010         .max_events             = ARRAY_SIZE(intel_perfmon_event_map),
2011         .apic                   = 1,
2012         /*
2013          * Intel PMCs cannot be accessed sanely above 32 bit width,
2014          * so we install an artificial 1<<31 period regardless of
2015          * the generic event period:
2016          */
2017         .max_period             = (1ULL << 31) - 1,
2018         .enable_bts             = intel_pmu_enable_bts,
2019         .disable_bts            = intel_pmu_disable_bts,
2020         .get_event_idx          = intel_get_event_idx,
2021 };
2022
2023 static __initconst struct x86_pmu amd_pmu = {
2024         .name                   = "AMD",
2025         .handle_irq             = amd_pmu_handle_irq,
2026         .disable_all            = amd_pmu_disable_all,
2027         .enable_all             = amd_pmu_enable_all,
2028         .enable                 = amd_pmu_enable_event,
2029         .disable                = amd_pmu_disable_event,
2030         .eventsel               = MSR_K7_EVNTSEL0,
2031         .perfctr                = MSR_K7_PERFCTR0,
2032         .event_map              = amd_pmu_event_map,
2033         .raw_event              = amd_pmu_raw_event,
2034         .max_events             = ARRAY_SIZE(amd_perfmon_event_map),
2035         .num_events             = 4,
2036         .event_bits             = 48,
2037         .event_mask             = (1ULL << 48) - 1,
2038         .apic                   = 1,
2039         /* use highest bit to detect overflow */
2040         .max_period             = (1ULL << 47) - 1,
2041         .get_event_idx          = gen_get_event_idx,
2042 };
2043
2044 static __init int p6_pmu_init(void)
2045 {
2046         switch (boot_cpu_data.x86_model) {
2047         case 1:
2048         case 3:  /* Pentium Pro */
2049         case 5:
2050         case 6:  /* Pentium II */
2051         case 7:
2052         case 8:
2053         case 11: /* Pentium III */
2054                 event_constraints = intel_p6_event_constraints;
2055                 break;
2056         case 9:
2057         case 13:
2058                 /* Pentium M */
2059                 event_constraints = intel_p6_event_constraints;
2060                 break;
2061         default:
2062                 pr_cont("unsupported p6 CPU model %d ",
2063                         boot_cpu_data.x86_model);
2064                 return -ENODEV;
2065         }
2066
2067         x86_pmu = p6_pmu;
2068
2069         return 0;
2070 }
2071
2072 static __init int intel_pmu_init(void)
2073 {
2074         union cpuid10_edx edx;
2075         union cpuid10_eax eax;
2076         unsigned int unused;
2077         unsigned int ebx;
2078         int version;
2079
2080         if (!cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON)) {
2081                 /* check for P6 processor family */
2082            if (boot_cpu_data.x86 == 6) {
2083                 return p6_pmu_init();
2084            } else {
2085                 return -ENODEV;
2086            }
2087         }
2088
2089         /*
2090          * Check whether the Architectural PerfMon supports
2091          * Branch Misses Retired hw_event or not.
2092          */
2093         cpuid(10, &eax.full, &ebx, &unused, &edx.full);
2094         if (eax.split.mask_length <= ARCH_PERFMON_BRANCH_MISSES_RETIRED)
2095                 return -ENODEV;
2096
2097         version = eax.split.version_id;
2098         if (version < 2)
2099                 return -ENODEV;
2100
2101         x86_pmu                         = intel_pmu;
2102         x86_pmu.version                 = version;
2103         x86_pmu.num_events              = eax.split.num_events;
2104         x86_pmu.event_bits              = eax.split.bit_width;
2105         x86_pmu.event_mask              = (1ULL << eax.split.bit_width) - 1;
2106
2107         /*
2108          * Quirk: v2 perfmon does not report fixed-purpose events, so
2109          * assume at least 3 events:
2110          */
2111         x86_pmu.num_events_fixed        = max((int)edx.split.num_events_fixed, 3);
2112
2113         /*
2114          * Install the hw-cache-events table:
2115          */
2116         switch (boot_cpu_data.x86_model) {
2117         case 15: /* original 65 nm celeron/pentium/core2/xeon, "Merom"/"Conroe" */
2118         case 22: /* single-core 65 nm celeron/core2solo "Merom-L"/"Conroe-L" */
2119         case 23: /* current 45 nm celeron/core2/xeon "Penryn"/"Wolfdale" */
2120         case 29: /* six-core 45 nm xeon "Dunnington" */
2121                 memcpy(hw_cache_event_ids, core2_hw_cache_event_ids,
2122                        sizeof(hw_cache_event_ids));
2123
2124                 pr_cont("Core2 events, ");
2125                 event_constraints = intel_core_event_constraints;
2126                 break;
2127         default:
2128         case 26:
2129                 memcpy(hw_cache_event_ids, nehalem_hw_cache_event_ids,
2130                        sizeof(hw_cache_event_ids));
2131
2132                 event_constraints = intel_nehalem_event_constraints;
2133                 pr_cont("Nehalem/Corei7 events, ");
2134                 break;
2135         case 28:
2136                 memcpy(hw_cache_event_ids, atom_hw_cache_event_ids,
2137                        sizeof(hw_cache_event_ids));
2138
2139                 pr_cont("Atom events, ");
2140                 break;
2141         }
2142         return 0;
2143 }
2144
2145 static __init int amd_pmu_init(void)
2146 {
2147         /* Performance-monitoring supported from K7 and later: */
2148         if (boot_cpu_data.x86 < 6)
2149                 return -ENODEV;
2150
2151         x86_pmu = amd_pmu;
2152
2153         /* Events are common for all AMDs */
2154         memcpy(hw_cache_event_ids, amd_hw_cache_event_ids,
2155                sizeof(hw_cache_event_ids));
2156
2157         return 0;
2158 }
2159
2160 static void __init pmu_check_apic(void)
2161 {
2162         if (cpu_has_apic)
2163                 return;
2164
2165         x86_pmu.apic = 0;
2166         pr_info("no APIC, boot with the \"lapic\" boot parameter to force-enable it.\n");
2167         pr_info("no hardware sampling interrupt available.\n");
2168 }
2169
2170 void __init init_hw_perf_events(void)
2171 {
2172         int err;
2173
2174         pr_info("Performance Events: ");
2175
2176         switch (boot_cpu_data.x86_vendor) {
2177         case X86_VENDOR_INTEL:
2178                 err = intel_pmu_init();
2179                 break;
2180         case X86_VENDOR_AMD:
2181                 err = amd_pmu_init();
2182                 break;
2183         default:
2184                 return;
2185         }
2186         if (err != 0) {
2187                 pr_cont("no PMU driver, software events only.\n");
2188                 return;
2189         }
2190
2191         pmu_check_apic();
2192
2193         pr_cont("%s PMU driver.\n", x86_pmu.name);
2194
2195         if (x86_pmu.num_events > X86_PMC_MAX_GENERIC) {
2196                 WARN(1, KERN_ERR "hw perf events %d > max(%d), clipping!",
2197                      x86_pmu.num_events, X86_PMC_MAX_GENERIC);
2198                 x86_pmu.num_events = X86_PMC_MAX_GENERIC;
2199         }
2200         perf_event_mask = (1 << x86_pmu.num_events) - 1;
2201         perf_max_events = x86_pmu.num_events;
2202
2203         if (x86_pmu.num_events_fixed > X86_PMC_MAX_FIXED) {
2204                 WARN(1, KERN_ERR "hw perf events fixed %d > max(%d), clipping!",
2205                      x86_pmu.num_events_fixed, X86_PMC_MAX_FIXED);
2206                 x86_pmu.num_events_fixed = X86_PMC_MAX_FIXED;
2207         }
2208
2209         perf_event_mask |=
2210                 ((1LL << x86_pmu.num_events_fixed)-1) << X86_PMC_IDX_FIXED;
2211         x86_pmu.intel_ctrl = perf_event_mask;
2212
2213         perf_events_lapic_init();
2214         register_die_notifier(&perf_event_nmi_notifier);
2215
2216         pr_info("... version:                %d\n",     x86_pmu.version);
2217         pr_info("... bit width:              %d\n",     x86_pmu.event_bits);
2218         pr_info("... generic registers:      %d\n",     x86_pmu.num_events);
2219         pr_info("... value mask:             %016Lx\n", x86_pmu.event_mask);
2220         pr_info("... max period:             %016Lx\n", x86_pmu.max_period);
2221         pr_info("... fixed-purpose events:   %d\n",     x86_pmu.num_events_fixed);
2222         pr_info("... event mask:             %016Lx\n", perf_event_mask);
2223 }
2224
2225 static inline void x86_pmu_read(struct perf_event *event)
2226 {
2227         x86_perf_event_update(event, &event->hw, event->hw.idx);
2228 }
2229
2230 static const struct pmu pmu = {
2231         .enable         = x86_pmu_enable,
2232         .disable        = x86_pmu_disable,
2233         .read           = x86_pmu_read,
2234         .unthrottle     = x86_pmu_unthrottle,
2235 };
2236
2237 static int
2238 validate_event(struct cpu_hw_events *cpuc, struct perf_event *event)
2239 {
2240         struct hw_perf_event fake_event = event->hw;
2241
2242         if (event->pmu && event->pmu != &pmu)
2243                 return 0;
2244
2245         return x86_schedule_event(cpuc, &fake_event) >= 0;
2246 }
2247
2248 static int validate_group(struct perf_event *event)
2249 {
2250         struct perf_event *sibling, *leader = event->group_leader;
2251         struct cpu_hw_events fake_pmu;
2252
2253         memset(&fake_pmu, 0, sizeof(fake_pmu));
2254
2255         if (!validate_event(&fake_pmu, leader))
2256                 return -ENOSPC;
2257
2258         list_for_each_entry(sibling, &leader->sibling_list, group_entry) {
2259                 if (!validate_event(&fake_pmu, sibling))
2260                         return -ENOSPC;
2261         }
2262
2263         if (!validate_event(&fake_pmu, event))
2264                 return -ENOSPC;
2265
2266         return 0;
2267 }
2268
2269 const struct pmu *hw_perf_event_init(struct perf_event *event)
2270 {
2271         int err;
2272
2273         err = __hw_perf_event_init(event);
2274         if (!err) {
2275                 if (event->group_leader != event)
2276                         err = validate_group(event);
2277         }
2278         if (err) {
2279                 if (event->destroy)
2280                         event->destroy(event);
2281                 return ERR_PTR(err);
2282         }
2283
2284         return &pmu;
2285 }
2286
2287 /*
2288  * callchain support
2289  */
2290
2291 static inline
2292 void callchain_store(struct perf_callchain_entry *entry, u64 ip)
2293 {
2294         if (entry->nr < PERF_MAX_STACK_DEPTH)
2295                 entry->ip[entry->nr++] = ip;
2296 }
2297
2298 static DEFINE_PER_CPU(struct perf_callchain_entry, pmc_irq_entry);
2299 static DEFINE_PER_CPU(struct perf_callchain_entry, pmc_nmi_entry);
2300 static DEFINE_PER_CPU(int, in_ignored_frame);
2301
2302
2303 static void
2304 backtrace_warning_symbol(void *data, char *msg, unsigned long symbol)
2305 {
2306         /* Ignore warnings */
2307 }
2308
2309 static void backtrace_warning(void *data, char *msg)
2310 {
2311         /* Ignore warnings */
2312 }
2313
2314 static int backtrace_stack(void *data, char *name)
2315 {
2316         per_cpu(in_ignored_frame, smp_processor_id()) =
2317                         x86_is_stack_id(NMI_STACK, name) ||
2318                         x86_is_stack_id(DEBUG_STACK, name);
2319
2320         return 0;
2321 }
2322
2323 static void backtrace_address(void *data, unsigned long addr, int reliable)
2324 {
2325         struct perf_callchain_entry *entry = data;
2326
2327         if (per_cpu(in_ignored_frame, smp_processor_id()))
2328                 return;
2329
2330         if (reliable)
2331                 callchain_store(entry, addr);
2332 }
2333
2334 static const struct stacktrace_ops backtrace_ops = {
2335         .warning                = backtrace_warning,
2336         .warning_symbol         = backtrace_warning_symbol,
2337         .stack                  = backtrace_stack,
2338         .address                = backtrace_address,
2339         .walk_stack             = print_context_stack_bp,
2340 };
2341
2342 #include "../dumpstack.h"
2343
2344 static void
2345 perf_callchain_kernel(struct pt_regs *regs, struct perf_callchain_entry *entry)
2346 {
2347         callchain_store(entry, PERF_CONTEXT_KERNEL);
2348         callchain_store(entry, regs->ip);
2349
2350         dump_trace(NULL, regs, NULL, regs->bp, &backtrace_ops, entry);
2351 }
2352
2353 /*
2354  * best effort, GUP based copy_from_user() that assumes IRQ or NMI context
2355  */
2356 static unsigned long
2357 copy_from_user_nmi(void *to, const void __user *from, unsigned long n)
2358 {
2359         unsigned long offset, addr = (unsigned long)from;
2360         int type = in_nmi() ? KM_NMI : KM_IRQ0;
2361         unsigned long size, len = 0;
2362         struct page *page;
2363         void *map;
2364         int ret;
2365
2366         do {
2367                 ret = __get_user_pages_fast(addr, 1, 0, &page);
2368                 if (!ret)
2369                         break;
2370
2371                 offset = addr & (PAGE_SIZE - 1);
2372                 size = min(PAGE_SIZE - offset, n - len);
2373
2374                 map = kmap_atomic(page, type);
2375                 memcpy(to, map+offset, size);
2376                 kunmap_atomic(map, type);
2377                 put_page(page);
2378
2379                 len  += size;
2380                 to   += size;
2381                 addr += size;
2382
2383         } while (len < n);
2384
2385         return len;
2386 }
2387
2388 static int copy_stack_frame(const void __user *fp, struct stack_frame *frame)
2389 {
2390         unsigned long bytes;
2391
2392         bytes = copy_from_user_nmi(frame, fp, sizeof(*frame));
2393
2394         return bytes == sizeof(*frame);
2395 }
2396
2397 static void
2398 perf_callchain_user(struct pt_regs *regs, struct perf_callchain_entry *entry)
2399 {
2400         struct stack_frame frame;
2401         const void __user *fp;
2402
2403         if (!user_mode(regs))
2404                 regs = task_pt_regs(current);
2405
2406         fp = (void __user *)regs->bp;
2407
2408         callchain_store(entry, PERF_CONTEXT_USER);
2409         callchain_store(entry, regs->ip);
2410
2411         while (entry->nr < PERF_MAX_STACK_DEPTH) {
2412                 frame.next_frame             = NULL;
2413                 frame.return_address = 0;
2414
2415                 if (!copy_stack_frame(fp, &frame))
2416                         break;
2417
2418                 if ((unsigned long)fp < regs->sp)
2419                         break;
2420
2421                 callchain_store(entry, frame.return_address);
2422                 fp = frame.next_frame;
2423         }
2424 }
2425
2426 static void
2427 perf_do_callchain(struct pt_regs *regs, struct perf_callchain_entry *entry)
2428 {
2429         int is_user;
2430
2431         if (!regs)
2432                 return;
2433
2434         is_user = user_mode(regs);
2435
2436         if (!current || current->pid == 0)
2437                 return;
2438
2439         if (is_user && current->state != TASK_RUNNING)
2440                 return;
2441
2442         if (!is_user)
2443                 perf_callchain_kernel(regs, entry);
2444
2445         if (current->mm)
2446                 perf_callchain_user(regs, entry);
2447 }
2448
2449 struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
2450 {
2451         struct perf_callchain_entry *entry;
2452
2453         if (in_nmi())
2454                 entry = &__get_cpu_var(pmc_nmi_entry);
2455         else
2456                 entry = &__get_cpu_var(pmc_irq_entry);
2457
2458         entry->nr = 0;
2459
2460         perf_do_callchain(regs, entry);
2461
2462         return entry;
2463 }
2464
2465 void hw_perf_event_setup_online(int cpu)
2466 {
2467         init_debug_store_on_cpu(cpu);
2468 }