arch/x86/events/intel/uncore_snbep.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* SandyBridge-EP/IvyTown uncore support */
   3 #include "uncore.h"
   4 #include "uncore_discovery.h"
   5
   6 /* SNB-EP pci bus to socket mapping */
   7 #define SNBEP_CPUNODEID                 0x40
   8 #define SNBEP_GIDNIDMAP                 0x54
   9
  10 /* SNB-EP Box level control */
  11 #define SNBEP_PMON_BOX_CTL_RST_CTRL     (1 << 0)
  12 #define SNBEP_PMON_BOX_CTL_RST_CTRS     (1 << 1)
  13 #define SNBEP_PMON_BOX_CTL_FRZ          (1 << 8)
  14 #define SNBEP_PMON_BOX_CTL_FRZ_EN       (1 << 16)
  15 #define SNBEP_PMON_BOX_CTL_INT          (SNBEP_PMON_BOX_CTL_RST_CTRL | \
  16                                          SNBEP_PMON_BOX_CTL_RST_CTRS | \
  17                                          SNBEP_PMON_BOX_CTL_FRZ_EN)
  18 /* SNB-EP event control */
  19 #define SNBEP_PMON_CTL_EV_SEL_MASK      0x000000ff
  20 #define SNBEP_PMON_CTL_UMASK_MASK       0x0000ff00
  21 #define SNBEP_PMON_CTL_RST              (1 << 17)
  22 #define SNBEP_PMON_CTL_EDGE_DET         (1 << 18)
  23 #define SNBEP_PMON_CTL_EV_SEL_EXT       (1 << 21)
  24 #define SNBEP_PMON_CTL_EN               (1 << 22)
  25 #define SNBEP_PMON_CTL_INVERT           (1 << 23)
  26 #define SNBEP_PMON_CTL_TRESH_MASK       0xff000000
  27 #define SNBEP_PMON_RAW_EVENT_MASK       (SNBEP_PMON_CTL_EV_SEL_MASK | \
  28                                          SNBEP_PMON_CTL_UMASK_MASK | \
  29                                          SNBEP_PMON_CTL_EDGE_DET | \
  30                                          SNBEP_PMON_CTL_INVERT | \
  31                                          SNBEP_PMON_CTL_TRESH_MASK)
  32
  33 /* SNB-EP Ubox event control */
  34 #define SNBEP_U_MSR_PMON_CTL_TRESH_MASK         0x1f000000
  35 #define SNBEP_U_MSR_PMON_RAW_EVENT_MASK         \
  36                                 (SNBEP_PMON_CTL_EV_SEL_MASK | \
  37                                  SNBEP_PMON_CTL_UMASK_MASK | \
  38                                  SNBEP_PMON_CTL_EDGE_DET | \
  39                                  SNBEP_PMON_CTL_INVERT | \
  40                                  SNBEP_U_MSR_PMON_CTL_TRESH_MASK)
  41
  42 #define SNBEP_CBO_PMON_CTL_TID_EN               (1 << 19)
  43 #define SNBEP_CBO_MSR_PMON_RAW_EVENT_MASK       (SNBEP_PMON_RAW_EVENT_MASK | \
  44                                                  SNBEP_CBO_PMON_CTL_TID_EN)
  45
  46 /* SNB-EP PCU event control */
  47 #define SNBEP_PCU_MSR_PMON_CTL_OCC_SEL_MASK     0x0000c000
  48 #define SNBEP_PCU_MSR_PMON_CTL_TRESH_MASK       0x1f000000
  49 #define SNBEP_PCU_MSR_PMON_CTL_OCC_INVERT       (1 << 30)
  50 #define SNBEP_PCU_MSR_PMON_CTL_OCC_EDGE_DET     (1 << 31)
  51 #define SNBEP_PCU_MSR_PMON_RAW_EVENT_MASK       \
  52                                 (SNBEP_PMON_CTL_EV_SEL_MASK | \
  53                                  SNBEP_PCU_MSR_PMON_CTL_OCC_SEL_MASK | \
  54                                  SNBEP_PMON_CTL_EDGE_DET | \
  55                                  SNBEP_PMON_CTL_INVERT | \
  56                                  SNBEP_PCU_MSR_PMON_CTL_TRESH_MASK | \
  57                                  SNBEP_PCU_MSR_PMON_CTL_OCC_INVERT | \
  58                                  SNBEP_PCU_MSR_PMON_CTL_OCC_EDGE_DET)
  59
  60 #define SNBEP_QPI_PCI_PMON_RAW_EVENT_MASK       \
  61                                 (SNBEP_PMON_RAW_EVENT_MASK | \
  62                                  SNBEP_PMON_CTL_EV_SEL_EXT)
  63
  64 /* SNB-EP pci control register */
  65 #define SNBEP_PCI_PMON_BOX_CTL                  0xf4
  66 #define SNBEP_PCI_PMON_CTL0                     0xd8
  67 /* SNB-EP pci counter register */
  68 #define SNBEP_PCI_PMON_CTR0                     0xa0
  69
  70 /* SNB-EP home agent register */
  71 #define SNBEP_HA_PCI_PMON_BOX_ADDRMATCH0        0x40
  72 #define SNBEP_HA_PCI_PMON_BOX_ADDRMATCH1        0x44
  73 #define SNBEP_HA_PCI_PMON_BOX_OPCODEMATCH       0x48
  74 /* SNB-EP memory controller register */
  75 #define SNBEP_MC_CHy_PCI_PMON_FIXED_CTL         0xf0
  76 #define SNBEP_MC_CHy_PCI_PMON_FIXED_CTR         0xd0
  77 /* SNB-EP QPI register */
  78 #define SNBEP_Q_Py_PCI_PMON_PKT_MATCH0          0x228
  79 #define SNBEP_Q_Py_PCI_PMON_PKT_MATCH1          0x22c
  80 #define SNBEP_Q_Py_PCI_PMON_PKT_MASK0           0x238
  81 #define SNBEP_Q_Py_PCI_PMON_PKT_MASK1           0x23c
  82
  83 /* SNB-EP Ubox register */
  84 #define SNBEP_U_MSR_PMON_CTR0                   0xc16
  85 #define SNBEP_U_MSR_PMON_CTL0                   0xc10
  86
  87 #define SNBEP_U_MSR_PMON_UCLK_FIXED_CTL         0xc08
  88 #define SNBEP_U_MSR_PMON_UCLK_FIXED_CTR         0xc09
  89
  90 /* SNB-EP Cbo register */
  91 #define SNBEP_C0_MSR_PMON_CTR0                  0xd16
  92 #define SNBEP_C0_MSR_PMON_CTL0                  0xd10
  93 #define SNBEP_C0_MSR_PMON_BOX_CTL               0xd04
  94 #define SNBEP_C0_MSR_PMON_BOX_FILTER            0xd14
  95 #define SNBEP_CBO_MSR_OFFSET                    0x20
  96
  97 #define SNBEP_CB0_MSR_PMON_BOX_FILTER_TID       0x1f
  98 #define SNBEP_CB0_MSR_PMON_BOX_FILTER_NID       0x3fc00
  99 #define SNBEP_CB0_MSR_PMON_BOX_FILTER_STATE     0x7c0000
 100 #define SNBEP_CB0_MSR_PMON_BOX_FILTER_OPC       0xff800000
 101
 102 #define SNBEP_CBO_EVENT_EXTRA_REG(e, m, i) {    \
 103         .event = (e),                           \
 104         .msr = SNBEP_C0_MSR_PMON_BOX_FILTER,    \
 105         .config_mask = (m),                     \
 106         .idx = (i)                              \
 107 }
 108
 109 /* SNB-EP PCU register */
 110 #define SNBEP_PCU_MSR_PMON_CTR0                 0xc36
 111 #define SNBEP_PCU_MSR_PMON_CTL0                 0xc30
 112 #define SNBEP_PCU_MSR_PMON_BOX_CTL              0xc24
 113 #define SNBEP_PCU_MSR_PMON_BOX_FILTER           0xc34
 114 #define SNBEP_PCU_MSR_PMON_BOX_FILTER_MASK      0xffffffff
 115 #define SNBEP_PCU_MSR_CORE_C3_CTR               0x3fc
 116 #define SNBEP_PCU_MSR_CORE_C6_CTR               0x3fd
 117
 118 /* IVBEP event control */
 119 #define IVBEP_PMON_BOX_CTL_INT          (SNBEP_PMON_BOX_CTL_RST_CTRL | \
 120                                          SNBEP_PMON_BOX_CTL_RST_CTRS)
 121 #define IVBEP_PMON_RAW_EVENT_MASK               (SNBEP_PMON_CTL_EV_SEL_MASK | \
 122                                          SNBEP_PMON_CTL_UMASK_MASK | \
 123                                          SNBEP_PMON_CTL_EDGE_DET | \
 124                                          SNBEP_PMON_CTL_TRESH_MASK)
 125 /* IVBEP Ubox */
 126 #define IVBEP_U_MSR_PMON_GLOBAL_CTL             0xc00
 127 #define IVBEP_U_PMON_GLOBAL_FRZ_ALL             (1 << 31)
 128 #define IVBEP_U_PMON_GLOBAL_UNFRZ_ALL           (1 << 29)
 129
 130 #define IVBEP_U_MSR_PMON_RAW_EVENT_MASK \
 131                                 (SNBEP_PMON_CTL_EV_SEL_MASK | \
 132                                  SNBEP_PMON_CTL_UMASK_MASK | \
 133                                  SNBEP_PMON_CTL_EDGE_DET | \
 134                                  SNBEP_U_MSR_PMON_CTL_TRESH_MASK)
 135 /* IVBEP Cbo */
 136 #define IVBEP_CBO_MSR_PMON_RAW_EVENT_MASK               (IVBEP_PMON_RAW_EVENT_MASK | \
 137                                                  SNBEP_CBO_PMON_CTL_TID_EN)
 138
 139 #define IVBEP_CB0_MSR_PMON_BOX_FILTER_TID               (0x1fULL << 0)
 140 #define IVBEP_CB0_MSR_PMON_BOX_FILTER_LINK      (0xfULL << 5)
 141 #define IVBEP_CB0_MSR_PMON_BOX_FILTER_STATE     (0x3fULL << 17)
 142 #define IVBEP_CB0_MSR_PMON_BOX_FILTER_NID               (0xffffULL << 32)
 143 #define IVBEP_CB0_MSR_PMON_BOX_FILTER_OPC               (0x1ffULL << 52)
 144 #define IVBEP_CB0_MSR_PMON_BOX_FILTER_C6                (0x1ULL << 61)
 145 #define IVBEP_CB0_MSR_PMON_BOX_FILTER_NC                (0x1ULL << 62)
 146 #define IVBEP_CB0_MSR_PMON_BOX_FILTER_ISOC      (0x1ULL << 63)
 147
 148 /* IVBEP home agent */
 149 #define IVBEP_HA_PCI_PMON_CTL_Q_OCC_RST         (1 << 16)
 150 #define IVBEP_HA_PCI_PMON_RAW_EVENT_MASK                \
 151                                 (IVBEP_PMON_RAW_EVENT_MASK | \
 152                                  IVBEP_HA_PCI_PMON_CTL_Q_OCC_RST)
 153 /* IVBEP PCU */
 154 #define IVBEP_PCU_MSR_PMON_RAW_EVENT_MASK       \
 155                                 (SNBEP_PMON_CTL_EV_SEL_MASK | \
 156                                  SNBEP_PCU_MSR_PMON_CTL_OCC_SEL_MASK | \
 157                                  SNBEP_PMON_CTL_EDGE_DET | \
 158                                  SNBEP_PCU_MSR_PMON_CTL_TRESH_MASK | \
 159                                  SNBEP_PCU_MSR_PMON_CTL_OCC_INVERT | \
 160                                  SNBEP_PCU_MSR_PMON_CTL_OCC_EDGE_DET)
 161 /* IVBEP QPI */
 162 #define IVBEP_QPI_PCI_PMON_RAW_EVENT_MASK       \
 163                                 (IVBEP_PMON_RAW_EVENT_MASK | \
 164                                  SNBEP_PMON_CTL_EV_SEL_EXT)
 165
 166 #define __BITS_VALUE(x, i, n)  ((typeof(x))(((x) >> ((i) * (n))) & \
 167                                 ((1ULL << (n)) - 1)))
 168
 169 /* Haswell-EP Ubox */
 170 #define HSWEP_U_MSR_PMON_CTR0                   0x709
 171 #define HSWEP_U_MSR_PMON_CTL0                   0x705
 172 #define HSWEP_U_MSR_PMON_FILTER                 0x707
 173
 174 #define HSWEP_U_MSR_PMON_UCLK_FIXED_CTL         0x703
 175 #define HSWEP_U_MSR_PMON_UCLK_FIXED_CTR         0x704
 176
 177 #define HSWEP_U_MSR_PMON_BOX_FILTER_TID         (0x1 << 0)
 178 #define HSWEP_U_MSR_PMON_BOX_FILTER_CID         (0x1fULL << 1)
 179 #define HSWEP_U_MSR_PMON_BOX_FILTER_MASK \
 180                                         (HSWEP_U_MSR_PMON_BOX_FILTER_TID | \
 181                                          HSWEP_U_MSR_PMON_BOX_FILTER_CID)
 182
 183 /* Haswell-EP CBo */
 184 #define HSWEP_C0_MSR_PMON_CTR0                  0xe08
 185 #define HSWEP_C0_MSR_PMON_CTL0                  0xe01
 186 #define HSWEP_C0_MSR_PMON_BOX_CTL                       0xe00
 187 #define HSWEP_C0_MSR_PMON_BOX_FILTER0           0xe05
 188 #define HSWEP_CBO_MSR_OFFSET                    0x10
 189
 190
 191 #define HSWEP_CB0_MSR_PMON_BOX_FILTER_TID               (0x3fULL << 0)
 192 #define HSWEP_CB0_MSR_PMON_BOX_FILTER_LINK      (0xfULL << 6)
 193 #define HSWEP_CB0_MSR_PMON_BOX_FILTER_STATE     (0x7fULL << 17)
 194 #define HSWEP_CB0_MSR_PMON_BOX_FILTER_NID               (0xffffULL << 32)
 195 #define HSWEP_CB0_MSR_PMON_BOX_FILTER_OPC               (0x1ffULL << 52)
 196 #define HSWEP_CB0_MSR_PMON_BOX_FILTER_C6                (0x1ULL << 61)
 197 #define HSWEP_CB0_MSR_PMON_BOX_FILTER_NC                (0x1ULL << 62)
 198 #define HSWEP_CB0_MSR_PMON_BOX_FILTER_ISOC      (0x1ULL << 63)
 199
 200
 201 /* Haswell-EP Sbox */
 202 #define HSWEP_S0_MSR_PMON_CTR0                  0x726
 203 #define HSWEP_S0_MSR_PMON_CTL0                  0x721
 204 #define HSWEP_S0_MSR_PMON_BOX_CTL                       0x720
 205 #define HSWEP_SBOX_MSR_OFFSET                   0xa
 206 #define HSWEP_S_MSR_PMON_RAW_EVENT_MASK         (SNBEP_PMON_RAW_EVENT_MASK | \
 207                                                  SNBEP_CBO_PMON_CTL_TID_EN)
 208
 209 /* Haswell-EP PCU */
 210 #define HSWEP_PCU_MSR_PMON_CTR0                 0x717
 211 #define HSWEP_PCU_MSR_PMON_CTL0                 0x711
 212 #define HSWEP_PCU_MSR_PMON_BOX_CTL              0x710
 213 #define HSWEP_PCU_MSR_PMON_BOX_FILTER           0x715
 214
 215 /* KNL Ubox */
 216 #define KNL_U_MSR_PMON_RAW_EVENT_MASK \
 217                                         (SNBEP_U_MSR_PMON_RAW_EVENT_MASK | \
 218                                                 SNBEP_CBO_PMON_CTL_TID_EN)
 219 /* KNL CHA */
 220 #define KNL_CHA_MSR_OFFSET                      0xc
 221 #define KNL_CHA_MSR_PMON_CTL_QOR                (1 << 16)
 222 #define KNL_CHA_MSR_PMON_RAW_EVENT_MASK \
 223                                         (SNBEP_CBO_MSR_PMON_RAW_EVENT_MASK | \
 224                                          KNL_CHA_MSR_PMON_CTL_QOR)
 225 #define KNL_CHA_MSR_PMON_BOX_FILTER_TID         0x1ff
 226 #define KNL_CHA_MSR_PMON_BOX_FILTER_STATE       (7 << 18)
 227 #define KNL_CHA_MSR_PMON_BOX_FILTER_OP          (0xfffffe2aULL << 32)
 228 #define KNL_CHA_MSR_PMON_BOX_FILTER_REMOTE_NODE (0x1ULL << 32)
 229 #define KNL_CHA_MSR_PMON_BOX_FILTER_LOCAL_NODE  (0x1ULL << 33)
 230 #define KNL_CHA_MSR_PMON_BOX_FILTER_NNC         (0x1ULL << 37)
 231
 232 /* KNL EDC/MC UCLK */
 233 #define KNL_UCLK_MSR_PMON_CTR0_LOW              0x400
 234 #define KNL_UCLK_MSR_PMON_CTL0                  0x420
 235 #define KNL_UCLK_MSR_PMON_BOX_CTL               0x430
 236 #define KNL_UCLK_MSR_PMON_UCLK_FIXED_LOW        0x44c
 237 #define KNL_UCLK_MSR_PMON_UCLK_FIXED_CTL        0x454
 238 #define KNL_PMON_FIXED_CTL_EN                   0x1
 239
 240 /* KNL EDC */
 241 #define KNL_EDC0_ECLK_MSR_PMON_CTR0_LOW         0xa00
 242 #define KNL_EDC0_ECLK_MSR_PMON_CTL0             0xa20
 243 #define KNL_EDC0_ECLK_MSR_PMON_BOX_CTL          0xa30
 244 #define KNL_EDC0_ECLK_MSR_PMON_ECLK_FIXED_LOW   0xa3c
 245 #define KNL_EDC0_ECLK_MSR_PMON_ECLK_FIXED_CTL   0xa44
 246
 247 /* KNL MC */
 248 #define KNL_MC0_CH0_MSR_PMON_CTR0_LOW           0xb00
 249 #define KNL_MC0_CH0_MSR_PMON_CTL0               0xb20
 250 #define KNL_MC0_CH0_MSR_PMON_BOX_CTL            0xb30
 251 #define KNL_MC0_CH0_MSR_PMON_FIXED_LOW          0xb3c
 252 #define KNL_MC0_CH0_MSR_PMON_FIXED_CTL          0xb44
 253
 254 /* KNL IRP */
 255 #define KNL_IRP_PCI_PMON_BOX_CTL                0xf0
 256 #define KNL_IRP_PCI_PMON_RAW_EVENT_MASK         (SNBEP_PMON_RAW_EVENT_MASK | \
 257                                                  KNL_CHA_MSR_PMON_CTL_QOR)
 258 /* KNL PCU */
 259 #define KNL_PCU_PMON_CTL_EV_SEL_MASK            0x0000007f
 260 #define KNL_PCU_PMON_CTL_USE_OCC_CTR            (1 << 7)
 261 #define KNL_PCU_MSR_PMON_CTL_TRESH_MASK         0x3f000000
 262 #define KNL_PCU_MSR_PMON_RAW_EVENT_MASK \
 263                                 (KNL_PCU_PMON_CTL_EV_SEL_MASK | \
 264                                  KNL_PCU_PMON_CTL_USE_OCC_CTR | \
 265                                  SNBEP_PCU_MSR_PMON_CTL_OCC_SEL_MASK | \
 266                                  SNBEP_PMON_CTL_EDGE_DET | \
 267                                  SNBEP_CBO_PMON_CTL_TID_EN | \
 268                                  SNBEP_PMON_CTL_INVERT | \
 269                                  KNL_PCU_MSR_PMON_CTL_TRESH_MASK | \
 270                                  SNBEP_PCU_MSR_PMON_CTL_OCC_INVERT | \
 271                                  SNBEP_PCU_MSR_PMON_CTL_OCC_EDGE_DET)
 272
 273 /* SKX pci bus to socket mapping */
 274 #define SKX_CPUNODEID                   0xc0
 275 #define SKX_GIDNIDMAP                   0xd4
 276
 277 /*
 278  * The CPU_BUS_NUMBER MSR returns the values of the respective CPUBUSNO CSR
 279  * that BIOS programmed. MSR has package scope.
 280  * |  Bit  |  Default  |  Description
 281  * | [63]  |    00h    | VALID - When set, indicates the CPU bus
 282  *                       numbers have been initialized. (RO)
 283  * |[62:48]|    ---    | Reserved
 284  * |[47:40]|    00h    | BUS_NUM_5 - Return the bus number BIOS assigned
 285  *                       CPUBUSNO(5). (RO)
 286  * |[39:32]|    00h    | BUS_NUM_4 - Return the bus number BIOS assigned
 287  *                       CPUBUSNO(4). (RO)
 288  * |[31:24]|    00h    | BUS_NUM_3 - Return the bus number BIOS assigned
 289  *                       CPUBUSNO(3). (RO)
 290  * |[23:16]|    00h    | BUS_NUM_2 - Return the bus number BIOS assigned
 291  *                       CPUBUSNO(2). (RO)
 292  * |[15:8] |    00h    | BUS_NUM_1 - Return the bus number BIOS assigned
 293  *                       CPUBUSNO(1). (RO)
 294  * | [7:0] |    00h    | BUS_NUM_0 - Return the bus number BIOS assigned
 295  *                       CPUBUSNO(0). (RO)
 296  */
 297 #define SKX_MSR_CPU_BUS_NUMBER          0x300
 298 #define SKX_MSR_CPU_BUS_VALID_BIT       (1ULL << 63)
 299 #define BUS_NUM_STRIDE                  8
 300
 301 /* SKX CHA */
 302 #define SKX_CHA_MSR_PMON_BOX_FILTER_TID         (0x1ffULL << 0)
 303 #define SKX_CHA_MSR_PMON_BOX_FILTER_LINK        (0xfULL << 9)
 304 #define SKX_CHA_MSR_PMON_BOX_FILTER_STATE       (0x3ffULL << 17)
 305 #define SKX_CHA_MSR_PMON_BOX_FILTER_REM         (0x1ULL << 32)
 306 #define SKX_CHA_MSR_PMON_BOX_FILTER_LOC         (0x1ULL << 33)
 307 #define SKX_CHA_MSR_PMON_BOX_FILTER_ALL_OPC     (0x1ULL << 35)
 308 #define SKX_CHA_MSR_PMON_BOX_FILTER_NM          (0x1ULL << 36)
 309 #define SKX_CHA_MSR_PMON_BOX_FILTER_NOT_NM      (0x1ULL << 37)
 310 #define SKX_CHA_MSR_PMON_BOX_FILTER_OPC0        (0x3ffULL << 41)
 311 #define SKX_CHA_MSR_PMON_BOX_FILTER_OPC1        (0x3ffULL << 51)
 312 #define SKX_CHA_MSR_PMON_BOX_FILTER_C6          (0x1ULL << 61)
 313 #define SKX_CHA_MSR_PMON_BOX_FILTER_NC          (0x1ULL << 62)
 314 #define SKX_CHA_MSR_PMON_BOX_FILTER_ISOC        (0x1ULL << 63)
 315
 316 /* SKX IIO */
 317 #define SKX_IIO0_MSR_PMON_CTL0          0xa48
 318 #define SKX_IIO0_MSR_PMON_CTR0          0xa41
 319 #define SKX_IIO0_MSR_PMON_BOX_CTL       0xa40
 320 #define SKX_IIO_MSR_OFFSET              0x20
 321
 322 #define SKX_PMON_CTL_TRESH_MASK         (0xff << 24)
 323 #define SKX_PMON_CTL_TRESH_MASK_EXT     (0xf)
 324 #define SKX_PMON_CTL_CH_MASK            (0xff << 4)
 325 #define SKX_PMON_CTL_FC_MASK            (0x7 << 12)
 326 #define SKX_IIO_PMON_RAW_EVENT_MASK     (SNBEP_PMON_CTL_EV_SEL_MASK | \
 327                                          SNBEP_PMON_CTL_UMASK_MASK | \
 328                                          SNBEP_PMON_CTL_EDGE_DET | \
 329                                          SNBEP_PMON_CTL_INVERT | \
 330                                          SKX_PMON_CTL_TRESH_MASK)
 331 #define SKX_IIO_PMON_RAW_EVENT_MASK_EXT (SKX_PMON_CTL_TRESH_MASK_EXT | \
 332                                          SKX_PMON_CTL_CH_MASK | \
 333                                          SKX_PMON_CTL_FC_MASK)
 334
 335 /* SKX IRP */
 336 #define SKX_IRP0_MSR_PMON_CTL0          0xa5b
 337 #define SKX_IRP0_MSR_PMON_CTR0          0xa59
 338 #define SKX_IRP0_MSR_PMON_BOX_CTL       0xa58
 339 #define SKX_IRP_MSR_OFFSET              0x20
 340
 341 /* SKX UPI */
 342 #define SKX_UPI_PCI_PMON_CTL0           0x350
 343 #define SKX_UPI_PCI_PMON_CTR0           0x318
 344 #define SKX_UPI_PCI_PMON_BOX_CTL        0x378
 345 #define SKX_UPI_CTL_UMASK_EXT           0xffefff
 346
 347 /* SKX M2M */
 348 #define SKX_M2M_PCI_PMON_CTL0           0x228
 349 #define SKX_M2M_PCI_PMON_CTR0           0x200
 350 #define SKX_M2M_PCI_PMON_BOX_CTL        0x258
 351
 352 /* Memory Map registers device ID */
 353 #define SNR_ICX_MESH2IIO_MMAP_DID               0x9a2
 354 #define SNR_ICX_SAD_CONTROL_CFG         0x3f4
 355
 356 /* Getting I/O stack id in SAD_COTROL_CFG notation */
 357 #define SAD_CONTROL_STACK_ID(data)              (((data) >> 4) & 0x7)
 358
 359 /* SNR Ubox */
 360 #define SNR_U_MSR_PMON_CTR0                     0x1f98
 361 #define SNR_U_MSR_PMON_CTL0                     0x1f91
 362 #define SNR_U_MSR_PMON_UCLK_FIXED_CTL           0x1f93
 363 #define SNR_U_MSR_PMON_UCLK_FIXED_CTR           0x1f94
 364
 365 /* SNR CHA */
 366 #define SNR_CHA_RAW_EVENT_MASK_EXT              0x3ffffff
 367 #define SNR_CHA_MSR_PMON_CTL0                   0x1c01
 368 #define SNR_CHA_MSR_PMON_CTR0                   0x1c08
 369 #define SNR_CHA_MSR_PMON_BOX_CTL                0x1c00
 370 #define SNR_C0_MSR_PMON_BOX_FILTER0             0x1c05
 371
 372
 373 /* SNR IIO */
 374 #define SNR_IIO_MSR_PMON_CTL0                   0x1e08
 375 #define SNR_IIO_MSR_PMON_CTR0                   0x1e01
 376 #define SNR_IIO_MSR_PMON_BOX_CTL                0x1e00
 377 #define SNR_IIO_MSR_OFFSET                      0x10
 378 #define SNR_IIO_PMON_RAW_EVENT_MASK_EXT         0x7ffff
 379
 380 /* SNR IRP */
 381 #define SNR_IRP0_MSR_PMON_CTL0                  0x1ea8
 382 #define SNR_IRP0_MSR_PMON_CTR0                  0x1ea1
 383 #define SNR_IRP0_MSR_PMON_BOX_CTL               0x1ea0
 384 #define SNR_IRP_MSR_OFFSET                      0x10
 385
 386 /* SNR M2PCIE */
 387 #define SNR_M2PCIE_MSR_PMON_CTL0                0x1e58
 388 #define SNR_M2PCIE_MSR_PMON_CTR0                0x1e51
 389 #define SNR_M2PCIE_MSR_PMON_BOX_CTL             0x1e50
 390 #define SNR_M2PCIE_MSR_OFFSET                   0x10
 391
 392 /* SNR PCU */
 393 #define SNR_PCU_MSR_PMON_CTL0                   0x1ef1
 394 #define SNR_PCU_MSR_PMON_CTR0                   0x1ef8
 395 #define SNR_PCU_MSR_PMON_BOX_CTL                0x1ef0
 396 #define SNR_PCU_MSR_PMON_BOX_FILTER             0x1efc
 397
 398 /* SNR M2M */
 399 #define SNR_M2M_PCI_PMON_CTL0                   0x468
 400 #define SNR_M2M_PCI_PMON_CTR0                   0x440
 401 #define SNR_M2M_PCI_PMON_BOX_CTL                0x438
 402 #define SNR_M2M_PCI_PMON_UMASK_EXT              0xff
 403
 404 /* SNR PCIE3 */
 405 #define SNR_PCIE3_PCI_PMON_CTL0                 0x508
 406 #define SNR_PCIE3_PCI_PMON_CTR0                 0x4e8
 407 #define SNR_PCIE3_PCI_PMON_BOX_CTL              0x4e0
 408
 409 /* SNR IMC */
 410 #define SNR_IMC_MMIO_PMON_FIXED_CTL             0x54
 411 #define SNR_IMC_MMIO_PMON_FIXED_CTR             0x38
 412 #define SNR_IMC_MMIO_PMON_CTL0                  0x40
 413 #define SNR_IMC_MMIO_PMON_CTR0                  0x8
 414 #define SNR_IMC_MMIO_PMON_BOX_CTL               0x22800
 415 #define SNR_IMC_MMIO_OFFSET                     0x4000
 416 #define SNR_IMC_MMIO_SIZE                       0x4000
 417 #define SNR_IMC_MMIO_BASE_OFFSET                0xd0
 418 #define SNR_IMC_MMIO_BASE_MASK                  0x1FFFFFFF
 419 #define SNR_IMC_MMIO_MEM0_OFFSET                0xd8
 420 #define SNR_IMC_MMIO_MEM0_MASK                  0x7FF
 421
 422 /* ICX CHA */
 423 #define ICX_C34_MSR_PMON_CTR0                   0xb68
 424 #define ICX_C34_MSR_PMON_CTL0                   0xb61
 425 #define ICX_C34_MSR_PMON_BOX_CTL                0xb60
 426 #define ICX_C34_MSR_PMON_BOX_FILTER0            0xb65
 427
 428 /* ICX IIO */
 429 #define ICX_IIO_MSR_PMON_CTL0                   0xa58
 430 #define ICX_IIO_MSR_PMON_CTR0                   0xa51
 431 #define ICX_IIO_MSR_PMON_BOX_CTL                0xa50
 432
 433 /* ICX IRP */
 434 #define ICX_IRP0_MSR_PMON_CTL0                  0xa4d
 435 #define ICX_IRP0_MSR_PMON_CTR0                  0xa4b
 436 #define ICX_IRP0_MSR_PMON_BOX_CTL               0xa4a
 437
 438 /* ICX M2PCIE */
 439 #define ICX_M2PCIE_MSR_PMON_CTL0                0xa46
 440 #define ICX_M2PCIE_MSR_PMON_CTR0                0xa41
 441 #define ICX_M2PCIE_MSR_PMON_BOX_CTL             0xa40
 442
 443 /* ICX UPI */
 444 #define ICX_UPI_PCI_PMON_CTL0                   0x350
 445 #define ICX_UPI_PCI_PMON_CTR0                   0x320
 446 #define ICX_UPI_PCI_PMON_BOX_CTL                0x318
 447 #define ICX_UPI_CTL_UMASK_EXT                   0xffffff
 448 #define ICX_UBOX_DID                            0x3450
 449
 450 /* ICX M3UPI*/
 451 #define ICX_M3UPI_PCI_PMON_CTL0                 0xd8
 452 #define ICX_M3UPI_PCI_PMON_CTR0                 0xa8
 453 #define ICX_M3UPI_PCI_PMON_BOX_CTL              0xa0
 454
 455 /* ICX IMC */
 456 #define ICX_NUMBER_IMC_CHN                      3
 457 #define ICX_IMC_MEM_STRIDE                      0x4
 458
 459 /* SPR */
 460 #define SPR_RAW_EVENT_MASK_EXT                  0xffffff
 461 #define SPR_UBOX_DID                            0x3250
 462
 463 /* SPR CHA */
 464 #define SPR_CHA_PMON_CTL_TID_EN                 (1 << 16)
 465 #define SPR_CHA_PMON_EVENT_MASK                 (SNBEP_PMON_RAW_EVENT_MASK | \
 466                                                  SPR_CHA_PMON_CTL_TID_EN)
 467 #define SPR_CHA_PMON_BOX_FILTER_TID             0x3ff
 468
 469 #define SPR_C0_MSR_PMON_BOX_FILTER0             0x200e
 470
 471 DEFINE_UNCORE_FORMAT_ATTR(event, event, "config:0-7");
 472 DEFINE_UNCORE_FORMAT_ATTR(event2, event, "config:0-6");
 473 DEFINE_UNCORE_FORMAT_ATTR(event_ext, event, "config:0-7,21");
 474 DEFINE_UNCORE_FORMAT_ATTR(use_occ_ctr, use_occ_ctr, "config:7");
 475 DEFINE_UNCORE_FORMAT_ATTR(umask, umask, "config:8-15");
 476 DEFINE_UNCORE_FORMAT_ATTR(umask_ext, umask, "config:8-15,32-43,45-55");
 477 DEFINE_UNCORE_FORMAT_ATTR(umask_ext2, umask, "config:8-15,32-57");
 478 DEFINE_UNCORE_FORMAT_ATTR(umask_ext3, umask, "config:8-15,32-39");
 479 DEFINE_UNCORE_FORMAT_ATTR(umask_ext4, umask, "config:8-15,32-55");
 480 DEFINE_UNCORE_FORMAT_ATTR(qor, qor, "config:16");
 481 DEFINE_UNCORE_FORMAT_ATTR(edge, edge, "config:18");
 482 DEFINE_UNCORE_FORMAT_ATTR(tid_en, tid_en, "config:19");
 483 DEFINE_UNCORE_FORMAT_ATTR(tid_en2, tid_en, "config:16");
 484 DEFINE_UNCORE_FORMAT_ATTR(inv, inv, "config:23");
 485 DEFINE_UNCORE_FORMAT_ATTR(thresh9, thresh, "config:24-35");
 486 DEFINE_UNCORE_FORMAT_ATTR(thresh8, thresh, "config:24-31");
 487 DEFINE_UNCORE_FORMAT_ATTR(thresh6, thresh, "config:24-29");
 488 DEFINE_UNCORE_FORMAT_ATTR(thresh5, thresh, "config:24-28");
 489 DEFINE_UNCORE_FORMAT_ATTR(occ_sel, occ_sel, "config:14-15");
 490 DEFINE_UNCORE_FORMAT_ATTR(occ_invert, occ_invert, "config:30");
 491 DEFINE_UNCORE_FORMAT_ATTR(occ_edge, occ_edge, "config:14-51");
 492 DEFINE_UNCORE_FORMAT_ATTR(occ_edge_det, occ_edge_det, "config:31");
 493 DEFINE_UNCORE_FORMAT_ATTR(ch_mask, ch_mask, "config:36-43");
 494 DEFINE_UNCORE_FORMAT_ATTR(ch_mask2, ch_mask, "config:36-47");
 495 DEFINE_UNCORE_FORMAT_ATTR(fc_mask, fc_mask, "config:44-46");
 496 DEFINE_UNCORE_FORMAT_ATTR(fc_mask2, fc_mask, "config:48-50");
 497 DEFINE_UNCORE_FORMAT_ATTR(filter_tid, filter_tid, "config1:0-4");
 498 DEFINE_UNCORE_FORMAT_ATTR(filter_tid2, filter_tid, "config1:0");
 499 DEFINE_UNCORE_FORMAT_ATTR(filter_tid3, filter_tid, "config1:0-5");
 500 DEFINE_UNCORE_FORMAT_ATTR(filter_tid4, filter_tid, "config1:0-8");
 501 DEFINE_UNCORE_FORMAT_ATTR(filter_tid5, filter_tid, "config1:0-9");
 502 DEFINE_UNCORE_FORMAT_ATTR(filter_cid, filter_cid, "config1:5");
 503 DEFINE_UNCORE_FORMAT_ATTR(filter_link, filter_link, "config1:5-8");
 504 DEFINE_UNCORE_FORMAT_ATTR(filter_link2, filter_link, "config1:6-8");
 505 DEFINE_UNCORE_FORMAT_ATTR(filter_link3, filter_link, "config1:12");
 506 DEFINE_UNCORE_FORMAT_ATTR(filter_nid, filter_nid, "config1:10-17");
 507 DEFINE_UNCORE_FORMAT_ATTR(filter_nid2, filter_nid, "config1:32-47");
 508 DEFINE_UNCORE_FORMAT_ATTR(filter_state, filter_state, "config1:18-22");
 509 DEFINE_UNCORE_FORMAT_ATTR(filter_state2, filter_state, "config1:17-22");
 510 DEFINE_UNCORE_FORMAT_ATTR(filter_state3, filter_state, "config1:17-23");
 511 DEFINE_UNCORE_FORMAT_ATTR(filter_state4, filter_state, "config1:18-20");
 512 DEFINE_UNCORE_FORMAT_ATTR(filter_state5, filter_state, "config1:17-26");
 513 DEFINE_UNCORE_FORMAT_ATTR(filter_rem, filter_rem, "config1:32");
 514 DEFINE_UNCORE_FORMAT_ATTR(filter_loc, filter_loc, "config1:33");
 515 DEFINE_UNCORE_FORMAT_ATTR(filter_nm, filter_nm, "config1:36");
 516 DEFINE_UNCORE_FORMAT_ATTR(filter_not_nm, filter_not_nm, "config1:37");
 517 DEFINE_UNCORE_FORMAT_ATTR(filter_local, filter_local, "config1:33");
 518 DEFINE_UNCORE_FORMAT_ATTR(filter_all_op, filter_all_op, "config1:35");
 519 DEFINE_UNCORE_FORMAT_ATTR(filter_nnm, filter_nnm, "config1:37");
 520 DEFINE_UNCORE_FORMAT_ATTR(filter_opc, filter_opc, "config1:23-31");
 521 DEFINE_UNCORE_FORMAT_ATTR(filter_opc2, filter_opc, "config1:52-60");
 522 DEFINE_UNCORE_FORMAT_ATTR(filter_opc3, filter_opc, "config1:41-60");
 523 DEFINE_UNCORE_FORMAT_ATTR(filter_opc_0, filter_opc0, "config1:41-50");
 524 DEFINE_UNCORE_FORMAT_ATTR(filter_opc_1, filter_opc1, "config1:51-60");
 525 DEFINE_UNCORE_FORMAT_ATTR(filter_nc, filter_nc, "config1:62");
 526 DEFINE_UNCORE_FORMAT_ATTR(filter_c6, filter_c6, "config1:61");
 527 DEFINE_UNCORE_FORMAT_ATTR(filter_isoc, filter_isoc, "config1:63");
 528 DEFINE_UNCORE_FORMAT_ATTR(filter_band0, filter_band0, "config1:0-7");
 529 DEFINE_UNCORE_FORMAT_ATTR(filter_band1, filter_band1, "config1:8-15");
 530 DEFINE_UNCORE_FORMAT_ATTR(filter_band2, filter_band2, "config1:16-23");
 531 DEFINE_UNCORE_FORMAT_ATTR(filter_band3, filter_band3, "config1:24-31");
 532 DEFINE_UNCORE_FORMAT_ATTR(match_rds, match_rds, "config1:48-51");
 533 DEFINE_UNCORE_FORMAT_ATTR(match_rnid30, match_rnid30, "config1:32-35");
 534 DEFINE_UNCORE_FORMAT_ATTR(match_rnid4, match_rnid4, "config1:31");
 535 DEFINE_UNCORE_FORMAT_ATTR(match_dnid, match_dnid, "config1:13-17");
 536 DEFINE_UNCORE_FORMAT_ATTR(match_mc, match_mc, "config1:9-12");
 537 DEFINE_UNCORE_FORMAT_ATTR(match_opc, match_opc, "config1:5-8");
 538 DEFINE_UNCORE_FORMAT_ATTR(match_vnw, match_vnw, "config1:3-4");
 539 DEFINE_UNCORE_FORMAT_ATTR(match0, match0, "config1:0-31");
 540 DEFINE_UNCORE_FORMAT_ATTR(match1, match1, "config1:32-63");
 541 DEFINE_UNCORE_FORMAT_ATTR(mask_rds, mask_rds, "config2:48-51");
 542 DEFINE_UNCORE_FORMAT_ATTR(mask_rnid30, mask_rnid30, "config2:32-35");
 543 DEFINE_UNCORE_FORMAT_ATTR(mask_rnid4, mask_rnid4, "config2:31");
 544 DEFINE_UNCORE_FORMAT_ATTR(mask_dnid, mask_dnid, "config2:13-17");
 545 DEFINE_UNCORE_FORMAT_ATTR(mask_mc, mask_mc, "config2:9-12");
 546 DEFINE_UNCORE_FORMAT_ATTR(mask_opc, mask_opc, "config2:5-8");
 547 DEFINE_UNCORE_FORMAT_ATTR(mask_vnw, mask_vnw, "config2:3-4");
 548 DEFINE_UNCORE_FORMAT_ATTR(mask0, mask0, "config2:0-31");
 549 DEFINE_UNCORE_FORMAT_ATTR(mask1, mask1, "config2:32-63");
 550
 551 static void snbep_uncore_pci_disable_box(struct intel_uncore_box *box)
 552 {
 553         struct pci_dev *pdev = box->pci_dev;
 554         int box_ctl = uncore_pci_box_ctl(box);
 555         u32 config = 0;
 556
 557         if (!pci_read_config_dword(pdev, box_ctl, &config)) {
 558                 config |= SNBEP_PMON_BOX_CTL_FRZ;
 559                 pci_write_config_dword(pdev, box_ctl, config);
 560         }
 561 }
 562
 563 static void snbep_uncore_pci_enable_box(struct intel_uncore_box *box)
 564 {
 565         struct pci_dev *pdev = box->pci_dev;
 566         int box_ctl = uncore_pci_box_ctl(box);
 567         u32 config = 0;
 568
 569         if (!pci_read_config_dword(pdev, box_ctl, &config)) {
 570                 config &= ~SNBEP_PMON_BOX_CTL_FRZ;
 571                 pci_write_config_dword(pdev, box_ctl, config);
 572         }
 573 }
 574
 575 static void snbep_uncore_pci_enable_event(struct intel_uncore_box *box, struct perf_event *event)
 576 {
 577         struct pci_dev *pdev = box->pci_dev;
 578         struct hw_perf_event *hwc = &event->hw;
 579
 580         pci_write_config_dword(pdev, hwc->config_base, hwc->config | SNBEP_PMON_CTL_EN);
 581 }
 582
 583 static void snbep_uncore_pci_disable_event(struct intel_uncore_box *box, struct perf_event *event)
 584 {
 585         struct pci_dev *pdev = box->pci_dev;
 586         struct hw_perf_event *hwc = &event->hw;
 587
 588         pci_write_config_dword(pdev, hwc->config_base, hwc->config);
 589 }
 590
 591 static u64 snbep_uncore_pci_read_counter(struct intel_uncore_box *box, struct perf_event *event)
 592 {
 593         struct pci_dev *pdev = box->pci_dev;
 594         struct hw_perf_event *hwc = &event->hw;
 595         u64 count = 0;
 596
 597         pci_read_config_dword(pdev, hwc->event_base, (u32 *)&count);
 598         pci_read_config_dword(pdev, hwc->event_base + 4, (u32 *)&count + 1);
 599
 600         return count;
 601 }
 602
 603 static void snbep_uncore_pci_init_box(struct intel_uncore_box *box)
 604 {
 605         struct pci_dev *pdev = box->pci_dev;
 606         int box_ctl = uncore_pci_box_ctl(box);
 607
 608         pci_write_config_dword(pdev, box_ctl, SNBEP_PMON_BOX_CTL_INT);
 609 }
 610
 611 static void snbep_uncore_msr_disable_box(struct intel_uncore_box *box)
 612 {
 613         u64 config;
 614         unsigned msr;
 615
 616         msr = uncore_msr_box_ctl(box);
 617         if (msr) {
 618                 rdmsrl(msr, config);
 619                 config |= SNBEP_PMON_BOX_CTL_FRZ;
 620                 wrmsrl(msr, config);
 621         }
 622 }
 623
 624 static void snbep_uncore_msr_enable_box(struct intel_uncore_box *box)
 625 {
 626         u64 config;
 627         unsigned msr;
 628
 629         msr = uncore_msr_box_ctl(box);
 630         if (msr) {
 631                 rdmsrl(msr, config);
 632                 config &= ~SNBEP_PMON_BOX_CTL_FRZ;
 633                 wrmsrl(msr, config);
 634         }
 635 }
 636
 637 static void snbep_uncore_msr_enable_event(struct intel_uncore_box *box, struct perf_event *event)
 638 {
 639         struct hw_perf_event *hwc = &event->hw;
 640         struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
 641
 642         if (reg1->idx != EXTRA_REG_NONE)
 643                 wrmsrl(reg1->reg, uncore_shared_reg_config(box, 0));
 644
 645         wrmsrl(hwc->config_base, hwc->config | SNBEP_PMON_CTL_EN);
 646 }
 647
 648 static void snbep_uncore_msr_disable_event(struct intel_uncore_box *box,
 649                                         struct perf_event *event)
 650 {
 651         struct hw_perf_event *hwc = &event->hw;
 652
 653         wrmsrl(hwc->config_base, hwc->config);
 654 }
 655
 656 static void snbep_uncore_msr_init_box(struct intel_uncore_box *box)
 657 {
 658         unsigned msr = uncore_msr_box_ctl(box);
 659
 660         if (msr)
 661                 wrmsrl(msr, SNBEP_PMON_BOX_CTL_INT);
 662 }
 663
 664 static struct attribute *snbep_uncore_formats_attr[] = {
 665         &format_attr_event.attr,
 666         &format_attr_umask.attr,
 667         &format_attr_edge.attr,
 668         &format_attr_inv.attr,
 669         &format_attr_thresh8.attr,
 670         NULL,
 671 };
 672
 673 static struct attribute *snbep_uncore_ubox_formats_attr[] = {
 674         &format_attr_event.attr,
 675         &format_attr_umask.attr,
 676         &format_attr_edge.attr,
 677         &format_attr_inv.attr,
 678         &format_attr_thresh5.attr,
 679         NULL,
 680 };
 681
 682 static struct attribute *snbep_uncore_cbox_formats_attr[] = {
 683         &format_attr_event.attr,
 684         &format_attr_umask.attr,
 685         &format_attr_edge.attr,
 686         &format_attr_tid_en.attr,
 687         &format_attr_inv.attr,
 688         &format_attr_thresh8.attr,
 689         &format_attr_filter_tid.attr,
 690         &format_attr_filter_nid.attr,
 691         &format_attr_filter_state.attr,
 692         &format_attr_filter_opc.attr,
 693         NULL,
 694 };
 695
 696 static struct attribute *snbep_uncore_pcu_formats_attr[] = {
 697         &format_attr_event.attr,
 698         &format_attr_occ_sel.attr,
 699         &format_attr_edge.attr,
 700         &format_attr_inv.attr,
 701         &format_attr_thresh5.attr,
 702         &format_attr_occ_invert.attr,
 703         &format_attr_occ_edge.attr,
 704         &format_attr_filter_band0.attr,
 705         &format_attr_filter_band1.attr,
 706         &format_attr_filter_band2.attr,
 707         &format_attr_filter_band3.attr,
 708         NULL,
 709 };
 710
 711 static struct attribute *snbep_uncore_qpi_formats_attr[] = {
 712         &format_attr_event_ext.attr,
 713         &format_attr_umask.attr,
 714         &format_attr_edge.attr,
 715         &format_attr_inv.attr,
 716         &format_attr_thresh8.attr,
 717         &format_attr_match_rds.attr,
 718         &format_attr_match_rnid30.attr,
 719         &format_attr_match_rnid4.attr,
 720         &format_attr_match_dnid.attr,
 721         &format_attr_match_mc.attr,
 722         &format_attr_match_opc.attr,
 723         &format_attr_match_vnw.attr,
 724         &format_attr_match0.attr,
 725         &format_attr_match1.attr,
 726         &format_attr_mask_rds.attr,
 727         &format_attr_mask_rnid30.attr,
 728         &format_attr_mask_rnid4.attr,
 729         &format_attr_mask_dnid.attr,
 730         &format_attr_mask_mc.attr,
 731         &format_attr_mask_opc.attr,
 732         &format_attr_mask_vnw.attr,
 733         &format_attr_mask0.attr,
 734         &format_attr_mask1.attr,
 735         NULL,
 736 };
 737
 738 static struct uncore_event_desc snbep_uncore_imc_events[] = {
 739         INTEL_UNCORE_EVENT_DESC(clockticks,      "event=0xff,umask=0x00"),
 740         INTEL_UNCORE_EVENT_DESC(cas_count_read,  "event=0x04,umask=0x03"),
 741         INTEL_UNCORE_EVENT_DESC(cas_count_read.scale, "6.103515625e-5"),
 742         INTEL_UNCORE_EVENT_DESC(cas_count_read.unit, "MiB"),
 743         INTEL_UNCORE_EVENT_DESC(cas_count_write, "event=0x04,umask=0x0c"),
 744         INTEL_UNCORE_EVENT_DESC(cas_count_write.scale, "6.103515625e-5"),
 745         INTEL_UNCORE_EVENT_DESC(cas_count_write.unit, "MiB"),
 746         { /* end: all zeroes */ },
 747 };
 748
 749 static struct uncore_event_desc snbep_uncore_qpi_events[] = {
 750         INTEL_UNCORE_EVENT_DESC(clockticks,       "event=0x14"),
 751         INTEL_UNCORE_EVENT_DESC(txl_flits_active, "event=0x00,umask=0x06"),
 752         INTEL_UNCORE_EVENT_DESC(drs_data,         "event=0x102,umask=0x08"),
 753         INTEL_UNCORE_EVENT_DESC(ncb_data,         "event=0x103,umask=0x04"),
 754         { /* end: all zeroes */ },
 755 };
 756
 757 static const struct attribute_group snbep_uncore_format_group = {
 758         .name = "format",
 759         .attrs = snbep_uncore_formats_attr,
 760 };
 761
 762 static const struct attribute_group snbep_uncore_ubox_format_group = {
 763         .name = "format",
 764         .attrs = snbep_uncore_ubox_formats_attr,
 765 };
 766
 767 static const struct attribute_group snbep_uncore_cbox_format_group = {
 768         .name = "format",
 769         .attrs = snbep_uncore_cbox_formats_attr,
 770 };
 771
 772 static const struct attribute_group snbep_uncore_pcu_format_group = {
 773         .name = "format",
 774         .attrs = snbep_uncore_pcu_formats_attr,
 775 };
 776
 777 static const struct attribute_group snbep_uncore_qpi_format_group = {
 778         .name = "format",
 779         .attrs = snbep_uncore_qpi_formats_attr,
 780 };
 781
 782 #define __SNBEP_UNCORE_MSR_OPS_COMMON_INIT()                    \
 783         .disable_box    = snbep_uncore_msr_disable_box,         \
 784         .enable_box     = snbep_uncore_msr_enable_box,          \
 785         .disable_event  = snbep_uncore_msr_disable_event,       \
 786         .enable_event   = snbep_uncore_msr_enable_event,        \
 787         .read_counter   = uncore_msr_read_counter
 788
 789 #define SNBEP_UNCORE_MSR_OPS_COMMON_INIT()                      \
 790         __SNBEP_UNCORE_MSR_OPS_COMMON_INIT(),                   \
 791         .init_box       = snbep_uncore_msr_init_box             \
 792
 793 static struct intel_uncore_ops snbep_uncore_msr_ops = {
 794         SNBEP_UNCORE_MSR_OPS_COMMON_INIT(),
 795 };
 796
 797 #define SNBEP_UNCORE_PCI_OPS_COMMON_INIT()                      \
 798         .init_box       = snbep_uncore_pci_init_box,            \
 799         .disable_box    = snbep_uncore_pci_disable_box,         \
 800         .enable_box     = snbep_uncore_pci_enable_box,          \
 801         .disable_event  = snbep_uncore_pci_disable_event,       \
 802         .read_counter   = snbep_uncore_pci_read_counter
 803
 804 static struct intel_uncore_ops snbep_uncore_pci_ops = {
 805         SNBEP_UNCORE_PCI_OPS_COMMON_INIT(),
 806         .enable_event   = snbep_uncore_pci_enable_event,        \
 807 };
 808
 809 static struct event_constraint snbep_uncore_cbox_constraints[] = {
 810         UNCORE_EVENT_CONSTRAINT(0x01, 0x1),
 811         UNCORE_EVENT_CONSTRAINT(0x02, 0x3),
 812         UNCORE_EVENT_CONSTRAINT(0x04, 0x3),
 813         UNCORE_EVENT_CONSTRAINT(0x05, 0x3),
 814         UNCORE_EVENT_CONSTRAINT(0x07, 0x3),
 815         UNCORE_EVENT_CONSTRAINT(0x09, 0x3),
 816         UNCORE_EVENT_CONSTRAINT(0x11, 0x1),
 817         UNCORE_EVENT_CONSTRAINT(0x12, 0x3),
 818         UNCORE_EVENT_CONSTRAINT(0x13, 0x3),
 819         UNCORE_EVENT_CONSTRAINT(0x1b, 0xc),
 820         UNCORE_EVENT_CONSTRAINT(0x1c, 0xc),
 821         UNCORE_EVENT_CONSTRAINT(0x1d, 0xc),
 822         UNCORE_EVENT_CONSTRAINT(0x1e, 0xc),
 823         UNCORE_EVENT_CONSTRAINT(0x1f, 0xe),
 824         UNCORE_EVENT_CONSTRAINT(0x21, 0x3),
 825         UNCORE_EVENT_CONSTRAINT(0x23, 0x3),
 826         UNCORE_EVENT_CONSTRAINT(0x31, 0x3),
 827         UNCORE_EVENT_CONSTRAINT(0x32, 0x3),
 828         UNCORE_EVENT_CONSTRAINT(0x33, 0x3),
 829         UNCORE_EVENT_CONSTRAINT(0x34, 0x3),
 830         UNCORE_EVENT_CONSTRAINT(0x35, 0x3),
 831         UNCORE_EVENT_CONSTRAINT(0x36, 0x1),
 832         UNCORE_EVENT_CONSTRAINT(0x37, 0x3),
 833         UNCORE_EVENT_CONSTRAINT(0x38, 0x3),
 834         UNCORE_EVENT_CONSTRAINT(0x39, 0x3),
 835         UNCORE_EVENT_CONSTRAINT(0x3b, 0x1),
 836         EVENT_CONSTRAINT_END
 837 };
 838
 839 static struct event_constraint snbep_uncore_r2pcie_constraints[] = {
 840         UNCORE_EVENT_CONSTRAINT(0x10, 0x3),
 841         UNCORE_EVENT_CONSTRAINT(0x11, 0x3),
 842         UNCORE_EVENT_CONSTRAINT(0x12, 0x1),
 843         UNCORE_EVENT_CONSTRAINT(0x23, 0x3),
 844         UNCORE_EVENT_CONSTRAINT(0x24, 0x3),
 845         UNCORE_EVENT_CONSTRAINT(0x25, 0x3),
 846         UNCORE_EVENT_CONSTRAINT(0x26, 0x3),
 847         UNCORE_EVENT_CONSTRAINT(0x32, 0x3),
 848         UNCORE_EVENT_CONSTRAINT(0x33, 0x3),
 849         UNCORE_EVENT_CONSTRAINT(0x34, 0x3),
 850         EVENT_CONSTRAINT_END
 851 };
 852
 853 static struct event_constraint snbep_uncore_r3qpi_constraints[] = {
 854         UNCORE_EVENT_CONSTRAINT(0x10, 0x3),
 855         UNCORE_EVENT_CONSTRAINT(0x11, 0x3),
 856         UNCORE_EVENT_CONSTRAINT(0x12, 0x3),
 857         UNCORE_EVENT_CONSTRAINT(0x13, 0x1),
 858         UNCORE_EVENT_CONSTRAINT(0x20, 0x3),
 859         UNCORE_EVENT_CONSTRAINT(0x21, 0x3),
 860         UNCORE_EVENT_CONSTRAINT(0x22, 0x3),
 861         UNCORE_EVENT_CONSTRAINT(0x23, 0x3),
 862         UNCORE_EVENT_CONSTRAINT(0x24, 0x3),
 863         UNCORE_EVENT_CONSTRAINT(0x25, 0x3),
 864         UNCORE_EVENT_CONSTRAINT(0x26, 0x3),
 865         UNCORE_EVENT_CONSTRAINT(0x28, 0x3),
 866         UNCORE_EVENT_CONSTRAINT(0x29, 0x3),
 867         UNCORE_EVENT_CONSTRAINT(0x2a, 0x3),
 868         UNCORE_EVENT_CONSTRAINT(0x2b, 0x3),
 869         UNCORE_EVENT_CONSTRAINT(0x2c, 0x3),
 870         UNCORE_EVENT_CONSTRAINT(0x2d, 0x3),
 871         UNCORE_EVENT_CONSTRAINT(0x2e, 0x3),
 872         UNCORE_EVENT_CONSTRAINT(0x2f, 0x3),
 873         UNCORE_EVENT_CONSTRAINT(0x30, 0x3),
 874         UNCORE_EVENT_CONSTRAINT(0x31, 0x3),
 875         UNCORE_EVENT_CONSTRAINT(0x32, 0x3),
 876         UNCORE_EVENT_CONSTRAINT(0x33, 0x3),
 877         UNCORE_EVENT_CONSTRAINT(0x34, 0x3),
 878         UNCORE_EVENT_CONSTRAINT(0x36, 0x3),
 879         UNCORE_EVENT_CONSTRAINT(0x37, 0x3),
 880         UNCORE_EVENT_CONSTRAINT(0x38, 0x3),
 881         UNCORE_EVENT_CONSTRAINT(0x39, 0x3),
 882         EVENT_CONSTRAINT_END
 883 };
 884
 885 static struct intel_uncore_type snbep_uncore_ubox = {
 886         .name           = "ubox",
 887         .num_counters   = 2,
 888         .num_boxes      = 1,
 889         .perf_ctr_bits  = 44,
 890         .fixed_ctr_bits = 48,
 891         .perf_ctr       = SNBEP_U_MSR_PMON_CTR0,
 892         .event_ctl      = SNBEP_U_MSR_PMON_CTL0,
 893         .event_mask     = SNBEP_U_MSR_PMON_RAW_EVENT_MASK,
 894         .fixed_ctr      = SNBEP_U_MSR_PMON_UCLK_FIXED_CTR,
 895         .fixed_ctl      = SNBEP_U_MSR_PMON_UCLK_FIXED_CTL,
 896         .ops            = &snbep_uncore_msr_ops,
 897         .format_group   = &snbep_uncore_ubox_format_group,
 898 };
 899
 900 static struct extra_reg snbep_uncore_cbox_extra_regs[] = {
 901         SNBEP_CBO_EVENT_EXTRA_REG(SNBEP_CBO_PMON_CTL_TID_EN,
 902                                   SNBEP_CBO_PMON_CTL_TID_EN, 0x1),
 903         SNBEP_CBO_EVENT_EXTRA_REG(0x0334, 0xffff, 0x4),
 904         SNBEP_CBO_EVENT_EXTRA_REG(0x4334, 0xffff, 0x6),
 905         SNBEP_CBO_EVENT_EXTRA_REG(0x0534, 0xffff, 0x4),
 906         SNBEP_CBO_EVENT_EXTRA_REG(0x4534, 0xffff, 0x6),
 907         SNBEP_CBO_EVENT_EXTRA_REG(0x0934, 0xffff, 0x4),
 908         SNBEP_CBO_EVENT_EXTRA_REG(0x4934, 0xffff, 0x6),
 909         SNBEP_CBO_EVENT_EXTRA_REG(0x4134, 0xffff, 0x6),
 910         SNBEP_CBO_EVENT_EXTRA_REG(0x0135, 0xffff, 0x8),
 911         SNBEP_CBO_EVENT_EXTRA_REG(0x0335, 0xffff, 0x8),
 912         SNBEP_CBO_EVENT_EXTRA_REG(0x4135, 0xffff, 0xa),
 913         SNBEP_CBO_EVENT_EXTRA_REG(0x4335, 0xffff, 0xa),
 914         SNBEP_CBO_EVENT_EXTRA_REG(0x4435, 0xffff, 0x2),
 915         SNBEP_CBO_EVENT_EXTRA_REG(0x4835, 0xffff, 0x2),
 916         SNBEP_CBO_EVENT_EXTRA_REG(0x4a35, 0xffff, 0x2),
 917         SNBEP_CBO_EVENT_EXTRA_REG(0x5035, 0xffff, 0x2),
 918         SNBEP_CBO_EVENT_EXTRA_REG(0x0136, 0xffff, 0x8),
 919         SNBEP_CBO_EVENT_EXTRA_REG(0x0336, 0xffff, 0x8),
 920         SNBEP_CBO_EVENT_EXTRA_REG(0x4136, 0xffff, 0xa),
 921         SNBEP_CBO_EVENT_EXTRA_REG(0x4336, 0xffff, 0xa),
 922         SNBEP_CBO_EVENT_EXTRA_REG(0x4436, 0xffff, 0x2),
 923         SNBEP_CBO_EVENT_EXTRA_REG(0x4836, 0xffff, 0x2),
 924         SNBEP_CBO_EVENT_EXTRA_REG(0x4a36, 0xffff, 0x2),
 925         SNBEP_CBO_EVENT_EXTRA_REG(0x4037, 0x40ff, 0x2),
 926         EVENT_EXTRA_END
 927 };
 928
 929 static void snbep_cbox_put_constraint(struct intel_uncore_box *box, struct perf_event *event)
 930 {
 931         struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
 932         struct intel_uncore_extra_reg *er = &box->shared_regs[0];
 933         int i;
 934
 935         if (uncore_box_is_fake(box))
 936                 return;
 937
 938         for (i = 0; i < 5; i++) {
 939                 if (reg1->alloc & (0x1 << i))
 940                         atomic_sub(1 << (i * 6), &er->ref);
 941         }
 942         reg1->alloc = 0;
 943 }
 944
 945 static struct event_constraint *
 946 __snbep_cbox_get_constraint(struct intel_uncore_box *box, struct perf_event *event,
 947                             u64 (*cbox_filter_mask)(int fields))
 948 {
 949         struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
 950         struct intel_uncore_extra_reg *er = &box->shared_regs[0];
 951         int i, alloc = 0;
 952         unsigned long flags;
 953         u64 mask;
 954
 955         if (reg1->idx == EXTRA_REG_NONE)
 956                 return NULL;
 957
 958         raw_spin_lock_irqsave(&er->lock, flags);
 959         for (i = 0; i < 5; i++) {
 960                 if (!(reg1->idx & (0x1 << i)))
 961                         continue;
 962                 if (!uncore_box_is_fake(box) && (reg1->alloc & (0x1 << i)))
 963                         continue;
 964
 965                 mask = cbox_filter_mask(0x1 << i);
 966                 if (!__BITS_VALUE(atomic_read(&er->ref), i, 6) ||
 967                     !((reg1->config ^ er->config) & mask)) {
 968                         atomic_add(1 << (i * 6), &er->ref);
 969                         er->config &= ~mask;
 970                         er->config |= reg1->config & mask;
 971                         alloc |= (0x1 << i);
 972                 } else {
 973                         break;
 974                 }
 975         }
 976         raw_spin_unlock_irqrestore(&er->lock, flags);
 977         if (i < 5)
 978                 goto fail;
 979
 980         if (!uncore_box_is_fake(box))
 981                 reg1->alloc |= alloc;
 982
 983         return NULL;
 984 fail:
 985         for (; i >= 0; i--) {
 986                 if (alloc & (0x1 << i))
 987                         atomic_sub(1 << (i * 6), &er->ref);
 988         }
 989         return &uncore_constraint_empty;
 990 }
 991
 992 static u64 snbep_cbox_filter_mask(int fields)
 993 {
 994         u64 mask = 0;
 995
 996         if (fields & 0x1)
 997                 mask |= SNBEP_CB0_MSR_PMON_BOX_FILTER_TID;
 998         if (fields & 0x2)
 999                 mask |= SNBEP_CB0_MSR_PMON_BOX_FILTER_NID;
1000         if (fields & 0x4)
1001                 mask |= SNBEP_CB0_MSR_PMON_BOX_FILTER_STATE;
1002         if (fields & 0x8)
1003                 mask |= SNBEP_CB0_MSR_PMON_BOX_FILTER_OPC;
1004
1005         return mask;
1006 }
1007
1008 static struct event_constraint *
1009 snbep_cbox_get_constraint(struct intel_uncore_box *box, struct perf_event *event)
1010 {
1011         return __snbep_cbox_get_constraint(box, event, snbep_cbox_filter_mask);
1012 }
1013
1014 static int snbep_cbox_hw_config(struct intel_uncore_box *box, struct perf_event *event)
1015 {
1016         struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
1017         struct extra_reg *er;
1018         int idx = 0;
1019
1020         for (er = snbep_uncore_cbox_extra_regs; er->msr; er++) {
1021                 if (er->event != (event->hw.config & er->config_mask))
1022                         continue;
1023                 idx |= er->idx;
1024         }
1025
1026         if (idx) {
1027                 reg1->reg = SNBEP_C0_MSR_PMON_BOX_FILTER +
1028                         SNBEP_CBO_MSR_OFFSET * box->pmu->pmu_idx;
1029                 reg1->config = event->attr.config1 & snbep_cbox_filter_mask(idx);
1030                 reg1->idx = idx;
1031         }
1032         return 0;
1033 }
1034
1035 static struct intel_uncore_ops snbep_uncore_cbox_ops = {
1036         SNBEP_UNCORE_MSR_OPS_COMMON_INIT(),
1037         .hw_config              = snbep_cbox_hw_config,
1038         .get_constraint         = snbep_cbox_get_constraint,
1039         .put_constraint         = snbep_cbox_put_constraint,
1040 };
1041
1042 static struct intel_uncore_type snbep_uncore_cbox = {
1043         .name                   = "cbox",
1044         .num_counters           = 4,
1045         .num_boxes              = 8,
1046         .perf_ctr_bits          = 44,
1047         .event_ctl              = SNBEP_C0_MSR_PMON_CTL0,
1048         .perf_ctr               = SNBEP_C0_MSR_PMON_CTR0,
1049         .event_mask             = SNBEP_CBO_MSR_PMON_RAW_EVENT_MASK,
1050         .box_ctl                = SNBEP_C0_MSR_PMON_BOX_CTL,
1051         .msr_offset             = SNBEP_CBO_MSR_OFFSET,
1052         .num_shared_regs        = 1,
1053         .constraints            = snbep_uncore_cbox_constraints,
1054         .ops                    = &snbep_uncore_cbox_ops,
1055         .format_group           = &snbep_uncore_cbox_format_group,
1056 };
1057
1058 static u64 snbep_pcu_alter_er(struct perf_event *event, int new_idx, bool modify)
1059 {
1060         struct hw_perf_event *hwc = &event->hw;
1061         struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
1062         u64 config = reg1->config;
1063
1064         if (new_idx > reg1->idx)
1065                 config <<= 8 * (new_idx - reg1->idx);
1066         else
1067                 config >>= 8 * (reg1->idx - new_idx);
1068
1069         if (modify) {
1070                 hwc->config += new_idx - reg1->idx;
1071                 reg1->config = config;
1072                 reg1->idx = new_idx;
1073         }
1074         return config;
1075 }
1076
1077 static struct event_constraint *
1078 snbep_pcu_get_constraint(struct intel_uncore_box *box, struct perf_event *event)
1079 {
1080         struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
1081         struct intel_uncore_extra_reg *er = &box->shared_regs[0];
1082         unsigned long flags;
1083         int idx = reg1->idx;
1084         u64 mask, config1 = reg1->config;
1085         bool ok = false;
1086
1087         if (reg1->idx == EXTRA_REG_NONE ||
1088             (!uncore_box_is_fake(box) && reg1->alloc))
1089                 return NULL;
1090 again:
1091         mask = 0xffULL << (idx * 8);
1092         raw_spin_lock_irqsave(&er->lock, flags);
1093         if (!__BITS_VALUE(atomic_read(&er->ref), idx, 8) ||
1094             !((config1 ^ er->config) & mask)) {
1095                 atomic_add(1 << (idx * 8), &er->ref);
1096                 er->config &= ~mask;
1097                 er->config |= config1 & mask;
1098                 ok = true;
1099         }
1100         raw_spin_unlock_irqrestore(&er->lock, flags);
1101
1102         if (!ok) {
1103                 idx = (idx + 1) % 4;
1104                 if (idx != reg1->idx) {
1105                         config1 = snbep_pcu_alter_er(event, idx, false);
1106                         goto again;
1107                 }
1108                 return &uncore_constraint_empty;
1109         }
1110
1111         if (!uncore_box_is_fake(box)) {
1112                 if (idx != reg1->idx)
1113                         snbep_pcu_alter_er(event, idx, true);
1114                 reg1->alloc = 1;
1115         }
1116         return NULL;
1117 }
1118
1119 static void snbep_pcu_put_constraint(struct intel_uncore_box *box, struct perf_event *event)
1120 {
1121         struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
1122         struct intel_uncore_extra_reg *er = &box->shared_regs[0];
1123
1124         if (uncore_box_is_fake(box) || !reg1->alloc)
1125                 return;
1126
1127         atomic_sub(1 << (reg1->idx * 8), &er->ref);
1128         reg1->alloc = 0;
1129 }
1130
1131 static int snbep_pcu_hw_config(struct intel_uncore_box *box, struct perf_event *event)
1132 {
1133         struct hw_perf_event *hwc = &event->hw;
1134         struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
1135         int ev_sel = hwc->config & SNBEP_PMON_CTL_EV_SEL_MASK;
1136
1137         if (ev_sel >= 0xb && ev_sel <= 0xe) {
1138                 reg1->reg = SNBEP_PCU_MSR_PMON_BOX_FILTER;
1139                 reg1->idx = ev_sel - 0xb;
1140                 reg1->config = event->attr.config1 & (0xff << (reg1->idx * 8));
1141         }
1142         return 0;
1143 }
1144
1145 static struct intel_uncore_ops snbep_uncore_pcu_ops = {
1146         SNBEP_UNCORE_MSR_OPS_COMMON_INIT(),
1147         .hw_config              = snbep_pcu_hw_config,
1148         .get_constraint         = snbep_pcu_get_constraint,
1149         .put_constraint         = snbep_pcu_put_constraint,
1150 };
1151
1152 static struct intel_uncore_type snbep_uncore_pcu = {
1153         .name                   = "pcu",
1154         .num_counters           = 4,
1155         .num_boxes              = 1,
1156         .perf_ctr_bits          = 48,
1157         .perf_ctr               = SNBEP_PCU_MSR_PMON_CTR0,
1158         .event_ctl              = SNBEP_PCU_MSR_PMON_CTL0,
1159         .event_mask             = SNBEP_PCU_MSR_PMON_RAW_EVENT_MASK,
1160         .box_ctl                = SNBEP_PCU_MSR_PMON_BOX_CTL,
1161         .num_shared_regs        = 1,
1162         .ops                    = &snbep_uncore_pcu_ops,
1163         .format_group           = &snbep_uncore_pcu_format_group,
1164 };
1165
1166 static struct intel_uncore_type *snbep_msr_uncores[] = {
1167         &snbep_uncore_ubox,
1168         &snbep_uncore_cbox,
1169         &snbep_uncore_pcu,
1170         NULL,
1171 };
1172
1173 void snbep_uncore_cpu_init(void)
1174 {
1175         if (snbep_uncore_cbox.num_boxes > boot_cpu_data.x86_max_cores)
1176                 snbep_uncore_cbox.num_boxes = boot_cpu_data.x86_max_cores;
1177         uncore_msr_uncores = snbep_msr_uncores;
1178 }
1179
1180 enum {
1181         SNBEP_PCI_QPI_PORT0_FILTER,
1182         SNBEP_PCI_QPI_PORT1_FILTER,
1183         BDX_PCI_QPI_PORT2_FILTER,
1184 };
1185
1186 static int snbep_qpi_hw_config(struct intel_uncore_box *box, struct perf_event *event)
1187 {
1188         struct hw_perf_event *hwc = &event->hw;
1189         struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
1190         struct hw_perf_event_extra *reg2 = &hwc->branch_reg;
1191
1192         if ((hwc->config & SNBEP_PMON_CTL_EV_SEL_MASK) == 0x38) {
1193                 reg1->idx = 0;
1194                 reg1->reg = SNBEP_Q_Py_PCI_PMON_PKT_MATCH0;
1195                 reg1->config = event->attr.config1;
1196                 reg2->reg = SNBEP_Q_Py_PCI_PMON_PKT_MASK0;
1197                 reg2->config = event->attr.config2;
1198         }
1199         return 0;
1200 }
1201
1202 static void snbep_qpi_enable_event(struct intel_uncore_box *box, struct perf_event *event)
1203 {
1204         struct pci_dev *pdev = box->pci_dev;
1205         struct hw_perf_event *hwc = &event->hw;
1206         struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
1207         struct hw_perf_event_extra *reg2 = &hwc->branch_reg;
1208
1209         if (reg1->idx != EXTRA_REG_NONE) {
1210                 int idx = box->pmu->pmu_idx + SNBEP_PCI_QPI_PORT0_FILTER;
1211                 int die = box->dieid;
1212                 struct pci_dev *filter_pdev = uncore_extra_pci_dev[die].dev[idx];
1213
1214                 if (filter_pdev) {
1215                         pci_write_config_dword(filter_pdev, reg1->reg,
1216                                                 (u32)reg1->config);
1217                         pci_write_config_dword(filter_pdev, reg1->reg + 4,
1218                                                 (u32)(reg1->config >> 32));
1219                         pci_write_config_dword(filter_pdev, reg2->reg,
1220                                                 (u32)reg2->config);
1221                         pci_write_config_dword(filter_pdev, reg2->reg + 4,
1222                                                 (u32)(reg2->config >> 32));
1223                 }
1224         }
1225
1226         pci_write_config_dword(pdev, hwc->config_base, hwc->config | SNBEP_PMON_CTL_EN);
1227 }
1228
1229 static struct intel_uncore_ops snbep_uncore_qpi_ops = {
1230         SNBEP_UNCORE_PCI_OPS_COMMON_INIT(),
1231         .enable_event           = snbep_qpi_enable_event,
1232         .hw_config              = snbep_qpi_hw_config,
1233         .get_constraint         = uncore_get_constraint,
1234         .put_constraint         = uncore_put_constraint,
1235 };
1236
1237 #define SNBEP_UNCORE_PCI_COMMON_INIT()                          \
1238         .perf_ctr       = SNBEP_PCI_PMON_CTR0,                  \
1239         .event_ctl      = SNBEP_PCI_PMON_CTL0,                  \
1240         .event_mask     = SNBEP_PMON_RAW_EVENT_MASK,            \
1241         .box_ctl        = SNBEP_PCI_PMON_BOX_CTL,               \
1242         .ops            = &snbep_uncore_pci_ops,                \
1243         .format_group   = &snbep_uncore_format_group
1244
1245 static struct intel_uncore_type snbep_uncore_ha = {
1246         .name           = "ha",
1247         .num_counters   = 4,
1248         .num_boxes      = 1,
1249         .perf_ctr_bits  = 48,
1250         SNBEP_UNCORE_PCI_COMMON_INIT(),
1251 };
1252
1253 static struct intel_uncore_type snbep_uncore_imc = {
1254         .name           = "imc",
1255         .num_counters   = 4,
1256         .num_boxes      = 4,
1257         .perf_ctr_bits  = 48,
1258         .fixed_ctr_bits = 48,
1259         .fixed_ctr      = SNBEP_MC_CHy_PCI_PMON_FIXED_CTR,
1260         .fixed_ctl      = SNBEP_MC_CHy_PCI_PMON_FIXED_CTL,
1261         .event_descs    = snbep_uncore_imc_events,
1262         SNBEP_UNCORE_PCI_COMMON_INIT(),
1263 };
1264
1265 static struct intel_uncore_type snbep_uncore_qpi = {
1266         .name                   = "qpi",
1267         .num_counters           = 4,
1268         .num_boxes              = 2,
1269         .perf_ctr_bits          = 48,
1270         .perf_ctr               = SNBEP_PCI_PMON_CTR0,
1271         .event_ctl              = SNBEP_PCI_PMON_CTL0,
1272         .event_mask             = SNBEP_QPI_PCI_PMON_RAW_EVENT_MASK,
1273         .box_ctl                = SNBEP_PCI_PMON_BOX_CTL,
1274         .num_shared_regs        = 1,
1275         .ops                    = &snbep_uncore_qpi_ops,
1276         .event_descs            = snbep_uncore_qpi_events,
1277         .format_group           = &snbep_uncore_qpi_format_group,
1278 };
1279
1280
1281 static struct intel_uncore_type snbep_uncore_r2pcie = {
1282         .name           = "r2pcie",
1283         .num_counters   = 4,
1284         .num_boxes      = 1,
1285         .perf_ctr_bits  = 44,
1286         .constraints    = snbep_uncore_r2pcie_constraints,
1287         SNBEP_UNCORE_PCI_COMMON_INIT(),
1288 };
1289
1290 static struct intel_uncore_type snbep_uncore_r3qpi = {
1291         .name           = "r3qpi",
1292         .num_counters   = 3,
1293         .num_boxes      = 2,
1294         .perf_ctr_bits  = 44,
1295         .constraints    = snbep_uncore_r3qpi_constraints,
1296         SNBEP_UNCORE_PCI_COMMON_INIT(),
1297 };
1298
1299 enum {
1300         SNBEP_PCI_UNCORE_HA,
1301         SNBEP_PCI_UNCORE_IMC,
1302         SNBEP_PCI_UNCORE_QPI,
1303         SNBEP_PCI_UNCORE_R2PCIE,
1304         SNBEP_PCI_UNCORE_R3QPI,
1305 };
1306
1307 static struct intel_uncore_type *snbep_pci_uncores[] = {
1308         [SNBEP_PCI_UNCORE_HA]           = &snbep_uncore_ha,
1309         [SNBEP_PCI_UNCORE_IMC]          = &snbep_uncore_imc,
1310         [SNBEP_PCI_UNCORE_QPI]          = &snbep_uncore_qpi,
1311         [SNBEP_PCI_UNCORE_R2PCIE]       = &snbep_uncore_r2pcie,
1312         [SNBEP_PCI_UNCORE_R3QPI]        = &snbep_uncore_r3qpi,
1313         NULL,
1314 };
1315
1316 static const struct pci_device_id snbep_uncore_pci_ids[] = {
1317         { /* Home Agent */
1318                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_HA),
1319                 .driver_data = UNCORE_PCI_DEV_DATA(SNBEP_PCI_UNCORE_HA, 0),
1320         },
1321         { /* MC Channel 0 */
1322                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_IMC0),
1323                 .driver_data = UNCORE_PCI_DEV_DATA(SNBEP_PCI_UNCORE_IMC, 0),
1324         },
1325         { /* MC Channel 1 */
1326                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_IMC1),
1327                 .driver_data = UNCORE_PCI_DEV_DATA(SNBEP_PCI_UNCORE_IMC, 1),
1328         },
1329         { /* MC Channel 2 */
1330                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_IMC2),
1331                 .driver_data = UNCORE_PCI_DEV_DATA(SNBEP_PCI_UNCORE_IMC, 2),
1332         },
1333         { /* MC Channel 3 */
1334                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_IMC3),
1335                 .driver_data = UNCORE_PCI_DEV_DATA(SNBEP_PCI_UNCORE_IMC, 3),
1336         },
1337         { /* QPI Port 0 */
1338                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_QPI0),
1339                 .driver_data = UNCORE_PCI_DEV_DATA(SNBEP_PCI_UNCORE_QPI, 0),
1340         },
1341         { /* QPI Port 1 */
1342                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_QPI1),
1343                 .driver_data = UNCORE_PCI_DEV_DATA(SNBEP_PCI_UNCORE_QPI, 1),
1344         },
1345         { /* R2PCIe */
1346                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_R2PCIE),
1347                 .driver_data = UNCORE_PCI_DEV_DATA(SNBEP_PCI_UNCORE_R2PCIE, 0),
1348         },
1349         { /* R3QPI Link 0 */
1350                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_R3QPI0),
1351                 .driver_data = UNCORE_PCI_DEV_DATA(SNBEP_PCI_UNCORE_R3QPI, 0),
1352         },
1353         { /* R3QPI Link 1 */
1354                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_R3QPI1),
1355                 .driver_data = UNCORE_PCI_DEV_DATA(SNBEP_PCI_UNCORE_R3QPI, 1),
1356         },
1357         { /* QPI Port 0 filter  */
1358                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x3c86),
1359                 .driver_data = UNCORE_PCI_DEV_DATA(UNCORE_EXTRA_PCI_DEV,
1360                                                    SNBEP_PCI_QPI_PORT0_FILTER),
1361         },
1362         { /* QPI Port 0 filter  */
1363                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x3c96),
1364                 .driver_data = UNCORE_PCI_DEV_DATA(UNCORE_EXTRA_PCI_DEV,
1365                                                    SNBEP_PCI_QPI_PORT1_FILTER),
1366         },
1367         { /* end: all zeroes */ }
1368 };
1369
1370 static struct pci_driver snbep_uncore_pci_driver = {
1371         .name           = "snbep_uncore",
1372         .id_table       = snbep_uncore_pci_ids,
1373 };
1374
1375 #define NODE_ID_MASK    0x7
1376
1377 /* Each three bits from 0 to 23 of GIDNIDMAP register correspond Node ID. */
1378 #define GIDNIDMAP(config, id)   (((config) >> (3 * (id))) & 0x7)
1379
1380 static int upi_nodeid_groupid(struct pci_dev *ubox_dev, int nodeid_loc, int idmap_loc,
1381                               int *nodeid, int *groupid)
1382 {
1383         int ret;
1384
1385         /* get the Node ID of the local register */
1386         ret = pci_read_config_dword(ubox_dev, nodeid_loc, nodeid);
1387         if (ret)
1388                 goto err;
1389
1390         *nodeid = *nodeid & NODE_ID_MASK;
1391         /* get the Node ID mapping */
1392         ret = pci_read_config_dword(ubox_dev, idmap_loc, groupid);
1393         if (ret)
1394                 goto err;
1395 err:
1396         return ret;
1397 }
1398
1399 /*
1400  * build pci bus to socket mapping
1401  */
1402 static int snbep_pci2phy_map_init(int devid, int nodeid_loc, int idmap_loc, bool reverse)
1403 {
1404         struct pci_dev *ubox_dev = NULL;
1405         int i, bus, nodeid, segment, die_id;
1406         struct pci2phy_map *map;
1407         int err = 0;
1408         u32 config = 0;
1409
1410         while (1) {
1411                 /* find the UBOX device */
1412                 ubox_dev = pci_get_device(PCI_VENDOR_ID_INTEL, devid, ubox_dev);
1413                 if (!ubox_dev)
1414                         break;
1415                 bus = ubox_dev->bus->number;
1416                 /*
1417                  * The nodeid and idmap registers only contain enough
1418                  * information to handle 8 nodes.  On systems with more
1419                  * than 8 nodes, we need to rely on NUMA information,
1420                  * filled in from BIOS supplied information, to determine
1421                  * the topology.
1422                  */
1423                 if (nr_node_ids <= 8) {
1424                         err = upi_nodeid_groupid(ubox_dev, nodeid_loc, idmap_loc,
1425                                                  &nodeid, &config);
1426                         if (err)
1427                                 break;
1428
1429                         segment = pci_domain_nr(ubox_dev->bus);
1430                         raw_spin_lock(&pci2phy_map_lock);
1431                         map = __find_pci2phy_map(segment);
1432                         if (!map) {
1433                                 raw_spin_unlock(&pci2phy_map_lock);
1434                                 err = -ENOMEM;
1435                                 break;
1436                         }
1437
1438                         /*
1439                          * every three bits in the Node ID mapping register maps
1440                          * to a particular node.
1441                          */
1442                         for (i = 0; i < 8; i++) {
1443                                 if (nodeid == GIDNIDMAP(config, i)) {
1444                                         if (topology_max_die_per_package() > 1)
1445                                                 die_id = i;
1446                                         else
1447                                                 die_id = topology_phys_to_logical_pkg(i);
1448                                         if (die_id < 0)
1449                                                 die_id = -ENODEV;
1450                                         map->pbus_to_dieid[bus] = die_id;
1451                                         break;
1452                                 }
1453                         }
1454                         raw_spin_unlock(&pci2phy_map_lock);
1455                 } else {
1456                         segment = pci_domain_nr(ubox_dev->bus);
1457                         raw_spin_lock(&pci2phy_map_lock);
1458                         map = __find_pci2phy_map(segment);
1459                         if (!map) {
1460                                 raw_spin_unlock(&pci2phy_map_lock);
1461                                 err = -ENOMEM;
1462                                 break;
1463                         }
1464
1465                         map->pbus_to_dieid[bus] = die_id = uncore_device_to_die(ubox_dev);
1466
1467                         raw_spin_unlock(&pci2phy_map_lock);
1468
1469                         if (WARN_ON_ONCE(die_id == -1)) {
1470                                 err = -EINVAL;
1471                                 break;
1472                         }
1473                 }
1474         }
1475
1476         if (!err) {
1477                 /*
1478                  * For PCI bus with no UBOX device, find the next bus
1479                  * that has UBOX device and use its mapping.
1480                  */
1481                 raw_spin_lock(&pci2phy_map_lock);
1482                 list_for_each_entry(map, &pci2phy_map_head, list) {
1483                         i = -1;
1484                         if (reverse) {
1485                                 for (bus = 255; bus >= 0; bus--) {
1486                                         if (map->pbus_to_dieid[bus] != -1)
1487                                                 i = map->pbus_to_dieid[bus];
1488                                         else
1489                                                 map->pbus_to_dieid[bus] = i;
1490                                 }
1491                         } else {
1492                                 for (bus = 0; bus <= 255; bus++) {
1493                                         if (map->pbus_to_dieid[bus] != -1)
1494                                                 i = map->pbus_to_dieid[bus];
1495                                         else
1496                                                 map->pbus_to_dieid[bus] = i;
1497                                 }
1498                         }
1499                 }
1500                 raw_spin_unlock(&pci2phy_map_lock);
1501         }
1502
1503         pci_dev_put(ubox_dev);
1504
1505         return err ? pcibios_err_to_errno(err) : 0;
1506 }
1507
1508 int snbep_uncore_pci_init(void)
1509 {
1510         int ret = snbep_pci2phy_map_init(0x3ce0, SNBEP_CPUNODEID, SNBEP_GIDNIDMAP, true);
1511         if (ret)
1512                 return ret;
1513         uncore_pci_uncores = snbep_pci_uncores;
1514         uncore_pci_driver = &snbep_uncore_pci_driver;
1515         return 0;
1516 }
1517 /* end of Sandy Bridge-EP uncore support */
1518
1519 /* IvyTown uncore support */
1520 static void ivbep_uncore_msr_init_box(struct intel_uncore_box *box)
1521 {
1522         unsigned msr = uncore_msr_box_ctl(box);
1523         if (msr)
1524                 wrmsrl(msr, IVBEP_PMON_BOX_CTL_INT);
1525 }
1526
1527 static void ivbep_uncore_pci_init_box(struct intel_uncore_box *box)
1528 {
1529         struct pci_dev *pdev = box->pci_dev;
1530
1531         pci_write_config_dword(pdev, SNBEP_PCI_PMON_BOX_CTL, IVBEP_PMON_BOX_CTL_INT);
1532 }
1533
1534 #define IVBEP_UNCORE_MSR_OPS_COMMON_INIT()                      \
1535         .init_box       = ivbep_uncore_msr_init_box,            \
1536         .disable_box    = snbep_uncore_msr_disable_box,         \
1537         .enable_box     = snbep_uncore_msr_enable_box,          \
1538         .disable_event  = snbep_uncore_msr_disable_event,       \
1539         .enable_event   = snbep_uncore_msr_enable_event,        \
1540         .read_counter   = uncore_msr_read_counter
1541
1542 static struct intel_uncore_ops ivbep_uncore_msr_ops = {
1543         IVBEP_UNCORE_MSR_OPS_COMMON_INIT(),
1544 };
1545
1546 static struct intel_uncore_ops ivbep_uncore_pci_ops = {
1547         .init_box       = ivbep_uncore_pci_init_box,
1548         .disable_box    = snbep_uncore_pci_disable_box,
1549         .enable_box     = snbep_uncore_pci_enable_box,
1550         .disable_event  = snbep_uncore_pci_disable_event,
1551         .enable_event   = snbep_uncore_pci_enable_event,
1552         .read_counter   = snbep_uncore_pci_read_counter,
1553 };
1554
1555 #define IVBEP_UNCORE_PCI_COMMON_INIT()                          \
1556         .perf_ctr       = SNBEP_PCI_PMON_CTR0,                  \
1557         .event_ctl      = SNBEP_PCI_PMON_CTL0,                  \
1558         .event_mask     = IVBEP_PMON_RAW_EVENT_MASK,            \
1559         .box_ctl        = SNBEP_PCI_PMON_BOX_CTL,               \
1560         .ops            = &ivbep_uncore_pci_ops,                        \
1561         .format_group   = &ivbep_uncore_format_group
1562
1563 static struct attribute *ivbep_uncore_formats_attr[] = {
1564         &format_attr_event.attr,
1565         &format_attr_umask.attr,
1566         &format_attr_edge.attr,
1567         &format_attr_inv.attr,
1568         &format_attr_thresh8.attr,
1569         NULL,
1570 };
1571
1572 static struct attribute *ivbep_uncore_ubox_formats_attr[] = {
1573         &format_attr_event.attr,
1574         &format_attr_umask.attr,
1575         &format_attr_edge.attr,
1576         &format_attr_inv.attr,
1577         &format_attr_thresh5.attr,
1578         NULL,
1579 };
1580
1581 static struct attribute *ivbep_uncore_cbox_formats_attr[] = {
1582         &format_attr_event.attr,
1583         &format_attr_umask.attr,
1584         &format_attr_edge.attr,
1585         &format_attr_tid_en.attr,
1586         &format_attr_thresh8.attr,
1587         &format_attr_filter_tid.attr,
1588         &format_attr_filter_link.attr,
1589         &format_attr_filter_state2.attr,
1590         &format_attr_filter_nid2.attr,
1591         &format_attr_filter_opc2.attr,
1592         &format_attr_filter_nc.attr,
1593         &format_attr_filter_c6.attr,
1594         &format_attr_filter_isoc.attr,
1595         NULL,
1596 };
1597
1598 static struct attribute *ivbep_uncore_pcu_formats_attr[] = {
1599         &format_attr_event.attr,
1600         &format_attr_occ_sel.attr,
1601         &format_attr_edge.attr,
1602         &format_attr_thresh5.attr,
1603         &format_attr_occ_invert.attr,
1604         &format_attr_occ_edge.attr,
1605         &format_attr_filter_band0.attr,
1606         &format_attr_filter_band1.attr,
1607         &format_attr_filter_band2.attr,
1608         &format_attr_filter_band3.attr,
1609         NULL,
1610 };
1611
1612 static struct attribute *ivbep_uncore_qpi_formats_attr[] = {
1613         &format_attr_event_ext.attr,
1614         &format_attr_umask.attr,
1615         &format_attr_edge.attr,
1616         &format_attr_thresh8.attr,
1617         &format_attr_match_rds.attr,
1618         &format_attr_match_rnid30.attr,
1619         &format_attr_match_rnid4.attr,
1620         &format_attr_match_dnid.attr,
1621         &format_attr_match_mc.attr,
1622         &format_attr_match_opc.attr,
1623         &format_attr_match_vnw.attr,
1624         &format_attr_match0.attr,
1625         &format_attr_match1.attr,
1626         &format_attr_mask_rds.attr,
1627         &format_attr_mask_rnid30.attr,
1628         &format_attr_mask_rnid4.attr,
1629         &format_attr_mask_dnid.attr,
1630         &format_attr_mask_mc.attr,
1631         &format_attr_mask_opc.attr,
1632         &format_attr_mask_vnw.attr,
1633         &format_attr_mask0.attr,
1634         &format_attr_mask1.attr,
1635         NULL,
1636 };
1637
1638 static const struct attribute_group ivbep_uncore_format_group = {
1639         .name = "format",
1640         .attrs = ivbep_uncore_formats_attr,
1641 };
1642
1643 static const struct attribute_group ivbep_uncore_ubox_format_group = {
1644         .name = "format",
1645         .attrs = ivbep_uncore_ubox_formats_attr,
1646 };
1647
1648 static const struct attribute_group ivbep_uncore_cbox_format_group = {
1649         .name = "format",
1650         .attrs = ivbep_uncore_cbox_formats_attr,
1651 };
1652
1653 static const struct attribute_group ivbep_uncore_pcu_format_group = {
1654         .name = "format",
1655         .attrs = ivbep_uncore_pcu_formats_attr,
1656 };
1657
1658 static const struct attribute_group ivbep_uncore_qpi_format_group = {
1659         .name = "format",
1660         .attrs = ivbep_uncore_qpi_formats_attr,
1661 };
1662
1663 static struct intel_uncore_type ivbep_uncore_ubox = {
1664         .name           = "ubox",
1665         .num_counters   = 2,
1666         .num_boxes      = 1,
1667         .perf_ctr_bits  = 44,
1668         .fixed_ctr_bits = 48,
1669         .perf_ctr       = SNBEP_U_MSR_PMON_CTR0,
1670         .event_ctl      = SNBEP_U_MSR_PMON_CTL0,
1671         .event_mask     = IVBEP_U_MSR_PMON_RAW_EVENT_MASK,
1672         .fixed_ctr      = SNBEP_U_MSR_PMON_UCLK_FIXED_CTR,
1673         .fixed_ctl      = SNBEP_U_MSR_PMON_UCLK_FIXED_CTL,
1674         .ops            = &ivbep_uncore_msr_ops,
1675         .format_group   = &ivbep_uncore_ubox_format_group,
1676 };
1677
1678 static struct extra_reg ivbep_uncore_cbox_extra_regs[] = {
1679         SNBEP_CBO_EVENT_EXTRA_REG(SNBEP_CBO_PMON_CTL_TID_EN,
1680                                   SNBEP_CBO_PMON_CTL_TID_EN, 0x1),
1681         SNBEP_CBO_EVENT_EXTRA_REG(0x1031, 0x10ff, 0x2),
1682         SNBEP_CBO_EVENT_EXTRA_REG(0x1134, 0xffff, 0x4),
1683         SNBEP_CBO_EVENT_EXTRA_REG(0x4134, 0xffff, 0xc),
1684         SNBEP_CBO_EVENT_EXTRA_REG(0x5134, 0xffff, 0xc),
1685         SNBEP_CBO_EVENT_EXTRA_REG(0x0334, 0xffff, 0x4),
1686         SNBEP_CBO_EVENT_EXTRA_REG(0x4334, 0xffff, 0xc),
1687         SNBEP_CBO_EVENT_EXTRA_REG(0x0534, 0xffff, 0x4),
1688         SNBEP_CBO_EVENT_EXTRA_REG(0x4534, 0xffff, 0xc),
1689         SNBEP_CBO_EVENT_EXTRA_REG(0x0934, 0xffff, 0x4),
1690         SNBEP_CBO_EVENT_EXTRA_REG(0x4934, 0xffff, 0xc),
1691         SNBEP_CBO_EVENT_EXTRA_REG(0x0135, 0xffff, 0x10),
1692         SNBEP_CBO_EVENT_EXTRA_REG(0x0335, 0xffff, 0x10),
1693         SNBEP_CBO_EVENT_EXTRA_REG(0x2135, 0xffff, 0x10),
1694         SNBEP_CBO_EVENT_EXTRA_REG(0x2335, 0xffff, 0x10),
1695         SNBEP_CBO_EVENT_EXTRA_REG(0x4135, 0xffff, 0x18),
1696         SNBEP_CBO_EVENT_EXTRA_REG(0x4335, 0xffff, 0x18),
1697         SNBEP_CBO_EVENT_EXTRA_REG(0x4435, 0xffff, 0x8),
1698         SNBEP_CBO_EVENT_EXTRA_REG(0x4835, 0xffff, 0x8),
1699         SNBEP_CBO_EVENT_EXTRA_REG(0x4a35, 0xffff, 0x8),
1700         SNBEP_CBO_EVENT_EXTRA_REG(0x5035, 0xffff, 0x8),
1701         SNBEP_CBO_EVENT_EXTRA_REG(0x8135, 0xffff, 0x10),
1702         SNBEP_CBO_EVENT_EXTRA_REG(0x8335, 0xffff, 0x10),
1703         SNBEP_CBO_EVENT_EXTRA_REG(0x0136, 0xffff, 0x10),
1704         SNBEP_CBO_EVENT_EXTRA_REG(0x0336, 0xffff, 0x10),
1705         SNBEP_CBO_EVENT_EXTRA_REG(0x2136, 0xffff, 0x10),
1706         SNBEP_CBO_EVENT_EXTRA_REG(0x2336, 0xffff, 0x10),
1707         SNBEP_CBO_EVENT_EXTRA_REG(0x4136, 0xffff, 0x18),
1708         SNBEP_CBO_EVENT_EXTRA_REG(0x4336, 0xffff, 0x18),
1709         SNBEP_CBO_EVENT_EXTRA_REG(0x4436, 0xffff, 0x8),
1710         SNBEP_CBO_EVENT_EXTRA_REG(0x4836, 0xffff, 0x8),
1711         SNBEP_CBO_EVENT_EXTRA_REG(0x4a36, 0xffff, 0x8),
1712         SNBEP_CBO_EVENT_EXTRA_REG(0x5036, 0xffff, 0x8),
1713         SNBEP_CBO_EVENT_EXTRA_REG(0x8136, 0xffff, 0x10),
1714         SNBEP_CBO_EVENT_EXTRA_REG(0x8336, 0xffff, 0x10),
1715         SNBEP_CBO_EVENT_EXTRA_REG(0x4037, 0x40ff, 0x8),
1716         EVENT_EXTRA_END
1717 };
1718
1719 static u64 ivbep_cbox_filter_mask(int fields)
1720 {
1721         u64 mask = 0;
1722
1723         if (fields & 0x1)
1724                 mask |= IVBEP_CB0_MSR_PMON_BOX_FILTER_TID;
1725         if (fields & 0x2)
1726                 mask |= IVBEP_CB0_MSR_PMON_BOX_FILTER_LINK;
1727         if (fields & 0x4)
1728                 mask |= IVBEP_CB0_MSR_PMON_BOX_FILTER_STATE;
1729         if (fields & 0x8)
1730                 mask |= IVBEP_CB0_MSR_PMON_BOX_FILTER_NID;
1731         if (fields & 0x10) {
1732                 mask |= IVBEP_CB0_MSR_PMON_BOX_FILTER_OPC;
1733                 mask |= IVBEP_CB0_MSR_PMON_BOX_FILTER_NC;
1734                 mask |= IVBEP_CB0_MSR_PMON_BOX_FILTER_C6;
1735                 mask |= IVBEP_CB0_MSR_PMON_BOX_FILTER_ISOC;
1736         }
1737
1738         return mask;
1739 }
1740
1741 static struct event_constraint *
1742 ivbep_cbox_get_constraint(struct intel_uncore_box *box, struct perf_event *event)
1743 {
1744         return __snbep_cbox_get_constraint(box, event, ivbep_cbox_filter_mask);
1745 }
1746
1747 static int ivbep_cbox_hw_config(struct intel_uncore_box *box, struct perf_event *event)
1748 {
1749         struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
1750         struct extra_reg *er;
1751         int idx = 0;
1752
1753         for (er = ivbep_uncore_cbox_extra_regs; er->msr; er++) {
1754                 if (er->event != (event->hw.config & er->config_mask))
1755                         continue;
1756                 idx |= er->idx;
1757         }
1758
1759         if (idx) {
1760                 reg1->reg = SNBEP_C0_MSR_PMON_BOX_FILTER +
1761                         SNBEP_CBO_MSR_OFFSET * box->pmu->pmu_idx;
1762                 reg1->config = event->attr.config1 & ivbep_cbox_filter_mask(idx);
1763                 reg1->idx = idx;
1764         }
1765         return 0;
1766 }
1767
1768 static void ivbep_cbox_enable_event(struct intel_uncore_box *box, struct perf_event *event)
1769 {
1770         struct hw_perf_event *hwc = &event->hw;
1771         struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
1772
1773         if (reg1->idx != EXTRA_REG_NONE) {
1774                 u64 filter = uncore_shared_reg_config(box, 0);
1775                 wrmsrl(reg1->reg, filter & 0xffffffff);
1776                 wrmsrl(reg1->reg + 6, filter >> 32);
1777         }
1778
1779         wrmsrl(hwc->config_base, hwc->config | SNBEP_PMON_CTL_EN);
1780 }
1781
1782 static struct intel_uncore_ops ivbep_uncore_cbox_ops = {
1783         .init_box               = ivbep_uncore_msr_init_box,
1784         .disable_box            = snbep_uncore_msr_disable_box,
1785         .enable_box             = snbep_uncore_msr_enable_box,
1786         .disable_event          = snbep_uncore_msr_disable_event,
1787         .enable_event           = ivbep_cbox_enable_event,
1788         .read_counter           = uncore_msr_read_counter,
1789         .hw_config              = ivbep_cbox_hw_config,
1790         .get_constraint         = ivbep_cbox_get_constraint,
1791         .put_constraint         = snbep_cbox_put_constraint,
1792 };
1793
1794 static struct intel_uncore_type ivbep_uncore_cbox = {
1795         .name                   = "cbox",
1796         .num_counters           = 4,
1797         .num_boxes              = 15,
1798         .perf_ctr_bits          = 44,
1799         .event_ctl              = SNBEP_C0_MSR_PMON_CTL0,
1800         .perf_ctr               = SNBEP_C0_MSR_PMON_CTR0,
1801         .event_mask             = IVBEP_CBO_MSR_PMON_RAW_EVENT_MASK,
1802         .box_ctl                = SNBEP_C0_MSR_PMON_BOX_CTL,
1803         .msr_offset             = SNBEP_CBO_MSR_OFFSET,
1804         .num_shared_regs        = 1,
1805         .constraints            = snbep_uncore_cbox_constraints,
1806         .ops                    = &ivbep_uncore_cbox_ops,
1807         .format_group           = &ivbep_uncore_cbox_format_group,
1808 };
1809
1810 static struct intel_uncore_ops ivbep_uncore_pcu_ops = {
1811         IVBEP_UNCORE_MSR_OPS_COMMON_INIT(),
1812         .hw_config              = snbep_pcu_hw_config,
1813         .get_constraint         = snbep_pcu_get_constraint,
1814         .put_constraint         = snbep_pcu_put_constraint,
1815 };
1816
1817 static struct intel_uncore_type ivbep_uncore_pcu = {
1818         .name                   = "pcu",
1819         .num_counters           = 4,
1820         .num_boxes              = 1,
1821         .perf_ctr_bits          = 48,
1822         .perf_ctr               = SNBEP_PCU_MSR_PMON_CTR0,
1823         .event_ctl              = SNBEP_PCU_MSR_PMON_CTL0,
1824         .event_mask             = IVBEP_PCU_MSR_PMON_RAW_EVENT_MASK,
1825         .box_ctl                = SNBEP_PCU_MSR_PMON_BOX_CTL,
1826         .num_shared_regs        = 1,
1827         .ops                    = &ivbep_uncore_pcu_ops,
1828         .format_group           = &ivbep_uncore_pcu_format_group,
1829 };
1830
1831 static struct intel_uncore_type *ivbep_msr_uncores[] = {
1832         &ivbep_uncore_ubox,
1833         &ivbep_uncore_cbox,
1834         &ivbep_uncore_pcu,
1835         NULL,
1836 };
1837
1838 void ivbep_uncore_cpu_init(void)
1839 {
1840         if (ivbep_uncore_cbox.num_boxes > boot_cpu_data.x86_max_cores)
1841                 ivbep_uncore_cbox.num_boxes = boot_cpu_data.x86_max_cores;
1842         uncore_msr_uncores = ivbep_msr_uncores;
1843 }
1844
1845 static struct intel_uncore_type ivbep_uncore_ha = {
1846         .name           = "ha",
1847         .num_counters   = 4,
1848         .num_boxes      = 2,
1849         .perf_ctr_bits  = 48,
1850         IVBEP_UNCORE_PCI_COMMON_INIT(),
1851 };
1852
1853 static struct intel_uncore_type ivbep_uncore_imc = {
1854         .name           = "imc",
1855         .num_counters   = 4,
1856         .num_boxes      = 8,
1857         .perf_ctr_bits  = 48,
1858         .fixed_ctr_bits = 48,
1859         .fixed_ctr      = SNBEP_MC_CHy_PCI_PMON_FIXED_CTR,
1860         .fixed_ctl      = SNBEP_MC_CHy_PCI_PMON_FIXED_CTL,
1861         .event_descs    = snbep_uncore_imc_events,
1862         IVBEP_UNCORE_PCI_COMMON_INIT(),
1863 };
1864
1865 /* registers in IRP boxes are not properly aligned */
1866 static unsigned ivbep_uncore_irp_ctls[] = {0xd8, 0xdc, 0xe0, 0xe4};
1867 static unsigned ivbep_uncore_irp_ctrs[] = {0xa0, 0xb0, 0xb8, 0xc0};
1868
1869 static void ivbep_uncore_irp_enable_event(struct intel_uncore_box *box, struct perf_event *event)
1870 {
1871         struct pci_dev *pdev = box->pci_dev;
1872         struct hw_perf_event *hwc = &event->hw;
1873
1874         pci_write_config_dword(pdev, ivbep_uncore_irp_ctls[hwc->idx],
1875                                hwc->config | SNBEP_PMON_CTL_EN);
1876 }
1877
1878 static void ivbep_uncore_irp_disable_event(struct intel_uncore_box *box, struct perf_event *event)
1879 {
1880         struct pci_dev *pdev = box->pci_dev;
1881         struct hw_perf_event *hwc = &event->hw;
1882
1883         pci_write_config_dword(pdev, ivbep_uncore_irp_ctls[hwc->idx], hwc->config);
1884 }
1885
1886 static u64 ivbep_uncore_irp_read_counter(struct intel_uncore_box *box, struct perf_event *event)
1887 {
1888         struct pci_dev *pdev = box->pci_dev;
1889         struct hw_perf_event *hwc = &event->hw;
1890         u64 count = 0;
1891
1892         pci_read_config_dword(pdev, ivbep_uncore_irp_ctrs[hwc->idx], (u32 *)&count);
1893         pci_read_config_dword(pdev, ivbep_uncore_irp_ctrs[hwc->idx] + 4, (u32 *)&count + 1);
1894
1895         return count;
1896 }
1897
1898 static struct intel_uncore_ops ivbep_uncore_irp_ops = {
1899         .init_box       = ivbep_uncore_pci_init_box,
1900         .disable_box    = snbep_uncore_pci_disable_box,
1901         .enable_box     = snbep_uncore_pci_enable_box,
1902         .disable_event  = ivbep_uncore_irp_disable_event,
1903         .enable_event   = ivbep_uncore_irp_enable_event,
1904         .read_counter   = ivbep_uncore_irp_read_counter,
1905 };
1906
1907 static struct intel_uncore_type ivbep_uncore_irp = {
1908         .name                   = "irp",
1909         .num_counters           = 4,
1910         .num_boxes              = 1,
1911         .perf_ctr_bits          = 48,
1912         .event_mask             = IVBEP_PMON_RAW_EVENT_MASK,
1913         .box_ctl                = SNBEP_PCI_PMON_BOX_CTL,
1914         .ops                    = &ivbep_uncore_irp_ops,
1915         .format_group           = &ivbep_uncore_format_group,
1916 };
1917
1918 static struct intel_uncore_ops ivbep_uncore_qpi_ops = {
1919         .init_box       = ivbep_uncore_pci_init_box,
1920         .disable_box    = snbep_uncore_pci_disable_box,
1921         .enable_box     = snbep_uncore_pci_enable_box,
1922         .disable_event  = snbep_uncore_pci_disable_event,
1923         .enable_event   = snbep_qpi_enable_event,
1924         .read_counter   = snbep_uncore_pci_read_counter,
1925         .hw_config      = snbep_qpi_hw_config,
1926         .get_constraint = uncore_get_constraint,
1927         .put_constraint = uncore_put_constraint,
1928 };
1929
1930 static struct intel_uncore_type ivbep_uncore_qpi = {
1931         .name                   = "qpi",
1932         .num_counters           = 4,
1933         .num_boxes              = 3,
1934         .perf_ctr_bits          = 48,
1935         .perf_ctr               = SNBEP_PCI_PMON_CTR0,
1936         .event_ctl              = SNBEP_PCI_PMON_CTL0,
1937         .event_mask             = IVBEP_QPI_PCI_PMON_RAW_EVENT_MASK,
1938         .box_ctl                = SNBEP_PCI_PMON_BOX_CTL,
1939         .num_shared_regs        = 1,
1940         .ops                    = &ivbep_uncore_qpi_ops,
1941         .format_group           = &ivbep_uncore_qpi_format_group,
1942 };
1943
1944 static struct intel_uncore_type ivbep_uncore_r2pcie = {
1945         .name           = "r2pcie",
1946         .num_counters   = 4,
1947         .num_boxes      = 1,
1948         .perf_ctr_bits  = 44,
1949         .constraints    = snbep_uncore_r2pcie_constraints,
1950         IVBEP_UNCORE_PCI_COMMON_INIT(),
1951 };
1952
1953 static struct intel_uncore_type ivbep_uncore_r3qpi = {
1954         .name           = "r3qpi",
1955         .num_counters   = 3,
1956         .num_boxes      = 2,
1957         .perf_ctr_bits  = 44,
1958         .constraints    = snbep_uncore_r3qpi_constraints,
1959         IVBEP_UNCORE_PCI_COMMON_INIT(),
1960 };
1961
1962 enum {
1963         IVBEP_PCI_UNCORE_HA,
1964         IVBEP_PCI_UNCORE_IMC,
1965         IVBEP_PCI_UNCORE_IRP,
1966         IVBEP_PCI_UNCORE_QPI,
1967         IVBEP_PCI_UNCORE_R2PCIE,
1968         IVBEP_PCI_UNCORE_R3QPI,
1969 };
1970
1971 static struct intel_uncore_type *ivbep_pci_uncores[] = {
1972         [IVBEP_PCI_UNCORE_HA]   = &ivbep_uncore_ha,
1973         [IVBEP_PCI_UNCORE_IMC]  = &ivbep_uncore_imc,
1974         [IVBEP_PCI_UNCORE_IRP]  = &ivbep_uncore_irp,
1975         [IVBEP_PCI_UNCORE_QPI]  = &ivbep_uncore_qpi,
1976         [IVBEP_PCI_UNCORE_R2PCIE]       = &ivbep_uncore_r2pcie,
1977         [IVBEP_PCI_UNCORE_R3QPI]        = &ivbep_uncore_r3qpi,
1978         NULL,
1979 };
1980
1981 static const struct pci_device_id ivbep_uncore_pci_ids[] = {
1982         { /* Home Agent 0 */
1983                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe30),
1984                 .driver_data = UNCORE_PCI_DEV_DATA(IVBEP_PCI_UNCORE_HA, 0),
1985         },
1986         { /* Home Agent 1 */
1987                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe38),
1988                 .driver_data = UNCORE_PCI_DEV_DATA(IVBEP_PCI_UNCORE_HA, 1),
1989         },
1990         { /* MC0 Channel 0 */
1991                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xeb4),
1992                 .driver_data = UNCORE_PCI_DEV_DATA(IVBEP_PCI_UNCORE_IMC, 0),
1993         },
1994         { /* MC0 Channel 1 */
1995                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xeb5),
1996                 .driver_data = UNCORE_PCI_DEV_DATA(IVBEP_PCI_UNCORE_IMC, 1),
1997         },
1998         { /* MC0 Channel 3 */
1999                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xeb0),
2000                 .driver_data = UNCORE_PCI_DEV_DATA(IVBEP_PCI_UNCORE_IMC, 2),
2001         },
2002         { /* MC0 Channel 4 */
2003                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xeb1),
2004                 .driver_data = UNCORE_PCI_DEV_DATA(IVBEP_PCI_UNCORE_IMC, 3),
2005         },
2006         { /* MC1 Channel 0 */
2007                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xef4),
2008                 .driver_data = UNCORE_PCI_DEV_DATA(IVBEP_PCI_UNCORE_IMC, 4),
2009         },
2010         { /* MC1 Channel 1 */
2011                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xef5),
2012                 .driver_data = UNCORE_PCI_DEV_DATA(IVBEP_PCI_UNCORE_IMC, 5),
2013         },
2014         { /* MC1 Channel 3 */
2015                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xef0),
2016                 .driver_data = UNCORE_PCI_DEV_DATA(IVBEP_PCI_UNCORE_IMC, 6),
2017         },
2018         { /* MC1 Channel 4 */
2019                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xef1),
2020                 .driver_data = UNCORE_PCI_DEV_DATA(IVBEP_PCI_UNCORE_IMC, 7),
2021         },
2022         { /* IRP */
2023                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe39),
2024                 .driver_data = UNCORE_PCI_DEV_DATA(IVBEP_PCI_UNCORE_IRP, 0),
2025         },
2026         { /* QPI0 Port 0 */
2027                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe32),
2028                 .driver_data = UNCORE_PCI_DEV_DATA(IVBEP_PCI_UNCORE_QPI, 0),
2029         },
2030         { /* QPI0 Port 1 */
2031                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe33),
2032                 .driver_data = UNCORE_PCI_DEV_DATA(IVBEP_PCI_UNCORE_QPI, 1),
2033         },
2034         { /* QPI1 Port 2 */
2035                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe3a),
2036                 .driver_data = UNCORE_PCI_DEV_DATA(IVBEP_PCI_UNCORE_QPI, 2),
2037         },
2038         { /* R2PCIe */
2039                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe34),
2040                 .driver_data = UNCORE_PCI_DEV_DATA(IVBEP_PCI_UNCORE_R2PCIE, 0),
2041         },
2042         { /* R3QPI0 Link 0 */
2043                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe36),
2044                 .driver_data = UNCORE_PCI_DEV_DATA(IVBEP_PCI_UNCORE_R3QPI, 0),
2045         },
2046         { /* R3QPI0 Link 1 */
2047                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe37),
2048                 .driver_data = UNCORE_PCI_DEV_DATA(IVBEP_PCI_UNCORE_R3QPI, 1),
2049         },
2050         { /* R3QPI1 Link 2 */
2051                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe3e),
2052                 .driver_data = UNCORE_PCI_DEV_DATA(IVBEP_PCI_UNCORE_R3QPI, 2),
2053         },
2054         { /* QPI Port 0 filter  */
2055                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe86),
2056                 .driver_data = UNCORE_PCI_DEV_DATA(UNCORE_EXTRA_PCI_DEV,
2057                                                    SNBEP_PCI_QPI_PORT0_FILTER),
2058         },
2059         { /* QPI Port 0 filter  */
2060                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe96),
2061                 .driver_data = UNCORE_PCI_DEV_DATA(UNCORE_EXTRA_PCI_DEV,
2062                                                    SNBEP_PCI_QPI_PORT1_FILTER),
2063         },
2064         { /* end: all zeroes */ }
2065 };
2066
2067 static struct pci_driver ivbep_uncore_pci_driver = {
2068         .name           = "ivbep_uncore",
2069         .id_table       = ivbep_uncore_pci_ids,
2070 };
2071
2072 int ivbep_uncore_pci_init(void)
2073 {
2074         int ret = snbep_pci2phy_map_init(0x0e1e, SNBEP_CPUNODEID, SNBEP_GIDNIDMAP, true);
2075         if (ret)
2076                 return ret;
2077         uncore_pci_uncores = ivbep_pci_uncores;
2078         uncore_pci_driver = &ivbep_uncore_pci_driver;
2079         return 0;
2080 }
2081 /* end of IvyTown uncore support */
2082
2083 /* KNL uncore support */
2084 static struct attribute *knl_uncore_ubox_formats_attr[] = {
2085         &format_attr_event.attr,
2086         &format_attr_umask.attr,
2087         &format_attr_edge.attr,
2088         &format_attr_tid_en.attr,
2089         &format_attr_inv.attr,
2090         &format_attr_thresh5.attr,
2091         NULL,
2092 };
2093
2094 static const struct attribute_group knl_uncore_ubox_format_group = {
2095         .name = "format",
2096         .attrs = knl_uncore_ubox_formats_attr,
2097 };
2098
2099 static struct intel_uncore_type knl_uncore_ubox = {
2100         .name                   = "ubox",
2101         .num_counters           = 2,
2102         .num_boxes              = 1,
2103         .perf_ctr_bits          = 48,
2104         .fixed_ctr_bits         = 48,
2105         .perf_ctr               = HSWEP_U_MSR_PMON_CTR0,
2106         .event_ctl              = HSWEP_U_MSR_PMON_CTL0,
2107         .event_mask             = KNL_U_MSR_PMON_RAW_EVENT_MASK,
2108         .fixed_ctr              = HSWEP_U_MSR_PMON_UCLK_FIXED_CTR,
2109         .fixed_ctl              = HSWEP_U_MSR_PMON_UCLK_FIXED_CTL,
2110         .ops                    = &snbep_uncore_msr_ops,
2111         .format_group           = &knl_uncore_ubox_format_group,
2112 };
2113
2114 static struct attribute *knl_uncore_cha_formats_attr[] = {
2115         &format_attr_event.attr,
2116         &format_attr_umask.attr,
2117         &format_attr_qor.attr,
2118         &format_attr_edge.attr,
2119         &format_attr_tid_en.attr,
2120         &format_attr_inv.attr,
2121         &format_attr_thresh8.attr,
2122         &format_attr_filter_tid4.attr,
2123         &format_attr_filter_link3.attr,
2124         &format_attr_filter_state4.attr,
2125         &format_attr_filter_local.attr,
2126         &format_attr_filter_all_op.attr,
2127         &format_attr_filter_nnm.attr,
2128         &format_attr_filter_opc3.attr,
2129         &format_attr_filter_nc.attr,
2130         &format_attr_filter_isoc.attr,
2131         NULL,
2132 };
2133
2134 static const struct attribute_group knl_uncore_cha_format_group = {
2135         .name = "format",
2136         .attrs = knl_uncore_cha_formats_attr,
2137 };
2138
2139 static struct event_constraint knl_uncore_cha_constraints[] = {
2140         UNCORE_EVENT_CONSTRAINT(0x11, 0x1),
2141         UNCORE_EVENT_CONSTRAINT(0x1f, 0x1),
2142         UNCORE_EVENT_CONSTRAINT(0x36, 0x1),
2143         EVENT_CONSTRAINT_END
2144 };
2145
2146 static struct extra_reg knl_uncore_cha_extra_regs[] = {
2147         SNBEP_CBO_EVENT_EXTRA_REG(SNBEP_CBO_PMON_CTL_TID_EN,
2148                                   SNBEP_CBO_PMON_CTL_TID_EN, 0x1),
2149         SNBEP_CBO_EVENT_EXTRA_REG(0x3d, 0xff, 0x2),
2150         SNBEP_CBO_EVENT_EXTRA_REG(0x35, 0xff, 0x4),
2151         SNBEP_CBO_EVENT_EXTRA_REG(0x36, 0xff, 0x4),
2152         EVENT_EXTRA_END
2153 };
2154
2155 static u64 knl_cha_filter_mask(int fields)
2156 {
2157         u64 mask = 0;
2158
2159         if (fields & 0x1)
2160                 mask |= KNL_CHA_MSR_PMON_BOX_FILTER_TID;
2161         if (fields & 0x2)
2162                 mask |= KNL_CHA_MSR_PMON_BOX_FILTER_STATE;
2163         if (fields & 0x4)
2164                 mask |= KNL_CHA_MSR_PMON_BOX_FILTER_OP;
2165         return mask;
2166 }
2167
2168 static struct event_constraint *
2169 knl_cha_get_constraint(struct intel_uncore_box *box, struct perf_event *event)
2170 {
2171         return __snbep_cbox_get_constraint(box, event, knl_cha_filter_mask);
2172 }
2173
2174 static int knl_cha_hw_config(struct intel_uncore_box *box,
2175                              struct perf_event *event)
2176 {
2177         struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
2178         struct extra_reg *er;
2179         int idx = 0;
2180
2181         for (er = knl_uncore_cha_extra_regs; er->msr; er++) {
2182                 if (er->event != (event->hw.config & er->config_mask))
2183                         continue;
2184                 idx |= er->idx;
2185         }
2186
2187         if (idx) {
2188                 reg1->reg = HSWEP_C0_MSR_PMON_BOX_FILTER0 +
2189                             KNL_CHA_MSR_OFFSET * box->pmu->pmu_idx;
2190                 reg1->config = event->attr.config1 & knl_cha_filter_mask(idx);
2191
2192                 reg1->config |= KNL_CHA_MSR_PMON_BOX_FILTER_REMOTE_NODE;
2193                 reg1->config |= KNL_CHA_MSR_PMON_BOX_FILTER_LOCAL_NODE;
2194                 reg1->config |= KNL_CHA_MSR_PMON_BOX_FILTER_NNC;
2195                 reg1->idx = idx;
2196         }
2197         return 0;
2198 }
2199
2200 static void hswep_cbox_enable_event(struct intel_uncore_box *box,
2201                                     struct perf_event *event);
2202
2203 static struct intel_uncore_ops knl_uncore_cha_ops = {
2204         .init_box               = snbep_uncore_msr_init_box,
2205         .disable_box            = snbep_uncore_msr_disable_box,
2206         .enable_box             = snbep_uncore_msr_enable_box,
2207         .disable_event          = snbep_uncore_msr_disable_event,
2208         .enable_event           = hswep_cbox_enable_event,
2209         .read_counter           = uncore_msr_read_counter,
2210         .hw_config              = knl_cha_hw_config,
2211         .get_constraint         = knl_cha_get_constraint,
2212         .put_constraint         = snbep_cbox_put_constraint,
2213 };
2214
2215 static struct intel_uncore_type knl_uncore_cha = {
2216         .name                   = "cha",
2217         .num_counters           = 4,
2218         .num_boxes              = 38,
2219         .perf_ctr_bits          = 48,
2220         .event_ctl              = HSWEP_C0_MSR_PMON_CTL0,
2221         .perf_ctr               = HSWEP_C0_MSR_PMON_CTR0,
2222         .event_mask             = KNL_CHA_MSR_PMON_RAW_EVENT_MASK,
2223         .box_ctl                = HSWEP_C0_MSR_PMON_BOX_CTL,
2224         .msr_offset             = KNL_CHA_MSR_OFFSET,
2225         .num_shared_regs        = 1,
2226         .constraints            = knl_uncore_cha_constraints,
2227         .ops                    = &knl_uncore_cha_ops,
2228         .format_group           = &knl_uncore_cha_format_group,
2229 };
2230
2231 static struct attribute *knl_uncore_pcu_formats_attr[] = {
2232         &format_attr_event2.attr,
2233         &format_attr_use_occ_ctr.attr,
2234         &format_attr_occ_sel.attr,
2235         &format_attr_edge.attr,
2236         &format_attr_tid_en.attr,
2237         &format_attr_inv.attr,
2238         &format_attr_thresh6.attr,
2239         &format_attr_occ_invert.attr,
2240         &format_attr_occ_edge_det.attr,
2241         NULL,
2242 };
2243
2244 static const struct attribute_group knl_uncore_pcu_format_group = {
2245         .name = "format",
2246         .attrs = knl_uncore_pcu_formats_attr,
2247 };
2248
2249 static struct intel_uncore_type knl_uncore_pcu = {
2250         .name                   = "pcu",
2251         .num_counters           = 4,
2252         .num_boxes              = 1,
2253         .perf_ctr_bits          = 48,
2254         .perf_ctr               = HSWEP_PCU_MSR_PMON_CTR0,
2255         .event_ctl              = HSWEP_PCU_MSR_PMON_CTL0,
2256         .event_mask             = KNL_PCU_MSR_PMON_RAW_EVENT_MASK,
2257         .box_ctl                = HSWEP_PCU_MSR_PMON_BOX_CTL,
2258         .ops                    = &snbep_uncore_msr_ops,
2259         .format_group           = &knl_uncore_pcu_format_group,
2260 };
2261
2262 static struct intel_uncore_type *knl_msr_uncores[] = {
2263         &knl_uncore_ubox,
2264         &knl_uncore_cha,
2265         &knl_uncore_pcu,
2266         NULL,
2267 };
2268
2269 void knl_uncore_cpu_init(void)
2270 {
2271         uncore_msr_uncores = knl_msr_uncores;
2272 }
2273
2274 static void knl_uncore_imc_enable_box(struct intel_uncore_box *box)
2275 {
2276         struct pci_dev *pdev = box->pci_dev;
2277         int box_ctl = uncore_pci_box_ctl(box);
2278
2279         pci_write_config_dword(pdev, box_ctl, 0);
2280 }
2281
2282 static void knl_uncore_imc_enable_event(struct intel_uncore_box *box,
2283                                         struct perf_event *event)
2284 {
2285         struct pci_dev *pdev = box->pci_dev;
2286         struct hw_perf_event *hwc = &event->hw;
2287
2288         if ((event->attr.config & SNBEP_PMON_CTL_EV_SEL_MASK)
2289                                                         == UNCORE_FIXED_EVENT)
2290                 pci_write_config_dword(pdev, hwc->config_base,
2291                                        hwc->config | KNL_PMON_FIXED_CTL_EN);
2292         else
2293                 pci_write_config_dword(pdev, hwc->config_base,
2294                                        hwc->config | SNBEP_PMON_CTL_EN);
2295 }
2296
2297 static struct intel_uncore_ops knl_uncore_imc_ops = {
2298         .init_box       = snbep_uncore_pci_init_box,
2299         .disable_box    = snbep_uncore_pci_disable_box,
2300         .enable_box     = knl_uncore_imc_enable_box,
2301         .read_counter   = snbep_uncore_pci_read_counter,
2302         .enable_event   = knl_uncore_imc_enable_event,
2303         .disable_event  = snbep_uncore_pci_disable_event,
2304 };
2305
2306 static struct intel_uncore_type knl_uncore_imc_uclk = {
2307         .name                   = "imc_uclk",
2308         .num_counters           = 4,
2309         .num_boxes              = 2,
2310         .perf_ctr_bits          = 48,
2311         .fixed_ctr_bits         = 48,
2312         .perf_ctr               = KNL_UCLK_MSR_PMON_CTR0_LOW,
2313         .event_ctl              = KNL_UCLK_MSR_PMON_CTL0,
2314         .event_mask             = SNBEP_PMON_RAW_EVENT_MASK,
2315         .fixed_ctr              = KNL_UCLK_MSR_PMON_UCLK_FIXED_LOW,
2316         .fixed_ctl              = KNL_UCLK_MSR_PMON_UCLK_FIXED_CTL,
2317         .box_ctl                = KNL_UCLK_MSR_PMON_BOX_CTL,
2318         .ops                    = &knl_uncore_imc_ops,
2319         .format_group           = &snbep_uncore_format_group,
2320 };
2321
2322 static struct intel_uncore_type knl_uncore_imc_dclk = {
2323         .name                   = "imc",
2324         .num_counters           = 4,
2325         .num_boxes              = 6,
2326         .perf_ctr_bits          = 48,
2327         .fixed_ctr_bits         = 48,
2328         .perf_ctr               = KNL_MC0_CH0_MSR_PMON_CTR0_LOW,
2329         .event_ctl              = KNL_MC0_CH0_MSR_PMON_CTL0,
2330         .event_mask             = SNBEP_PMON_RAW_EVENT_MASK,
2331         .fixed_ctr              = KNL_MC0_CH0_MSR_PMON_FIXED_LOW,
2332         .fixed_ctl              = KNL_MC0_CH0_MSR_PMON_FIXED_CTL,
2333         .box_ctl                = KNL_MC0_CH0_MSR_PMON_BOX_CTL,
2334         .ops                    = &knl_uncore_imc_ops,
2335         .format_group           = &snbep_uncore_format_group,
2336 };
2337
2338 static struct intel_uncore_type knl_uncore_edc_uclk = {
2339         .name                   = "edc_uclk",
2340         .num_counters           = 4,
2341         .num_boxes              = 8,
2342         .perf_ctr_bits          = 48,
2343         .fixed_ctr_bits         = 48,
2344         .perf_ctr               = KNL_UCLK_MSR_PMON_CTR0_LOW,
2345         .event_ctl              = KNL_UCLK_MSR_PMON_CTL0,
2346         .event_mask             = SNBEP_PMON_RAW_EVENT_MASK,
2347         .fixed_ctr              = KNL_UCLK_MSR_PMON_UCLK_FIXED_LOW,
2348         .fixed_ctl              = KNL_UCLK_MSR_PMON_UCLK_FIXED_CTL,
2349         .box_ctl                = KNL_UCLK_MSR_PMON_BOX_CTL,
2350         .ops                    = &knl_uncore_imc_ops,
2351         .format_group           = &snbep_uncore_format_group,
2352 };
2353
2354 static struct intel_uncore_type knl_uncore_edc_eclk = {
2355         .name                   = "edc_eclk",
2356         .num_counters           = 4,
2357         .num_boxes              = 8,
2358         .perf_ctr_bits          = 48,
2359         .fixed_ctr_bits         = 48,
2360         .perf_ctr               = KNL_EDC0_ECLK_MSR_PMON_CTR0_LOW,
2361         .event_ctl              = KNL_EDC0_ECLK_MSR_PMON_CTL0,
2362         .event_mask             = SNBEP_PMON_RAW_EVENT_MASK,
2363         .fixed_ctr              = KNL_EDC0_ECLK_MSR_PMON_ECLK_FIXED_LOW,
2364         .fixed_ctl              = KNL_EDC0_ECLK_MSR_PMON_ECLK_FIXED_CTL,
2365         .box_ctl                = KNL_EDC0_ECLK_MSR_PMON_BOX_CTL,
2366         .ops                    = &knl_uncore_imc_ops,
2367         .format_group           = &snbep_uncore_format_group,
2368 };
2369
2370 static struct event_constraint knl_uncore_m2pcie_constraints[] = {
2371         UNCORE_EVENT_CONSTRAINT(0x23, 0x3),
2372         EVENT_CONSTRAINT_END
2373 };
2374
2375 static struct intel_uncore_type knl_uncore_m2pcie = {
2376         .name           = "m2pcie",
2377         .num_counters   = 4,
2378         .num_boxes      = 1,
2379         .perf_ctr_bits  = 48,
2380         .constraints    = knl_uncore_m2pcie_constraints,
2381         SNBEP_UNCORE_PCI_COMMON_INIT(),
2382 };
2383
2384 static struct attribute *knl_uncore_irp_formats_attr[] = {
2385         &format_attr_event.attr,
2386         &format_attr_umask.attr,
2387         &format_attr_qor.attr,
2388         &format_attr_edge.attr,
2389         &format_attr_inv.attr,
2390         &format_attr_thresh8.attr,
2391         NULL,
2392 };
2393
2394 static const struct attribute_group knl_uncore_irp_format_group = {
2395         .name = "format",
2396         .attrs = knl_uncore_irp_formats_attr,
2397 };
2398
2399 static struct intel_uncore_type knl_uncore_irp = {
2400         .name                   = "irp",
2401         .num_counters           = 2,
2402         .num_boxes              = 1,
2403         .perf_ctr_bits          = 48,
2404         .perf_ctr               = SNBEP_PCI_PMON_CTR0,
2405         .event_ctl              = SNBEP_PCI_PMON_CTL0,
2406         .event_mask             = KNL_IRP_PCI_PMON_RAW_EVENT_MASK,
2407         .box_ctl                = KNL_IRP_PCI_PMON_BOX_CTL,
2408         .ops                    = &snbep_uncore_pci_ops,
2409         .format_group           = &knl_uncore_irp_format_group,
2410 };
2411
2412 enum {
2413         KNL_PCI_UNCORE_MC_UCLK,
2414         KNL_PCI_UNCORE_MC_DCLK,
2415         KNL_PCI_UNCORE_EDC_UCLK,
2416         KNL_PCI_UNCORE_EDC_ECLK,
2417         KNL_PCI_UNCORE_M2PCIE,
2418         KNL_PCI_UNCORE_IRP,
2419 };
2420
2421 static struct intel_uncore_type *knl_pci_uncores[] = {
2422         [KNL_PCI_UNCORE_MC_UCLK]        = &knl_uncore_imc_uclk,
2423         [KNL_PCI_UNCORE_MC_DCLK]        = &knl_uncore_imc_dclk,
2424         [KNL_PCI_UNCORE_EDC_UCLK]       = &knl_uncore_edc_uclk,
2425         [KNL_PCI_UNCORE_EDC_ECLK]       = &knl_uncore_edc_eclk,
2426         [KNL_PCI_UNCORE_M2PCIE]         = &knl_uncore_m2pcie,
2427         [KNL_PCI_UNCORE_IRP]            = &knl_uncore_irp,
2428         NULL,
2429 };
2430
2431 /*
2432  * KNL uses a common PCI device ID for multiple instances of an Uncore PMU
2433  * device type. prior to KNL, each instance of a PMU device type had a unique
2434  * device ID.
2435  *
2436  *      PCI Device ID   Uncore PMU Devices
2437  *      ----------------------------------
2438  *      0x7841          MC0 UClk, MC1 UClk
2439  *      0x7843          MC0 DClk CH 0, MC0 DClk CH 1, MC0 DClk CH 2,
2440  *                      MC1 DClk CH 0, MC1 DClk CH 1, MC1 DClk CH 2
2441  *      0x7833          EDC0 UClk, EDC1 UClk, EDC2 UClk, EDC3 UClk,
2442  *                      EDC4 UClk, EDC5 UClk, EDC6 UClk, EDC7 UClk
2443  *      0x7835          EDC0 EClk, EDC1 EClk, EDC2 EClk, EDC3 EClk,
2444  *                      EDC4 EClk, EDC5 EClk, EDC6 EClk, EDC7 EClk
2445  *      0x7817          M2PCIe
2446  *      0x7814          IRP
2447 */
2448
2449 static const struct pci_device_id knl_uncore_pci_ids[] = {
2450         { /* MC0 UClk */
2451                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x7841),
2452                 .driver_data = UNCORE_PCI_DEV_FULL_DATA(10, 0, KNL_PCI_UNCORE_MC_UCLK, 0),
2453         },
2454         { /* MC1 UClk */
2455                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x7841),
2456                 .driver_data = UNCORE_PCI_DEV_FULL_DATA(11, 0, KNL_PCI_UNCORE_MC_UCLK, 1),
2457         },
2458         { /* MC0 DClk CH 0 */
2459                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x7843),
2460                 .driver_data = UNCORE_PCI_DEV_FULL_DATA(8, 2, KNL_PCI_UNCORE_MC_DCLK, 0),
2461         },
2462         { /* MC0 DClk CH 1 */
2463                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x7843),
2464                 .driver_data = UNCORE_PCI_DEV_FULL_DATA(8, 3, KNL_PCI_UNCORE_MC_DCLK, 1),
2465         },
2466         { /* MC0 DClk CH 2 */
2467                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x7843),
2468                 .driver_data = UNCORE_PCI_DEV_FULL_DATA(8, 4, KNL_PCI_UNCORE_MC_DCLK, 2),
2469         },
2470         { /* MC1 DClk CH 0 */
2471                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x7843),
2472                 .driver_data = UNCORE_PCI_DEV_FULL_DATA(9, 2, KNL_PCI_UNCORE_MC_DCLK, 3),
2473         },
2474         { /* MC1 DClk CH 1 */
2475                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x7843),
2476                 .driver_data = UNCORE_PCI_DEV_FULL_DATA(9, 3, KNL_PCI_UNCORE_MC_DCLK, 4),
2477         },
2478         { /* MC1 DClk CH 2 */
2479                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x7843),
2480                 .driver_data = UNCORE_PCI_DEV_FULL_DATA(9, 4, KNL_PCI_UNCORE_MC_DCLK, 5),
2481         },
2482         { /* EDC0 UClk */
2483                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x7833),
2484                 .driver_data = UNCORE_PCI_DEV_FULL_DATA(15, 0, KNL_PCI_UNCORE_EDC_UCLK, 0),
2485         },
2486         { /* EDC1 UClk */
2487                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x7833),
2488                 .driver_data = UNCORE_PCI_DEV_FULL_DATA(16, 0, KNL_PCI_UNCORE_EDC_UCLK, 1),
2489         },
2490         { /* EDC2 UClk */
2491                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x7833),
2492                 .driver_data = UNCORE_PCI_DEV_FULL_DATA(17, 0, KNL_PCI_UNCORE_EDC_UCLK, 2),
2493         },
2494         { /* EDC3 UClk */
2495                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x7833),
2496                 .driver_data = UNCORE_PCI_DEV_FULL_DATA(18, 0, KNL_PCI_UNCORE_EDC_UCLK, 3),
2497         },
2498         { /* EDC4 UClk */
2499                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x7833),
2500                 .driver_data = UNCORE_PCI_DEV_FULL_DATA(19, 0, KNL_PCI_UNCORE_EDC_UCLK, 4),
2501         },
2502         { /* EDC5 UClk */
2503                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x7833),
2504                 .driver_data = UNCORE_PCI_DEV_FULL_DATA(20, 0, KNL_PCI_UNCORE_EDC_UCLK, 5),
2505         },
2506         { /* EDC6 UClk */
2507                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x7833),
2508                 .driver_data = UNCORE_PCI_DEV_FULL_DATA(21, 0, KNL_PCI_UNCORE_EDC_UCLK, 6),
2509         },
2510         { /* EDC7 UClk */
2511                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x7833),
2512                 .driver_data = UNCORE_PCI_DEV_FULL_DATA(22, 0, KNL_PCI_UNCORE_EDC_UCLK, 7),
2513         },
2514         { /* EDC0 EClk */
2515                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x7835),
2516                 .driver_data = UNCORE_PCI_DEV_FULL_DATA(24, 2, KNL_PCI_UNCORE_EDC_ECLK, 0),
2517         },
2518         { /* EDC1 EClk */
2519                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x7835),
2520                 .driver_data = UNCORE_PCI_DEV_FULL_DATA(25, 2, KNL_PCI_UNCORE_EDC_ECLK, 1),
2521         },
2522         { /* EDC2 EClk */
2523                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x7835),
2524                 .driver_data = UNCORE_PCI_DEV_FULL_DATA(26, 2, KNL_PCI_UNCORE_EDC_ECLK, 2),
2525         },
2526         { /* EDC3 EClk */
2527                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x7835),
2528                 .driver_data = UNCORE_PCI_DEV_FULL_DATA(27, 2, KNL_PCI_UNCORE_EDC_ECLK, 3),
2529         },
2530         { /* EDC4 EClk */
2531                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x7835),
2532                 .driver_data = UNCORE_PCI_DEV_FULL_DATA(28, 2, KNL_PCI_UNCORE_EDC_ECLK, 4),
2533         },
2534         { /* EDC5 EClk */
2535                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x7835),
2536                 .driver_data = UNCORE_PCI_DEV_FULL_DATA(29, 2, KNL_PCI_UNCORE_EDC_ECLK, 5),
2537         },
2538         { /* EDC6 EClk */
2539                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x7835),
2540                 .driver_data = UNCORE_PCI_DEV_FULL_DATA(30, 2, KNL_PCI_UNCORE_EDC_ECLK, 6),
2541         },
2542         { /* EDC7 EClk */
2543                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x7835),
2544                 .driver_data = UNCORE_PCI_DEV_FULL_DATA(31, 2, KNL_PCI_UNCORE_EDC_ECLK, 7),
2545         },
2546         { /* M2PCIe */
2547                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x7817),
2548                 .driver_data = UNCORE_PCI_DEV_DATA(KNL_PCI_UNCORE_M2PCIE, 0),
2549         },
2550         { /* IRP */
2551                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x7814),
2552                 .driver_data = UNCORE_PCI_DEV_DATA(KNL_PCI_UNCORE_IRP, 0),
2553         },
2554         { /* end: all zeroes */ }
2555 };
2556
2557 static struct pci_driver knl_uncore_pci_driver = {
2558         .name           = "knl_uncore",
2559         .id_table       = knl_uncore_pci_ids,
2560 };
2561
2562 int knl_uncore_pci_init(void)
2563 {
2564         int ret;
2565
2566         /* All KNL PCI based PMON units are on the same PCI bus except IRP */
2567         ret = snb_pci2phy_map_init(0x7814); /* IRP */
2568         if (ret)
2569                 return ret;
2570         ret = snb_pci2phy_map_init(0x7817); /* M2PCIe */
2571         if (ret)
2572                 return ret;
2573         uncore_pci_uncores = knl_pci_uncores;
2574         uncore_pci_driver = &knl_uncore_pci_driver;
2575         return 0;
2576 }
2577
2578 /* end of KNL uncore support */
2579
2580 /* Haswell-EP uncore support */
2581 static struct attribute *hswep_uncore_ubox_formats_attr[] = {
2582         &format_attr_event.attr,
2583         &format_attr_umask.attr,
2584         &format_attr_edge.attr,
2585         &format_attr_inv.attr,
2586         &format_attr_thresh5.attr,
2587         &format_attr_filter_tid2.attr,
2588         &format_attr_filter_cid.attr,
2589         NULL,
2590 };
2591
2592 static const struct attribute_group hswep_uncore_ubox_format_group = {
2593         .name = "format",
2594         .attrs = hswep_uncore_ubox_formats_attr,
2595 };
2596
2597 static int hswep_ubox_hw_config(struct intel_uncore_box *box, struct perf_event *event)
2598 {
2599         struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
2600         reg1->reg = HSWEP_U_MSR_PMON_FILTER;
2601         reg1->config = event->attr.config1 & HSWEP_U_MSR_PMON_BOX_FILTER_MASK;
2602         reg1->idx = 0;
2603         return 0;
2604 }
2605
2606 static struct intel_uncore_ops hswep_uncore_ubox_ops = {
2607         SNBEP_UNCORE_MSR_OPS_COMMON_INIT(),
2608         .hw_config              = hswep_ubox_hw_config,
2609         .get_constraint         = uncore_get_constraint,
2610         .put_constraint         = uncore_put_constraint,
2611 };
2612
2613 static struct intel_uncore_type hswep_uncore_ubox = {
2614         .name                   = "ubox",
2615         .num_counters           = 2,
2616         .num_boxes              = 1,
2617         .perf_ctr_bits          = 44,
2618         .fixed_ctr_bits         = 48,
2619         .perf_ctr               = HSWEP_U_MSR_PMON_CTR0,
2620         .event_ctl              = HSWEP_U_MSR_PMON_CTL0,
2621         .event_mask             = SNBEP_U_MSR_PMON_RAW_EVENT_MASK,
2622         .fixed_ctr              = HSWEP_U_MSR_PMON_UCLK_FIXED_CTR,
2623         .fixed_ctl              = HSWEP_U_MSR_PMON_UCLK_FIXED_CTL,
2624         .num_shared_regs        = 1,
2625         .ops                    = &hswep_uncore_ubox_ops,
2626         .format_group           = &hswep_uncore_ubox_format_group,
2627 };
2628
2629 static struct attribute *hswep_uncore_cbox_formats_attr[] = {
2630         &format_attr_event.attr,
2631         &format_attr_umask.attr,
2632         &format_attr_edge.attr,
2633         &format_attr_tid_en.attr,
2634         &format_attr_thresh8.attr,
2635         &format_attr_filter_tid3.attr,
2636         &format_attr_filter_link2.attr,
2637         &format_attr_filter_state3.attr,
2638         &format_attr_filter_nid2.attr,
2639         &format_attr_filter_opc2.attr,
2640         &format_attr_filter_nc.attr,
2641         &format_attr_filter_c6.attr,
2642         &format_attr_filter_isoc.attr,
2643         NULL,
2644 };
2645
2646 static const struct attribute_group hswep_uncore_cbox_format_group = {
2647         .name = "format",
2648         .attrs = hswep_uncore_cbox_formats_attr,
2649 };
2650
2651 static struct event_constraint hswep_uncore_cbox_constraints[] = {
2652         UNCORE_EVENT_CONSTRAINT(0x01, 0x1),
2653         UNCORE_EVENT_CONSTRAINT(0x09, 0x1),
2654         UNCORE_EVENT_CONSTRAINT(0x11, 0x1),
2655         UNCORE_EVENT_CONSTRAINT(0x36, 0x1),
2656         UNCORE_EVENT_CONSTRAINT(0x38, 0x3),
2657         UNCORE_EVENT_CONSTRAINT(0x3b, 0x1),
2658         UNCORE_EVENT_CONSTRAINT(0x3e, 0x1),
2659         EVENT_CONSTRAINT_END
2660 };
2661
2662 static struct extra_reg hswep_uncore_cbox_extra_regs[] = {
2663         SNBEP_CBO_EVENT_EXTRA_REG(SNBEP_CBO_PMON_CTL_TID_EN,
2664                                   SNBEP_CBO_PMON_CTL_TID_EN, 0x1),
2665         SNBEP_CBO_EVENT_EXTRA_REG(0x0334, 0xffff, 0x4),
2666         SNBEP_CBO_EVENT_EXTRA_REG(0x0534, 0xffff, 0x4),
2667         SNBEP_CBO_EVENT_EXTRA_REG(0x0934, 0xffff, 0x4),
2668         SNBEP_CBO_EVENT_EXTRA_REG(0x1134, 0xffff, 0x4),
2669         SNBEP_CBO_EVENT_EXTRA_REG(0x2134, 0xffff, 0x4),
2670         SNBEP_CBO_EVENT_EXTRA_REG(0x4134, 0xffff, 0x4),
2671         SNBEP_CBO_EVENT_EXTRA_REG(0x4037, 0x40ff, 0x8),
2672         SNBEP_CBO_EVENT_EXTRA_REG(0x4028, 0x40ff, 0x8),
2673         SNBEP_CBO_EVENT_EXTRA_REG(0x4032, 0x40ff, 0x8),
2674         SNBEP_CBO_EVENT_EXTRA_REG(0x4029, 0x40ff, 0x8),
2675         SNBEP_CBO_EVENT_EXTRA_REG(0x4033, 0x40ff, 0x8),
2676         SNBEP_CBO_EVENT_EXTRA_REG(0x402A, 0x40ff, 0x8),
2677         SNBEP_CBO_EVENT_EXTRA_REG(0x0135, 0xffff, 0x12),
2678         SNBEP_CBO_EVENT_EXTRA_REG(0x0335, 0xffff, 0x10),
2679         SNBEP_CBO_EVENT_EXTRA_REG(0x4135, 0xffff, 0x18),
2680         SNBEP_CBO_EVENT_EXTRA_REG(0x4435, 0xffff, 0x8),
2681         SNBEP_CBO_EVENT_EXTRA_REG(0x4835, 0xffff, 0x8),
2682         SNBEP_CBO_EVENT_EXTRA_REG(0x5035, 0xffff, 0x8),
2683         SNBEP_CBO_EVENT_EXTRA_REG(0x4335, 0xffff, 0x18),
2684         SNBEP_CBO_EVENT_EXTRA_REG(0x4a35, 0xffff, 0x8),
2685         SNBEP_CBO_EVENT_EXTRA_REG(0x2335, 0xffff, 0x10),
2686         SNBEP_CBO_EVENT_EXTRA_REG(0x8335, 0xffff, 0x10),
2687         SNBEP_CBO_EVENT_EXTRA_REG(0x2135, 0xffff, 0x10),
2688         SNBEP_CBO_EVENT_EXTRA_REG(0x8135, 0xffff, 0x10),
2689         SNBEP_CBO_EVENT_EXTRA_REG(0x0136, 0xffff, 0x10),
2690         SNBEP_CBO_EVENT_EXTRA_REG(0x0336, 0xffff, 0x10),
2691         SNBEP_CBO_EVENT_EXTRA_REG(0x4136, 0xffff, 0x18),
2692         SNBEP_CBO_EVENT_EXTRA_REG(0x4436, 0xffff, 0x8),
2693         SNBEP_CBO_EVENT_EXTRA_REG(0x4836, 0xffff, 0x8),
2694         SNBEP_CBO_EVENT_EXTRA_REG(0x4336, 0xffff, 0x18),
2695         SNBEP_CBO_EVENT_EXTRA_REG(0x4a36, 0xffff, 0x8),
2696         SNBEP_CBO_EVENT_EXTRA_REG(0x2336, 0xffff, 0x10),
2697         SNBEP_CBO_EVENT_EXTRA_REG(0x8336, 0xffff, 0x10),
2698         SNBEP_CBO_EVENT_EXTRA_REG(0x2136, 0xffff, 0x10),
2699         SNBEP_CBO_EVENT_EXTRA_REG(0x8136, 0xffff, 0x10),
2700         SNBEP_CBO_EVENT_EXTRA_REG(0x5036, 0xffff, 0x8),
2701         EVENT_EXTRA_END
2702 };
2703
2704 static u64 hswep_cbox_filter_mask(int fields)
2705 {
2706         u64 mask = 0;
2707         if (fields & 0x1)
2708                 mask |= HSWEP_CB0_MSR_PMON_BOX_FILTER_TID;
2709         if (fields & 0x2)
2710                 mask |= HSWEP_CB0_MSR_PMON_BOX_FILTER_LINK;
2711         if (fields & 0x4)
2712                 mask |= HSWEP_CB0_MSR_PMON_BOX_FILTER_STATE;
2713         if (fields & 0x8)
2714                 mask |= HSWEP_CB0_MSR_PMON_BOX_FILTER_NID;
2715         if (fields & 0x10) {
2716                 mask |= HSWEP_CB0_MSR_PMON_BOX_FILTER_OPC;
2717                 mask |= HSWEP_CB0_MSR_PMON_BOX_FILTER_NC;
2718                 mask |= HSWEP_CB0_MSR_PMON_BOX_FILTER_C6;
2719                 mask |= HSWEP_CB0_MSR_PMON_BOX_FILTER_ISOC;
2720         }
2721         return mask;
2722 }
2723
2724 static struct event_constraint *
2725 hswep_cbox_get_constraint(struct intel_uncore_box *box, struct perf_event *event)
2726 {
2727         return __snbep_cbox_get_constraint(box, event, hswep_cbox_filter_mask);
2728 }
2729
2730 static int hswep_cbox_hw_config(struct intel_uncore_box *box, struct perf_event *event)
2731 {
2732         struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
2733         struct extra_reg *er;
2734         int idx = 0;
2735
2736         for (er = hswep_uncore_cbox_extra_regs; er->msr; er++) {
2737                 if (er->event != (event->hw.config & er->config_mask))
2738                         continue;
2739                 idx |= er->idx;
2740         }
2741
2742         if (idx) {
2743                 reg1->reg = HSWEP_C0_MSR_PMON_BOX_FILTER0 +
2744                             HSWEP_CBO_MSR_OFFSET * box->pmu->pmu_idx;
2745                 reg1->config = event->attr.config1 & hswep_cbox_filter_mask(idx);
2746                 reg1->idx = idx;
2747         }
2748         return 0;
2749 }
2750
2751 static void hswep_cbox_enable_event(struct intel_uncore_box *box,
2752                                   struct perf_event *event)
2753 {
2754         struct hw_perf_event *hwc = &event->hw;
2755         struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
2756
2757         if (reg1->idx != EXTRA_REG_NONE) {
2758                 u64 filter = uncore_shared_reg_config(box, 0);
2759                 wrmsrl(reg1->reg, filter & 0xffffffff);
2760                 wrmsrl(reg1->reg + 1, filter >> 32);
2761         }
2762
2763         wrmsrl(hwc->config_base, hwc->config | SNBEP_PMON_CTL_EN);
2764 }
2765
2766 static struct intel_uncore_ops hswep_uncore_cbox_ops = {
2767         .init_box               = snbep_uncore_msr_init_box,
2768         .disable_box            = snbep_uncore_msr_disable_box,
2769         .enable_box             = snbep_uncore_msr_enable_box,
2770         .disable_event          = snbep_uncore_msr_disable_event,
2771         .enable_event           = hswep_cbox_enable_event,
2772         .read_counter           = uncore_msr_read_counter,
2773         .hw_config              = hswep_cbox_hw_config,
2774         .get_constraint         = hswep_cbox_get_constraint,
2775         .put_constraint         = snbep_cbox_put_constraint,
2776 };
2777
2778 static struct intel_uncore_type hswep_uncore_cbox = {
2779         .name                   = "cbox",
2780         .num_counters           = 4,
2781         .num_boxes              = 18,
2782         .perf_ctr_bits          = 48,
2783         .event_ctl              = HSWEP_C0_MSR_PMON_CTL0,
2784         .perf_ctr               = HSWEP_C0_MSR_PMON_CTR0,
2785         .event_mask             = SNBEP_CBO_MSR_PMON_RAW_EVENT_MASK,
2786         .box_ctl                = HSWEP_C0_MSR_PMON_BOX_CTL,
2787         .msr_offset             = HSWEP_CBO_MSR_OFFSET,
2788         .num_shared_regs        = 1,
2789         .constraints            = hswep_uncore_cbox_constraints,
2790         .ops                    = &hswep_uncore_cbox_ops,
2791         .format_group           = &hswep_uncore_cbox_format_group,
2792 };
2793
2794 /*
2795  * Write SBOX Initialization register bit by bit to avoid spurious #GPs
2796  */
2797 static void hswep_uncore_sbox_msr_init_box(struct intel_uncore_box *box)
2798 {
2799         unsigned msr = uncore_msr_box_ctl(box);
2800
2801         if (msr) {
2802                 u64 init = SNBEP_PMON_BOX_CTL_INT;
2803                 u64 flags = 0;
2804                 int i;
2805
2806                 for_each_set_bit(i, (unsigned long *)&init, 64) {
2807                         flags |= (1ULL << i);
2808                         wrmsrl(msr, flags);
2809                 }
2810         }
2811 }
2812
2813 static struct intel_uncore_ops hswep_uncore_sbox_msr_ops = {
2814         __SNBEP_UNCORE_MSR_OPS_COMMON_INIT(),
2815         .init_box               = hswep_uncore_sbox_msr_init_box
2816 };
2817
2818 static struct attribute *hswep_uncore_sbox_formats_attr[] = {
2819         &format_attr_event.attr,
2820         &format_attr_umask.attr,
2821         &format_attr_edge.attr,
2822         &format_attr_tid_en.attr,
2823         &format_attr_inv.attr,
2824         &format_attr_thresh8.attr,
2825         NULL,
2826 };
2827
2828 static const struct attribute_group hswep_uncore_sbox_format_group = {
2829         .name = "format",
2830         .attrs = hswep_uncore_sbox_formats_attr,
2831 };
2832
2833 static struct intel_uncore_type hswep_uncore_sbox = {
2834         .name                   = "sbox",
2835         .num_counters           = 4,
2836         .num_boxes              = 4,
2837         .perf_ctr_bits          = 44,
2838         .event_ctl              = HSWEP_S0_MSR_PMON_CTL0,
2839         .perf_ctr               = HSWEP_S0_MSR_PMON_CTR0,
2840         .event_mask             = HSWEP_S_MSR_PMON_RAW_EVENT_MASK,
2841         .box_ctl                = HSWEP_S0_MSR_PMON_BOX_CTL,
2842         .msr_offset             = HSWEP_SBOX_MSR_OFFSET,
2843         .ops                    = &hswep_uncore_sbox_msr_ops,
2844         .format_group           = &hswep_uncore_sbox_format_group,
2845 };
2846
2847 static int hswep_pcu_hw_config(struct intel_uncore_box *box, struct perf_event *event)
2848 {
2849         struct hw_perf_event *hwc = &event->hw;
2850         struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
2851         int ev_sel = hwc->config & SNBEP_PMON_CTL_EV_SEL_MASK;
2852
2853         if (ev_sel >= 0xb && ev_sel <= 0xe) {
2854                 reg1->reg = HSWEP_PCU_MSR_PMON_BOX_FILTER;
2855                 reg1->idx = ev_sel - 0xb;
2856                 reg1->config = event->attr.config1 & (0xff << reg1->idx);
2857         }
2858         return 0;
2859 }
2860
2861 static struct intel_uncore_ops hswep_uncore_pcu_ops = {
2862         SNBEP_UNCORE_MSR_OPS_COMMON_INIT(),
2863         .hw_config              = hswep_pcu_hw_config,
2864         .get_constraint         = snbep_pcu_get_constraint,
2865         .put_constraint         = snbep_pcu_put_constraint,
2866 };
2867
2868 static struct intel_uncore_type hswep_uncore_pcu = {
2869         .name                   = "pcu",
2870         .num_counters           = 4,
2871         .num_boxes              = 1,
2872         .perf_ctr_bits          = 48,
2873         .perf_ctr               = HSWEP_PCU_MSR_PMON_CTR0,
2874         .event_ctl              = HSWEP_PCU_MSR_PMON_CTL0,
2875         .event_mask             = SNBEP_PCU_MSR_PMON_RAW_EVENT_MASK,
2876         .box_ctl                = HSWEP_PCU_MSR_PMON_BOX_CTL,
2877         .num_shared_regs        = 1,
2878         .ops                    = &hswep_uncore_pcu_ops,
2879         .format_group           = &snbep_uncore_pcu_format_group,
2880 };
2881
2882 static struct intel_uncore_type *hswep_msr_uncores[] = {
2883         &hswep_uncore_ubox,
2884         &hswep_uncore_cbox,
2885         &hswep_uncore_sbox,
2886         &hswep_uncore_pcu,
2887         NULL,
2888 };
2889
2890 #define HSWEP_PCU_DID                   0x2fc0
2891 #define HSWEP_PCU_CAPID4_OFFET          0x94
2892 #define hswep_get_chop(_cap)            (((_cap) >> 6) & 0x3)
2893
2894 static bool hswep_has_limit_sbox(unsigned int device)
2895 {
2896         struct pci_dev *dev = pci_get_device(PCI_VENDOR_ID_INTEL, device, NULL);
2897         u32 capid4;
2898
2899         if (!dev)
2900                 return false;
2901
2902         pci_read_config_dword(dev, HSWEP_PCU_CAPID4_OFFET, &capid4);
2903         pci_dev_put(dev);
2904         if (!hswep_get_chop(capid4))
2905                 return true;
2906
2907         return false;
2908 }
2909
2910 void hswep_uncore_cpu_init(void)
2911 {
2912         if (hswep_uncore_cbox.num_boxes > boot_cpu_data.x86_max_cores)
2913                 hswep_uncore_cbox.num_boxes = boot_cpu_data.x86_max_cores;
2914
2915         /* Detect 6-8 core systems with only two SBOXes */
2916         if (hswep_has_limit_sbox(HSWEP_PCU_DID))
2917                 hswep_uncore_sbox.num_boxes = 2;
2918
2919         uncore_msr_uncores = hswep_msr_uncores;
2920 }
2921
2922 static struct intel_uncore_type hswep_uncore_ha = {
2923         .name           = "ha",
2924         .num_counters   = 4,
2925         .num_boxes      = 2,
2926         .perf_ctr_bits  = 48,
2927         SNBEP_UNCORE_PCI_COMMON_INIT(),
2928 };
2929
2930 static struct uncore_event_desc hswep_uncore_imc_events[] = {
2931         INTEL_UNCORE_EVENT_DESC(clockticks,      "event=0x00,umask=0x00"),
2932         INTEL_UNCORE_EVENT_DESC(cas_count_read,  "event=0x04,umask=0x03"),
2933         INTEL_UNCORE_EVENT_DESC(cas_count_read.scale, "6.103515625e-5"),
2934         INTEL_UNCORE_EVENT_DESC(cas_count_read.unit, "MiB"),
2935         INTEL_UNCORE_EVENT_DESC(cas_count_write, "event=0x04,umask=0x0c"),
2936         INTEL_UNCORE_EVENT_DESC(cas_count_write.scale, "6.103515625e-5"),
2937         INTEL_UNCORE_EVENT_DESC(cas_count_write.unit, "MiB"),
2938         { /* end: all zeroes */ },
2939 };
2940
2941 static struct intel_uncore_type hswep_uncore_imc = {
2942         .name           = "imc",
2943         .num_counters   = 4,
2944         .num_boxes      = 8,
2945         .perf_ctr_bits  = 48,
2946         .fixed_ctr_bits = 48,
2947         .fixed_ctr      = SNBEP_MC_CHy_PCI_PMON_FIXED_CTR,
2948         .fixed_ctl      = SNBEP_MC_CHy_PCI_PMON_FIXED_CTL,
2949         .event_descs    = hswep_uncore_imc_events,
2950         SNBEP_UNCORE_PCI_COMMON_INIT(),
2951 };
2952
2953 static unsigned hswep_uncore_irp_ctrs[] = {0xa0, 0xa8, 0xb0, 0xb8};
2954
2955 static u64 hswep_uncore_irp_read_counter(struct intel_uncore_box *box, struct perf_event *event)
2956 {
2957         struct pci_dev *pdev = box->pci_dev;
2958         struct hw_perf_event *hwc = &event->hw;
2959         u64 count = 0;
2960
2961         pci_read_config_dword(pdev, hswep_uncore_irp_ctrs[hwc->idx], (u32 *)&count);
2962         pci_read_config_dword(pdev, hswep_uncore_irp_ctrs[hwc->idx] + 4, (u32 *)&count + 1);
2963
2964         return count;
2965 }
2966
2967 static struct intel_uncore_ops hswep_uncore_irp_ops = {
2968         .init_box       = snbep_uncore_pci_init_box,
2969         .disable_box    = snbep_uncore_pci_disable_box,
2970         .enable_box     = snbep_uncore_pci_enable_box,
2971         .disable_event  = ivbep_uncore_irp_disable_event,
2972         .enable_event   = ivbep_uncore_irp_enable_event,
2973         .read_counter   = hswep_uncore_irp_read_counter,
2974 };
2975
2976 static struct intel_uncore_type hswep_uncore_irp = {
2977         .name                   = "irp",
2978         .num_counters           = 4,
2979         .num_boxes              = 1,
2980         .perf_ctr_bits          = 48,
2981         .event_mask             = SNBEP_PMON_RAW_EVENT_MASK,
2982         .box_ctl                = SNBEP_PCI_PMON_BOX_CTL,
2983         .ops                    = &hswep_uncore_irp_ops,
2984         .format_group           = &snbep_uncore_format_group,
2985 };
2986
2987 static struct intel_uncore_type hswep_uncore_qpi = {
2988         .name                   = "qpi",
2989         .num_counters           = 4,
2990         .num_boxes              = 3,
2991         .perf_ctr_bits          = 48,
2992         .perf_ctr               = SNBEP_PCI_PMON_CTR0,
2993         .event_ctl              = SNBEP_PCI_PMON_CTL0,
2994         .event_mask             = SNBEP_QPI_PCI_PMON_RAW_EVENT_MASK,
2995         .box_ctl                = SNBEP_PCI_PMON_BOX_CTL,
2996         .num_shared_regs        = 1,
2997         .ops                    = &snbep_uncore_qpi_ops,
2998         .format_group           = &snbep_uncore_qpi_format_group,
2999 };
3000
3001 static struct event_constraint hswep_uncore_r2pcie_constraints[] = {
3002         UNCORE_EVENT_CONSTRAINT(0x10, 0x3),
3003         UNCORE_EVENT_CONSTRAINT(0x11, 0x3),
3004         UNCORE_EVENT_CONSTRAINT(0x13, 0x1),
3005         UNCORE_EVENT_CONSTRAINT(0x23, 0x1),
3006         UNCORE_EVENT_CONSTRAINT(0x24, 0x1),
3007         UNCORE_EVENT_CONSTRAINT(0x25, 0x1),
3008         UNCORE_EVENT_CONSTRAINT(0x26, 0x3),
3009         UNCORE_EVENT_CONSTRAINT(0x27, 0x1),
3010         UNCORE_EVENT_CONSTRAINT(0x28, 0x3),
3011         UNCORE_EVENT_CONSTRAINT(0x29, 0x3),
3012         UNCORE_EVENT_CONSTRAINT(0x2a, 0x1),
3013         UNCORE_EVENT_CONSTRAINT(0x2b, 0x3),
3014         UNCORE_EVENT_CONSTRAINT(0x2c, 0x3),
3015         UNCORE_EVENT_CONSTRAINT(0x2d, 0x3),
3016         UNCORE_EVENT_CONSTRAINT(0x32, 0x3),
3017         UNCORE_EVENT_CONSTRAINT(0x33, 0x3),
3018         UNCORE_EVENT_CONSTRAINT(0x34, 0x3),
3019         UNCORE_EVENT_CONSTRAINT(0x35, 0x3),
3020         EVENT_CONSTRAINT_END
3021 };
3022
3023 static struct intel_uncore_type hswep_uncore_r2pcie = {
3024         .name           = "r2pcie",
3025         .num_counters   = 4,
3026         .num_boxes      = 1,
3027         .perf_ctr_bits  = 48,
3028         .constraints    = hswep_uncore_r2pcie_constraints,
3029         SNBEP_UNCORE_PCI_COMMON_INIT(),
3030 };
3031
3032 static struct event_constraint hswep_uncore_r3qpi_constraints[] = {
3033         UNCORE_EVENT_CONSTRAINT(0x01, 0x3),
3034         UNCORE_EVENT_CONSTRAINT(0x07, 0x7),
3035         UNCORE_EVENT_CONSTRAINT(0x08, 0x7),
3036         UNCORE_EVENT_CONSTRAINT(0x09, 0x7),
3037         UNCORE_EVENT_CONSTRAINT(0x0a, 0x7),
3038         UNCORE_EVENT_CONSTRAINT(0x0e, 0x7),
3039         UNCORE_EVENT_CONSTRAINT(0x10, 0x3),
3040         UNCORE_EVENT_CONSTRAINT(0x11, 0x3),
3041         UNCORE_EVENT_CONSTRAINT(0x12, 0x3),
3042         UNCORE_EVENT_CONSTRAINT(0x13, 0x1),
3043         UNCORE_EVENT_CONSTRAINT(0x14, 0x3),
3044         UNCORE_EVENT_CONSTRAINT(0x15, 0x3),
3045         UNCORE_EVENT_CONSTRAINT(0x1f, 0x3),
3046         UNCORE_EVENT_CONSTRAINT(0x20, 0x3),
3047         UNCORE_EVENT_CONSTRAINT(0x21, 0x3),
3048         UNCORE_EVENT_CONSTRAINT(0x22, 0x3),
3049         UNCORE_EVENT_CONSTRAINT(0x23, 0x3),
3050         UNCORE_EVENT_CONSTRAINT(0x25, 0x3),
3051         UNCORE_EVENT_CONSTRAINT(0x26, 0x3),
3052         UNCORE_EVENT_CONSTRAINT(0x28, 0x3),
3053         UNCORE_EVENT_CONSTRAINT(0x29, 0x3),
3054         UNCORE_EVENT_CONSTRAINT(0x2c, 0x3),
3055         UNCORE_EVENT_CONSTRAINT(0x2d, 0x3),
3056         UNCORE_EVENT_CONSTRAINT(0x2e, 0x3),
3057         UNCORE_EVENT_CONSTRAINT(0x2f, 0x3),
3058         UNCORE_EVENT_CONSTRAINT(0x31, 0x3),
3059         UNCORE_EVENT_CONSTRAINT(0x32, 0x3),
3060         UNCORE_EVENT_CONSTRAINT(0x33, 0x3),
3061         UNCORE_EVENT_CONSTRAINT(0x34, 0x3),
3062         UNCORE_EVENT_CONSTRAINT(0x36, 0x3),
3063         UNCORE_EVENT_CONSTRAINT(0x37, 0x3),
3064         UNCORE_EVENT_CONSTRAINT(0x38, 0x3),
3065         UNCORE_EVENT_CONSTRAINT(0x39, 0x3),
3066         EVENT_CONSTRAINT_END
3067 };
3068
3069 static struct intel_uncore_type hswep_uncore_r3qpi = {
3070         .name           = "r3qpi",
3071         .num_counters   = 3,
3072         .num_boxes      = 3,
3073         .perf_ctr_bits  = 44,
3074         .constraints    = hswep_uncore_r3qpi_constraints,
3075         SNBEP_UNCORE_PCI_COMMON_INIT(),
3076 };
3077
3078 enum {
3079         HSWEP_PCI_UNCORE_HA,
3080         HSWEP_PCI_UNCORE_IMC,
3081         HSWEP_PCI_UNCORE_IRP,
3082         HSWEP_PCI_UNCORE_QPI,
3083         HSWEP_PCI_UNCORE_R2PCIE,
3084         HSWEP_PCI_UNCORE_R3QPI,
3085 };
3086
3087 static struct intel_uncore_type *hswep_pci_uncores[] = {
3088         [HSWEP_PCI_UNCORE_HA]   = &hswep_uncore_ha,
3089         [HSWEP_PCI_UNCORE_IMC]  = &hswep_uncore_imc,
3090         [HSWEP_PCI_UNCORE_IRP]  = &hswep_uncore_irp,
3091         [HSWEP_PCI_UNCORE_QPI]  = &hswep_uncore_qpi,
3092         [HSWEP_PCI_UNCORE_R2PCIE]       = &hswep_uncore_r2pcie,
3093         [HSWEP_PCI_UNCORE_R3QPI]        = &hswep_uncore_r3qpi,
3094         NULL,
3095 };
3096
3097 static const struct pci_device_id hswep_uncore_pci_ids[] = {
3098         { /* Home Agent 0 */
3099                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2f30),
3100                 .driver_data = UNCORE_PCI_DEV_DATA(HSWEP_PCI_UNCORE_HA, 0),
3101         },
3102         { /* Home Agent 1 */
3103                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2f38),
3104                 .driver_data = UNCORE_PCI_DEV_DATA(HSWEP_PCI_UNCORE_HA, 1),
3105         },
3106         { /* MC0 Channel 0 */
3107                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2fb0),
3108                 .driver_data = UNCORE_PCI_DEV_DATA(HSWEP_PCI_UNCORE_IMC, 0),
3109         },
3110         { /* MC0 Channel 1 */
3111                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2fb1),
3112                 .driver_data = UNCORE_PCI_DEV_DATA(HSWEP_PCI_UNCORE_IMC, 1),
3113         },
3114         { /* MC0 Channel 2 */
3115                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2fb4),
3116                 .driver_data = UNCORE_PCI_DEV_DATA(HSWEP_PCI_UNCORE_IMC, 2),
3117         },
3118         { /* MC0 Channel 3 */
3119                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2fb5),
3120                 .driver_data = UNCORE_PCI_DEV_DATA(HSWEP_PCI_UNCORE_IMC, 3),
3121         },
3122         { /* MC1 Channel 0 */
3123                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2fd0),
3124                 .driver_data = UNCORE_PCI_DEV_DATA(HSWEP_PCI_UNCORE_IMC, 4),
3125         },
3126         { /* MC1 Channel 1 */
3127                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2fd1),
3128                 .driver_data = UNCORE_PCI_DEV_DATA(HSWEP_PCI_UNCORE_IMC, 5),
3129         },
3130         { /* MC1 Channel 2 */
3131                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2fd4),
3132                 .driver_data = UNCORE_PCI_DEV_DATA(HSWEP_PCI_UNCORE_IMC, 6),
3133         },
3134         { /* MC1 Channel 3 */
3135                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2fd5),
3136                 .driver_data = UNCORE_PCI_DEV_DATA(HSWEP_PCI_UNCORE_IMC, 7),
3137         },
3138         { /* IRP */
3139                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2f39),
3140                 .driver_data = UNCORE_PCI_DEV_DATA(HSWEP_PCI_UNCORE_IRP, 0),
3141         },
3142         { /* QPI0 Port 0 */
3143                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2f32),
3144                 .driver_data = UNCORE_PCI_DEV_DATA(HSWEP_PCI_UNCORE_QPI, 0),
3145         },
3146         { /* QPI0 Port 1 */
3147                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2f33),
3148                 .driver_data = UNCORE_PCI_DEV_DATA(HSWEP_PCI_UNCORE_QPI, 1),
3149         },
3150         { /* QPI1 Port 2 */
3151                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2f3a),
3152                 .driver_data = UNCORE_PCI_DEV_DATA(HSWEP_PCI_UNCORE_QPI, 2),
3153         },
3154         { /* R2PCIe */
3155                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2f34),
3156                 .driver_data = UNCORE_PCI_DEV_DATA(HSWEP_PCI_UNCORE_R2PCIE, 0),
3157         },
3158         { /* R3QPI0 Link 0 */
3159                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2f36),
3160                 .driver_data = UNCORE_PCI_DEV_DATA(HSWEP_PCI_UNCORE_R3QPI, 0),
3161         },
3162         { /* R3QPI0 Link 1 */
3163                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2f37),
3164                 .driver_data = UNCORE_PCI_DEV_DATA(HSWEP_PCI_UNCORE_R3QPI, 1),
3165         },
3166         { /* R3QPI1 Link 2 */
3167                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2f3e),
3168                 .driver_data = UNCORE_PCI_DEV_DATA(HSWEP_PCI_UNCORE_R3QPI, 2),
3169         },
3170         { /* QPI Port 0 filter  */
3171                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2f86),
3172                 .driver_data = UNCORE_PCI_DEV_DATA(UNCORE_EXTRA_PCI_DEV,
3173                                                    SNBEP_PCI_QPI_PORT0_FILTER),
3174         },
3175         { /* QPI Port 1 filter  */
3176                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2f96),
3177                 .driver_data = UNCORE_PCI_DEV_DATA(UNCORE_EXTRA_PCI_DEV,
3178                                                    SNBEP_PCI_QPI_PORT1_FILTER),
3179         },
3180         { /* end: all zeroes */ }
3181 };
3182
3183 static struct pci_driver hswep_uncore_pci_driver = {
3184         .name           = "hswep_uncore",
3185         .id_table       = hswep_uncore_pci_ids,
3186 };
3187
3188 int hswep_uncore_pci_init(void)
3189 {
3190         int ret = snbep_pci2phy_map_init(0x2f1e, SNBEP_CPUNODEID, SNBEP_GIDNIDMAP, true);
3191         if (ret)
3192                 return ret;
3193         uncore_pci_uncores = hswep_pci_uncores;
3194         uncore_pci_driver = &hswep_uncore_pci_driver;
3195         return 0;
3196 }
3197 /* end of Haswell-EP uncore support */
3198
3199 /* BDX uncore support */
3200
3201 static struct intel_uncore_type bdx_uncore_ubox = {
3202         .name                   = "ubox",
3203         .num_counters           = 2,
3204         .num_boxes              = 1,
3205         .perf_ctr_bits          = 48,
3206         .fixed_ctr_bits         = 48,
3207         .perf_ctr               = HSWEP_U_MSR_PMON_CTR0,
3208         .event_ctl              = HSWEP_U_MSR_PMON_CTL0,
3209         .event_mask             = SNBEP_U_MSR_PMON_RAW_EVENT_MASK,
3210         .fixed_ctr              = HSWEP_U_MSR_PMON_UCLK_FIXED_CTR,
3211         .fixed_ctl              = HSWEP_U_MSR_PMON_UCLK_FIXED_CTL,
3212         .num_shared_regs        = 1,
3213         .ops                    = &ivbep_uncore_msr_ops,
3214         .format_group           = &ivbep_uncore_ubox_format_group,
3215 };
3216
3217 static struct event_constraint bdx_uncore_cbox_constraints[] = {
3218         UNCORE_EVENT_CONSTRAINT(0x09, 0x3),
3219         UNCORE_EVENT_CONSTRAINT(0x11, 0x1),
3220         UNCORE_EVENT_CONSTRAINT(0x36, 0x1),
3221         UNCORE_EVENT_CONSTRAINT(0x3e, 0x1),
3222         EVENT_CONSTRAINT_END
3223 };
3224
3225 static struct intel_uncore_type bdx_uncore_cbox = {
3226         .name                   = "cbox",
3227         .num_counters           = 4,
3228         .num_boxes              = 24,
3229         .perf_ctr_bits          = 48,
3230         .event_ctl              = HSWEP_C0_MSR_PMON_CTL0,
3231         .perf_ctr               = HSWEP_C0_MSR_PMON_CTR0,
3232         .event_mask             = SNBEP_CBO_MSR_PMON_RAW_EVENT_MASK,
3233         .box_ctl                = HSWEP_C0_MSR_PMON_BOX_CTL,
3234         .msr_offset             = HSWEP_CBO_MSR_OFFSET,
3235         .num_shared_regs        = 1,
3236         .constraints            = bdx_uncore_cbox_constraints,
3237         .ops                    = &hswep_uncore_cbox_ops,
3238         .format_group           = &hswep_uncore_cbox_format_group,
3239 };
3240
3241 static struct intel_uncore_type bdx_uncore_sbox = {
3242         .name                   = "sbox",
3243         .num_counters           = 4,
3244         .num_boxes              = 4,
3245         .perf_ctr_bits          = 48,
3246         .event_ctl              = HSWEP_S0_MSR_PMON_CTL0,
3247         .perf_ctr               = HSWEP_S0_MSR_PMON_CTR0,
3248         .event_mask             = HSWEP_S_MSR_PMON_RAW_EVENT_MASK,
3249         .box_ctl                = HSWEP_S0_MSR_PMON_BOX_CTL,
3250         .msr_offset             = HSWEP_SBOX_MSR_OFFSET,
3251         .ops                    = &hswep_uncore_sbox_msr_ops,
3252         .format_group           = &hswep_uncore_sbox_format_group,
3253 };
3254
3255 #define BDX_MSR_UNCORE_SBOX     3
3256
3257 static struct intel_uncore_type *bdx_msr_uncores[] = {
3258         &bdx_uncore_ubox,
3259         &bdx_uncore_cbox,
3260         &hswep_uncore_pcu,
3261         &bdx_uncore_sbox,
3262         NULL,
3263 };
3264
3265 /* Bit 7 'Use Occupancy' is not available for counter 0 on BDX */
3266 static struct event_constraint bdx_uncore_pcu_constraints[] = {
3267         EVENT_CONSTRAINT(0x80, 0xe, 0x80),
3268         EVENT_CONSTRAINT_END
3269 };
3270
3271 #define BDX_PCU_DID                     0x6fc0
3272
3273 void bdx_uncore_cpu_init(void)
3274 {
3275         if (bdx_uncore_cbox.num_boxes > boot_cpu_data.x86_max_cores)
3276                 bdx_uncore_cbox.num_boxes = boot_cpu_data.x86_max_cores;
3277         uncore_msr_uncores = bdx_msr_uncores;
3278
3279         /* Detect systems with no SBOXes */
3280         if ((boot_cpu_data.x86_model == 86) || hswep_has_limit_sbox(BDX_PCU_DID))
3281                 uncore_msr_uncores[BDX_MSR_UNCORE_SBOX] = NULL;
3282
3283         hswep_uncore_pcu.constraints = bdx_uncore_pcu_constraints;
3284 }
3285
3286 static struct intel_uncore_type bdx_uncore_ha = {
3287         .name           = "ha",
3288         .num_counters   = 4,
3289         .num_boxes      = 2,
3290         .perf_ctr_bits  = 48,
3291         SNBEP_UNCORE_PCI_COMMON_INIT(),
3292 };
3293
3294 static struct intel_uncore_type bdx_uncore_imc = {
3295         .name           = "imc",
3296         .num_counters   = 4,
3297         .num_boxes      = 8,
3298         .perf_ctr_bits  = 48,
3299         .fixed_ctr_bits = 48,
3300         .fixed_ctr      = SNBEP_MC_CHy_PCI_PMON_FIXED_CTR,
3301         .fixed_ctl      = SNBEP_MC_CHy_PCI_PMON_FIXED_CTL,
3302         .event_descs    = hswep_uncore_imc_events,
3303         SNBEP_UNCORE_PCI_COMMON_INIT(),
3304 };
3305
3306 static struct intel_uncore_type bdx_uncore_irp = {
3307         .name                   = "irp",
3308         .num_counters           = 4,
3309         .num_boxes              = 1,
3310         .perf_ctr_bits          = 48,
3311         .event_mask             = SNBEP_PMON_RAW_EVENT_MASK,
3312         .box_ctl                = SNBEP_PCI_PMON_BOX_CTL,
3313         .ops                    = &hswep_uncore_irp_ops,
3314         .format_group           = &snbep_uncore_format_group,
3315 };
3316
3317 static struct intel_uncore_type bdx_uncore_qpi = {
3318         .name                   = "qpi",
3319         .num_counters           = 4,
3320         .num_boxes              = 3,
3321         .perf_ctr_bits          = 48,
3322         .perf_ctr               = SNBEP_PCI_PMON_CTR0,
3323         .event_ctl              = SNBEP_PCI_PMON_CTL0,
3324         .event_mask             = SNBEP_QPI_PCI_PMON_RAW_EVENT_MASK,
3325         .box_ctl                = SNBEP_PCI_PMON_BOX_CTL,
3326         .num_shared_regs        = 1,
3327         .ops                    = &snbep_uncore_qpi_ops,
3328         .format_group           = &snbep_uncore_qpi_format_group,
3329 };
3330
3331 static struct event_constraint bdx_uncore_r2pcie_constraints[] = {
3332         UNCORE_EVENT_CONSTRAINT(0x10, 0x3),
3333         UNCORE_EVENT_CONSTRAINT(0x11, 0x3),
3334         UNCORE_EVENT_CONSTRAINT(0x13, 0x1),
3335         UNCORE_EVENT_CONSTRAINT(0x23, 0x1),
3336         UNCORE_EVENT_CONSTRAINT(0x25, 0x1),
3337         UNCORE_EVENT_CONSTRAINT(0x26, 0x3),
3338         UNCORE_EVENT_CONSTRAINT(0x28, 0x3),
3339         UNCORE_EVENT_CONSTRAINT(0x2c, 0x3),
3340         UNCORE_EVENT_CONSTRAINT(0x2d, 0x3),
3341         EVENT_CONSTRAINT_END
3342 };
3343
3344 static struct intel_uncore_type bdx_uncore_r2pcie = {
3345         .name           = "r2pcie",
3346         .num_counters   = 4,
3347         .num_boxes      = 1,
3348         .perf_ctr_bits  = 48,
3349         .constraints    = bdx_uncore_r2pcie_constraints,
3350         SNBEP_UNCORE_PCI_COMMON_INIT(),
3351 };
3352
3353 static struct event_constraint bdx_uncore_r3qpi_constraints[] = {
3354         UNCORE_EVENT_CONSTRAINT(0x01, 0x7),
3355         UNCORE_EVENT_CONSTRAINT(0x07, 0x7),
3356         UNCORE_EVENT_CONSTRAINT(0x08, 0x7),
3357         UNCORE_EVENT_CONSTRAINT(0x09, 0x7),
3358         UNCORE_EVENT_CONSTRAINT(0x0a, 0x7),
3359         UNCORE_EVENT_CONSTRAINT(0x0e, 0x7),
3360         UNCORE_EVENT_CONSTRAINT(0x10, 0x3),
3361         UNCORE_EVENT_CONSTRAINT(0x11, 0x3),
3362         UNCORE_EVENT_CONSTRAINT(0x13, 0x1),
3363         UNCORE_EVENT_CONSTRAINT(0x14, 0x3),
3364         UNCORE_EVENT_CONSTRAINT(0x15, 0x3),
3365         UNCORE_EVENT_CONSTRAINT(0x1f, 0x3),
3366         UNCORE_EVENT_CONSTRAINT(0x20, 0x3),
3367         UNCORE_EVENT_CONSTRAINT(0x21, 0x3),
3368         UNCORE_EVENT_CONSTRAINT(0x22, 0x3),
3369         UNCORE_EVENT_CONSTRAINT(0x23, 0x3),
3370         UNCORE_EVENT_CONSTRAINT(0x25, 0x3),
3371         UNCORE_EVENT_CONSTRAINT(0x26, 0x3),
3372         UNCORE_EVENT_CONSTRAINT(0x28, 0x3),
3373         UNCORE_EVENT_CONSTRAINT(0x29, 0x3),
3374         UNCORE_EVENT_CONSTRAINT(0x2c, 0x3),
3375         UNCORE_EVENT_CONSTRAINT(0x2d, 0x3),
3376         UNCORE_EVENT_CONSTRAINT(0x2e, 0x3),
3377         UNCORE_EVENT_CONSTRAINT(0x2f, 0x3),
3378         UNCORE_EVENT_CONSTRAINT(0x33, 0x3),
3379         UNCORE_EVENT_CONSTRAINT(0x34, 0x3),
3380         UNCORE_EVENT_CONSTRAINT(0x36, 0x3),
3381         UNCORE_EVENT_CONSTRAINT(0x37, 0x3),
3382         UNCORE_EVENT_CONSTRAINT(0x38, 0x3),
3383         UNCORE_EVENT_CONSTRAINT(0x39, 0x3),
3384         EVENT_CONSTRAINT_END
3385 };
3386
3387 static struct intel_uncore_type bdx_uncore_r3qpi = {
3388         .name           = "r3qpi",
3389         .num_counters   = 3,
3390         .num_boxes      = 3,
3391         .perf_ctr_bits  = 48,
3392         .constraints    = bdx_uncore_r3qpi_constraints,
3393         SNBEP_UNCORE_PCI_COMMON_INIT(),
3394 };
3395
3396 enum {
3397         BDX_PCI_UNCORE_HA,
3398         BDX_PCI_UNCORE_IMC,
3399         BDX_PCI_UNCORE_IRP,
3400         BDX_PCI_UNCORE_QPI,
3401         BDX_PCI_UNCORE_R2PCIE,
3402         BDX_PCI_UNCORE_R3QPI,
3403 };
3404
3405 static struct intel_uncore_type *bdx_pci_uncores[] = {
3406         [BDX_PCI_UNCORE_HA]     = &bdx_uncore_ha,
3407         [BDX_PCI_UNCORE_IMC]    = &bdx_uncore_imc,
3408         [BDX_PCI_UNCORE_IRP]    = &bdx_uncore_irp,
3409         [BDX_PCI_UNCORE_QPI]    = &bdx_uncore_qpi,
3410         [BDX_PCI_UNCORE_R2PCIE] = &bdx_uncore_r2pcie,
3411         [BDX_PCI_UNCORE_R3QPI]  = &bdx_uncore_r3qpi,
3412         NULL,
3413 };
3414
3415 static const struct pci_device_id bdx_uncore_pci_ids[] = {
3416         { /* Home Agent 0 */
3417                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x6f30),
3418                 .driver_data = UNCORE_PCI_DEV_DATA(BDX_PCI_UNCORE_HA, 0),
3419         },
3420         { /* Home Agent 1 */
3421                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x6f38),
3422                 .driver_data = UNCORE_PCI_DEV_DATA(BDX_PCI_UNCORE_HA, 1),
3423         },
3424         { /* MC0 Channel 0 */
3425                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x6fb0),
3426                 .driver_data = UNCORE_PCI_DEV_DATA(BDX_PCI_UNCORE_IMC, 0),
3427         },
3428         { /* MC0 Channel 1 */
3429                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x6fb1),
3430                 .driver_data = UNCORE_PCI_DEV_DATA(BDX_PCI_UNCORE_IMC, 1),
3431         },
3432         { /* MC0 Channel 2 */
3433                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x6fb4),
3434                 .driver_data = UNCORE_PCI_DEV_DATA(BDX_PCI_UNCORE_IMC, 2),
3435         },
3436         { /* MC0 Channel 3 */
3437                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x6fb5),
3438                 .driver_data = UNCORE_PCI_DEV_DATA(BDX_PCI_UNCORE_IMC, 3),
3439         },
3440         { /* MC1 Channel 0 */
3441                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x6fd0),
3442                 .driver_data = UNCORE_PCI_DEV_DATA(BDX_PCI_UNCORE_IMC, 4),
3443         },
3444         { /* MC1 Channel 1 */
3445                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x6fd1),
3446                 .driver_data = UNCORE_PCI_DEV_DATA(BDX_PCI_UNCORE_IMC, 5),
3447         },
3448         { /* MC1 Channel 2 */
3449                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x6fd4),
3450                 .driver_data = UNCORE_PCI_DEV_DATA(BDX_PCI_UNCORE_IMC, 6),
3451         },
3452         { /* MC1 Channel 3 */
3453                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x6fd5),
3454                 .driver_data = UNCORE_PCI_DEV_DATA(BDX_PCI_UNCORE_IMC, 7),
3455         },
3456         { /* IRP */
3457                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x6f39),
3458                 .driver_data = UNCORE_PCI_DEV_DATA(BDX_PCI_UNCORE_IRP, 0),
3459         },
3460         { /* QPI0 Port 0 */
3461                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x6f32),
3462                 .driver_data = UNCORE_PCI_DEV_DATA(BDX_PCI_UNCORE_QPI, 0),
3463         },
3464         { /* QPI0 Port 1 */
3465                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x6f33),
3466                 .driver_data = UNCORE_PCI_DEV_DATA(BDX_PCI_UNCORE_QPI, 1),
3467         },
3468         { /* QPI1 Port 2 */
3469                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x6f3a),
3470                 .driver_data = UNCORE_PCI_DEV_DATA(BDX_PCI_UNCORE_QPI, 2),
3471         },
3472         { /* R2PCIe */
3473                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x6f34),
3474                 .driver_data = UNCORE_PCI_DEV_DATA(BDX_PCI_UNCORE_R2PCIE, 0),
3475         },
3476         { /* R3QPI0 Link 0 */
3477                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x6f36),
3478                 .driver_data = UNCORE_PCI_DEV_DATA(BDX_PCI_UNCORE_R3QPI, 0),
3479         },
3480         { /* R3QPI0 Link 1 */
3481                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x6f37),
3482                 .driver_data = UNCORE_PCI_DEV_DATA(BDX_PCI_UNCORE_R3QPI, 1),
3483         },
3484         { /* R3QPI1 Link 2 */
3485                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x6f3e),
3486                 .driver_data = UNCORE_PCI_DEV_DATA(BDX_PCI_UNCORE_R3QPI, 2),
3487         },
3488         { /* QPI Port 0 filter  */
3489                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x6f86),
3490                 .driver_data = UNCORE_PCI_DEV_DATA(UNCORE_EXTRA_PCI_DEV,
3491                                                    SNBEP_PCI_QPI_PORT0_FILTER),
3492         },
3493         { /* QPI Port 1 filter  */
3494                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x6f96),
3495                 .driver_data = UNCORE_PCI_DEV_DATA(UNCORE_EXTRA_PCI_DEV,
3496                                                    SNBEP_PCI_QPI_PORT1_FILTER),
3497         },
3498         { /* QPI Port 2 filter  */
3499                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x6f46),
3500                 .driver_data = UNCORE_PCI_DEV_DATA(UNCORE_EXTRA_PCI_DEV,
3501                                                    BDX_PCI_QPI_PORT2_FILTER),
3502         },
3503         { /* end: all zeroes */ }
3504 };
3505
3506 static struct pci_driver bdx_uncore_pci_driver = {
3507         .name           = "bdx_uncore",
3508         .id_table       = bdx_uncore_pci_ids,
3509 };
3510
3511 int bdx_uncore_pci_init(void)
3512 {
3513         int ret = snbep_pci2phy_map_init(0x6f1e, SNBEP_CPUNODEID, SNBEP_GIDNIDMAP, true);
3514
3515         if (ret)
3516                 return ret;
3517         uncore_pci_uncores = bdx_pci_uncores;
3518         uncore_pci_driver = &bdx_uncore_pci_driver;
3519         return 0;
3520 }
3521
3522 /* end of BDX uncore support */
3523
3524 /* SKX uncore support */
3525
3526 static struct intel_uncore_type skx_uncore_ubox = {
3527         .name                   = "ubox",
3528         .num_counters           = 2,
3529         .num_boxes              = 1,
3530         .perf_ctr_bits          = 48,
3531         .fixed_ctr_bits         = 48,
3532         .perf_ctr               = HSWEP_U_MSR_PMON_CTR0,
3533         .event_ctl              = HSWEP_U_MSR_PMON_CTL0,
3534         .event_mask             = SNBEP_U_MSR_PMON_RAW_EVENT_MASK,
3535         .fixed_ctr              = HSWEP_U_MSR_PMON_UCLK_FIXED_CTR,
3536         .fixed_ctl              = HSWEP_U_MSR_PMON_UCLK_FIXED_CTL,
3537         .ops                    = &ivbep_uncore_msr_ops,
3538         .format_group           = &ivbep_uncore_ubox_format_group,
3539 };
3540
3541 static struct attribute *skx_uncore_cha_formats_attr[] = {
3542         &format_attr_event.attr,
3543         &format_attr_umask.attr,
3544         &format_attr_edge.attr,
3545         &format_attr_tid_en.attr,
3546         &format_attr_inv.attr,
3547         &format_attr_thresh8.attr,
3548         &format_attr_filter_tid4.attr,
3549         &format_attr_filter_state5.attr,
3550         &format_attr_filter_rem.attr,
3551         &format_attr_filter_loc.attr,
3552         &format_attr_filter_nm.attr,
3553         &format_attr_filter_all_op.attr,
3554         &format_attr_filter_not_nm.attr,
3555         &format_attr_filter_opc_0.attr,
3556         &format_attr_filter_opc_1.attr,
3557         &format_attr_filter_nc.attr,
3558         &format_attr_filter_isoc.attr,
3559         NULL,
3560 };
3561
3562 static const struct attribute_group skx_uncore_chabox_format_group = {
3563         .name = "format",
3564         .attrs = skx_uncore_cha_formats_attr,
3565 };
3566
3567 static struct event_constraint skx_uncore_chabox_constraints[] = {
3568         UNCORE_EVENT_CONSTRAINT(0x11, 0x1),
3569         UNCORE_EVENT_CONSTRAINT(0x36, 0x1),
3570         EVENT_CONSTRAINT_END
3571 };
3572
3573 static struct extra_reg skx_uncore_cha_extra_regs[] = {
3574         SNBEP_CBO_EVENT_EXTRA_REG(0x0334, 0xffff, 0x4),
3575         SNBEP_CBO_EVENT_EXTRA_REG(0x0534, 0xffff, 0x4),
3576         SNBEP_CBO_EVENT_EXTRA_REG(0x0934, 0xffff, 0x4),
3577         SNBEP_CBO_EVENT_EXTRA_REG(0x1134, 0xffff, 0x4),
3578         SNBEP_CBO_EVENT_EXTRA_REG(0x3134, 0xffff, 0x4),
3579         SNBEP_CBO_EVENT_EXTRA_REG(0x9134, 0xffff, 0x4),
3580         SNBEP_CBO_EVENT_EXTRA_REG(0x35, 0xff, 0x8),
3581         SNBEP_CBO_EVENT_EXTRA_REG(0x36, 0xff, 0x8),
3582         SNBEP_CBO_EVENT_EXTRA_REG(0x38, 0xff, 0x3),
3583         EVENT_EXTRA_END
3584 };
3585
3586 static u64 skx_cha_filter_mask(int fields)
3587 {
3588         u64 mask = 0;
3589
3590         if (fields & 0x1)
3591                 mask |= SKX_CHA_MSR_PMON_BOX_FILTER_TID;
3592         if (fields & 0x2)
3593                 mask |= SKX_CHA_MSR_PMON_BOX_FILTER_LINK;
3594         if (fields & 0x4)
3595                 mask |= SKX_CHA_MSR_PMON_BOX_FILTER_STATE;
3596         if (fields & 0x8) {
3597                 mask |= SKX_CHA_MSR_PMON_BOX_FILTER_REM;
3598                 mask |= SKX_CHA_MSR_PMON_BOX_FILTER_LOC;
3599                 mask |= SKX_CHA_MSR_PMON_BOX_FILTER_ALL_OPC;
3600                 mask |= SKX_CHA_MSR_PMON_BOX_FILTER_NM;
3601                 mask |= SKX_CHA_MSR_PMON_BOX_FILTER_NOT_NM;
3602                 mask |= SKX_CHA_MSR_PMON_BOX_FILTER_OPC0;
3603                 mask |= SKX_CHA_MSR_PMON_BOX_FILTER_OPC1;
3604                 mask |= SKX_CHA_MSR_PMON_BOX_FILTER_NC;
3605                 mask |= SKX_CHA_MSR_PMON_BOX_FILTER_ISOC;
3606         }
3607         return mask;
3608 }
3609
3610 static struct event_constraint *
3611 skx_cha_get_constraint(struct intel_uncore_box *box, struct perf_event *event)
3612 {
3613         return __snbep_cbox_get_constraint(box, event, skx_cha_filter_mask);
3614 }
3615
3616 static int skx_cha_hw_config(struct intel_uncore_box *box, struct perf_event *event)
3617 {
3618         struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
3619         struct extra_reg *er;
3620         int idx = 0;
3621         /* Any of the CHA events may be filtered by Thread/Core-ID.*/
3622         if (event->hw.config & SNBEP_CBO_PMON_CTL_TID_EN)
3623                 idx = SKX_CHA_MSR_PMON_BOX_FILTER_TID;
3624
3625         for (er = skx_uncore_cha_extra_regs; er->msr; er++) {
3626                 if (er->event != (event->hw.config & er->config_mask))
3627                         continue;
3628                 idx |= er->idx;
3629         }
3630
3631         if (idx) {
3632                 reg1->reg = HSWEP_C0_MSR_PMON_BOX_FILTER0 +
3633                             HSWEP_CBO_MSR_OFFSET * box->pmu->pmu_idx;
3634                 reg1->config = event->attr.config1 & skx_cha_filter_mask(idx);
3635                 reg1->idx = idx;
3636         }
3637         return 0;
3638 }
3639
3640 static struct intel_uncore_ops skx_uncore_chabox_ops = {
3641         /* There is no frz_en for chabox ctl */
3642         .init_box               = ivbep_uncore_msr_init_box,
3643         .disable_box            = snbep_uncore_msr_disable_box,
3644         .enable_box             = snbep_uncore_msr_enable_box,
3645         .disable_event          = snbep_uncore_msr_disable_event,
3646         .enable_event           = hswep_cbox_enable_event,
3647         .read_counter           = uncore_msr_read_counter,
3648         .hw_config              = skx_cha_hw_config,
3649         .get_constraint         = skx_cha_get_constraint,
3650         .put_constraint         = snbep_cbox_put_constraint,
3651 };
3652
3653 static struct intel_uncore_type skx_uncore_chabox = {
3654         .name                   = "cha",
3655         .num_counters           = 4,
3656         .perf_ctr_bits          = 48,
3657         .event_ctl              = HSWEP_C0_MSR_PMON_CTL0,
3658         .perf_ctr               = HSWEP_C0_MSR_PMON_CTR0,
3659         .event_mask             = HSWEP_S_MSR_PMON_RAW_EVENT_MASK,
3660         .box_ctl                = HSWEP_C0_MSR_PMON_BOX_CTL,
3661         .msr_offset             = HSWEP_CBO_MSR_OFFSET,
3662         .num_shared_regs        = 1,
3663         .constraints            = skx_uncore_chabox_constraints,
3664         .ops                    = &skx_uncore_chabox_ops,
3665         .format_group           = &skx_uncore_chabox_format_group,
3666 };
3667
3668 static struct attribute *skx_uncore_iio_formats_attr[] = {
3669         &format_attr_event.attr,
3670         &format_attr_umask.attr,
3671         &format_attr_edge.attr,
3672         &format_attr_inv.attr,
3673         &format_attr_thresh9.attr,
3674         &format_attr_ch_mask.attr,
3675         &format_attr_fc_mask.attr,
3676         NULL,
3677 };
3678
3679 static const struct attribute_group skx_uncore_iio_format_group = {
3680         .name = "format",
3681         .attrs = skx_uncore_iio_formats_attr,
3682 };
3683
3684 static struct event_constraint skx_uncore_iio_constraints[] = {
3685         UNCORE_EVENT_CONSTRAINT(0x83, 0x3),
3686         UNCORE_EVENT_CONSTRAINT(0x88, 0xc),
3687         UNCORE_EVENT_CONSTRAINT(0x95, 0xc),
3688         UNCORE_EVENT_CONSTRAINT(0xc0, 0xc),
3689         UNCORE_EVENT_CONSTRAINT(0xc5, 0xc),
3690         UNCORE_EVENT_CONSTRAINT(0xd4, 0xc),
3691         UNCORE_EVENT_CONSTRAINT(0xd5, 0xc),
3692         EVENT_CONSTRAINT_END
3693 };
3694
3695 static void skx_iio_enable_event(struct intel_uncore_box *box,
3696                                  struct perf_event *event)
3697 {
3698         struct hw_perf_event *hwc = &event->hw;
3699
3700         wrmsrl(hwc->config_base, hwc->config | SNBEP_PMON_CTL_EN);
3701 }
3702
3703 static struct intel_uncore_ops skx_uncore_iio_ops = {
3704         .init_box               = ivbep_uncore_msr_init_box,
3705         .disable_box            = snbep_uncore_msr_disable_box,
3706         .enable_box             = snbep_uncore_msr_enable_box,
3707         .disable_event          = snbep_uncore_msr_disable_event,
3708         .enable_event           = skx_iio_enable_event,
3709         .read_counter           = uncore_msr_read_counter,
3710 };
3711
3712 static struct intel_uncore_topology *pmu_topology(struct intel_uncore_pmu *pmu, int die)
3713 {
3714         int idx;
3715
3716         for (idx = 0; idx < pmu->type->num_boxes; idx++) {
3717                 if (pmu->type->topology[die][idx].pmu_idx == pmu->pmu_idx)
3718                         return &pmu->type->topology[die][idx];
3719         }
3720
3721         return NULL;
3722 }
3723
3724 static umode_t
3725 pmu_iio_mapping_visible(struct kobject *kobj, struct attribute *attr,
3726                          int die, int zero_bus_pmu)
3727 {
3728         struct intel_uncore_pmu *pmu = dev_to_uncore_pmu(kobj_to_dev(kobj));
3729         struct intel_uncore_topology *pmut = pmu_topology(pmu, die);
3730
3731         return (pmut && !pmut->iio->pci_bus_no && pmu->pmu_idx != zero_bus_pmu) ? 0 : attr->mode;
3732 }
3733
3734 static umode_t
3735 skx_iio_mapping_visible(struct kobject *kobj, struct attribute *attr, int die)
3736 {
3737         /* Root bus 0x00 is valid only for pmu_idx = 0. */
3738         return pmu_iio_mapping_visible(kobj, attr, die, 0);
3739 }
3740
3741 static ssize_t skx_iio_mapping_show(struct device *dev,
3742                                     struct device_attribute *attr, char *buf)
3743 {
3744         struct intel_uncore_pmu *pmu = dev_to_uncore_pmu(dev);
3745         struct dev_ext_attribute *ea = to_dev_ext_attribute(attr);
3746         long die = (long)ea->var;
3747         struct intel_uncore_topology *pmut = pmu_topology(pmu, die);
3748
3749         return sprintf(buf, "%04x:%02x\n", pmut ? pmut->iio->segment : 0,
3750                                            pmut ? pmut->iio->pci_bus_no : 0);
3751 }
3752
3753 static int skx_msr_cpu_bus_read(int cpu, u64 *topology)
3754 {
3755         u64 msr_value;
3756
3757         if (rdmsrl_on_cpu(cpu, SKX_MSR_CPU_BUS_NUMBER, &msr_value) ||
3758                         !(msr_value & SKX_MSR_CPU_BUS_VALID_BIT))
3759                 return -ENXIO;
3760
3761         *topology = msr_value;
3762
3763         return 0;
3764 }
3765
3766 static int die_to_cpu(int die)
3767 {
3768         int res = 0, cpu, current_die;
3769         /*
3770          * Using cpus_read_lock() to ensure cpu is not going down between
3771          * looking at cpu_online_mask.
3772          */
3773         cpus_read_lock();
3774         for_each_online_cpu(cpu) {
3775                 current_die = topology_logical_die_id(cpu);
3776                 if (current_die == die) {
3777                         res = cpu;
3778                         break;
3779                 }
3780         }
3781         cpus_read_unlock();
3782         return res;
3783 }
3784
3785 enum {
3786         IIO_TOPOLOGY_TYPE,
3787         UPI_TOPOLOGY_TYPE,
3788         TOPOLOGY_MAX
3789 };
3790
3791 static const size_t topology_size[TOPOLOGY_MAX] = {
3792         sizeof(*((struct intel_uncore_topology *)NULL)->iio),
3793         sizeof(*((struct intel_uncore_topology *)NULL)->upi)
3794 };
3795
3796 static int pmu_alloc_topology(struct intel_uncore_type *type, int topology_type)
3797 {
3798         int die, idx;
3799         struct intel_uncore_topology **topology;
3800
3801         if (!type->num_boxes)
3802                 return -EPERM;
3803
3804         topology = kcalloc(uncore_max_dies(), sizeof(*topology), GFP_KERNEL);
3805         if (!topology)
3806                 goto err;
3807
3808         for (die = 0; die < uncore_max_dies(); die++) {
3809                 topology[die] = kcalloc(type->num_boxes, sizeof(**topology), GFP_KERNEL);
3810                 if (!topology[die])
3811                         goto clear;
3812                 for (idx = 0; idx < type->num_boxes; idx++) {
3813                         topology[die][idx].untyped = kcalloc(type->num_boxes,
3814                                                              topology_size[topology_type],
3815                                                              GFP_KERNEL);
3816                         if (!topology[die][idx].untyped)
3817                                 goto clear;
3818                 }
3819         }
3820
3821         type->topology = topology;
3822
3823         return 0;
3824 clear:
3825         for (; die >= 0; die--) {
3826                 for (idx = 0; idx < type->num_boxes; idx++)
3827                         kfree(topology[die][idx].untyped);
3828                 kfree(topology[die]);
3829         }
3830         kfree(topology);
3831 err:
3832         return -ENOMEM;
3833 }
3834
3835 static void pmu_free_topology(struct intel_uncore_type *type)
3836 {
3837         int die, idx;
3838
3839         if (type->topology) {
3840                 for (die = 0; die < uncore_max_dies(); die++) {
3841                         for (idx = 0; idx < type->num_boxes; idx++)
3842                                 kfree(type->topology[die][idx].untyped);
3843                         kfree(type->topology[die]);
3844                 }
3845                 kfree(type->topology);
3846                 type->topology = NULL;
3847         }
3848 }
3849
3850 static int skx_pmu_get_topology(struct intel_uncore_type *type,
3851                                  int (*topology_cb)(struct intel_uncore_type*, int, int, u64))
3852 {
3853         int die, ret = -EPERM;
3854         u64 cpu_bus_msr;
3855
3856         for (die = 0; die < uncore_max_dies(); die++) {
3857                 ret = skx_msr_cpu_bus_read(die_to_cpu(die), &cpu_bus_msr);
3858                 if (ret)
3859                         break;
3860
3861                 ret = uncore_die_to_segment(die);
3862                 if (ret < 0)
3863                         break;
3864
3865                 ret = topology_cb(type, ret, die, cpu_bus_msr);
3866                 if (ret)
3867                         break;
3868         }
3869
3870         return ret;
3871 }
3872
3873 static int skx_iio_topology_cb(struct intel_uncore_type *type, int segment,
3874                                 int die, u64 cpu_bus_msr)
3875 {
3876         int idx;
3877         struct intel_uncore_topology *t;
3878
3879         for (idx = 0; idx < type->num_boxes; idx++) {
3880                 t = &type->topology[die][idx];
3881                 t->pmu_idx = idx;
3882                 t->iio->segment = segment;
3883                 t->iio->pci_bus_no = (cpu_bus_msr >> (idx * BUS_NUM_STRIDE)) & 0xff;
3884         }
3885
3886         return 0;
3887 }
3888
3889 static int skx_iio_get_topology(struct intel_uncore_type *type)
3890 {
3891         return skx_pmu_get_topology(type, skx_iio_topology_cb);
3892 }
3893
3894 static struct attribute_group skx_iio_mapping_group = {
3895         .is_visible     = skx_iio_mapping_visible,
3896 };
3897
3898 static const struct attribute_group *skx_iio_attr_update[] = {
3899         &skx_iio_mapping_group,
3900         NULL,
3901 };
3902
3903 static void pmu_clear_mapping_attr(const struct attribute_group **groups,
3904                                    struct attribute_group *ag)
3905 {
3906         int i;
3907
3908         for (i = 0; groups[i]; i++) {
3909                 if (groups[i] == ag) {
3910                         for (i++; groups[i]; i++)
3911                                 groups[i - 1] = groups[i];
3912                         groups[i - 1] = NULL;
3913                         break;
3914                 }
3915         }
3916 }
3917
3918 static void
3919 pmu_set_mapping(struct intel_uncore_type *type, struct attribute_group *ag,
3920                 ssize_t (*show)(struct device*, struct device_attribute*, char*),
3921                 int topology_type)
3922 {
3923         char buf[64];
3924         int ret;
3925         long die = -1;
3926         struct attribute **attrs = NULL;
3927         struct dev_ext_attribute *eas = NULL;
3928
3929         ret = pmu_alloc_topology(type, topology_type);
3930         if (ret < 0)
3931                 goto clear_attr_update;
3932
3933         ret = type->get_topology(type);
3934         if (ret < 0)
3935                 goto clear_topology;
3936
3937         /* One more for NULL. */
3938         attrs = kcalloc((uncore_max_dies() + 1), sizeof(*attrs), GFP_KERNEL);
3939         if (!attrs)
3940                 goto clear_topology;
3941
3942         eas = kcalloc(uncore_max_dies(), sizeof(*eas), GFP_KERNEL);
3943         if (!eas)
3944                 goto clear_attrs;
3945
3946         for (die = 0; die < uncore_max_dies(); die++) {
3947                 snprintf(buf, sizeof(buf), "die%ld", die);
3948                 sysfs_attr_init(&eas[die].attr.attr);
3949                 eas[die].attr.attr.name = kstrdup(buf, GFP_KERNEL);
3950                 if (!eas[die].attr.attr.name)
3951                         goto err;
3952                 eas[die].attr.attr.mode = 0444;
3953                 eas[die].attr.show = show;
3954                 eas[die].attr.store = NULL;
3955                 eas[die].var = (void *)die;
3956                 attrs[die] = &eas[die].attr.attr;
3957         }
3958         ag->attrs = attrs;
3959
3960         return;
3961 err:
3962         for (; die >= 0; die--)
3963                 kfree(eas[die].attr.attr.name);
3964         kfree(eas);
3965 clear_attrs:
3966         kfree(attrs);
3967 clear_topology:
3968         pmu_free_topology(type);
3969 clear_attr_update:
3970         pmu_clear_mapping_attr(type->attr_update, ag);
3971 }
3972
3973 static void
3974 pmu_cleanup_mapping(struct intel_uncore_type *type, struct attribute_group *ag)
3975 {
3976         struct attribute **attr = ag->attrs;
3977
3978         if (!attr)
3979                 return;
3980
3981         for (; *attr; attr++)
3982                 kfree((*attr)->name);
3983         kfree(attr_to_ext_attr(*ag->attrs));
3984         kfree(ag->attrs);
3985         ag->attrs = NULL;
3986         pmu_free_topology(type);
3987 }
3988
3989 static void
3990 pmu_iio_set_mapping(struct intel_uncore_type *type, struct attribute_group *ag)
3991 {
3992         pmu_set_mapping(type, ag, skx_iio_mapping_show, IIO_TOPOLOGY_TYPE);
3993 }
3994
3995 static void skx_iio_set_mapping(struct intel_uncore_type *type)
3996 {
3997         pmu_iio_set_mapping(type, &skx_iio_mapping_group);
3998 }
3999
4000 static void skx_iio_cleanup_mapping(struct intel_uncore_type *type)
4001 {
4002         pmu_cleanup_mapping(type, &skx_iio_mapping_group);
4003 }
4004
4005 static struct intel_uncore_type skx_uncore_iio = {
4006         .name                   = "iio",
4007         .num_counters           = 4,
4008         .num_boxes              = 6,
4009         .perf_ctr_bits          = 48,
4010         .event_ctl              = SKX_IIO0_MSR_PMON_CTL0,
4011         .perf_ctr               = SKX_IIO0_MSR_PMON_CTR0,
4012         .event_mask             = SKX_IIO_PMON_RAW_EVENT_MASK,
4013         .event_mask_ext         = SKX_IIO_PMON_RAW_EVENT_MASK_EXT,
4014         .box_ctl                = SKX_IIO0_MSR_PMON_BOX_CTL,
4015         .msr_offset             = SKX_IIO_MSR_OFFSET,
4016         .constraints            = skx_uncore_iio_constraints,
4017         .ops                    = &skx_uncore_iio_ops,
4018         .format_group           = &skx_uncore_iio_format_group,
4019         .attr_update            = skx_iio_attr_update,
4020         .get_topology           = skx_iio_get_topology,
4021         .set_mapping            = skx_iio_set_mapping,
4022         .cleanup_mapping        = skx_iio_cleanup_mapping,
4023 };
4024
4025 enum perf_uncore_iio_freerunning_type_id {
4026         SKX_IIO_MSR_IOCLK                       = 0,
4027         SKX_IIO_MSR_BW                          = 1,
4028         SKX_IIO_MSR_UTIL                        = 2,
4029
4030         SKX_IIO_FREERUNNING_TYPE_MAX,
4031 };
4032
4033
4034 static struct freerunning_counters skx_iio_freerunning[] = {
4035         [SKX_IIO_MSR_IOCLK]     = { 0xa45, 0x1, 0x20, 1, 36 },
4036         [SKX_IIO_MSR_BW]        = { 0xb00, 0x1, 0x10, 8, 36 },
4037         [SKX_IIO_MSR_UTIL]      = { 0xb08, 0x1, 0x10, 8, 36 },
4038 };
4039
4040 static struct uncore_event_desc skx_uncore_iio_freerunning_events[] = {
4041         /* Free-Running IO CLOCKS Counter */
4042         INTEL_UNCORE_EVENT_DESC(ioclk,                  "event=0xff,umask=0x10"),
4043         /* Free-Running IIO BANDWIDTH Counters */
4044         INTEL_UNCORE_EVENT_DESC(bw_in_port0,            "event=0xff,umask=0x20"),
4045         INTEL_UNCORE_EVENT_DESC(bw_in_port0.scale,      "3.814697266e-6"),
4046         INTEL_UNCORE_EVENT_DESC(bw_in_port0.unit,       "MiB"),
4047         INTEL_UNCORE_EVENT_DESC(bw_in_port1,            "event=0xff,umask=0x21"),
4048         INTEL_UNCORE_EVENT_DESC(bw_in_port1.scale,      "3.814697266e-6"),
4049         INTEL_UNCORE_EVENT_DESC(bw_in_port1.unit,       "MiB"),
4050         INTEL_UNCORE_EVENT_DESC(bw_in_port2,            "event=0xff,umask=0x22"),
4051         INTEL_UNCORE_EVENT_DESC(bw_in_port2.scale,      "3.814697266e-6"),
4052         INTEL_UNCORE_EVENT_DESC(bw_in_port2.unit,       "MiB"),
4053         INTEL_UNCORE_EVENT_DESC(bw_in_port3,            "event=0xff,umask=0x23"),
4054         INTEL_UNCORE_EVENT_DESC(bw_in_port3.scale,      "3.814697266e-6"),
4055         INTEL_UNCORE_EVENT_DESC(bw_in_port3.unit,       "MiB"),
4056         INTEL_UNCORE_EVENT_DESC(bw_out_port0,           "event=0xff,umask=0x24"),
4057         INTEL_UNCORE_EVENT_DESC(bw_out_port0.scale,     "3.814697266e-6"),
4058         INTEL_UNCORE_EVENT_DESC(bw_out_port0.unit,      "MiB"),
4059         INTEL_UNCORE_EVENT_DESC(bw_out_port1,           "event=0xff,umask=0x25"),
4060         INTEL_UNCORE_EVENT_DESC(bw_out_port1.scale,     "3.814697266e-6"),
4061         INTEL_UNCORE_EVENT_DESC(bw_out_port1.unit,      "MiB"),
4062         INTEL_UNCORE_EVENT_DESC(bw_out_port2,           "event=0xff,umask=0x26"),
4063         INTEL_UNCORE_EVENT_DESC(bw_out_port2.scale,     "3.814697266e-6"),
4064         INTEL_UNCORE_EVENT_DESC(bw_out_port2.unit,      "MiB"),
4065         INTEL_UNCORE_EVENT_DESC(bw_out_port3,           "event=0xff,umask=0x27"),
4066         INTEL_UNCORE_EVENT_DESC(bw_out_port3.scale,     "3.814697266e-6"),
4067         INTEL_UNCORE_EVENT_DESC(bw_out_port3.unit,      "MiB"),
4068         /* Free-running IIO UTILIZATION Counters */
4069         INTEL_UNCORE_EVENT_DESC(util_in_port0,          "event=0xff,umask=0x30"),
4070         INTEL_UNCORE_EVENT_DESC(util_out_port0,         "event=0xff,umask=0x31"),
4071         INTEL_UNCORE_EVENT_DESC(util_in_port1,          "event=0xff,umask=0x32"),
4072         INTEL_UNCORE_EVENT_DESC(util_out_port1,         "event=0xff,umask=0x33"),
4073         INTEL_UNCORE_EVENT_DESC(util_in_port2,          "event=0xff,umask=0x34"),
4074         INTEL_UNCORE_EVENT_DESC(util_out_port2,         "event=0xff,umask=0x35"),
4075         INTEL_UNCORE_EVENT_DESC(util_in_port3,          "event=0xff,umask=0x36"),
4076         INTEL_UNCORE_EVENT_DESC(util_out_port3,         "event=0xff,umask=0x37"),
4077         { /* end: all zeroes */ },
4078 };
4079
4080 static struct intel_uncore_ops skx_uncore_iio_freerunning_ops = {
4081         .read_counter           = uncore_msr_read_counter,
4082         .hw_config              = uncore_freerunning_hw_config,
4083 };
4084
4085 static struct attribute *skx_uncore_iio_freerunning_formats_attr[] = {
4086         &format_attr_event.attr,
4087         &format_attr_umask.attr,
4088         NULL,
4089 };
4090
4091 static const struct attribute_group skx_uncore_iio_freerunning_format_group = {
4092         .name = "format",
4093         .attrs = skx_uncore_iio_freerunning_formats_attr,
4094 };
4095
4096 static struct intel_uncore_type skx_uncore_iio_free_running = {
4097         .name                   = "iio_free_running",
4098         .num_counters           = 17,
4099         .num_boxes              = 6,
4100         .num_freerunning_types  = SKX_IIO_FREERUNNING_TYPE_MAX,
4101         .freerunning            = skx_iio_freerunning,
4102         .ops                    = &skx_uncore_iio_freerunning_ops,
4103         .event_descs            = skx_uncore_iio_freerunning_events,
4104         .format_group           = &skx_uncore_iio_freerunning_format_group,
4105 };
4106
4107 static struct attribute *skx_uncore_formats_attr[] = {
4108         &format_attr_event.attr,
4109         &format_attr_umask.attr,
4110         &format_attr_edge.attr,
4111         &format_attr_inv.attr,
4112         &format_attr_thresh8.attr,
4113         NULL,
4114 };
4115
4116 static const struct attribute_group skx_uncore_format_group = {
4117         .name = "format",
4118         .attrs = skx_uncore_formats_attr,
4119 };
4120
4121 static struct intel_uncore_type skx_uncore_irp = {
4122         .name                   = "irp",
4123         .num_counters           = 2,
4124         .num_boxes              = 6,
4125         .perf_ctr_bits          = 48,
4126         .event_ctl              = SKX_IRP0_MSR_PMON_CTL0,
4127         .perf_ctr               = SKX_IRP0_MSR_PMON_CTR0,
4128         .event_mask             = SNBEP_PMON_RAW_EVENT_MASK,
4129         .box_ctl                = SKX_IRP0_MSR_PMON_BOX_CTL,
4130         .msr_offset             = SKX_IRP_MSR_OFFSET,
4131         .ops                    = &skx_uncore_iio_ops,
4132         .format_group           = &skx_uncore_format_group,
4133 };
4134
4135 static struct attribute *skx_uncore_pcu_formats_attr[] = {
4136         &format_attr_event.attr,
4137         &format_attr_umask.attr,
4138         &format_attr_edge.attr,
4139         &format_attr_inv.attr,
4140         &format_attr_thresh8.attr,
4141         &format_attr_occ_invert.attr,
4142         &format_attr_occ_edge_det.attr,
4143         &format_attr_filter_band0.attr,
4144         &format_attr_filter_band1.attr,
4145         &format_attr_filter_band2.attr,
4146         &format_attr_filter_band3.attr,
4147         NULL,
4148 };
4149
4150 static struct attribute_group skx_uncore_pcu_format_group = {
4151         .name = "format",
4152         .attrs = skx_uncore_pcu_formats_attr,
4153 };
4154
4155 static struct intel_uncore_ops skx_uncore_pcu_ops = {
4156         IVBEP_UNCORE_MSR_OPS_COMMON_INIT(),
4157         .hw_config              = hswep_pcu_hw_config,
4158         .get_constraint         = snbep_pcu_get_constraint,
4159         .put_constraint         = snbep_pcu_put_constraint,
4160 };
4161
4162 static struct intel_uncore_type skx_uncore_pcu = {
4163         .name                   = "pcu",
4164         .num_counters           = 4,
4165         .num_boxes              = 1,
4166         .perf_ctr_bits          = 48,
4167         .perf_ctr               = HSWEP_PCU_MSR_PMON_CTR0,
4168         .event_ctl              = HSWEP_PCU_MSR_PMON_CTL0,
4169         .event_mask             = SNBEP_PCU_MSR_PMON_RAW_EVENT_MASK,
4170         .box_ctl                = HSWEP_PCU_MSR_PMON_BOX_CTL,
4171         .num_shared_regs        = 1,
4172         .ops                    = &skx_uncore_pcu_ops,
4173         .format_group           = &skx_uncore_pcu_format_group,
4174 };
4175
4176 static struct intel_uncore_type *skx_msr_uncores[] = {
4177         &skx_uncore_ubox,
4178         &skx_uncore_chabox,
4179         &skx_uncore_iio,
4180         &skx_uncore_iio_free_running,
4181         &skx_uncore_irp,
4182         &skx_uncore_pcu,
4183         NULL,
4184 };
4185
4186 /*
4187  * To determine the number of CHAs, it should read bits 27:0 in the CAPID6
4188  * register which located at Device 30, Function 3, Offset 0x9C. PCI ID 0x2083.
4189  */
4190 #define SKX_CAPID6              0x9c
4191 #define SKX_CHA_BIT_MASK        GENMASK(27, 0)
4192
4193 static int skx_count_chabox(void)
4194 {
4195         struct pci_dev *dev = NULL;
4196         u32 val = 0;
4197
4198         dev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x2083, dev);
4199         if (!dev)
4200                 goto out;
4201
4202         pci_read_config_dword(dev, SKX_CAPID6, &val);
4203         val &= SKX_CHA_BIT_MASK;
4204 out:
4205         pci_dev_put(dev);
4206         return hweight32(val);
4207 }
4208
4209 void skx_uncore_cpu_init(void)
4210 {
4211         skx_uncore_chabox.num_boxes = skx_count_chabox();
4212         uncore_msr_uncores = skx_msr_uncores;
4213 }
4214
4215 static struct intel_uncore_type skx_uncore_imc = {
4216         .name           = "imc",
4217         .num_counters   = 4,
4218         .num_boxes      = 6,
4219         .perf_ctr_bits  = 48,
4220         .fixed_ctr_bits = 48,
4221         .fixed_ctr      = SNBEP_MC_CHy_PCI_PMON_FIXED_CTR,
4222         .fixed_ctl      = SNBEP_MC_CHy_PCI_PMON_FIXED_CTL,
4223         .event_descs    = hswep_uncore_imc_events,
4224         .perf_ctr       = SNBEP_PCI_PMON_CTR0,
4225         .event_ctl      = SNBEP_PCI_PMON_CTL0,
4226         .event_mask     = SNBEP_PMON_RAW_EVENT_MASK,
4227         .box_ctl        = SNBEP_PCI_PMON_BOX_CTL,
4228         .ops            = &ivbep_uncore_pci_ops,
4229         .format_group   = &skx_uncore_format_group,
4230 };
4231
4232 static struct attribute *skx_upi_uncore_formats_attr[] = {
4233         &format_attr_event.attr,
4234         &format_attr_umask_ext.attr,
4235         &format_attr_edge.attr,
4236         &format_attr_inv.attr,
4237         &format_attr_thresh8.attr,
4238         NULL,
4239 };
4240
4241 static const struct attribute_group skx_upi_uncore_format_group = {
4242         .name = "format",
4243         .attrs = skx_upi_uncore_formats_attr,
4244 };
4245
4246 static void skx_upi_uncore_pci_init_box(struct intel_uncore_box *box)
4247 {
4248         struct pci_dev *pdev = box->pci_dev;
4249
4250         __set_bit(UNCORE_BOX_FLAG_CTL_OFFS8, &box->flags);
4251         pci_write_config_dword(pdev, SKX_UPI_PCI_PMON_BOX_CTL, IVBEP_PMON_BOX_CTL_INT);
4252 }
4253
4254 static struct intel_uncore_ops skx_upi_uncore_pci_ops = {
4255         .init_box       = skx_upi_uncore_pci_init_box,
4256         .disable_box    = snbep_uncore_pci_disable_box,
4257         .enable_box     = snbep_uncore_pci_enable_box,
4258         .disable_event  = snbep_uncore_pci_disable_event,
4259         .enable_event   = snbep_uncore_pci_enable_event,
4260         .read_counter   = snbep_uncore_pci_read_counter,
4261 };
4262
4263 static umode_t
4264 skx_upi_mapping_visible(struct kobject *kobj, struct attribute *attr, int die)
4265 {
4266         struct intel_uncore_pmu *pmu = dev_to_uncore_pmu(kobj_to_dev(kobj));
4267
4268         return pmu->type->topology[die][pmu->pmu_idx].upi->enabled ? attr->mode : 0;
4269 }
4270
4271 static ssize_t skx_upi_mapping_show(struct device *dev,
4272                                     struct device_attribute *attr, char *buf)
4273 {
4274         struct intel_uncore_pmu *pmu = dev_to_uncore_pmu(dev);
4275         struct dev_ext_attribute *ea = to_dev_ext_attribute(attr);
4276         long die = (long)ea->var;
4277         struct uncore_upi_topology *upi = pmu->type->topology[die][pmu->pmu_idx].upi;
4278
4279         return sysfs_emit(buf, "upi_%d,die_%d\n", upi->pmu_idx_to, upi->die_to);
4280 }
4281
4282 #define SKX_UPI_REG_DID                 0x2058
4283 #define SKX_UPI_REGS_ADDR_DEVICE_LINK0  0x0e
4284 #define SKX_UPI_REGS_ADDR_FUNCTION      0x00
4285
4286 /*
4287  * UPI Link Parameter 0
4288  * |  Bit  |  Default  |  Description
4289  * | 19:16 |     0h    | base_nodeid - The NodeID of the sending socket.
4290  * | 12:8  |    00h    | sending_port - The processor die port number of the sending port.
4291  */
4292 #define SKX_KTILP0_OFFSET       0x94
4293
4294 /*
4295  * UPI Pcode Status. This register is used by PCode to store the link training status.
4296  * |  Bit  |  Default  |  Description
4297  * |   4   |     0h    | ll_status_valid — Bit indicates the valid training status
4298  *                       logged from PCode to the BIOS.
4299  */
4300 #define SKX_KTIPCSTS_OFFSET     0x120
4301
4302 static int upi_fill_topology(struct pci_dev *dev, struct intel_uncore_topology *tp,
4303                              int pmu_idx)
4304 {
4305         int ret;
4306         u32 upi_conf;
4307         struct uncore_upi_topology *upi = tp->upi;
4308
4309         tp->pmu_idx = pmu_idx;
4310         ret = pci_read_config_dword(dev, SKX_KTIPCSTS_OFFSET, &upi_conf);
4311         if (ret) {
4312                 ret = pcibios_err_to_errno(ret);
4313                 goto err;
4314         }
4315         upi->enabled = (upi_conf >> 4) & 1;
4316         if (upi->enabled) {
4317                 ret = pci_read_config_dword(dev, SKX_KTILP0_OFFSET,
4318                                             &upi_conf);
4319                 if (ret) {
4320                         ret = pcibios_err_to_errno(ret);
4321                         goto err;
4322                 }
4323                 upi->die_to = (upi_conf >> 16) & 0xf;
4324                 upi->pmu_idx_to = (upi_conf >> 8) & 0x1f;
4325         }
4326 err:
4327         return ret;
4328 }
4329
4330 static int skx_upi_topology_cb(struct intel_uncore_type *type, int segment,
4331                                 int die, u64 cpu_bus_msr)
4332 {
4333         int idx, ret;
4334         struct intel_uncore_topology *upi;
4335         unsigned int devfn;
4336         struct pci_dev *dev = NULL;
4337         u8 bus = cpu_bus_msr >> (3 * BUS_NUM_STRIDE);
4338
4339         for (idx = 0; idx < type->num_boxes; idx++) {
4340                 upi = &type->topology[die][idx];
4341                 devfn = PCI_DEVFN(SKX_UPI_REGS_ADDR_DEVICE_LINK0 + idx,
4342                                   SKX_UPI_REGS_ADDR_FUNCTION);
4343                 dev = pci_get_domain_bus_and_slot(segment, bus, devfn);
4344                 if (dev) {
4345                         ret = upi_fill_topology(dev, upi, idx);
4346                         if (ret)
4347                                 break;
4348                 }
4349         }
4350
4351         pci_dev_put(dev);
4352         return ret;
4353 }
4354
4355 static int skx_upi_get_topology(struct intel_uncore_type *type)
4356 {
4357         /* CPX case is not supported */
4358         if (boot_cpu_data.x86_stepping == 11)
4359                 return -EPERM;
4360
4361         return skx_pmu_get_topology(type, skx_upi_topology_cb);
4362 }
4363
4364 static struct attribute_group skx_upi_mapping_group = {
4365         .is_visible     = skx_upi_mapping_visible,
4366 };
4367
4368 static const struct attribute_group *skx_upi_attr_update[] = {
4369         &skx_upi_mapping_group,
4370         NULL
4371 };
4372
4373 static void
4374 pmu_upi_set_mapping(struct intel_uncore_type *type, struct attribute_group *ag)
4375 {
4376         pmu_set_mapping(type, ag, skx_upi_mapping_show, UPI_TOPOLOGY_TYPE);
4377 }
4378
4379 static void skx_upi_set_mapping(struct intel_uncore_type *type)
4380 {
4381         pmu_upi_set_mapping(type, &skx_upi_mapping_group);
4382 }
4383
4384 static void skx_upi_cleanup_mapping(struct intel_uncore_type *type)
4385 {
4386         pmu_cleanup_mapping(type, &skx_upi_mapping_group);
4387 }
4388
4389 static struct intel_uncore_type skx_uncore_upi = {
4390         .name           = "upi",
4391         .num_counters   = 4,
4392         .num_boxes      = 3,
4393         .perf_ctr_bits  = 48,
4394         .perf_ctr       = SKX_UPI_PCI_PMON_CTR0,
4395         .event_ctl      = SKX_UPI_PCI_PMON_CTL0,
4396         .event_mask     = SNBEP_PMON_RAW_EVENT_MASK,
4397         .event_mask_ext = SKX_UPI_CTL_UMASK_EXT,
4398         .box_ctl        = SKX_UPI_PCI_PMON_BOX_CTL,
4399         .ops            = &skx_upi_uncore_pci_ops,
4400         .format_group   = &skx_upi_uncore_format_group,
4401         .attr_update    = skx_upi_attr_update,
4402         .get_topology   = skx_upi_get_topology,
4403         .set_mapping    = skx_upi_set_mapping,
4404         .cleanup_mapping = skx_upi_cleanup_mapping,
4405 };
4406
4407 static void skx_m2m_uncore_pci_init_box(struct intel_uncore_box *box)
4408 {
4409         struct pci_dev *pdev = box->pci_dev;
4410
4411         __set_bit(UNCORE_BOX_FLAG_CTL_OFFS8, &box->flags);
4412         pci_write_config_dword(pdev, SKX_M2M_PCI_PMON_BOX_CTL, IVBEP_PMON_BOX_CTL_INT);
4413 }
4414
4415 static struct intel_uncore_ops skx_m2m_uncore_pci_ops = {
4416         .init_box       = skx_m2m_uncore_pci_init_box,
4417         .disable_box    = snbep_uncore_pci_disable_box,
4418         .enable_box     = snbep_uncore_pci_enable_box,
4419         .disable_event  = snbep_uncore_pci_disable_event,
4420         .enable_event   = snbep_uncore_pci_enable_event,
4421         .read_counter   = snbep_uncore_pci_read_counter,
4422 };
4423
4424 static struct intel_uncore_type skx_uncore_m2m = {
4425         .name           = "m2m",
4426         .num_counters   = 4,
4427         .num_boxes      = 2,
4428         .perf_ctr_bits  = 48,
4429         .perf_ctr       = SKX_M2M_PCI_PMON_CTR0,
4430         .event_ctl      = SKX_M2M_PCI_PMON_CTL0,
4431         .event_mask     = SNBEP_PMON_RAW_EVENT_MASK,
4432         .box_ctl        = SKX_M2M_PCI_PMON_BOX_CTL,
4433         .ops            = &skx_m2m_uncore_pci_ops,
4434         .format_group   = &skx_uncore_format_group,
4435 };
4436
4437 static struct event_constraint skx_uncore_m2pcie_constraints[] = {
4438         UNCORE_EVENT_CONSTRAINT(0x23, 0x3),
4439         EVENT_CONSTRAINT_END
4440 };
4441
4442 static struct intel_uncore_type skx_uncore_m2pcie = {
4443         .name           = "m2pcie",
4444         .num_counters   = 4,
4445         .num_boxes      = 4,
4446         .perf_ctr_bits  = 48,
4447         .constraints    = skx_uncore_m2pcie_constraints,
4448         .perf_ctr       = SNBEP_PCI_PMON_CTR0,
4449         .event_ctl      = SNBEP_PCI_PMON_CTL0,
4450         .event_mask     = SNBEP_PMON_RAW_EVENT_MASK,
4451         .box_ctl        = SNBEP_PCI_PMON_BOX_CTL,
4452         .ops            = &ivbep_uncore_pci_ops,
4453         .format_group   = &skx_uncore_format_group,
4454 };
4455
4456 static struct event_constraint skx_uncore_m3upi_constraints[] = {
4457         UNCORE_EVENT_CONSTRAINT(0x1d, 0x1),
4458         UNCORE_EVENT_CONSTRAINT(0x1e, 0x1),
4459         UNCORE_EVENT_CONSTRAINT(0x40, 0x7),
4460         UNCORE_EVENT_CONSTRAINT(0x4e, 0x7),
4461         UNCORE_EVENT_CONSTRAINT(0x4f, 0x7),
4462         UNCORE_EVENT_CONSTRAINT(0x50, 0x7),
4463         UNCORE_EVENT_CONSTRAINT(0x51, 0x7),
4464         UNCORE_EVENT_CONSTRAINT(0x52, 0x7),
4465         EVENT_CONSTRAINT_END
4466 };
4467
4468 static struct intel_uncore_type skx_uncore_m3upi = {
4469         .name           = "m3upi",
4470         .num_counters   = 3,
4471         .num_boxes      = 3,
4472         .perf_ctr_bits  = 48,
4473         .constraints    = skx_uncore_m3upi_constraints,
4474         .perf_ctr       = SNBEP_PCI_PMON_CTR0,
4475         .event_ctl      = SNBEP_PCI_PMON_CTL0,
4476         .event_mask     = SNBEP_PMON_RAW_EVENT_MASK,
4477         .box_ctl        = SNBEP_PCI_PMON_BOX_CTL,
4478         .ops            = &ivbep_uncore_pci_ops,
4479         .format_group   = &skx_uncore_format_group,
4480 };
4481
4482 enum {
4483         SKX_PCI_UNCORE_IMC,
4484         SKX_PCI_UNCORE_M2M,
4485         SKX_PCI_UNCORE_UPI,
4486         SKX_PCI_UNCORE_M2PCIE,
4487         SKX_PCI_UNCORE_M3UPI,
4488 };
4489
4490 static struct intel_uncore_type *skx_pci_uncores[] = {
4491         [SKX_PCI_UNCORE_IMC]    = &skx_uncore_imc,
4492         [SKX_PCI_UNCORE_M2M]    = &skx_uncore_m2m,
4493         [SKX_PCI_UNCORE_UPI]    = &skx_uncore_upi,
4494         [SKX_PCI_UNCORE_M2PCIE] = &skx_uncore_m2pcie,
4495         [SKX_PCI_UNCORE_M3UPI]  = &skx_uncore_m3upi,
4496         NULL,
4497 };
4498
4499 static const struct pci_device_id skx_uncore_pci_ids[] = {
4500         { /* MC0 Channel 0 */
4501                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2042),
4502                 .driver_data = UNCORE_PCI_DEV_FULL_DATA(10, 2, SKX_PCI_UNCORE_IMC, 0),
4503         },
4504         { /* MC0 Channel 1 */
4505                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2046),
4506                 .driver_data = UNCORE_PCI_DEV_FULL_DATA(10, 6, SKX_PCI_UNCORE_IMC, 1),
4507         },
4508         { /* MC0 Channel 2 */
4509                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x204a),
4510                 .driver_data = UNCORE_PCI_DEV_FULL_DATA(11, 2, SKX_PCI_UNCORE_IMC, 2),
4511         },
4512         { /* MC1 Channel 0 */
4513                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2042),
4514                 .driver_data = UNCORE_PCI_DEV_FULL_DATA(12, 2, SKX_PCI_UNCORE_IMC, 3),
4515         },
4516         { /* MC1 Channel 1 */
4517                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2046),
4518                 .driver_data = UNCORE_PCI_DEV_FULL_DATA(12, 6, SKX_PCI_UNCORE_IMC, 4),
4519         },
4520         { /* MC1 Channel 2 */
4521                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x204a),
4522                 .driver_data = UNCORE_PCI_DEV_FULL_DATA(13, 2, SKX_PCI_UNCORE_IMC, 5),
4523         },
4524         { /* M2M0 */
4525                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2066),
4526                 .driver_data = UNCORE_PCI_DEV_FULL_DATA(8, 0, SKX_PCI_UNCORE_M2M, 0),
4527         },
4528         { /* M2M1 */
4529                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2066),
4530                 .driver_data = UNCORE_PCI_DEV_FULL_DATA(9, 0, SKX_PCI_UNCORE_M2M, 1),
4531         },
4532         { /* UPI0 Link 0 */
4533                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2058),
4534                 .driver_data = UNCORE_PCI_DEV_FULL_DATA(14, 0, SKX_PCI_UNCORE_UPI, 0),
4535         },
4536         { /* UPI0 Link 1 */
4537                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2058),
4538                 .driver_data = UNCORE_PCI_DEV_FULL_DATA(15, 0, SKX_PCI_UNCORE_UPI, 1),
4539         },
4540         { /* UPI1 Link 2 */
4541                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2058),
4542                 .driver_data = UNCORE_PCI_DEV_FULL_DATA(16, 0, SKX_PCI_UNCORE_UPI, 2),
4543         },
4544         { /* M2PCIe 0 */
4545                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2088),
4546                 .driver_data = UNCORE_PCI_DEV_FULL_DATA(21, 1, SKX_PCI_UNCORE_M2PCIE, 0),
4547         },
4548         { /* M2PCIe 1 */
4549                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2088),
4550                 .driver_data = UNCORE_PCI_DEV_FULL_DATA(22, 1, SKX_PCI_UNCORE_M2PCIE, 1),
4551         },
4552         { /* M2PCIe 2 */
4553                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2088),
4554                 .driver_data = UNCORE_PCI_DEV_FULL_DATA(23, 1, SKX_PCI_UNCORE_M2PCIE, 2),
4555         },
4556         { /* M2PCIe 3 */
4557                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x2088),
4558                 .driver_data = UNCORE_PCI_DEV_FULL_DATA(21, 5, SKX_PCI_UNCORE_M2PCIE, 3),
4559         },
4560         { /* M3UPI0 Link 0 */
4561                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x204D),
4562                 .driver_data = UNCORE_PCI_DEV_FULL_DATA(18, 1, SKX_PCI_UNCORE_M3UPI, 0),
4563         },
4564         { /* M3UPI0 Link 1 */
4565                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x204E),
4566                 .driver_data = UNCORE_PCI_DEV_FULL_DATA(18, 2, SKX_PCI_UNCORE_M3UPI, 1),
4567         },
4568         { /* M3UPI1 Link 2 */
4569                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x204D),
4570                 .driver_data = UNCORE_PCI_DEV_FULL_DATA(18, 5, SKX_PCI_UNCORE_M3UPI, 2),
4571         },
4572         { /* end: all zeroes */ }
4573 };
4574
4575
4576 static struct pci_driver skx_uncore_pci_driver = {
4577         .name           = "skx_uncore",
4578         .id_table       = skx_uncore_pci_ids,
4579 };
4580
4581 int skx_uncore_pci_init(void)
4582 {
4583         /* need to double check pci address */
4584         int ret = snbep_pci2phy_map_init(0x2014, SKX_CPUNODEID, SKX_GIDNIDMAP, false);
4585
4586         if (ret)
4587                 return ret;
4588
4589         uncore_pci_uncores = skx_pci_uncores;
4590         uncore_pci_driver = &skx_uncore_pci_driver;
4591         return 0;
4592 }
4593
4594 /* end of SKX uncore support */
4595
4596 /* SNR uncore support */
4597
4598 static struct intel_uncore_type snr_uncore_ubox = {
4599         .name                   = "ubox",
4600         .num_counters           = 2,
4601         .num_boxes              = 1,
4602         .perf_ctr_bits          = 48,
4603         .fixed_ctr_bits         = 48,
4604         .perf_ctr               = SNR_U_MSR_PMON_CTR0,
4605         .event_ctl              = SNR_U_MSR_PMON_CTL0,
4606         .event_mask             = SNBEP_PMON_RAW_EVENT_MASK,
4607         .fixed_ctr              = SNR_U_MSR_PMON_UCLK_FIXED_CTR,
4608         .fixed_ctl              = SNR_U_MSR_PMON_UCLK_FIXED_CTL,
4609         .ops                    = &ivbep_uncore_msr_ops,
4610         .format_group           = &ivbep_uncore_format_group,
4611 };
4612
4613 static struct attribute *snr_uncore_cha_formats_attr[] = {
4614         &format_attr_event.attr,
4615         &format_attr_umask_ext2.attr,
4616         &format_attr_edge.attr,
4617         &format_attr_tid_en.attr,
4618         &format_attr_inv.attr,
4619         &format_attr_thresh8.attr,
4620         &format_attr_filter_tid5.attr,
4621         NULL,
4622 };
4623 static const struct attribute_group snr_uncore_chabox_format_group = {
4624         .name = "format",
4625         .attrs = snr_uncore_cha_formats_attr,
4626 };
4627
4628 static int snr_cha_hw_config(struct intel_uncore_box *box, struct perf_event *event)
4629 {
4630         struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
4631
4632         reg1->reg = SNR_C0_MSR_PMON_BOX_FILTER0 +
4633                     box->pmu->type->msr_offset * box->pmu->pmu_idx;
4634         reg1->config = event->attr.config1 & SKX_CHA_MSR_PMON_BOX_FILTER_TID;
4635         reg1->idx = 0;
4636
4637         return 0;
4638 }
4639
4640 static void snr_cha_enable_event(struct intel_uncore_box *box,
4641                                    struct perf_event *event)
4642 {
4643         struct hw_perf_event *hwc = &event->hw;
4644         struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
4645
4646         if (reg1->idx != EXTRA_REG_NONE)
4647                 wrmsrl(reg1->reg, reg1->config);
4648
4649         wrmsrl(hwc->config_base, hwc->config | SNBEP_PMON_CTL_EN);
4650 }
4651
4652 static struct intel_uncore_ops snr_uncore_chabox_ops = {
4653         .init_box               = ivbep_uncore_msr_init_box,
4654         .disable_box            = snbep_uncore_msr_disable_box,
4655         .enable_box             = snbep_uncore_msr_enable_box,
4656         .disable_event          = snbep_uncore_msr_disable_event,
4657         .enable_event           = snr_cha_enable_event,
4658         .read_counter           = uncore_msr_read_counter,
4659         .hw_config              = snr_cha_hw_config,
4660 };
4661
4662 static struct intel_uncore_type snr_uncore_chabox = {
4663         .name                   = "cha",
4664         .num_counters           = 4,
4665         .num_boxes              = 6,
4666         .perf_ctr_bits          = 48,
4667         .event_ctl              = SNR_CHA_MSR_PMON_CTL0,
4668         .perf_ctr               = SNR_CHA_MSR_PMON_CTR0,
4669         .box_ctl                = SNR_CHA_MSR_PMON_BOX_CTL,
4670         .msr_offset             = HSWEP_CBO_MSR_OFFSET,
4671         .event_mask             = HSWEP_S_MSR_PMON_RAW_EVENT_MASK,
4672         .event_mask_ext         = SNR_CHA_RAW_EVENT_MASK_EXT,
4673         .ops                    = &snr_uncore_chabox_ops,
4674         .format_group           = &snr_uncore_chabox_format_group,
4675 };
4676
4677 static struct attribute *snr_uncore_iio_formats_attr[] = {
4678         &format_attr_event.attr,
4679         &format_attr_umask.attr,
4680         &format_attr_edge.attr,
4681         &format_attr_inv.attr,
4682         &format_attr_thresh9.attr,
4683         &format_attr_ch_mask2.attr,
4684         &format_attr_fc_mask2.attr,
4685         NULL,
4686 };
4687
4688 static const struct attribute_group snr_uncore_iio_format_group = {
4689         .name = "format",
4690         .attrs = snr_uncore_iio_formats_attr,
4691 };
4692
4693 static umode_t
4694 snr_iio_mapping_visible(struct kobject *kobj, struct attribute *attr, int die)
4695 {
4696         /* Root bus 0x00 is valid only for pmu_idx = 1. */
4697         return pmu_iio_mapping_visible(kobj, attr, die, 1);
4698 }
4699
4700 static struct attribute_group snr_iio_mapping_group = {
4701         .is_visible     = snr_iio_mapping_visible,
4702 };
4703
4704 static const struct attribute_group *snr_iio_attr_update[] = {
4705         &snr_iio_mapping_group,
4706         NULL,
4707 };
4708
4709 static int sad_cfg_iio_topology(struct intel_uncore_type *type, u8 *sad_pmon_mapping)
4710 {
4711         u32 sad_cfg;
4712         int die, stack_id, ret = -EPERM;
4713         struct pci_dev *dev = NULL;
4714
4715         while ((dev = pci_get_device(PCI_VENDOR_ID_INTEL, SNR_ICX_MESH2IIO_MMAP_DID, dev))) {
4716                 ret = pci_read_config_dword(dev, SNR_ICX_SAD_CONTROL_CFG, &sad_cfg);
4717                 if (ret) {
4718                         ret = pcibios_err_to_errno(ret);
4719                         break;
4720                 }
4721
4722                 die = uncore_pcibus_to_dieid(dev->bus);
4723                 stack_id = SAD_CONTROL_STACK_ID(sad_cfg);
4724                 if (die < 0 || stack_id >= type->num_boxes) {
4725                         ret = -EPERM;
4726                         break;
4727                 }
4728
4729                 /* Convert stack id from SAD_CONTROL to PMON notation. */
4730                 stack_id = sad_pmon_mapping[stack_id];
4731
4732                 type->topology[die][stack_id].iio->segment = pci_domain_nr(dev->bus);
4733                 type->topology[die][stack_id].pmu_idx = stack_id;
4734                 type->topology[die][stack_id].iio->pci_bus_no = dev->bus->number;
4735         }
4736
4737         pci_dev_put(dev);
4738
4739         return ret;
4740 }
4741
4742 /*
4743  * SNR has a static mapping of stack IDs from SAD_CONTROL_CFG notation to PMON
4744  */
4745 enum {
4746         SNR_QAT_PMON_ID,
4747         SNR_CBDMA_DMI_PMON_ID,
4748         SNR_NIS_PMON_ID,
4749         SNR_DLB_PMON_ID,
4750         SNR_PCIE_GEN3_PMON_ID
4751 };
4752
4753 static u8 snr_sad_pmon_mapping[] = {
4754         SNR_CBDMA_DMI_PMON_ID,
4755         SNR_PCIE_GEN3_PMON_ID,
4756         SNR_DLB_PMON_ID,
4757         SNR_NIS_PMON_ID,
4758         SNR_QAT_PMON_ID
4759 };
4760
4761 static int snr_iio_get_topology(struct intel_uncore_type *type)
4762 {
4763         return sad_cfg_iio_topology(type, snr_sad_pmon_mapping);
4764 }
4765
4766 static void snr_iio_set_mapping(struct intel_uncore_type *type)
4767 {
4768         pmu_iio_set_mapping(type, &snr_iio_mapping_group);
4769 }
4770
4771 static void snr_iio_cleanup_mapping(struct intel_uncore_type *type)
4772 {
4773         pmu_cleanup_mapping(type, &snr_iio_mapping_group);
4774 }
4775
4776 static struct event_constraint snr_uncore_iio_constraints[] = {
4777         UNCORE_EVENT_CONSTRAINT(0x83, 0x3),
4778         UNCORE_EVENT_CONSTRAINT(0xc0, 0xc),
4779         UNCORE_EVENT_CONSTRAINT(0xd5, 0xc),
4780         EVENT_CONSTRAINT_END
4781 };
4782
4783 static struct intel_uncore_type snr_uncore_iio = {
4784         .name                   = "iio",
4785         .num_counters           = 4,
4786         .num_boxes              = 5,
4787         .perf_ctr_bits          = 48,
4788         .event_ctl              = SNR_IIO_MSR_PMON_CTL0,
4789         .perf_ctr               = SNR_IIO_MSR_PMON_CTR0,
4790         .event_mask             = SNBEP_PMON_RAW_EVENT_MASK,
4791         .event_mask_ext         = SNR_IIO_PMON_RAW_EVENT_MASK_EXT,
4792         .box_ctl                = SNR_IIO_MSR_PMON_BOX_CTL,
4793         .msr_offset             = SNR_IIO_MSR_OFFSET,
4794         .constraints            = snr_uncore_iio_constraints,
4795         .ops                    = &ivbep_uncore_msr_ops,
4796         .format_group           = &snr_uncore_iio_format_group,
4797         .attr_update            = snr_iio_attr_update,
4798         .get_topology           = snr_iio_get_topology,
4799         .set_mapping            = snr_iio_set_mapping,
4800         .cleanup_mapping        = snr_iio_cleanup_mapping,
4801 };
4802
4803 static struct intel_uncore_type snr_uncore_irp = {
4804         .name                   = "irp",
4805         .num_counters           = 2,
4806         .num_boxes              = 5,
4807         .perf_ctr_bits          = 48,
4808         .event_ctl              = SNR_IRP0_MSR_PMON_CTL0,
4809         .perf_ctr               = SNR_IRP0_MSR_PMON_CTR0,
4810         .event_mask             = SNBEP_PMON_RAW_EVENT_MASK,
4811         .box_ctl                = SNR_IRP0_MSR_PMON_BOX_CTL,
4812         .msr_offset             = SNR_IRP_MSR_OFFSET,
4813         .ops                    = &ivbep_uncore_msr_ops,
4814         .format_group           = &ivbep_uncore_format_group,
4815 };
4816
4817 static struct intel_uncore_type snr_uncore_m2pcie = {
4818         .name           = "m2pcie",
4819         .num_counters   = 4,
4820         .num_boxes      = 5,
4821         .perf_ctr_bits  = 48,
4822         .event_ctl      = SNR_M2PCIE_MSR_PMON_CTL0,
4823         .perf_ctr       = SNR_M2PCIE_MSR_PMON_CTR0,
4824         .box_ctl        = SNR_M2PCIE_MSR_PMON_BOX_CTL,
4825         .msr_offset     = SNR_M2PCIE_MSR_OFFSET,
4826         .event_mask     = SNBEP_PMON_RAW_EVENT_MASK,
4827         .ops            = &ivbep_uncore_msr_ops,
4828         .format_group   = &ivbep_uncore_format_group,
4829 };
4830
4831 static int snr_pcu_hw_config(struct intel_uncore_box *box, struct perf_event *event)
4832 {
4833         struct hw_perf_event *hwc = &event->hw;
4834         struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
4835         int ev_sel = hwc->config & SNBEP_PMON_CTL_EV_SEL_MASK;
4836
4837         if (ev_sel >= 0xb && ev_sel <= 0xe) {
4838                 reg1->reg = SNR_PCU_MSR_PMON_BOX_FILTER;
4839                 reg1->idx = ev_sel - 0xb;
4840                 reg1->config = event->attr.config1 & (0xff << reg1->idx);
4841         }
4842         return 0;
4843 }
4844
4845 static struct intel_uncore_ops snr_uncore_pcu_ops = {
4846         IVBEP_UNCORE_MSR_OPS_COMMON_INIT(),
4847         .hw_config              = snr_pcu_hw_config,
4848         .get_constraint         = snbep_pcu_get_constraint,
4849         .put_constraint         = snbep_pcu_put_constraint,
4850 };
4851
4852 static struct intel_uncore_type snr_uncore_pcu = {
4853         .name                   = "pcu",
4854         .num_counters           = 4,
4855         .num_boxes              = 1,
4856         .perf_ctr_bits          = 48,
4857         .perf_ctr               = SNR_PCU_MSR_PMON_CTR0,
4858         .event_ctl              = SNR_PCU_MSR_PMON_CTL0,
4859         .event_mask             = SNBEP_PMON_RAW_EVENT_MASK,
4860         .box_ctl                = SNR_PCU_MSR_PMON_BOX_CTL,
4861         .num_shared_regs        = 1,
4862         .ops                    = &snr_uncore_pcu_ops,
4863         .format_group           = &skx_uncore_pcu_format_group,
4864 };
4865
4866 enum perf_uncore_snr_iio_freerunning_type_id {
4867         SNR_IIO_MSR_IOCLK,
4868         SNR_IIO_MSR_BW_IN,
4869
4870         SNR_IIO_FREERUNNING_TYPE_MAX,
4871 };
4872
4873 static struct freerunning_counters snr_iio_freerunning[] = {
4874         [SNR_IIO_MSR_IOCLK]     = { 0x1eac, 0x1, 0x10, 1, 48 },
4875         [SNR_IIO_MSR_BW_IN]     = { 0x1f00, 0x1, 0x10, 8, 48 },
4876 };
4877
4878 static struct uncore_event_desc snr_uncore_iio_freerunning_events[] = {
4879         /* Free-Running IIO CLOCKS Counter */
4880         INTEL_UNCORE_EVENT_DESC(ioclk,                  "event=0xff,umask=0x10"),
4881         /* Free-Running IIO BANDWIDTH IN Counters */
4882         INTEL_UNCORE_EVENT_DESC(bw_in_port0,            "event=0xff,umask=0x20"),
4883         INTEL_UNCORE_EVENT_DESC(bw_in_port0.scale,      "3.814697266e-6"),
4884         INTEL_UNCORE_EVENT_DESC(bw_in_port0.unit,       "MiB"),
4885         INTEL_UNCORE_EVENT_DESC(bw_in_port1,            "event=0xff,umask=0x21"),
4886         INTEL_UNCORE_EVENT_DESC(bw_in_port1.scale,      "3.814697266e-6"),
4887         INTEL_UNCORE_EVENT_DESC(bw_in_port1.unit,       "MiB"),
4888         INTEL_UNCORE_EVENT_DESC(bw_in_port2,            "event=0xff,umask=0x22"),
4889         INTEL_UNCORE_EVENT_DESC(bw_in_port2.scale,      "3.814697266e-6"),
4890         INTEL_UNCORE_EVENT_DESC(bw_in_port2.unit,       "MiB"),
4891         INTEL_UNCORE_EVENT_DESC(bw_in_port3,            "event=0xff,umask=0x23"),
4892         INTEL_UNCORE_EVENT_DESC(bw_in_port3.scale,      "3.814697266e-6"),
4893         INTEL_UNCORE_EVENT_DESC(bw_in_port3.unit,       "MiB"),
4894         INTEL_UNCORE_EVENT_DESC(bw_in_port4,            "event=0xff,umask=0x24"),
4895         INTEL_UNCORE_EVENT_DESC(bw_in_port4.scale,      "3.814697266e-6"),
4896         INTEL_UNCORE_EVENT_DESC(bw_in_port4.unit,       "MiB"),
4897         INTEL_UNCORE_EVENT_DESC(bw_in_port5,            "event=0xff,umask=0x25"),
4898         INTEL_UNCORE_EVENT_DESC(bw_in_port5.scale,      "3.814697266e-6"),
4899         INTEL_UNCORE_EVENT_DESC(bw_in_port5.unit,       "MiB"),
4900         INTEL_UNCORE_EVENT_DESC(bw_in_port6,            "event=0xff,umask=0x26"),
4901         INTEL_UNCORE_EVENT_DESC(bw_in_port6.scale,      "3.814697266e-6"),
4902         INTEL_UNCORE_EVENT_DESC(bw_in_port6.unit,       "MiB"),
4903         INTEL_UNCORE_EVENT_DESC(bw_in_port7,            "event=0xff,umask=0x27"),
4904         INTEL_UNCORE_EVENT_DESC(bw_in_port7.scale,      "3.814697266e-6"),
4905         INTEL_UNCORE_EVENT_DESC(bw_in_port7.unit,       "MiB"),
4906         { /* end: all zeroes */ },
4907 };
4908
4909 static struct intel_uncore_type snr_uncore_iio_free_running = {
4910         .name                   = "iio_free_running",
4911         .num_counters           = 9,
4912         .num_boxes              = 5,
4913         .num_freerunning_types  = SNR_IIO_FREERUNNING_TYPE_MAX,
4914         .freerunning            = snr_iio_freerunning,
4915         .ops                    = &skx_uncore_iio_freerunning_ops,
4916         .event_descs            = snr_uncore_iio_freerunning_events,
4917         .format_group           = &skx_uncore_iio_freerunning_format_group,
4918 };
4919
4920 static struct intel_uncore_type *snr_msr_uncores[] = {
4921         &snr_uncore_ubox,
4922         &snr_uncore_chabox,
4923         &snr_uncore_iio,
4924         &snr_uncore_irp,
4925         &snr_uncore_m2pcie,
4926         &snr_uncore_pcu,
4927         &snr_uncore_iio_free_running,
4928         NULL,
4929 };
4930
4931 void snr_uncore_cpu_init(void)
4932 {
4933         uncore_msr_uncores = snr_msr_uncores;
4934 }
4935
4936 static void snr_m2m_uncore_pci_init_box(struct intel_uncore_box *box)
4937 {
4938         struct pci_dev *pdev = box->pci_dev;
4939         int box_ctl = uncore_pci_box_ctl(box);
4940
4941         __set_bit(UNCORE_BOX_FLAG_CTL_OFFS8, &box->flags);
4942         pci_write_config_dword(pdev, box_ctl, IVBEP_PMON_BOX_CTL_INT);
4943 }
4944
4945 static struct intel_uncore_ops snr_m2m_uncore_pci_ops = {
4946         .init_box       = snr_m2m_uncore_pci_init_box,
4947         .disable_box    = snbep_uncore_pci_disable_box,
4948         .enable_box     = snbep_uncore_pci_enable_box,
4949         .disable_event  = snbep_uncore_pci_disable_event,
4950         .enable_event   = snbep_uncore_pci_enable_event,
4951         .read_counter   = snbep_uncore_pci_read_counter,
4952 };
4953
4954 static struct attribute *snr_m2m_uncore_formats_attr[] = {
4955         &format_attr_event.attr,
4956         &format_attr_umask_ext3.attr,
4957         &format_attr_edge.attr,
4958         &format_attr_inv.attr,
4959         &format_attr_thresh8.attr,
4960         NULL,
4961 };
4962
4963 static const struct attribute_group snr_m2m_uncore_format_group = {
4964         .name = "format",
4965         .attrs = snr_m2m_uncore_formats_attr,
4966 };
4967
4968 static struct intel_uncore_type snr_uncore_m2m = {
4969         .name           = "m2m",
4970         .num_counters   = 4,
4971         .num_boxes      = 1,
4972         .perf_ctr_bits  = 48,
4973         .perf_ctr       = SNR_M2M_PCI_PMON_CTR0,
4974         .event_ctl      = SNR_M2M_PCI_PMON_CTL0,
4975         .event_mask     = SNBEP_PMON_RAW_EVENT_MASK,
4976         .event_mask_ext = SNR_M2M_PCI_PMON_UMASK_EXT,
4977         .box_ctl        = SNR_M2M_PCI_PMON_BOX_CTL,
4978         .ops            = &snr_m2m_uncore_pci_ops,
4979         .format_group   = &snr_m2m_uncore_format_group,
4980 };
4981
4982 static void snr_uncore_pci_enable_event(struct intel_uncore_box *box, struct perf_event *event)
4983 {
4984         struct pci_dev *pdev = box->pci_dev;
4985         struct hw_perf_event *hwc = &event->hw;
4986
4987         pci_write_config_dword(pdev, hwc->config_base, (u32)(hwc->config | SNBEP_PMON_CTL_EN));
4988         pci_write_config_dword(pdev, hwc->config_base + 4, (u32)(hwc->config >> 32));
4989 }
4990
4991 static struct intel_uncore_ops snr_pcie3_uncore_pci_ops = {
4992         .init_box       = snr_m2m_uncore_pci_init_box,
4993         .disable_box    = snbep_uncore_pci_disable_box,
4994         .enable_box     = snbep_uncore_pci_enable_box,
4995         .disable_event  = snbep_uncore_pci_disable_event,
4996         .enable_event   = snr_uncore_pci_enable_event,
4997         .read_counter   = snbep_uncore_pci_read_counter,
4998 };
4999
5000 static struct intel_uncore_type snr_uncore_pcie3 = {
5001         .name           = "pcie3",
5002         .num_counters   = 4,
5003         .num_boxes      = 1,
5004         .perf_ctr_bits  = 48,
5005         .perf_ctr       = SNR_PCIE3_PCI_PMON_CTR0,
5006         .event_ctl      = SNR_PCIE3_PCI_PMON_CTL0,
5007         .event_mask     = SKX_IIO_PMON_RAW_EVENT_MASK,
5008         .event_mask_ext = SKX_IIO_PMON_RAW_EVENT_MASK_EXT,
5009         .box_ctl        = SNR_PCIE3_PCI_PMON_BOX_CTL,
5010         .ops            = &snr_pcie3_uncore_pci_ops,
5011         .format_group   = &skx_uncore_iio_format_group,
5012 };
5013
5014 enum {
5015         SNR_PCI_UNCORE_M2M,
5016         SNR_PCI_UNCORE_PCIE3,
5017 };
5018
5019 static struct intel_uncore_type *snr_pci_uncores[] = {
5020         [SNR_PCI_UNCORE_M2M]            = &snr_uncore_m2m,
5021         [SNR_PCI_UNCORE_PCIE3]          = &snr_uncore_pcie3,
5022         NULL,
5023 };
5024
5025 static const struct pci_device_id snr_uncore_pci_ids[] = {
5026         { /* M2M */
5027                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x344a),
5028                 .driver_data = UNCORE_PCI_DEV_FULL_DATA(12, 0, SNR_PCI_UNCORE_M2M, 0),
5029         },
5030         { /* end: all zeroes */ }
5031 };
5032
5033 static struct pci_driver snr_uncore_pci_driver = {
5034         .name           = "snr_uncore",
5035         .id_table       = snr_uncore_pci_ids,
5036 };
5037
5038 static const struct pci_device_id snr_uncore_pci_sub_ids[] = {
5039         { /* PCIe3 RP */
5040                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x334a),
5041                 .driver_data = UNCORE_PCI_DEV_FULL_DATA(4, 0, SNR_PCI_UNCORE_PCIE3, 0),
5042         },
5043         { /* end: all zeroes */ }
5044 };
5045
5046 static struct pci_driver snr_uncore_pci_sub_driver = {
5047         .name           = "snr_uncore_sub",
5048         .id_table       = snr_uncore_pci_sub_ids,
5049 };
5050
5051 int snr_uncore_pci_init(void)
5052 {
5053         /* SNR UBOX DID */
5054         int ret = snbep_pci2phy_map_init(0x3460, SKX_CPUNODEID,
5055                                          SKX_GIDNIDMAP, true);
5056
5057         if (ret)
5058                 return ret;
5059
5060         uncore_pci_uncores = snr_pci_uncores;
5061         uncore_pci_driver = &snr_uncore_pci_driver;
5062         uncore_pci_sub_driver = &snr_uncore_pci_sub_driver;
5063         return 0;
5064 }
5065
5066 #define SNR_MC_DEVICE_ID        0x3451
5067
5068 static struct pci_dev *snr_uncore_get_mc_dev(unsigned int device, int id)
5069 {
5070         struct pci_dev *mc_dev = NULL;
5071         int pkg;
5072
5073         while (1) {
5074                 mc_dev = pci_get_device(PCI_VENDOR_ID_INTEL, device, mc_dev);
5075                 if (!mc_dev)
5076                         break;
5077                 pkg = uncore_pcibus_to_dieid(mc_dev->bus);
5078                 if (pkg == id)
5079                         break;
5080         }
5081         return mc_dev;
5082 }
5083
5084 static int snr_uncore_mmio_map(struct intel_uncore_box *box,
5085                                unsigned int box_ctl, int mem_offset,
5086                                unsigned int device)
5087 {
5088         struct pci_dev *pdev = snr_uncore_get_mc_dev(device, box->dieid);
5089         struct intel_uncore_type *type = box->pmu->type;
5090         resource_size_t addr;
5091         u32 pci_dword;
5092
5093         if (!pdev)
5094                 return -ENODEV;
5095
5096         pci_read_config_dword(pdev, SNR_IMC_MMIO_BASE_OFFSET, &pci_dword);
5097         addr = ((resource_size_t)pci_dword & SNR_IMC_MMIO_BASE_MASK) << 23;
5098
5099         pci_read_config_dword(pdev, mem_offset, &pci_dword);
5100         addr |= (pci_dword & SNR_IMC_MMIO_MEM0_MASK) << 12;
5101
5102         addr += box_ctl;
5103
5104         pci_dev_put(pdev);
5105
5106         box->io_addr = ioremap(addr, type->mmio_map_size);
5107         if (!box->io_addr) {
5108                 pr_warn("perf uncore: Failed to ioremap for %s.\n", type->name);
5109                 return -EINVAL;
5110         }
5111
5112         return 0;
5113 }
5114
5115 static void __snr_uncore_mmio_init_box(struct intel_uncore_box *box,
5116                                        unsigned int box_ctl, int mem_offset,
5117                                        unsigned int device)
5118 {
5119         if (!snr_uncore_mmio_map(box, box_ctl, mem_offset, device))
5120                 writel(IVBEP_PMON_BOX_CTL_INT, box->io_addr);
5121 }
5122
5123 static void snr_uncore_mmio_init_box(struct intel_uncore_box *box)
5124 {
5125         __snr_uncore_mmio_init_box(box, uncore_mmio_box_ctl(box),
5126                                    SNR_IMC_MMIO_MEM0_OFFSET,
5127                                    SNR_MC_DEVICE_ID);
5128 }
5129
5130 static void snr_uncore_mmio_disable_box(struct intel_uncore_box *box)
5131 {
5132         u32 config;
5133
5134         if (!box->io_addr)
5135                 return;
5136
5137         config = readl(box->io_addr);
5138         config |= SNBEP_PMON_BOX_CTL_FRZ;
5139         writel(config, box->io_addr);
5140 }
5141
5142 static void snr_uncore_mmio_enable_box(struct intel_uncore_box *box)
5143 {
5144         u32 config;
5145
5146         if (!box->io_addr)
5147                 return;
5148
5149         config = readl(box->io_addr);
5150         config &= ~SNBEP_PMON_BOX_CTL_FRZ;
5151         writel(config, box->io_addr);
5152 }
5153
5154 static void snr_uncore_mmio_enable_event(struct intel_uncore_box *box,
5155                                            struct perf_event *event)
5156 {
5157         struct hw_perf_event *hwc = &event->hw;
5158
5159         if (!box->io_addr)
5160                 return;
5161
5162         if (!uncore_mmio_is_valid_offset(box, hwc->config_base))
5163                 return;
5164
5165         writel(hwc->config | SNBEP_PMON_CTL_EN,
5166                box->io_addr + hwc->config_base);
5167 }
5168
5169 static void snr_uncore_mmio_disable_event(struct intel_uncore_box *box,
5170                                             struct perf_event *event)
5171 {
5172         struct hw_perf_event *hwc = &event->hw;
5173
5174         if (!box->io_addr)
5175                 return;
5176
5177         if (!uncore_mmio_is_valid_offset(box, hwc->config_base))
5178                 return;
5179
5180         writel(hwc->config, box->io_addr + hwc->config_base);
5181 }
5182
5183 static struct intel_uncore_ops snr_uncore_mmio_ops = {
5184         .init_box       = snr_uncore_mmio_init_box,
5185         .exit_box       = uncore_mmio_exit_box,
5186         .disable_box    = snr_uncore_mmio_disable_box,
5187         .enable_box     = snr_uncore_mmio_enable_box,
5188         .disable_event  = snr_uncore_mmio_disable_event,
5189         .enable_event   = snr_uncore_mmio_enable_event,
5190         .read_counter   = uncore_mmio_read_counter,
5191 };
5192
5193 static struct uncore_event_desc snr_uncore_imc_events[] = {
5194         INTEL_UNCORE_EVENT_DESC(clockticks,      "event=0x00,umask=0x00"),
5195         INTEL_UNCORE_EVENT_DESC(cas_count_read,  "event=0x04,umask=0x0f"),
5196         INTEL_UNCORE_EVENT_DESC(cas_count_read.scale, "6.103515625e-5"),
5197         INTEL_UNCORE_EVENT_DESC(cas_count_read.unit, "MiB"),
5198         INTEL_UNCORE_EVENT_DESC(cas_count_write, "event=0x04,umask=0x30"),
5199         INTEL_UNCORE_EVENT_DESC(cas_count_write.scale, "6.103515625e-5"),
5200         INTEL_UNCORE_EVENT_DESC(cas_count_write.unit, "MiB"),
5201         { /* end: all zeroes */ },
5202 };
5203
5204 static struct intel_uncore_type snr_uncore_imc = {
5205         .name           = "imc",
5206         .num_counters   = 4,
5207         .num_boxes      = 2,
5208         .perf_ctr_bits  = 48,
5209         .fixed_ctr_bits = 48,
5210         .fixed_ctr      = SNR_IMC_MMIO_PMON_FIXED_CTR,
5211         .fixed_ctl      = SNR_IMC_MMIO_PMON_FIXED_CTL,
5212         .event_descs    = snr_uncore_imc_events,
5213         .perf_ctr       = SNR_IMC_MMIO_PMON_CTR0,
5214         .event_ctl      = SNR_IMC_MMIO_PMON_CTL0,
5215         .event_mask     = SNBEP_PMON_RAW_EVENT_MASK,
5216         .box_ctl        = SNR_IMC_MMIO_PMON_BOX_CTL,
5217         .mmio_offset    = SNR_IMC_MMIO_OFFSET,
5218         .mmio_map_size  = SNR_IMC_MMIO_SIZE,
5219         .ops            = &snr_uncore_mmio_ops,
5220         .format_group   = &skx_uncore_format_group,
5221 };
5222
5223 enum perf_uncore_snr_imc_freerunning_type_id {
5224         SNR_IMC_DCLK,
5225         SNR_IMC_DDR,
5226
5227         SNR_IMC_FREERUNNING_TYPE_MAX,
5228 };
5229
5230 static struct freerunning_counters snr_imc_freerunning[] = {
5231         [SNR_IMC_DCLK]  = { 0x22b0, 0x0, 0, 1, 48 },
5232         [SNR_IMC_DDR]   = { 0x2290, 0x8, 0, 2, 48 },
5233 };
5234
5235 static struct uncore_event_desc snr_uncore_imc_freerunning_events[] = {
5236         INTEL_UNCORE_EVENT_DESC(dclk,           "event=0xff,umask=0x10"),
5237
5238         INTEL_UNCORE_EVENT_DESC(read,           "event=0xff,umask=0x20"),
5239         INTEL_UNCORE_EVENT_DESC(read.scale,     "6.103515625e-5"),
5240         INTEL_UNCORE_EVENT_DESC(read.unit,      "MiB"),
5241         INTEL_UNCORE_EVENT_DESC(write,          "event=0xff,umask=0x21"),
5242         INTEL_UNCORE_EVENT_DESC(write.scale,    "6.103515625e-5"),
5243         INTEL_UNCORE_EVENT_DESC(write.unit,     "MiB"),
5244         { /* end: all zeroes */ },
5245 };
5246
5247 static struct intel_uncore_ops snr_uncore_imc_freerunning_ops = {
5248         .init_box       = snr_uncore_mmio_init_box,
5249         .exit_box       = uncore_mmio_exit_box,
5250         .read_counter   = uncore_mmio_read_counter,
5251         .hw_config      = uncore_freerunning_hw_config,
5252 };
5253
5254 static struct intel_uncore_type snr_uncore_imc_free_running = {
5255         .name                   = "imc_free_running",
5256         .num_counters           = 3,
5257         .num_boxes              = 1,
5258         .num_freerunning_types  = SNR_IMC_FREERUNNING_TYPE_MAX,
5259         .mmio_map_size          = SNR_IMC_MMIO_SIZE,
5260         .freerunning            = snr_imc_freerunning,
5261         .ops                    = &snr_uncore_imc_freerunning_ops,
5262         .event_descs            = snr_uncore_imc_freerunning_events,
5263         .format_group           = &skx_uncore_iio_freerunning_format_group,
5264 };
5265
5266 static struct intel_uncore_type *snr_mmio_uncores[] = {
5267         &snr_uncore_imc,
5268         &snr_uncore_imc_free_running,
5269         NULL,
5270 };
5271
5272 void snr_uncore_mmio_init(void)
5273 {
5274         uncore_mmio_uncores = snr_mmio_uncores;
5275 }
5276
5277 /* end of SNR uncore support */
5278
5279 /* ICX uncore support */
5280
5281 static unsigned icx_cha_msr_offsets[] = {
5282         0x2a0, 0x2ae, 0x2bc, 0x2ca, 0x2d8, 0x2e6, 0x2f4, 0x302, 0x310,
5283         0x31e, 0x32c, 0x33a, 0x348, 0x356, 0x364, 0x372, 0x380, 0x38e,
5284         0x3aa, 0x3b8, 0x3c6, 0x3d4, 0x3e2, 0x3f0, 0x3fe, 0x40c, 0x41a,
5285         0x428, 0x436, 0x444, 0x452, 0x460, 0x46e, 0x47c, 0x0,   0xe,
5286         0x1c,  0x2a,  0x38,  0x46,
5287 };
5288
5289 static int icx_cha_hw_config(struct intel_uncore_box *box, struct perf_event *event)
5290 {
5291         struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
5292         bool tie_en = !!(event->hw.config & SNBEP_CBO_PMON_CTL_TID_EN);
5293
5294         if (tie_en) {
5295                 reg1->reg = ICX_C34_MSR_PMON_BOX_FILTER0 +
5296                             icx_cha_msr_offsets[box->pmu->pmu_idx];
5297                 reg1->config = event->attr.config1 & SKX_CHA_MSR_PMON_BOX_FILTER_TID;
5298                 reg1->idx = 0;
5299         }
5300
5301         return 0;
5302 }
5303
5304 static struct intel_uncore_ops icx_uncore_chabox_ops = {
5305         .init_box               = ivbep_uncore_msr_init_box,
5306         .disable_box            = snbep_uncore_msr_disable_box,
5307         .enable_box             = snbep_uncore_msr_enable_box,
5308         .disable_event          = snbep_uncore_msr_disable_event,
5309         .enable_event           = snr_cha_enable_event,
5310         .read_counter           = uncore_msr_read_counter,
5311         .hw_config              = icx_cha_hw_config,
5312 };
5313
5314 static struct intel_uncore_type icx_uncore_chabox = {
5315         .name                   = "cha",
5316         .num_counters           = 4,
5317         .perf_ctr_bits          = 48,
5318         .event_ctl              = ICX_C34_MSR_PMON_CTL0,
5319         .perf_ctr               = ICX_C34_MSR_PMON_CTR0,
5320         .box_ctl                = ICX_C34_MSR_PMON_BOX_CTL,
5321         .msr_offsets            = icx_cha_msr_offsets,
5322         .event_mask             = HSWEP_S_MSR_PMON_RAW_EVENT_MASK,
5323         .event_mask_ext         = SNR_CHA_RAW_EVENT_MASK_EXT,
5324         .constraints            = skx_uncore_chabox_constraints,
5325         .ops                    = &icx_uncore_chabox_ops,
5326         .format_group           = &snr_uncore_chabox_format_group,
5327 };
5328
5329 static unsigned icx_msr_offsets[] = {
5330         0x0, 0x20, 0x40, 0x90, 0xb0, 0xd0,
5331 };
5332
5333 static struct event_constraint icx_uncore_iio_constraints[] = {
5334         UNCORE_EVENT_CONSTRAINT(0x02, 0x3),
5335         UNCORE_EVENT_CONSTRAINT(0x03, 0x3),
5336         UNCORE_EVENT_CONSTRAINT(0x83, 0x3),
5337         UNCORE_EVENT_CONSTRAINT(0x88, 0xc),
5338         UNCORE_EVENT_CONSTRAINT(0xc0, 0xc),
5339         UNCORE_EVENT_CONSTRAINT(0xc5, 0xc),
5340         UNCORE_EVENT_CONSTRAINT(0xd5, 0xc),
5341         EVENT_CONSTRAINT_END
5342 };
5343
5344 static umode_t
5345 icx_iio_mapping_visible(struct kobject *kobj, struct attribute *attr, int die)
5346 {
5347         /* Root bus 0x00 is valid only for pmu_idx = 5. */
5348         return pmu_iio_mapping_visible(kobj, attr, die, 5);
5349 }
5350
5351 static struct attribute_group icx_iio_mapping_group = {
5352         .is_visible     = icx_iio_mapping_visible,
5353 };
5354
5355 static const struct attribute_group *icx_iio_attr_update[] = {
5356         &icx_iio_mapping_group,
5357         NULL,
5358 };
5359
5360 /*
5361  * ICX has a static mapping of stack IDs from SAD_CONTROL_CFG notation to PMON
5362  */
5363 enum {
5364         ICX_PCIE1_PMON_ID,
5365         ICX_PCIE2_PMON_ID,
5366         ICX_PCIE3_PMON_ID,
5367         ICX_PCIE4_PMON_ID,
5368         ICX_PCIE5_PMON_ID,
5369         ICX_CBDMA_DMI_PMON_ID
5370 };
5371
5372 static u8 icx_sad_pmon_mapping[] = {
5373         ICX_CBDMA_DMI_PMON_ID,
5374         ICX_PCIE1_PMON_ID,
5375         ICX_PCIE2_PMON_ID,
5376         ICX_PCIE3_PMON_ID,
5377         ICX_PCIE4_PMON_ID,
5378         ICX_PCIE5_PMON_ID,
5379 };
5380
5381 static int icx_iio_get_topology(struct intel_uncore_type *type)
5382 {
5383         return sad_cfg_iio_topology(type, icx_sad_pmon_mapping);
5384 }
5385
5386 static void icx_iio_set_mapping(struct intel_uncore_type *type)
5387 {
5388         /* Detect ICX-D system. This case is not supported */
5389         if (boot_cpu_data.x86_model == INTEL_FAM6_ICELAKE_D) {
5390                 pmu_clear_mapping_attr(type->attr_update, &icx_iio_mapping_group);
5391                 return;
5392         }
5393         pmu_iio_set_mapping(type, &icx_iio_mapping_group);
5394 }
5395
5396 static void icx_iio_cleanup_mapping(struct intel_uncore_type *type)
5397 {
5398         pmu_cleanup_mapping(type, &icx_iio_mapping_group);
5399 }
5400
5401 static struct intel_uncore_type icx_uncore_iio = {
5402         .name                   = "iio",
5403         .num_counters           = 4,
5404         .num_boxes              = 6,
5405         .perf_ctr_bits          = 48,
5406         .event_ctl              = ICX_IIO_MSR_PMON_CTL0,
5407         .perf_ctr               = ICX_IIO_MSR_PMON_CTR0,
5408         .event_mask             = SNBEP_PMON_RAW_EVENT_MASK,
5409         .event_mask_ext         = SNR_IIO_PMON_RAW_EVENT_MASK_EXT,
5410         .box_ctl                = ICX_IIO_MSR_PMON_BOX_CTL,
5411         .msr_offsets            = icx_msr_offsets,
5412         .constraints            = icx_uncore_iio_constraints,
5413         .ops                    = &skx_uncore_iio_ops,
5414         .format_group           = &snr_uncore_iio_format_group,
5415         .attr_update            = icx_iio_attr_update,
5416         .get_topology           = icx_iio_get_topology,
5417         .set_mapping            = icx_iio_set_mapping,
5418         .cleanup_mapping        = icx_iio_cleanup_mapping,
5419 };
5420
5421 static struct intel_uncore_type icx_uncore_irp = {
5422         .name                   = "irp",
5423         .num_counters           = 2,
5424         .num_boxes              = 6,
5425         .perf_ctr_bits          = 48,
5426         .event_ctl              = ICX_IRP0_MSR_PMON_CTL0,
5427         .perf_ctr               = ICX_IRP0_MSR_PMON_CTR0,
5428         .event_mask             = SNBEP_PMON_RAW_EVENT_MASK,
5429         .box_ctl                = ICX_IRP0_MSR_PMON_BOX_CTL,
5430         .msr_offsets            = icx_msr_offsets,
5431         .ops                    = &ivbep_uncore_msr_ops,
5432         .format_group           = &ivbep_uncore_format_group,
5433 };
5434
5435 static struct event_constraint icx_uncore_m2pcie_constraints[] = {
5436         UNCORE_EVENT_CONSTRAINT(0x14, 0x3),
5437         UNCORE_EVENT_CONSTRAINT(0x23, 0x3),
5438         UNCORE_EVENT_CONSTRAINT(0x2d, 0x3),
5439         EVENT_CONSTRAINT_END
5440 };
5441
5442 static struct intel_uncore_type icx_uncore_m2pcie = {
5443         .name           = "m2pcie",
5444         .num_counters   = 4,
5445         .num_boxes      = 6,
5446         .perf_ctr_bits  = 48,
5447         .event_ctl      = ICX_M2PCIE_MSR_PMON_CTL0,
5448         .perf_ctr       = ICX_M2PCIE_MSR_PMON_CTR0,
5449         .box_ctl        = ICX_M2PCIE_MSR_PMON_BOX_CTL,
5450         .msr_offsets    = icx_msr_offsets,
5451         .constraints    = icx_uncore_m2pcie_constraints,
5452         .event_mask     = SNBEP_PMON_RAW_EVENT_MASK,
5453         .ops            = &ivbep_uncore_msr_ops,
5454         .format_group   = &ivbep_uncore_format_group,
5455 };
5456
5457 enum perf_uncore_icx_iio_freerunning_type_id {
5458         ICX_IIO_MSR_IOCLK,
5459         ICX_IIO_MSR_BW_IN,
5460
5461         ICX_IIO_FREERUNNING_TYPE_MAX,
5462 };
5463
5464 static unsigned icx_iio_clk_freerunning_box_offsets[] = {
5465         0x0, 0x20, 0x40, 0x90, 0xb0, 0xd0,
5466 };
5467
5468 static unsigned icx_iio_bw_freerunning_box_offsets[] = {
5469         0x0, 0x10, 0x20, 0x90, 0xa0, 0xb0,
5470 };
5471
5472 static struct freerunning_counters icx_iio_freerunning[] = {
5473         [ICX_IIO_MSR_IOCLK]     = { 0xa55, 0x1, 0x20, 1, 48, icx_iio_clk_freerunning_box_offsets },
5474         [ICX_IIO_MSR_BW_IN]     = { 0xaa0, 0x1, 0x10, 8, 48, icx_iio_bw_freerunning_box_offsets },
5475 };
5476
5477 static struct uncore_event_desc icx_uncore_iio_freerunning_events[] = {
5478         /* Free-Running IIO CLOCKS Counter */
5479         INTEL_UNCORE_EVENT_DESC(ioclk,                  "event=0xff,umask=0x10"),
5480         /* Free-Running IIO BANDWIDTH IN Counters */
5481         INTEL_UNCORE_EVENT_DESC(bw_in_port0,            "event=0xff,umask=0x20"),
5482         INTEL_UNCORE_EVENT_DESC(bw_in_port0.scale,      "3.814697266e-6"),
5483         INTEL_UNCORE_EVENT_DESC(bw_in_port0.unit,       "MiB"),
5484         INTEL_UNCORE_EVENT_DESC(bw_in_port1,            "event=0xff,umask=0x21"),
5485         INTEL_UNCORE_EVENT_DESC(bw_in_port1.scale,      "3.814697266e-6"),
5486         INTEL_UNCORE_EVENT_DESC(bw_in_port1.unit,       "MiB"),
5487         INTEL_UNCORE_EVENT_DESC(bw_in_port2,            "event=0xff,umask=0x22"),
5488         INTEL_UNCORE_EVENT_DESC(bw_in_port2.scale,      "3.814697266e-6"),
5489         INTEL_UNCORE_EVENT_DESC(bw_in_port2.unit,       "MiB"),
5490         INTEL_UNCORE_EVENT_DESC(bw_in_port3,            "event=0xff,umask=0x23"),
5491         INTEL_UNCORE_EVENT_DESC(bw_in_port3.scale,      "3.814697266e-6"),
5492         INTEL_UNCORE_EVENT_DESC(bw_in_port3.unit,       "MiB"),
5493         INTEL_UNCORE_EVENT_DESC(bw_in_port4,            "event=0xff,umask=0x24"),
5494         INTEL_UNCORE_EVENT_DESC(bw_in_port4.scale,      "3.814697266e-6"),
5495         INTEL_UNCORE_EVENT_DESC(bw_in_port4.unit,       "MiB"),
5496         INTEL_UNCORE_EVENT_DESC(bw_in_port5,            "event=0xff,umask=0x25"),
5497         INTEL_UNCORE_EVENT_DESC(bw_in_port5.scale,      "3.814697266e-6"),
5498         INTEL_UNCORE_EVENT_DESC(bw_in_port5.unit,       "MiB"),
5499         INTEL_UNCORE_EVENT_DESC(bw_in_port6,            "event=0xff,umask=0x26"),
5500         INTEL_UNCORE_EVENT_DESC(bw_in_port6.scale,      "3.814697266e-6"),
5501         INTEL_UNCORE_EVENT_DESC(bw_in_port6.unit,       "MiB"),
5502         INTEL_UNCORE_EVENT_DESC(bw_in_port7,            "event=0xff,umask=0x27"),
5503         INTEL_UNCORE_EVENT_DESC(bw_in_port7.scale,      "3.814697266e-6"),
5504         INTEL_UNCORE_EVENT_DESC(bw_in_port7.unit,       "MiB"),
5505         { /* end: all zeroes */ },
5506 };
5507
5508 static struct intel_uncore_type icx_uncore_iio_free_running = {
5509         .name                   = "iio_free_running",
5510         .num_counters           = 9,
5511         .num_boxes              = 6,
5512         .num_freerunning_types  = ICX_IIO_FREERUNNING_TYPE_MAX,
5513         .freerunning            = icx_iio_freerunning,
5514         .ops                    = &skx_uncore_iio_freerunning_ops,
5515         .event_descs            = icx_uncore_iio_freerunning_events,
5516         .format_group           = &skx_uncore_iio_freerunning_format_group,
5517 };
5518
5519 static struct intel_uncore_type *icx_msr_uncores[] = {
5520         &skx_uncore_ubox,
5521         &icx_uncore_chabox,
5522         &icx_uncore_iio,
5523         &icx_uncore_irp,
5524         &icx_uncore_m2pcie,
5525         &skx_uncore_pcu,
5526         &icx_uncore_iio_free_running,
5527         NULL,
5528 };
5529
5530 /*
5531  * To determine the number of CHAs, it should read CAPID6(Low) and CAPID7 (High)
5532  * registers which located at Device 30, Function 3
5533  */
5534 #define ICX_CAPID6              0x9c
5535 #define ICX_CAPID7              0xa0
5536
5537 static u64 icx_count_chabox(void)
5538 {
5539         struct pci_dev *dev = NULL;
5540         u64 caps = 0;
5541
5542         dev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x345b, dev);
5543         if (!dev)
5544                 goto out;
5545
5546         pci_read_config_dword(dev, ICX_CAPID6, (u32 *)&caps);
5547         pci_read_config_dword(dev, ICX_CAPID7, (u32 *)&caps + 1);
5548 out:
5549         pci_dev_put(dev);
5550         return hweight64(caps);
5551 }
5552
5553 void icx_uncore_cpu_init(void)
5554 {
5555         u64 num_boxes = icx_count_chabox();
5556
5557         if (WARN_ON(num_boxes > ARRAY_SIZE(icx_cha_msr_offsets)))
5558                 return;
5559         icx_uncore_chabox.num_boxes = num_boxes;
5560         uncore_msr_uncores = icx_msr_uncores;
5561 }
5562
5563 static struct intel_uncore_type icx_uncore_m2m = {
5564         .name           = "m2m",
5565         .num_counters   = 4,
5566         .num_boxes      = 4,
5567         .perf_ctr_bits  = 48,
5568         .perf_ctr       = SNR_M2M_PCI_PMON_CTR0,
5569         .event_ctl      = SNR_M2M_PCI_PMON_CTL0,
5570         .event_mask     = SNBEP_PMON_RAW_EVENT_MASK,
5571         .event_mask_ext = SNR_M2M_PCI_PMON_UMASK_EXT,
5572         .box_ctl        = SNR_M2M_PCI_PMON_BOX_CTL,
5573         .ops            = &snr_m2m_uncore_pci_ops,
5574         .format_group   = &snr_m2m_uncore_format_group,
5575 };
5576
5577 static struct attribute *icx_upi_uncore_formats_attr[] = {
5578         &format_attr_event.attr,
5579         &format_attr_umask_ext4.attr,
5580         &format_attr_edge.attr,
5581         &format_attr_inv.attr,
5582         &format_attr_thresh8.attr,
5583         NULL,
5584 };
5585
5586 static const struct attribute_group icx_upi_uncore_format_group = {
5587         .name = "format",
5588         .attrs = icx_upi_uncore_formats_attr,
5589 };
5590
5591 #define ICX_UPI_REGS_ADDR_DEVICE_LINK0  0x02
5592 #define ICX_UPI_REGS_ADDR_FUNCTION      0x01
5593
5594 static int discover_upi_topology(struct intel_uncore_type *type, int ubox_did, int dev_link0)
5595 {
5596         struct pci_dev *ubox = NULL;
5597         struct pci_dev *dev = NULL;
5598         u32 nid, gid;
5599         int i, idx, ret = -EPERM;
5600         struct intel_uncore_topology *upi;
5601         unsigned int devfn;
5602
5603         /* GIDNIDMAP method supports machines which have less than 8 sockets. */
5604         if (uncore_max_dies() > 8)
5605                 goto err;
5606
5607         while ((ubox = pci_get_device(PCI_VENDOR_ID_INTEL, ubox_did, ubox))) {
5608                 ret = upi_nodeid_groupid(ubox, SKX_CPUNODEID, SKX_GIDNIDMAP, &nid, &gid);
5609                 if (ret) {
5610                         ret = pcibios_err_to_errno(ret);
5611                         break;
5612                 }
5613
5614                 for (i = 0; i < 8; i++) {
5615                         if (nid != GIDNIDMAP(gid, i))
5616                                 continue;
5617                         for (idx = 0; idx < type->num_boxes; idx++) {
5618                                 upi = &type->topology[nid][idx];
5619                                 devfn = PCI_DEVFN(dev_link0 + idx, ICX_UPI_REGS_ADDR_FUNCTION);
5620                                 dev = pci_get_domain_bus_and_slot(pci_domain_nr(ubox->bus),
5621                                                                   ubox->bus->number,
5622                                                                   devfn);
5623                                 if (dev) {
5624                                         ret = upi_fill_topology(dev, upi, idx);
5625                                         if (ret)
5626                                                 goto err;
5627                                 }
5628                         }
5629                 }
5630         }
5631 err:
5632         pci_dev_put(ubox);
5633         pci_dev_put(dev);
5634         return ret;
5635 }
5636
5637 static int icx_upi_get_topology(struct intel_uncore_type *type)
5638 {
5639         return discover_upi_topology(type, ICX_UBOX_DID, ICX_UPI_REGS_ADDR_DEVICE_LINK0);
5640 }
5641
5642 static struct attribute_group icx_upi_mapping_group = {
5643         .is_visible     = skx_upi_mapping_visible,
5644 };
5645
5646 static const struct attribute_group *icx_upi_attr_update[] = {
5647         &icx_upi_mapping_group,
5648         NULL
5649 };
5650
5651 static void icx_upi_set_mapping(struct intel_uncore_type *type)
5652 {
5653         pmu_upi_set_mapping(type, &icx_upi_mapping_group);
5654 }
5655
5656 static void icx_upi_cleanup_mapping(struct intel_uncore_type *type)
5657 {
5658         pmu_cleanup_mapping(type, &icx_upi_mapping_group);
5659 }
5660
5661 static struct intel_uncore_type icx_uncore_upi = {
5662         .name           = "upi",
5663         .num_counters   = 4,
5664         .num_boxes      = 3,
5665         .perf_ctr_bits  = 48,
5666         .perf_ctr       = ICX_UPI_PCI_PMON_CTR0,
5667         .event_ctl      = ICX_UPI_PCI_PMON_CTL0,
5668         .event_mask     = SNBEP_PMON_RAW_EVENT_MASK,
5669         .event_mask_ext = ICX_UPI_CTL_UMASK_EXT,
5670         .box_ctl        = ICX_UPI_PCI_PMON_BOX_CTL,
5671         .ops            = &skx_upi_uncore_pci_ops,
5672         .format_group   = &icx_upi_uncore_format_group,
5673         .attr_update    = icx_upi_attr_update,
5674         .get_topology   = icx_upi_get_topology,
5675         .set_mapping    = icx_upi_set_mapping,
5676         .cleanup_mapping = icx_upi_cleanup_mapping,
5677 };
5678
5679 static struct event_constraint icx_uncore_m3upi_constraints[] = {
5680         UNCORE_EVENT_CONSTRAINT(0x1c, 0x1),
5681         UNCORE_EVENT_CONSTRAINT(0x1d, 0x1),
5682         UNCORE_EVENT_CONSTRAINT(0x1e, 0x1),
5683         UNCORE_EVENT_CONSTRAINT(0x1f, 0x1),
5684         UNCORE_EVENT_CONSTRAINT(0x40, 0x7),
5685         UNCORE_EVENT_CONSTRAINT(0x4e, 0x7),
5686         UNCORE_EVENT_CONSTRAINT(0x4f, 0x7),
5687         UNCORE_EVENT_CONSTRAINT(0x50, 0x7),
5688         EVENT_CONSTRAINT_END
5689 };
5690
5691 static struct intel_uncore_type icx_uncore_m3upi = {
5692         .name           = "m3upi",
5693         .num_counters   = 4,
5694         .num_boxes      = 3,
5695         .perf_ctr_bits  = 48,
5696         .perf_ctr       = ICX_M3UPI_PCI_PMON_CTR0,
5697         .event_ctl      = ICX_M3UPI_PCI_PMON_CTL0,
5698         .event_mask     = SNBEP_PMON_RAW_EVENT_MASK,
5699         .box_ctl        = ICX_M3UPI_PCI_PMON_BOX_CTL,
5700         .constraints    = icx_uncore_m3upi_constraints,
5701         .ops            = &ivbep_uncore_pci_ops,
5702         .format_group   = &skx_uncore_format_group,
5703 };
5704
5705 enum {
5706         ICX_PCI_UNCORE_M2M,
5707         ICX_PCI_UNCORE_UPI,
5708         ICX_PCI_UNCORE_M3UPI,
5709 };
5710
5711 static struct intel_uncore_type *icx_pci_uncores[] = {
5712         [ICX_PCI_UNCORE_M2M]            = &icx_uncore_m2m,
5713         [ICX_PCI_UNCORE_UPI]            = &icx_uncore_upi,
5714         [ICX_PCI_UNCORE_M3UPI]          = &icx_uncore_m3upi,
5715         NULL,
5716 };
5717
5718 static const struct pci_device_id icx_uncore_pci_ids[] = {
5719         { /* M2M 0 */
5720                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x344a),
5721                 .driver_data = UNCORE_PCI_DEV_FULL_DATA(12, 0, ICX_PCI_UNCORE_M2M, 0),
5722         },
5723         { /* M2M 1 */
5724                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x344a),
5725                 .driver_data = UNCORE_PCI_DEV_FULL_DATA(13, 0, ICX_PCI_UNCORE_M2M, 1),
5726         },
5727         { /* M2M 2 */
5728                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x344a),
5729                 .driver_data = UNCORE_PCI_DEV_FULL_DATA(14, 0, ICX_PCI_UNCORE_M2M, 2),
5730         },
5731         { /* M2M 3 */
5732                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x344a),
5733                 .driver_data = UNCORE_PCI_DEV_FULL_DATA(15, 0, ICX_PCI_UNCORE_M2M, 3),
5734         },
5735         { /* UPI Link 0 */
5736                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x3441),
5737                 .driver_data = UNCORE_PCI_DEV_FULL_DATA(2, 1, ICX_PCI_UNCORE_UPI, 0),
5738         },
5739         { /* UPI Link 1 */
5740                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x3441),
5741                 .driver_data = UNCORE_PCI_DEV_FULL_DATA(3, 1, ICX_PCI_UNCORE_UPI, 1),
5742         },
5743         { /* UPI Link 2 */
5744                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x3441),
5745                 .driver_data = UNCORE_PCI_DEV_FULL_DATA(4, 1, ICX_PCI_UNCORE_UPI, 2),
5746         },
5747         { /* M3UPI Link 0 */
5748                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x3446),
5749                 .driver_data = UNCORE_PCI_DEV_FULL_DATA(5, 1, ICX_PCI_UNCORE_M3UPI, 0),
5750         },
5751         { /* M3UPI Link 1 */
5752                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x3446),
5753                 .driver_data = UNCORE_PCI_DEV_FULL_DATA(6, 1, ICX_PCI_UNCORE_M3UPI, 1),
5754         },
5755         { /* M3UPI Link 2 */
5756                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x3446),
5757                 .driver_data = UNCORE_PCI_DEV_FULL_DATA(7, 1, ICX_PCI_UNCORE_M3UPI, 2),
5758         },
5759         { /* end: all zeroes */ }
5760 };
5761
5762 static struct pci_driver icx_uncore_pci_driver = {
5763         .name           = "icx_uncore",
5764         .id_table       = icx_uncore_pci_ids,
5765 };
5766
5767 int icx_uncore_pci_init(void)
5768 {
5769         /* ICX UBOX DID */
5770         int ret = snbep_pci2phy_map_init(0x3450, SKX_CPUNODEID,
5771                                          SKX_GIDNIDMAP, true);
5772
5773         if (ret)
5774                 return ret;
5775
5776         uncore_pci_uncores = icx_pci_uncores;
5777         uncore_pci_driver = &icx_uncore_pci_driver;
5778         return 0;
5779 }
5780
5781 static void icx_uncore_imc_init_box(struct intel_uncore_box *box)
5782 {
5783         unsigned int box_ctl = box->pmu->type->box_ctl +
5784                                box->pmu->type->mmio_offset * (box->pmu->pmu_idx % ICX_NUMBER_IMC_CHN);
5785         int mem_offset = (box->pmu->pmu_idx / ICX_NUMBER_IMC_CHN) * ICX_IMC_MEM_STRIDE +
5786                          SNR_IMC_MMIO_MEM0_OFFSET;
5787
5788         __snr_uncore_mmio_init_box(box, box_ctl, mem_offset,
5789                                    SNR_MC_DEVICE_ID);
5790 }
5791
5792 static struct intel_uncore_ops icx_uncore_mmio_ops = {
5793         .init_box       = icx_uncore_imc_init_box,
5794         .exit_box       = uncore_mmio_exit_box,
5795         .disable_box    = snr_uncore_mmio_disable_box,
5796         .enable_box     = snr_uncore_mmio_enable_box,
5797         .disable_event  = snr_uncore_mmio_disable_event,
5798         .enable_event   = snr_uncore_mmio_enable_event,
5799         .read_counter   = uncore_mmio_read_counter,
5800 };
5801
5802 static struct intel_uncore_type icx_uncore_imc = {
5803         .name           = "imc",
5804         .num_counters   = 4,
5805         .num_boxes      = 12,
5806         .perf_ctr_bits  = 48,
5807         .fixed_ctr_bits = 48,
5808         .fixed_ctr      = SNR_IMC_MMIO_PMON_FIXED_CTR,
5809         .fixed_ctl      = SNR_IMC_MMIO_PMON_FIXED_CTL,
5810         .event_descs    = snr_uncore_imc_events,
5811         .perf_ctr       = SNR_IMC_MMIO_PMON_CTR0,
5812         .event_ctl      = SNR_IMC_MMIO_PMON_CTL0,
5813         .event_mask     = SNBEP_PMON_RAW_EVENT_MASK,
5814         .box_ctl        = SNR_IMC_MMIO_PMON_BOX_CTL,
5815         .mmio_offset    = SNR_IMC_MMIO_OFFSET,
5816         .mmio_map_size  = SNR_IMC_MMIO_SIZE,
5817         .ops            = &icx_uncore_mmio_ops,
5818         .format_group   = &skx_uncore_format_group,
5819 };
5820
5821 enum perf_uncore_icx_imc_freerunning_type_id {
5822         ICX_IMC_DCLK,
5823         ICX_IMC_DDR,
5824         ICX_IMC_DDRT,
5825
5826         ICX_IMC_FREERUNNING_TYPE_MAX,
5827 };
5828
5829 static struct freerunning_counters icx_imc_freerunning[] = {
5830         [ICX_IMC_DCLK]  = { 0x22b0, 0x0, 0, 1, 48 },
5831         [ICX_IMC_DDR]   = { 0x2290, 0x8, 0, 2, 48 },
5832         [ICX_IMC_DDRT]  = { 0x22a0, 0x8, 0, 2, 48 },
5833 };
5834
5835 static struct uncore_event_desc icx_uncore_imc_freerunning_events[] = {
5836         INTEL_UNCORE_EVENT_DESC(dclk,                   "event=0xff,umask=0x10"),
5837
5838         INTEL_UNCORE_EVENT_DESC(read,                   "event=0xff,umask=0x20"),
5839         INTEL_UNCORE_EVENT_DESC(read.scale,             "6.103515625e-5"),
5840         INTEL_UNCORE_EVENT_DESC(read.unit,              "MiB"),
5841         INTEL_UNCORE_EVENT_DESC(write,                  "event=0xff,umask=0x21"),
5842         INTEL_UNCORE_EVENT_DESC(write.scale,            "6.103515625e-5"),
5843         INTEL_UNCORE_EVENT_DESC(write.unit,             "MiB"),
5844
5845         INTEL_UNCORE_EVENT_DESC(ddrt_read,              "event=0xff,umask=0x30"),
5846         INTEL_UNCORE_EVENT_DESC(ddrt_read.scale,        "6.103515625e-5"),
5847         INTEL_UNCORE_EVENT_DESC(ddrt_read.unit,         "MiB"),
5848         INTEL_UNCORE_EVENT_DESC(ddrt_write,             "event=0xff,umask=0x31"),
5849         INTEL_UNCORE_EVENT_DESC(ddrt_write.scale,       "6.103515625e-5"),
5850         INTEL_UNCORE_EVENT_DESC(ddrt_write.unit,        "MiB"),
5851         { /* end: all zeroes */ },
5852 };
5853
5854 static void icx_uncore_imc_freerunning_init_box(struct intel_uncore_box *box)
5855 {
5856         int mem_offset = box->pmu->pmu_idx * ICX_IMC_MEM_STRIDE +
5857                          SNR_IMC_MMIO_MEM0_OFFSET;
5858
5859         snr_uncore_mmio_map(box, uncore_mmio_box_ctl(box),
5860                             mem_offset, SNR_MC_DEVICE_ID);
5861 }
5862
5863 static struct intel_uncore_ops icx_uncore_imc_freerunning_ops = {
5864         .init_box       = icx_uncore_imc_freerunning_init_box,
5865         .exit_box       = uncore_mmio_exit_box,
5866         .read_counter   = uncore_mmio_read_counter,
5867         .hw_config      = uncore_freerunning_hw_config,
5868 };
5869
5870 static struct intel_uncore_type icx_uncore_imc_free_running = {
5871         .name                   = "imc_free_running",
5872         .num_counters           = 5,
5873         .num_boxes              = 4,
5874         .num_freerunning_types  = ICX_IMC_FREERUNNING_TYPE_MAX,
5875         .mmio_map_size          = SNR_IMC_MMIO_SIZE,
5876         .freerunning            = icx_imc_freerunning,
5877         .ops                    = &icx_uncore_imc_freerunning_ops,
5878         .event_descs            = icx_uncore_imc_freerunning_events,
5879         .format_group           = &skx_uncore_iio_freerunning_format_group,
5880 };
5881
5882 static struct intel_uncore_type *icx_mmio_uncores[] = {
5883         &icx_uncore_imc,
5884         &icx_uncore_imc_free_running,
5885         NULL,
5886 };
5887
5888 void icx_uncore_mmio_init(void)
5889 {
5890         uncore_mmio_uncores = icx_mmio_uncores;
5891 }
5892
5893 /* end of ICX uncore support */
5894
5895 /* SPR uncore support */
5896
5897 static void spr_uncore_msr_enable_event(struct intel_uncore_box *box,
5898                                         struct perf_event *event)
5899 {
5900         struct hw_perf_event *hwc = &event->hw;
5901         struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
5902
5903         if (reg1->idx != EXTRA_REG_NONE)
5904                 wrmsrl(reg1->reg, reg1->config);
5905
5906         wrmsrl(hwc->config_base, hwc->config);
5907 }
5908
5909 static void spr_uncore_msr_disable_event(struct intel_uncore_box *box,
5910                                          struct perf_event *event)
5911 {
5912         struct hw_perf_event *hwc = &event->hw;
5913         struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
5914
5915         if (reg1->idx != EXTRA_REG_NONE)
5916                 wrmsrl(reg1->reg, 0);
5917
5918         wrmsrl(hwc->config_base, 0);
5919 }
5920
5921 static int spr_cha_hw_config(struct intel_uncore_box *box, struct perf_event *event)
5922 {
5923         struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
5924         bool tie_en = !!(event->hw.config & SPR_CHA_PMON_CTL_TID_EN);
5925         struct intel_uncore_type *type = box->pmu->type;
5926
5927         if (tie_en) {
5928                 reg1->reg = SPR_C0_MSR_PMON_BOX_FILTER0 +
5929                             HSWEP_CBO_MSR_OFFSET * type->box_ids[box->pmu->pmu_idx];
5930                 reg1->config = event->attr.config1 & SPR_CHA_PMON_BOX_FILTER_TID;
5931                 reg1->idx = 0;
5932         }
5933
5934         return 0;
5935 }
5936
5937 static struct intel_uncore_ops spr_uncore_chabox_ops = {
5938         .init_box               = intel_generic_uncore_msr_init_box,
5939         .disable_box            = intel_generic_uncore_msr_disable_box,
5940         .enable_box             = intel_generic_uncore_msr_enable_box,
5941         .disable_event          = spr_uncore_msr_disable_event,
5942         .enable_event           = spr_uncore_msr_enable_event,
5943         .read_counter           = uncore_msr_read_counter,
5944         .hw_config              = spr_cha_hw_config,
5945         .get_constraint         = uncore_get_constraint,
5946         .put_constraint         = uncore_put_constraint,
5947 };
5948
5949 static struct attribute *spr_uncore_cha_formats_attr[] = {
5950         &format_attr_event.attr,
5951         &format_attr_umask_ext4.attr,
5952         &format_attr_tid_en2.attr,
5953         &format_attr_edge.attr,
5954         &format_attr_inv.attr,
5955         &format_attr_thresh8.attr,
5956         &format_attr_filter_tid5.attr,
5957         NULL,
5958 };
5959 static const struct attribute_group spr_uncore_chabox_format_group = {
5960         .name = "format",
5961         .attrs = spr_uncore_cha_formats_attr,
5962 };
5963
5964 static ssize_t alias_show(struct device *dev,
5965                           struct device_attribute *attr,
5966                           char *buf)
5967 {
5968         struct intel_uncore_pmu *pmu = dev_to_uncore_pmu(dev);
5969         char pmu_name[UNCORE_PMU_NAME_LEN];
5970
5971         uncore_get_alias_name(pmu_name, pmu);
5972         return sysfs_emit(buf, "%s\n", pmu_name);
5973 }
5974
5975 static DEVICE_ATTR_RO(alias);
5976
5977 static struct attribute *uncore_alias_attrs[] = {
5978         &dev_attr_alias.attr,
5979         NULL
5980 };
5981
5982 ATTRIBUTE_GROUPS(uncore_alias);
5983
5984 static struct intel_uncore_type spr_uncore_chabox = {
5985         .name                   = "cha",
5986         .event_mask             = SPR_CHA_PMON_EVENT_MASK,
5987         .event_mask_ext         = SPR_RAW_EVENT_MASK_EXT,
5988         .num_shared_regs        = 1,
5989         .constraints            = skx_uncore_chabox_constraints,
5990         .ops                    = &spr_uncore_chabox_ops,
5991         .format_group           = &spr_uncore_chabox_format_group,
5992         .attr_update            = uncore_alias_groups,
5993 };
5994
5995 static struct intel_uncore_type spr_uncore_iio = {
5996         .name                   = "iio",
5997         .event_mask             = SNBEP_PMON_RAW_EVENT_MASK,
5998         .event_mask_ext         = SNR_IIO_PMON_RAW_EVENT_MASK_EXT,
5999         .format_group           = &snr_uncore_iio_format_group,
6000         .attr_update            = uncore_alias_groups,
6001         .constraints            = icx_uncore_iio_constraints,
6002 };
6003
6004 static struct attribute *spr_uncore_raw_formats_attr[] = {
6005         &format_attr_event.attr,
6006         &format_attr_umask_ext4.attr,
6007         &format_attr_edge.attr,
6008         &format_attr_inv.attr,
6009         &format_attr_thresh8.attr,
6010         NULL,
6011 };
6012
6013 static const struct attribute_group spr_uncore_raw_format_group = {
6014         .name                   = "format",
6015         .attrs                  = spr_uncore_raw_formats_attr,
6016 };
6017
6018 #define SPR_UNCORE_COMMON_FORMAT()                              \
6019         .event_mask             = SNBEP_PMON_RAW_EVENT_MASK,    \
6020         .event_mask_ext         = SPR_RAW_EVENT_MASK_EXT,       \
6021         .format_group           = &spr_uncore_raw_format_group, \
6022         .attr_update            = uncore_alias_groups
6023
6024 static struct intel_uncore_type spr_uncore_irp = {
6025         SPR_UNCORE_COMMON_FORMAT(),
6026         .name                   = "irp",
6027
6028 };
6029
6030 static struct event_constraint spr_uncore_m2pcie_constraints[] = {
6031         UNCORE_EVENT_CONSTRAINT(0x14, 0x3),
6032         UNCORE_EVENT_CONSTRAINT(0x2d, 0x3),
6033         EVENT_CONSTRAINT_END
6034 };
6035
6036 static struct intel_uncore_type spr_uncore_m2pcie = {
6037         SPR_UNCORE_COMMON_FORMAT(),
6038         .name                   = "m2pcie",
6039         .constraints            = spr_uncore_m2pcie_constraints,
6040 };
6041
6042 static struct intel_uncore_type spr_uncore_pcu = {
6043         .name                   = "pcu",
6044         .attr_update            = uncore_alias_groups,
6045 };
6046
6047 static void spr_uncore_mmio_enable_event(struct intel_uncore_box *box,
6048                                          struct perf_event *event)
6049 {
6050         struct hw_perf_event *hwc = &event->hw;
6051
6052         if (!box->io_addr)
6053                 return;
6054
6055         if (uncore_pmc_fixed(hwc->idx))
6056                 writel(SNBEP_PMON_CTL_EN, box->io_addr + hwc->config_base);
6057         else
6058                 writel(hwc->config, box->io_addr + hwc->config_base);
6059 }
6060
6061 static struct intel_uncore_ops spr_uncore_mmio_ops = {
6062         .init_box               = intel_generic_uncore_mmio_init_box,
6063         .exit_box               = uncore_mmio_exit_box,
6064         .disable_box            = intel_generic_uncore_mmio_disable_box,
6065         .enable_box             = intel_generic_uncore_mmio_enable_box,
6066         .disable_event          = intel_generic_uncore_mmio_disable_event,
6067         .enable_event           = spr_uncore_mmio_enable_event,
6068         .read_counter           = uncore_mmio_read_counter,
6069 };
6070
6071 static struct uncore_event_desc spr_uncore_imc_events[] = {
6072         INTEL_UNCORE_EVENT_DESC(clockticks,      "event=0x01,umask=0x00"),
6073         INTEL_UNCORE_EVENT_DESC(cas_count_read,  "event=0x05,umask=0xcf"),
6074         INTEL_UNCORE_EVENT_DESC(cas_count_read.scale, "6.103515625e-5"),
6075         INTEL_UNCORE_EVENT_DESC(cas_count_read.unit, "MiB"),
6076         INTEL_UNCORE_EVENT_DESC(cas_count_write, "event=0x05,umask=0xf0"),
6077         INTEL_UNCORE_EVENT_DESC(cas_count_write.scale, "6.103515625e-5"),
6078         INTEL_UNCORE_EVENT_DESC(cas_count_write.unit, "MiB"),
6079         { /* end: all zeroes */ },
6080 };
6081
6082 static struct intel_uncore_type spr_uncore_imc = {
6083         SPR_UNCORE_COMMON_FORMAT(),
6084         .name                   = "imc",
6085         .fixed_ctr_bits         = 48,
6086         .fixed_ctr              = SNR_IMC_MMIO_PMON_FIXED_CTR,
6087         .fixed_ctl              = SNR_IMC_MMIO_PMON_FIXED_CTL,
6088         .ops                    = &spr_uncore_mmio_ops,
6089         .event_descs            = spr_uncore_imc_events,
6090 };
6091
6092 static void spr_uncore_pci_enable_event(struct intel_uncore_box *box,
6093                                         struct perf_event *event)
6094 {
6095         struct pci_dev *pdev = box->pci_dev;
6096         struct hw_perf_event *hwc = &event->hw;
6097
6098         pci_write_config_dword(pdev, hwc->config_base + 4, (u32)(hwc->config >> 32));
6099         pci_write_config_dword(pdev, hwc->config_base, (u32)hwc->config);
6100 }
6101
6102 static struct intel_uncore_ops spr_uncore_pci_ops = {
6103         .init_box               = intel_generic_uncore_pci_init_box,
6104         .disable_box            = intel_generic_uncore_pci_disable_box,
6105         .enable_box             = intel_generic_uncore_pci_enable_box,
6106         .disable_event          = intel_generic_uncore_pci_disable_event,
6107         .enable_event           = spr_uncore_pci_enable_event,
6108         .read_counter           = intel_generic_uncore_pci_read_counter,
6109 };
6110
6111 #define SPR_UNCORE_PCI_COMMON_FORMAT()                  \
6112         SPR_UNCORE_COMMON_FORMAT(),                     \
6113         .ops                    = &spr_uncore_pci_ops
6114
6115 static struct intel_uncore_type spr_uncore_m2m = {
6116         SPR_UNCORE_PCI_COMMON_FORMAT(),
6117         .name                   = "m2m",
6118 };
6119
6120 static struct attribute_group spr_upi_mapping_group = {
6121         .is_visible     = skx_upi_mapping_visible,
6122 };
6123
6124 static const struct attribute_group *spr_upi_attr_update[] = {
6125         &uncore_alias_group,
6126         &spr_upi_mapping_group,
6127         NULL
6128 };
6129
6130 #define SPR_UPI_REGS_ADDR_DEVICE_LINK0  0x01
6131
6132 static void spr_upi_set_mapping(struct intel_uncore_type *type)
6133 {
6134         pmu_upi_set_mapping(type, &spr_upi_mapping_group);
6135 }
6136
6137 static void spr_upi_cleanup_mapping(struct intel_uncore_type *type)
6138 {
6139         pmu_cleanup_mapping(type, &spr_upi_mapping_group);
6140 }
6141
6142 static int spr_upi_get_topology(struct intel_uncore_type *type)
6143 {
6144         return discover_upi_topology(type, SPR_UBOX_DID, SPR_UPI_REGS_ADDR_DEVICE_LINK0);
6145 }
6146
6147 static struct intel_uncore_type spr_uncore_mdf = {
6148         SPR_UNCORE_COMMON_FORMAT(),
6149         .name                   = "mdf",
6150 };
6151
6152 #define UNCORE_SPR_NUM_UNCORE_TYPES             12
6153 #define UNCORE_SPR_IIO                          1
6154 #define UNCORE_SPR_IMC                          6
6155 #define UNCORE_SPR_UPI                          8
6156 #define UNCORE_SPR_M3UPI                        9
6157
6158 /*
6159  * The uncore units, which are supported by the discovery table,
6160  * are defined here.
6161  */
6162 static struct intel_uncore_type *spr_uncores[UNCORE_SPR_NUM_UNCORE_TYPES] = {
6163         &spr_uncore_chabox,
6164         &spr_uncore_iio,
6165         &spr_uncore_irp,
6166         &spr_uncore_m2pcie,
6167         &spr_uncore_pcu,
6168         NULL,
6169         &spr_uncore_imc,
6170         &spr_uncore_m2m,
6171         NULL,
6172         NULL,
6173         NULL,
6174         &spr_uncore_mdf,
6175 };
6176
6177 /*
6178  * The uncore units, which are not supported by the discovery table,
6179  * are implemented from here.
6180  */
6181 #define SPR_UNCORE_UPI_NUM_BOXES        4
6182
6183 static unsigned int spr_upi_pci_offsets[SPR_UNCORE_UPI_NUM_BOXES] = {
6184         0, 0x8000, 0x10000, 0x18000
6185 };
6186
6187 static struct intel_uncore_type spr_uncore_upi = {
6188         .event_mask             = SNBEP_PMON_RAW_EVENT_MASK,
6189         .event_mask_ext         = SPR_RAW_EVENT_MASK_EXT,
6190         .format_group           = &spr_uncore_raw_format_group,
6191         .ops                    = &spr_uncore_pci_ops,
6192         .name                   = "upi",
6193         .attr_update            = spr_upi_attr_update,
6194         .get_topology           = spr_upi_get_topology,
6195         .set_mapping            = spr_upi_set_mapping,
6196         .cleanup_mapping        = spr_upi_cleanup_mapping,
6197         .type_id                = UNCORE_SPR_UPI,
6198         .num_counters           = 4,
6199         .num_boxes              = SPR_UNCORE_UPI_NUM_BOXES,
6200         .perf_ctr_bits          = 48,
6201         .perf_ctr               = ICX_UPI_PCI_PMON_CTR0,
6202         .event_ctl              = ICX_UPI_PCI_PMON_CTL0,
6203         .box_ctl                = ICX_UPI_PCI_PMON_BOX_CTL,
6204         .pci_offsets            = spr_upi_pci_offsets,
6205 };
6206
6207 static struct intel_uncore_type spr_uncore_m3upi = {
6208         SPR_UNCORE_PCI_COMMON_FORMAT(),
6209         .name                   = "m3upi",
6210         .type_id                = UNCORE_SPR_M3UPI,
6211         .num_counters           = 4,
6212         .num_boxes              = SPR_UNCORE_UPI_NUM_BOXES,
6213         .perf_ctr_bits          = 48,
6214         .perf_ctr               = ICX_M3UPI_PCI_PMON_CTR0,
6215         .event_ctl              = ICX_M3UPI_PCI_PMON_CTL0,
6216         .box_ctl                = ICX_M3UPI_PCI_PMON_BOX_CTL,
6217         .pci_offsets            = spr_upi_pci_offsets,
6218         .constraints            = icx_uncore_m3upi_constraints,
6219 };
6220
6221 enum perf_uncore_spr_iio_freerunning_type_id {
6222         SPR_IIO_MSR_IOCLK,
6223         SPR_IIO_MSR_BW_IN,
6224         SPR_IIO_MSR_BW_OUT,
6225
6226         SPR_IIO_FREERUNNING_TYPE_MAX,
6227 };
6228
6229 static struct freerunning_counters spr_iio_freerunning[] = {
6230         [SPR_IIO_MSR_IOCLK]     = { 0x340e, 0x1, 0x10, 1, 48 },
6231         [SPR_IIO_MSR_BW_IN]     = { 0x3800, 0x1, 0x10, 8, 48 },
6232         [SPR_IIO_MSR_BW_OUT]    = { 0x3808, 0x1, 0x10, 8, 48 },
6233 };
6234
6235 static struct uncore_event_desc spr_uncore_iio_freerunning_events[] = {
6236         /* Free-Running IIO CLOCKS Counter */
6237         INTEL_UNCORE_EVENT_DESC(ioclk,                  "event=0xff,umask=0x10"),
6238         /* Free-Running IIO BANDWIDTH IN Counters */
6239         INTEL_UNCORE_EVENT_DESC(bw_in_port0,            "event=0xff,umask=0x20"),
6240         INTEL_UNCORE_EVENT_DESC(bw_in_port0.scale,      "3.814697266e-6"),
6241         INTEL_UNCORE_EVENT_DESC(bw_in_port0.unit,       "MiB"),
6242         INTEL_UNCORE_EVENT_DESC(bw_in_port1,            "event=0xff,umask=0x21"),
6243         INTEL_UNCORE_EVENT_DESC(bw_in_port1.scale,      "3.814697266e-6"),
6244         INTEL_UNCORE_EVENT_DESC(bw_in_port1.unit,       "MiB"),
6245         INTEL_UNCORE_EVENT_DESC(bw_in_port2,            "event=0xff,umask=0x22"),
6246         INTEL_UNCORE_EVENT_DESC(bw_in_port2.scale,      "3.814697266e-6"),
6247         INTEL_UNCORE_EVENT_DESC(bw_in_port2.unit,       "MiB"),
6248         INTEL_UNCORE_EVENT_DESC(bw_in_port3,            "event=0xff,umask=0x23"),
6249         INTEL_UNCORE_EVENT_DESC(bw_in_port3.scale,      "3.814697266e-6"),
6250         INTEL_UNCORE_EVENT_DESC(bw_in_port3.unit,       "MiB"),
6251         INTEL_UNCORE_EVENT_DESC(bw_in_port4,            "event=0xff,umask=0x24"),
6252         INTEL_UNCORE_EVENT_DESC(bw_in_port4.scale,      "3.814697266e-6"),
6253         INTEL_UNCORE_EVENT_DESC(bw_in_port4.unit,       "MiB"),
6254         INTEL_UNCORE_EVENT_DESC(bw_in_port5,            "event=0xff,umask=0x25"),
6255         INTEL_UNCORE_EVENT_DESC(bw_in_port5.scale,      "3.814697266e-6"),
6256         INTEL_UNCORE_EVENT_DESC(bw_in_port5.unit,       "MiB"),
6257         INTEL_UNCORE_EVENT_DESC(bw_in_port6,            "event=0xff,umask=0x26"),
6258         INTEL_UNCORE_EVENT_DESC(bw_in_port6.scale,      "3.814697266e-6"),
6259         INTEL_UNCORE_EVENT_DESC(bw_in_port6.unit,       "MiB"),
6260         INTEL_UNCORE_EVENT_DESC(bw_in_port7,            "event=0xff,umask=0x27"),
6261         INTEL_UNCORE_EVENT_DESC(bw_in_port7.scale,      "3.814697266e-6"),
6262         INTEL_UNCORE_EVENT_DESC(bw_in_port7.unit,       "MiB"),
6263         /* Free-Running IIO BANDWIDTH OUT Counters */
6264         INTEL_UNCORE_EVENT_DESC(bw_out_port0,           "event=0xff,umask=0x30"),
6265         INTEL_UNCORE_EVENT_DESC(bw_out_port0.scale,     "3.814697266e-6"),
6266         INTEL_UNCORE_EVENT_DESC(bw_out_port0.unit,      "MiB"),
6267         INTEL_UNCORE_EVENT_DESC(bw_out_port1,           "event=0xff,umask=0x31"),
6268         INTEL_UNCORE_EVENT_DESC(bw_out_port1.scale,     "3.814697266e-6"),
6269         INTEL_UNCORE_EVENT_DESC(bw_out_port1.unit,      "MiB"),
6270         INTEL_UNCORE_EVENT_DESC(bw_out_port2,           "event=0xff,umask=0x32"),
6271         INTEL_UNCORE_EVENT_DESC(bw_out_port2.scale,     "3.814697266e-6"),
6272         INTEL_UNCORE_EVENT_DESC(bw_out_port2.unit,      "MiB"),
6273         INTEL_UNCORE_EVENT_DESC(bw_out_port3,           "event=0xff,umask=0x33"),
6274         INTEL_UNCORE_EVENT_DESC(bw_out_port3.scale,     "3.814697266e-6"),
6275         INTEL_UNCORE_EVENT_DESC(bw_out_port3.unit,      "MiB"),
6276         INTEL_UNCORE_EVENT_DESC(bw_out_port4,           "event=0xff,umask=0x34"),
6277         INTEL_UNCORE_EVENT_DESC(bw_out_port4.scale,     "3.814697266e-6"),
6278         INTEL_UNCORE_EVENT_DESC(bw_out_port4.unit,      "MiB"),
6279         INTEL_UNCORE_EVENT_DESC(bw_out_port5,           "event=0xff,umask=0x35"),
6280         INTEL_UNCORE_EVENT_DESC(bw_out_port5.scale,     "3.814697266e-6"),
6281         INTEL_UNCORE_EVENT_DESC(bw_out_port5.unit,      "MiB"),
6282         INTEL_UNCORE_EVENT_DESC(bw_out_port6,           "event=0xff,umask=0x36"),
6283         INTEL_UNCORE_EVENT_DESC(bw_out_port6.scale,     "3.814697266e-6"),
6284         INTEL_UNCORE_EVENT_DESC(bw_out_port6.unit,      "MiB"),
6285         INTEL_UNCORE_EVENT_DESC(bw_out_port7,           "event=0xff,umask=0x37"),
6286         INTEL_UNCORE_EVENT_DESC(bw_out_port7.scale,     "3.814697266e-6"),
6287         INTEL_UNCORE_EVENT_DESC(bw_out_port7.unit,      "MiB"),
6288         { /* end: all zeroes */ },
6289 };
6290
6291 static struct intel_uncore_type spr_uncore_iio_free_running = {
6292         .name                   = "iio_free_running",
6293         .num_counters           = 17,
6294         .num_freerunning_types  = SPR_IIO_FREERUNNING_TYPE_MAX,
6295         .freerunning            = spr_iio_freerunning,
6296         .ops                    = &skx_uncore_iio_freerunning_ops,
6297         .event_descs            = spr_uncore_iio_freerunning_events,
6298         .format_group           = &skx_uncore_iio_freerunning_format_group,
6299 };
6300
6301 enum perf_uncore_spr_imc_freerunning_type_id {
6302         SPR_IMC_DCLK,
6303         SPR_IMC_PQ_CYCLES,
6304
6305         SPR_IMC_FREERUNNING_TYPE_MAX,
6306 };
6307
6308 static struct freerunning_counters spr_imc_freerunning[] = {
6309         [SPR_IMC_DCLK]          = { 0x22b0, 0x0, 0, 1, 48 },
6310         [SPR_IMC_PQ_CYCLES]     = { 0x2318, 0x8, 0, 2, 48 },
6311 };
6312
6313 static struct uncore_event_desc spr_uncore_imc_freerunning_events[] = {
6314         INTEL_UNCORE_EVENT_DESC(dclk,                   "event=0xff,umask=0x10"),
6315
6316         INTEL_UNCORE_EVENT_DESC(rpq_cycles,             "event=0xff,umask=0x20"),
6317         INTEL_UNCORE_EVENT_DESC(wpq_cycles,             "event=0xff,umask=0x21"),
6318         { /* end: all zeroes */ },
6319 };
6320
6321 #define SPR_MC_DEVICE_ID        0x3251
6322
6323 static void spr_uncore_imc_freerunning_init_box(struct intel_uncore_box *box)
6324 {
6325         int mem_offset = box->pmu->pmu_idx * ICX_IMC_MEM_STRIDE + SNR_IMC_MMIO_MEM0_OFFSET;
6326
6327         snr_uncore_mmio_map(box, uncore_mmio_box_ctl(box),
6328                             mem_offset, SPR_MC_DEVICE_ID);
6329 }
6330
6331 static struct intel_uncore_ops spr_uncore_imc_freerunning_ops = {
6332         .init_box       = spr_uncore_imc_freerunning_init_box,
6333         .exit_box       = uncore_mmio_exit_box,
6334         .read_counter   = uncore_mmio_read_counter,
6335         .hw_config      = uncore_freerunning_hw_config,
6336 };
6337
6338 static struct intel_uncore_type spr_uncore_imc_free_running = {
6339         .name                   = "imc_free_running",
6340         .num_counters           = 3,
6341         .mmio_map_size          = SNR_IMC_MMIO_SIZE,
6342         .num_freerunning_types  = SPR_IMC_FREERUNNING_TYPE_MAX,
6343         .freerunning            = spr_imc_freerunning,
6344         .ops                    = &spr_uncore_imc_freerunning_ops,
6345         .event_descs            = spr_uncore_imc_freerunning_events,
6346         .format_group           = &skx_uncore_iio_freerunning_format_group,
6347 };
6348
6349 #define UNCORE_SPR_MSR_EXTRA_UNCORES            1
6350 #define UNCORE_SPR_MMIO_EXTRA_UNCORES           1
6351 #define UNCORE_SPR_PCI_EXTRA_UNCORES            2
6352
6353 static struct intel_uncore_type *spr_msr_uncores[UNCORE_SPR_MSR_EXTRA_UNCORES] = {
6354         &spr_uncore_iio_free_running,
6355 };
6356
6357 static struct intel_uncore_type *spr_mmio_uncores[UNCORE_SPR_MMIO_EXTRA_UNCORES] = {
6358         &spr_uncore_imc_free_running,
6359 };
6360
6361 static struct intel_uncore_type *spr_pci_uncores[UNCORE_SPR_PCI_EXTRA_UNCORES] = {
6362         &spr_uncore_upi,
6363         &spr_uncore_m3upi
6364 };
6365
6366 int spr_uncore_units_ignore[] = {
6367         UNCORE_SPR_UPI,
6368         UNCORE_SPR_M3UPI,
6369         UNCORE_IGNORE_END
6370 };
6371
6372 static void uncore_type_customized_copy(struct intel_uncore_type *to_type,
6373                                         struct intel_uncore_type *from_type)
6374 {
6375         if (!to_type || !from_type)
6376                 return;
6377
6378         if (from_type->name)
6379                 to_type->name = from_type->name;
6380         if (from_type->fixed_ctr_bits)
6381                 to_type->fixed_ctr_bits = from_type->fixed_ctr_bits;
6382         if (from_type->event_mask)
6383                 to_type->event_mask = from_type->event_mask;
6384         if (from_type->event_mask_ext)
6385                 to_type->event_mask_ext = from_type->event_mask_ext;
6386         if (from_type->fixed_ctr)
6387                 to_type->fixed_ctr = from_type->fixed_ctr;
6388         if (from_type->fixed_ctl)
6389                 to_type->fixed_ctl = from_type->fixed_ctl;
6390         if (from_type->fixed_ctr_bits)
6391                 to_type->fixed_ctr_bits = from_type->fixed_ctr_bits;
6392         if (from_type->num_shared_regs)
6393                 to_type->num_shared_regs = from_type->num_shared_regs;
6394         if (from_type->constraints)
6395                 to_type->constraints = from_type->constraints;
6396         if (from_type->ops)
6397                 to_type->ops = from_type->ops;
6398         if (from_type->event_descs)
6399                 to_type->event_descs = from_type->event_descs;
6400         if (from_type->format_group)
6401                 to_type->format_group = from_type->format_group;
6402         if (from_type->attr_update)
6403                 to_type->attr_update = from_type->attr_update;
6404         if (from_type->set_mapping)
6405                 to_type->set_mapping = from_type->set_mapping;
6406         if (from_type->get_topology)
6407                 to_type->get_topology = from_type->get_topology;
6408         if (from_type->cleanup_mapping)
6409                 to_type->cleanup_mapping = from_type->cleanup_mapping;
6410 }
6411
6412 static struct intel_uncore_type **
6413 uncore_get_uncores(enum uncore_access_type type_id, int num_extra,
6414                     struct intel_uncore_type **extra)
6415 {
6416         struct intel_uncore_type **types, **start_types;
6417         int i;
6418
6419         start_types = types = intel_uncore_generic_init_uncores(type_id, num_extra);
6420
6421         /* Only copy the customized features */
6422         for (; *types; types++) {
6423                 if ((*types)->type_id >= UNCORE_SPR_NUM_UNCORE_TYPES)
6424                         continue;
6425                 uncore_type_customized_copy(*types, spr_uncores[(*types)->type_id]);
6426         }
6427
6428         for (i = 0; i < num_extra; i++, types++)
6429                 *types = extra[i];
6430
6431         return start_types;
6432 }
6433
6434 static struct intel_uncore_type *
6435 uncore_find_type_by_id(struct intel_uncore_type **types, int type_id)
6436 {
6437         for (; *types; types++) {
6438                 if (type_id == (*types)->type_id)
6439                         return *types;
6440         }
6441
6442         return NULL;
6443 }
6444
6445 static int uncore_type_max_boxes(struct intel_uncore_type **types,
6446                                  int type_id)
6447 {
6448         struct intel_uncore_type *type;
6449         int i, max = 0;
6450
6451         type = uncore_find_type_by_id(types, type_id);
6452         if (!type)
6453                 return 0;
6454
6455         for (i = 0; i < type->num_boxes; i++) {
6456                 if (type->box_ids[i] > max)
6457                         max = type->box_ids[i];
6458         }
6459
6460         return max + 1;
6461 }
6462
6463 void spr_uncore_cpu_init(void)
6464 {
6465         uncore_msr_uncores = uncore_get_uncores(UNCORE_ACCESS_MSR,
6466                                                 UNCORE_SPR_MSR_EXTRA_UNCORES,
6467                                                 spr_msr_uncores);
6468
6469         spr_uncore_iio_free_running.num_boxes = uncore_type_max_boxes(uncore_msr_uncores, UNCORE_SPR_IIO);
6470 }
6471
6472 #define SPR_UNCORE_UPI_PCIID            0x3241
6473 #define SPR_UNCORE_UPI0_DEVFN           0x9
6474 #define SPR_UNCORE_M3UPI_PCIID          0x3246
6475 #define SPR_UNCORE_M3UPI0_DEVFN         0x29
6476
6477 static void spr_update_device_location(int type_id)
6478 {
6479         struct intel_uncore_type *type;
6480         struct pci_dev *dev = NULL;
6481         u32 device, devfn;
6482         u64 *ctls;
6483         int die;
6484
6485         if (type_id == UNCORE_SPR_UPI) {
6486                 type = &spr_uncore_upi;
6487                 device = SPR_UNCORE_UPI_PCIID;
6488                 devfn = SPR_UNCORE_UPI0_DEVFN;
6489         } else if (type_id == UNCORE_SPR_M3UPI) {
6490                 type = &spr_uncore_m3upi;
6491                 device = SPR_UNCORE_M3UPI_PCIID;
6492                 devfn = SPR_UNCORE_M3UPI0_DEVFN;
6493         } else
6494                 return;
6495
6496         ctls = kcalloc(__uncore_max_dies, sizeof(u64), GFP_KERNEL);
6497         if (!ctls) {
6498                 type->num_boxes = 0;
6499                 return;
6500         }
6501
6502         while ((dev = pci_get_device(PCI_VENDOR_ID_INTEL, device, dev)) != NULL) {
6503                 if (devfn != dev->devfn)
6504                         continue;
6505
6506                 die = uncore_device_to_die(dev);
6507                 if (die < 0)
6508                         continue;
6509
6510                 ctls[die] = pci_domain_nr(dev->bus) << UNCORE_DISCOVERY_PCI_DOMAIN_OFFSET |
6511                             dev->bus->number << UNCORE_DISCOVERY_PCI_BUS_OFFSET |
6512                             devfn << UNCORE_DISCOVERY_PCI_DEVFN_OFFSET |
6513                             type->box_ctl;
6514         }
6515
6516         type->box_ctls = ctls;
6517 }
6518
6519 int spr_uncore_pci_init(void)
6520 {
6521         /*
6522          * The discovery table of UPI on some SPR variant is broken,
6523          * which impacts the detection of both UPI and M3UPI uncore PMON.
6524          * Use the pre-defined UPI and M3UPI table to replace.
6525          *
6526          * The accurate location, e.g., domain and BUS number,
6527          * can only be retrieved at load time.
6528          * Update the location of UPI and M3UPI.
6529          */
6530         spr_update_device_location(UNCORE_SPR_UPI);
6531         spr_update_device_location(UNCORE_SPR_M3UPI);
6532         uncore_pci_uncores = uncore_get_uncores(UNCORE_ACCESS_PCI,
6533                                                 UNCORE_SPR_PCI_EXTRA_UNCORES,
6534                                                 spr_pci_uncores);
6535         return 0;
6536 }
6537
6538 void spr_uncore_mmio_init(void)
6539 {
6540         int ret = snbep_pci2phy_map_init(0x3250, SKX_CPUNODEID, SKX_GIDNIDMAP, true);
6541
6542         if (ret)
6543                 uncore_mmio_uncores = uncore_get_uncores(UNCORE_ACCESS_MMIO, 0, NULL);
6544         else {
6545                 uncore_mmio_uncores = uncore_get_uncores(UNCORE_ACCESS_MMIO,
6546                                                          UNCORE_SPR_MMIO_EXTRA_UNCORES,
6547                                                          spr_mmio_uncores);
6548
6549                 spr_uncore_imc_free_running.num_boxes = uncore_type_max_boxes(uncore_mmio_uncores, UNCORE_SPR_IMC) / 2;
6550         }
6551 }
6552
6553 /* end of SPR uncore support */