arch/x86/kernel/cpu/perf_event_intel_uncore.c

   1 #include "perf_event_intel_uncore.h"
   2
   3 static struct intel_uncore_type *empty_uncore[] = { NULL, };
   4 static struct intel_uncore_type **msr_uncores = empty_uncore;
   5 static struct intel_uncore_type **pci_uncores = empty_uncore;
   6 /* pci bus to socket mapping */
   7 static int pcibus_to_physid[256] = { [0 ... 255] = -1, };
   8
   9 static struct pci_dev *extra_pci_dev[UNCORE_SOCKET_MAX][UNCORE_EXTRA_PCI_DEV_MAX];
  10
  11 static DEFINE_RAW_SPINLOCK(uncore_box_lock);
  12
  13 /* mask of cpus that collect uncore events */
  14 static cpumask_t uncore_cpu_mask;
  15
  16 /* constraint for the fixed counter */
  17 static struct event_constraint constraint_fixed =
  18         EVENT_CONSTRAINT(~0ULL, 1 << UNCORE_PMC_IDX_FIXED, ~0ULL);
  19 static struct event_constraint constraint_empty =
  20         EVENT_CONSTRAINT(0, 0, 0);
  21
  22 #define __BITS_VALUE(x, i, n)  ((typeof(x))(((x) >> ((i) * (n))) & \
  23                                 ((1ULL << (n)) - 1)))
  24
  25 DEFINE_UNCORE_FORMAT_ATTR(event, event, "config:0-7");
  26 DEFINE_UNCORE_FORMAT_ATTR(event_ext, event, "config:0-7,21");
  27 DEFINE_UNCORE_FORMAT_ATTR(umask, umask, "config:8-15");
  28 DEFINE_UNCORE_FORMAT_ATTR(edge, edge, "config:18");
  29 DEFINE_UNCORE_FORMAT_ATTR(tid_en, tid_en, "config:19");
  30 DEFINE_UNCORE_FORMAT_ATTR(inv, inv, "config:23");
  31 DEFINE_UNCORE_FORMAT_ATTR(cmask5, cmask, "config:24-28");
  32 DEFINE_UNCORE_FORMAT_ATTR(cmask8, cmask, "config:24-31");
  33 DEFINE_UNCORE_FORMAT_ATTR(thresh8, thresh, "config:24-31");
  34 DEFINE_UNCORE_FORMAT_ATTR(thresh5, thresh, "config:24-28");
  35 DEFINE_UNCORE_FORMAT_ATTR(occ_sel, occ_sel, "config:14-15");
  36 DEFINE_UNCORE_FORMAT_ATTR(occ_invert, occ_invert, "config:30");
  37 DEFINE_UNCORE_FORMAT_ATTR(occ_edge, occ_edge, "config:14-51");
  38 DEFINE_UNCORE_FORMAT_ATTR(filter_tid, filter_tid, "config1:0-4");
  39 DEFINE_UNCORE_FORMAT_ATTR(filter_link, filter_link, "config1:5-8");
  40 DEFINE_UNCORE_FORMAT_ATTR(filter_nid, filter_nid, "config1:10-17");
  41 DEFINE_UNCORE_FORMAT_ATTR(filter_nid2, filter_nid, "config1:32-47");
  42 DEFINE_UNCORE_FORMAT_ATTR(filter_state, filter_state, "config1:18-22");
  43 DEFINE_UNCORE_FORMAT_ATTR(filter_state2, filter_state, "config1:17-22");
  44 DEFINE_UNCORE_FORMAT_ATTR(filter_opc, filter_opc, "config1:23-31");
  45 DEFINE_UNCORE_FORMAT_ATTR(filter_opc2, filter_opc, "config1:52-60");
  46 DEFINE_UNCORE_FORMAT_ATTR(filter_band0, filter_band0, "config1:0-7");
  47 DEFINE_UNCORE_FORMAT_ATTR(filter_band1, filter_band1, "config1:8-15");
  48 DEFINE_UNCORE_FORMAT_ATTR(filter_band2, filter_band2, "config1:16-23");
  49 DEFINE_UNCORE_FORMAT_ATTR(filter_band3, filter_band3, "config1:24-31");
  50
  51 static u64 uncore_msr_read_counter(struct intel_uncore_box *box, struct perf_event *event)
  52 {
  53         u64 count;
  54
  55         rdmsrl(event->hw.event_base, count);
  56
  57         return count;
  58 }
  59
  60 /*
  61  * generic get constraint function for shared match/mask registers.
  62  */
  63 static struct event_constraint *
  64 uncore_get_constraint(struct intel_uncore_box *box, struct perf_event *event)
  65 {
  66         struct intel_uncore_extra_reg *er;
  67         struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
  68         struct hw_perf_event_extra *reg2 = &event->hw.branch_reg;
  69         unsigned long flags;
  70         bool ok = false;
  71
  72         /*
  73          * reg->alloc can be set due to existing state, so for fake box we
  74          * need to ignore this, otherwise we might fail to allocate proper
  75          * fake state for this extra reg constraint.
  76          */
  77         if (reg1->idx == EXTRA_REG_NONE ||
  78             (!uncore_box_is_fake(box) && reg1->alloc))
  79                 return NULL;
  80
  81         er = &box->shared_regs[reg1->idx];
  82         raw_spin_lock_irqsave(&er->lock, flags);
  83         if (!atomic_read(&er->ref) ||
  84             (er->config1 == reg1->config && er->config2 == reg2->config)) {
  85                 atomic_inc(&er->ref);
  86                 er->config1 = reg1->config;
  87                 er->config2 = reg2->config;
  88                 ok = true;
  89         }
  90         raw_spin_unlock_irqrestore(&er->lock, flags);
  91
  92         if (ok) {
  93                 if (!uncore_box_is_fake(box))
  94                         reg1->alloc = 1;
  95                 return NULL;
  96         }
  97
  98         return &constraint_empty;
  99 }
 100
 101 static void uncore_put_constraint(struct intel_uncore_box *box, struct perf_event *event)
 102 {
 103         struct intel_uncore_extra_reg *er;
 104         struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
 105
 106         /*
 107          * Only put constraint if extra reg was actually allocated. Also
 108          * takes care of event which do not use an extra shared reg.
 109          *
 110          * Also, if this is a fake box we shouldn't touch any event state
 111          * (reg->alloc) and we don't care about leaving inconsistent box
 112          * state either since it will be thrown out.
 113          */
 114         if (uncore_box_is_fake(box) || !reg1->alloc)
 115                 return;
 116
 117         er = &box->shared_regs[reg1->idx];
 118         atomic_dec(&er->ref);
 119         reg1->alloc = 0;
 120 }
 121
 122 static u64 uncore_shared_reg_config(struct intel_uncore_box *box, int idx)
 123 {
 124         struct intel_uncore_extra_reg *er;
 125         unsigned long flags;
 126         u64 config;
 127
 128         er = &box->shared_regs[idx];
 129
 130         raw_spin_lock_irqsave(&er->lock, flags);
 131         config = er->config;
 132         raw_spin_unlock_irqrestore(&er->lock, flags);
 133
 134         return config;
 135 }
 136
 137 /* Sandy Bridge-EP uncore support */
 138 static struct intel_uncore_type snbep_uncore_cbox;
 139 static struct intel_uncore_type snbep_uncore_pcu;
 140
 141 static void snbep_uncore_pci_disable_box(struct intel_uncore_box *box)
 142 {
 143         struct pci_dev *pdev = box->pci_dev;
 144         int box_ctl = uncore_pci_box_ctl(box);
 145         u32 config = 0;
 146
 147         if (!pci_read_config_dword(pdev, box_ctl, &config)) {
 148                 config |= SNBEP_PMON_BOX_CTL_FRZ;
 149                 pci_write_config_dword(pdev, box_ctl, config);
 150         }
 151 }
 152
 153 static void snbep_uncore_pci_enable_box(struct intel_uncore_box *box)
 154 {
 155         struct pci_dev *pdev = box->pci_dev;
 156         int box_ctl = uncore_pci_box_ctl(box);
 157         u32 config = 0;
 158
 159         if (!pci_read_config_dword(pdev, box_ctl, &config)) {
 160                 config &= ~SNBEP_PMON_BOX_CTL_FRZ;
 161                 pci_write_config_dword(pdev, box_ctl, config);
 162         }
 163 }
 164
 165 static void snbep_uncore_pci_enable_event(struct intel_uncore_box *box, struct perf_event *event)
 166 {
 167         struct pci_dev *pdev = box->pci_dev;
 168         struct hw_perf_event *hwc = &event->hw;
 169
 170         pci_write_config_dword(pdev, hwc->config_base, hwc->config | SNBEP_PMON_CTL_EN);
 171 }
 172
 173 static void snbep_uncore_pci_disable_event(struct intel_uncore_box *box, struct perf_event *event)
 174 {
 175         struct pci_dev *pdev = box->pci_dev;
 176         struct hw_perf_event *hwc = &event->hw;
 177
 178         pci_write_config_dword(pdev, hwc->config_base, hwc->config);
 179 }
 180
 181 static u64 snbep_uncore_pci_read_counter(struct intel_uncore_box *box, struct perf_event *event)
 182 {
 183         struct pci_dev *pdev = box->pci_dev;
 184         struct hw_perf_event *hwc = &event->hw;
 185         u64 count = 0;
 186
 187         pci_read_config_dword(pdev, hwc->event_base, (u32 *)&count);
 188         pci_read_config_dword(pdev, hwc->event_base + 4, (u32 *)&count + 1);
 189
 190         return count;
 191 }
 192
 193 static void snbep_uncore_pci_init_box(struct intel_uncore_box *box)
 194 {
 195         struct pci_dev *pdev = box->pci_dev;
 196
 197         pci_write_config_dword(pdev, SNBEP_PCI_PMON_BOX_CTL, SNBEP_PMON_BOX_CTL_INT);
 198 }
 199
 200 static void snbep_uncore_msr_disable_box(struct intel_uncore_box *box)
 201 {
 202         u64 config;
 203         unsigned msr;
 204
 205         msr = uncore_msr_box_ctl(box);
 206         if (msr) {
 207                 rdmsrl(msr, config);
 208                 config |= SNBEP_PMON_BOX_CTL_FRZ;
 209                 wrmsrl(msr, config);
 210         }
 211 }
 212
 213 static void snbep_uncore_msr_enable_box(struct intel_uncore_box *box)
 214 {
 215         u64 config;
 216         unsigned msr;
 217
 218         msr = uncore_msr_box_ctl(box);
 219         if (msr) {
 220                 rdmsrl(msr, config);
 221                 config &= ~SNBEP_PMON_BOX_CTL_FRZ;
 222                 wrmsrl(msr, config);
 223         }
 224 }
 225
 226 static void snbep_uncore_msr_enable_event(struct intel_uncore_box *box, struct perf_event *event)
 227 {
 228         struct hw_perf_event *hwc = &event->hw;
 229         struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
 230
 231         if (reg1->idx != EXTRA_REG_NONE)
 232                 wrmsrl(reg1->reg, uncore_shared_reg_config(box, 0));
 233
 234         wrmsrl(hwc->config_base, hwc->config | SNBEP_PMON_CTL_EN);
 235 }
 236
 237 static void snbep_uncore_msr_disable_event(struct intel_uncore_box *box,
 238                                         struct perf_event *event)
 239 {
 240         struct hw_perf_event *hwc = &event->hw;
 241
 242         wrmsrl(hwc->config_base, hwc->config);
 243 }
 244
 245 static void snbep_uncore_msr_init_box(struct intel_uncore_box *box)
 246 {
 247         unsigned msr = uncore_msr_box_ctl(box);
 248
 249         if (msr)
 250                 wrmsrl(msr, SNBEP_PMON_BOX_CTL_INT);
 251 }
 252
 253 static struct attribute *snbep_uncore_formats_attr[] = {
 254         &format_attr_event.attr,
 255         &format_attr_umask.attr,
 256         &format_attr_edge.attr,
 257         &format_attr_inv.attr,
 258         &format_attr_thresh8.attr,
 259         NULL,
 260 };
 261
 262 static struct attribute *snbep_uncore_ubox_formats_attr[] = {
 263         &format_attr_event.attr,
 264         &format_attr_umask.attr,
 265         &format_attr_edge.attr,
 266         &format_attr_inv.attr,
 267         &format_attr_thresh5.attr,
 268         NULL,
 269 };
 270
 271 static struct attribute *snbep_uncore_cbox_formats_attr[] = {
 272         &format_attr_event.attr,
 273         &format_attr_umask.attr,
 274         &format_attr_edge.attr,
 275         &format_attr_tid_en.attr,
 276         &format_attr_inv.attr,
 277         &format_attr_thresh8.attr,
 278         &format_attr_filter_tid.attr,
 279         &format_attr_filter_nid.attr,
 280         &format_attr_filter_state.attr,
 281         &format_attr_filter_opc.attr,
 282         NULL,
 283 };
 284
 285 static struct attribute *snbep_uncore_pcu_formats_attr[] = {
 286         &format_attr_event.attr,
 287         &format_attr_occ_sel.attr,
 288         &format_attr_edge.attr,
 289         &format_attr_inv.attr,
 290         &format_attr_thresh5.attr,
 291         &format_attr_occ_invert.attr,
 292         &format_attr_occ_edge.attr,
 293         &format_attr_filter_band0.attr,
 294         &format_attr_filter_band1.attr,
 295         &format_attr_filter_band2.attr,
 296         &format_attr_filter_band3.attr,
 297         NULL,
 298 };
 299
 300 static struct attribute *snbep_uncore_qpi_formats_attr[] = {
 301         &format_attr_event_ext.attr,
 302         &format_attr_umask.attr,
 303         &format_attr_edge.attr,
 304         &format_attr_inv.attr,
 305         &format_attr_thresh8.attr,
 306         NULL,
 307 };
 308
 309 static struct uncore_event_desc snbep_uncore_imc_events[] = {
 310         INTEL_UNCORE_EVENT_DESC(clockticks,      "event=0xff,umask=0x00"),
 311         INTEL_UNCORE_EVENT_DESC(cas_count_read,  "event=0x04,umask=0x03"),
 312         INTEL_UNCORE_EVENT_DESC(cas_count_write, "event=0x04,umask=0x0c"),
 313         { /* end: all zeroes */ },
 314 };
 315
 316 static struct uncore_event_desc snbep_uncore_qpi_events[] = {
 317         INTEL_UNCORE_EVENT_DESC(clockticks,       "event=0x14"),
 318         INTEL_UNCORE_EVENT_DESC(txl_flits_active, "event=0x00,umask=0x06"),
 319         INTEL_UNCORE_EVENT_DESC(drs_data,         "event=0x02,umask=0x08"),
 320         INTEL_UNCORE_EVENT_DESC(ncb_data,         "event=0x03,umask=0x04"),
 321         { /* end: all zeroes */ },
 322 };
 323
 324 static struct attribute_group snbep_uncore_format_group = {
 325         .name = "format",
 326         .attrs = snbep_uncore_formats_attr,
 327 };
 328
 329 static struct attribute_group snbep_uncore_ubox_format_group = {
 330         .name = "format",
 331         .attrs = snbep_uncore_ubox_formats_attr,
 332 };
 333
 334 static struct attribute_group snbep_uncore_cbox_format_group = {
 335         .name = "format",
 336         .attrs = snbep_uncore_cbox_formats_attr,
 337 };
 338
 339 static struct attribute_group snbep_uncore_pcu_format_group = {
 340         .name = "format",
 341         .attrs = snbep_uncore_pcu_formats_attr,
 342 };
 343
 344 static struct attribute_group snbep_uncore_qpi_format_group = {
 345         .name = "format",
 346         .attrs = snbep_uncore_qpi_formats_attr,
 347 };
 348
 349 #define SNBEP_UNCORE_MSR_OPS_COMMON_INIT()                      \
 350         .init_box       = snbep_uncore_msr_init_box,            \
 351         .disable_box    = snbep_uncore_msr_disable_box,         \
 352         .enable_box     = snbep_uncore_msr_enable_box,          \
 353         .disable_event  = snbep_uncore_msr_disable_event,       \
 354         .enable_event   = snbep_uncore_msr_enable_event,        \
 355         .read_counter   = uncore_msr_read_counter
 356
 357 static struct intel_uncore_ops snbep_uncore_msr_ops = {
 358         SNBEP_UNCORE_MSR_OPS_COMMON_INIT(),
 359 };
 360
 361 static struct intel_uncore_ops snbep_uncore_pci_ops = {
 362         .init_box       = snbep_uncore_pci_init_box,
 363         .disable_box    = snbep_uncore_pci_disable_box,
 364         .enable_box     = snbep_uncore_pci_enable_box,
 365         .disable_event  = snbep_uncore_pci_disable_event,
 366         .enable_event   = snbep_uncore_pci_enable_event,
 367         .read_counter   = snbep_uncore_pci_read_counter,
 368 };
 369
 370 static struct event_constraint snbep_uncore_cbox_constraints[] = {
 371         UNCORE_EVENT_CONSTRAINT(0x01, 0x1),
 372         UNCORE_EVENT_CONSTRAINT(0x02, 0x3),
 373         UNCORE_EVENT_CONSTRAINT(0x04, 0x3),
 374         UNCORE_EVENT_CONSTRAINT(0x05, 0x3),
 375         UNCORE_EVENT_CONSTRAINT(0x07, 0x3),
 376         UNCORE_EVENT_CONSTRAINT(0x09, 0x3),
 377         UNCORE_EVENT_CONSTRAINT(0x11, 0x1),
 378         UNCORE_EVENT_CONSTRAINT(0x12, 0x3),
 379         UNCORE_EVENT_CONSTRAINT(0x13, 0x3),
 380         UNCORE_EVENT_CONSTRAINT(0x1b, 0xc),
 381         UNCORE_EVENT_CONSTRAINT(0x1c, 0xc),
 382         UNCORE_EVENT_CONSTRAINT(0x1d, 0xc),
 383         UNCORE_EVENT_CONSTRAINT(0x1e, 0xc),
 384         EVENT_CONSTRAINT_OVERLAP(0x1f, 0xe, 0xff),
 385         UNCORE_EVENT_CONSTRAINT(0x21, 0x3),
 386         UNCORE_EVENT_CONSTRAINT(0x23, 0x3),
 387         UNCORE_EVENT_CONSTRAINT(0x31, 0x3),
 388         UNCORE_EVENT_CONSTRAINT(0x32, 0x3),
 389         UNCORE_EVENT_CONSTRAINT(0x33, 0x3),
 390         UNCORE_EVENT_CONSTRAINT(0x34, 0x3),
 391         UNCORE_EVENT_CONSTRAINT(0x35, 0x3),
 392         UNCORE_EVENT_CONSTRAINT(0x36, 0x1),
 393         UNCORE_EVENT_CONSTRAINT(0x37, 0x3),
 394         UNCORE_EVENT_CONSTRAINT(0x38, 0x3),
 395         UNCORE_EVENT_CONSTRAINT(0x39, 0x3),
 396         UNCORE_EVENT_CONSTRAINT(0x3b, 0x1),
 397         EVENT_CONSTRAINT_END
 398 };
 399
 400 static struct event_constraint snbep_uncore_r2pcie_constraints[] = {
 401         UNCORE_EVENT_CONSTRAINT(0x10, 0x3),
 402         UNCORE_EVENT_CONSTRAINT(0x11, 0x3),
 403         UNCORE_EVENT_CONSTRAINT(0x12, 0x1),
 404         UNCORE_EVENT_CONSTRAINT(0x23, 0x3),
 405         UNCORE_EVENT_CONSTRAINT(0x24, 0x3),
 406         UNCORE_EVENT_CONSTRAINT(0x25, 0x3),
 407         UNCORE_EVENT_CONSTRAINT(0x26, 0x3),
 408         UNCORE_EVENT_CONSTRAINT(0x32, 0x3),
 409         UNCORE_EVENT_CONSTRAINT(0x33, 0x3),
 410         UNCORE_EVENT_CONSTRAINT(0x34, 0x3),
 411         EVENT_CONSTRAINT_END
 412 };
 413
 414 static struct event_constraint snbep_uncore_r3qpi_constraints[] = {
 415         UNCORE_EVENT_CONSTRAINT(0x10, 0x3),
 416         UNCORE_EVENT_CONSTRAINT(0x11, 0x3),
 417         UNCORE_EVENT_CONSTRAINT(0x12, 0x3),
 418         UNCORE_EVENT_CONSTRAINT(0x13, 0x1),
 419         UNCORE_EVENT_CONSTRAINT(0x20, 0x3),
 420         UNCORE_EVENT_CONSTRAINT(0x21, 0x3),
 421         UNCORE_EVENT_CONSTRAINT(0x22, 0x3),
 422         UNCORE_EVENT_CONSTRAINT(0x23, 0x3),
 423         UNCORE_EVENT_CONSTRAINT(0x24, 0x3),
 424         UNCORE_EVENT_CONSTRAINT(0x25, 0x3),
 425         UNCORE_EVENT_CONSTRAINT(0x26, 0x3),
 426         UNCORE_EVENT_CONSTRAINT(0x28, 0x3),
 427         UNCORE_EVENT_CONSTRAINT(0x29, 0x3),
 428         UNCORE_EVENT_CONSTRAINT(0x2a, 0x3),
 429         UNCORE_EVENT_CONSTRAINT(0x2b, 0x3),
 430         UNCORE_EVENT_CONSTRAINT(0x2c, 0x3),
 431         UNCORE_EVENT_CONSTRAINT(0x2d, 0x3),
 432         UNCORE_EVENT_CONSTRAINT(0x2e, 0x3),
 433         UNCORE_EVENT_CONSTRAINT(0x2f, 0x3),
 434         UNCORE_EVENT_CONSTRAINT(0x30, 0x3),
 435         UNCORE_EVENT_CONSTRAINT(0x31, 0x3),
 436         UNCORE_EVENT_CONSTRAINT(0x32, 0x3),
 437         UNCORE_EVENT_CONSTRAINT(0x33, 0x3),
 438         UNCORE_EVENT_CONSTRAINT(0x34, 0x3),
 439         UNCORE_EVENT_CONSTRAINT(0x36, 0x3),
 440         UNCORE_EVENT_CONSTRAINT(0x37, 0x3),
 441         UNCORE_EVENT_CONSTRAINT(0x38, 0x3),
 442         UNCORE_EVENT_CONSTRAINT(0x39, 0x3),
 443         EVENT_CONSTRAINT_END
 444 };
 445
 446 static struct intel_uncore_type snbep_uncore_ubox = {
 447         .name           = "ubox",
 448         .num_counters   = 2,
 449         .num_boxes      = 1,
 450         .perf_ctr_bits  = 44,
 451         .fixed_ctr_bits = 48,
 452         .perf_ctr       = SNBEP_U_MSR_PMON_CTR0,
 453         .event_ctl      = SNBEP_U_MSR_PMON_CTL0,
 454         .event_mask     = SNBEP_U_MSR_PMON_RAW_EVENT_MASK,
 455         .fixed_ctr      = SNBEP_U_MSR_PMON_UCLK_FIXED_CTR,
 456         .fixed_ctl      = SNBEP_U_MSR_PMON_UCLK_FIXED_CTL,
 457         .ops            = &snbep_uncore_msr_ops,
 458         .format_group   = &snbep_uncore_ubox_format_group,
 459 };
 460
 461 static struct extra_reg snbep_uncore_cbox_extra_regs[] = {
 462         SNBEP_CBO_EVENT_EXTRA_REG(SNBEP_CBO_PMON_CTL_TID_EN,
 463                                   SNBEP_CBO_PMON_CTL_TID_EN, 0x1),
 464         SNBEP_CBO_EVENT_EXTRA_REG(0x0334, 0xffff, 0x4),
 465         SNBEP_CBO_EVENT_EXTRA_REG(0x0534, 0xffff, 0x4),
 466         SNBEP_CBO_EVENT_EXTRA_REG(0x0934, 0xffff, 0x4),
 467         SNBEP_CBO_EVENT_EXTRA_REG(0x4134, 0xffff, 0x6),
 468         SNBEP_CBO_EVENT_EXTRA_REG(0x0135, 0xffff, 0x8),
 469         SNBEP_CBO_EVENT_EXTRA_REG(0x0335, 0xffff, 0x8),
 470         SNBEP_CBO_EVENT_EXTRA_REG(0x4135, 0xffff, 0xc),
 471         SNBEP_CBO_EVENT_EXTRA_REG(0x4335, 0xffff, 0xc),
 472         SNBEP_CBO_EVENT_EXTRA_REG(0x4435, 0xffff, 0x2),
 473         SNBEP_CBO_EVENT_EXTRA_REG(0x4835, 0xffff, 0x2),
 474         SNBEP_CBO_EVENT_EXTRA_REG(0x4a35, 0xffff, 0x2),
 475         SNBEP_CBO_EVENT_EXTRA_REG(0x5035, 0xffff, 0x2),
 476         SNBEP_CBO_EVENT_EXTRA_REG(0x0136, 0xffff, 0x8),
 477         SNBEP_CBO_EVENT_EXTRA_REG(0x0336, 0xffff, 0x8),
 478         SNBEP_CBO_EVENT_EXTRA_REG(0x4136, 0xffff, 0xc),
 479         SNBEP_CBO_EVENT_EXTRA_REG(0x4336, 0xffff, 0xc),
 480         SNBEP_CBO_EVENT_EXTRA_REG(0x4436, 0xffff, 0x2),
 481         SNBEP_CBO_EVENT_EXTRA_REG(0x4836, 0xffff, 0x2),
 482         SNBEP_CBO_EVENT_EXTRA_REG(0x4a36, 0xffff, 0x2),
 483         SNBEP_CBO_EVENT_EXTRA_REG(0x4037, 0x40ff, 0x2),
 484         EVENT_EXTRA_END
 485 };
 486
 487 static void snbep_cbox_put_constraint(struct intel_uncore_box *box, struct perf_event *event)
 488 {
 489         struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
 490         struct intel_uncore_extra_reg *er = &box->shared_regs[0];
 491         int i;
 492
 493         if (uncore_box_is_fake(box))
 494                 return;
 495
 496         for (i = 0; i < 5; i++) {
 497                 if (reg1->alloc & (0x1 << i))
 498                         atomic_sub(1 << (i * 6), &er->ref);
 499         }
 500         reg1->alloc = 0;
 501 }
 502
 503 static struct event_constraint *
 504 __snbep_cbox_get_constraint(struct intel_uncore_box *box, struct perf_event *event,
 505                             u64 (*cbox_filter_mask)(int fields))
 506 {
 507         struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
 508         struct intel_uncore_extra_reg *er = &box->shared_regs[0];
 509         int i, alloc = 0;
 510         unsigned long flags;
 511         u64 mask;
 512
 513         if (reg1->idx == EXTRA_REG_NONE)
 514                 return NULL;
 515
 516         raw_spin_lock_irqsave(&er->lock, flags);
 517         for (i = 0; i < 5; i++) {
 518                 if (!(reg1->idx & (0x1 << i)))
 519                         continue;
 520                 if (!uncore_box_is_fake(box) && (reg1->alloc & (0x1 << i)))
 521                         continue;
 522
 523                 mask = cbox_filter_mask(0x1 << i);
 524                 if (!__BITS_VALUE(atomic_read(&er->ref), i, 6) ||
 525                     !((reg1->config ^ er->config) & mask)) {
 526                         atomic_add(1 << (i * 6), &er->ref);
 527                         er->config &= ~mask;
 528                         er->config |= reg1->config & mask;
 529                         alloc |= (0x1 << i);
 530                 } else {
 531                         break;
 532                 }
 533         }
 534         raw_spin_unlock_irqrestore(&er->lock, flags);
 535         if (i < 5)
 536                 goto fail;
 537
 538         if (!uncore_box_is_fake(box))
 539                 reg1->alloc |= alloc;
 540
 541         return NULL;
 542 fail:
 543         for (; i >= 0; i--) {
 544                 if (alloc & (0x1 << i))
 545                         atomic_sub(1 << (i * 6), &er->ref);
 546         }
 547         return &constraint_empty;
 548 }
 549
 550 static u64 snbep_cbox_filter_mask(int fields)
 551 {
 552         u64 mask = 0;
 553
 554         if (fields & 0x1)
 555                 mask |= SNBEP_CB0_MSR_PMON_BOX_FILTER_TID;
 556         if (fields & 0x2)
 557                 mask |= SNBEP_CB0_MSR_PMON_BOX_FILTER_NID;
 558         if (fields & 0x4)
 559                 mask |= SNBEP_CB0_MSR_PMON_BOX_FILTER_STATE;
 560         if (fields & 0x8)
 561                 mask |= SNBEP_CB0_MSR_PMON_BOX_FILTER_OPC;
 562
 563         return mask;
 564 }
 565
 566 static struct event_constraint *
 567 snbep_cbox_get_constraint(struct intel_uncore_box *box, struct perf_event *event)
 568 {
 569         return __snbep_cbox_get_constraint(box, event, snbep_cbox_filter_mask);
 570 }
 571
 572 static int snbep_cbox_hw_config(struct intel_uncore_box *box, struct perf_event *event)
 573 {
 574         struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
 575         struct extra_reg *er;
 576         int idx = 0;
 577
 578         for (er = snbep_uncore_cbox_extra_regs; er->msr; er++) {
 579                 if (er->event != (event->hw.config & er->config_mask))
 580                         continue;
 581                 idx |= er->idx;
 582         }
 583
 584         if (idx) {
 585                 reg1->reg = SNBEP_C0_MSR_PMON_BOX_FILTER +
 586                         SNBEP_CBO_MSR_OFFSET * box->pmu->pmu_idx;
 587                 reg1->config = event->attr.config1 & snbep_cbox_filter_mask(idx);
 588                 reg1->idx = idx;
 589         }
 590         return 0;
 591 }
 592
 593 static struct intel_uncore_ops snbep_uncore_cbox_ops = {
 594         SNBEP_UNCORE_MSR_OPS_COMMON_INIT(),
 595         .hw_config              = snbep_cbox_hw_config,
 596         .get_constraint         = snbep_cbox_get_constraint,
 597         .put_constraint         = snbep_cbox_put_constraint,
 598 };
 599
 600 static struct intel_uncore_type snbep_uncore_cbox = {
 601         .name                   = "cbox",
 602         .num_counters           = 4,
 603         .num_boxes              = 8,
 604         .perf_ctr_bits          = 44,
 605         .event_ctl              = SNBEP_C0_MSR_PMON_CTL0,
 606         .perf_ctr               = SNBEP_C0_MSR_PMON_CTR0,
 607         .event_mask             = SNBEP_CBO_MSR_PMON_RAW_EVENT_MASK,
 608         .box_ctl                = SNBEP_C0_MSR_PMON_BOX_CTL,
 609         .msr_offset             = SNBEP_CBO_MSR_OFFSET,
 610         .num_shared_regs        = 1,
 611         .constraints            = snbep_uncore_cbox_constraints,
 612         .ops                    = &snbep_uncore_cbox_ops,
 613         .format_group           = &snbep_uncore_cbox_format_group,
 614 };
 615
 616 static u64 snbep_pcu_alter_er(struct perf_event *event, int new_idx, bool modify)
 617 {
 618         struct hw_perf_event *hwc = &event->hw;
 619         struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
 620         u64 config = reg1->config;
 621
 622         if (new_idx > reg1->idx)
 623                 config <<= 8 * (new_idx - reg1->idx);
 624         else
 625                 config >>= 8 * (reg1->idx - new_idx);
 626
 627         if (modify) {
 628                 hwc->config += new_idx - reg1->idx;
 629                 reg1->config = config;
 630                 reg1->idx = new_idx;
 631         }
 632         return config;
 633 }
 634
 635 static struct event_constraint *
 636 snbep_pcu_get_constraint(struct intel_uncore_box *box, struct perf_event *event)
 637 {
 638         struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
 639         struct intel_uncore_extra_reg *er = &box->shared_regs[0];
 640         unsigned long flags;
 641         int idx = reg1->idx;
 642         u64 mask, config1 = reg1->config;
 643         bool ok = false;
 644
 645         if (reg1->idx == EXTRA_REG_NONE ||
 646             (!uncore_box_is_fake(box) && reg1->alloc))
 647                 return NULL;
 648 again:
 649         mask = 0xffULL << (idx * 8);
 650         raw_spin_lock_irqsave(&er->lock, flags);
 651         if (!__BITS_VALUE(atomic_read(&er->ref), idx, 8) ||
 652             !((config1 ^ er->config) & mask)) {
 653                 atomic_add(1 << (idx * 8), &er->ref);
 654                 er->config &= ~mask;
 655                 er->config |= config1 & mask;
 656                 ok = true;
 657         }
 658         raw_spin_unlock_irqrestore(&er->lock, flags);
 659
 660         if (!ok) {
 661                 idx = (idx + 1) % 4;
 662                 if (idx != reg1->idx) {
 663                         config1 = snbep_pcu_alter_er(event, idx, false);
 664                         goto again;
 665                 }
 666                 return &constraint_empty;
 667         }
 668
 669         if (!uncore_box_is_fake(box)) {
 670                 if (idx != reg1->idx)
 671                         snbep_pcu_alter_er(event, idx, true);
 672                 reg1->alloc = 1;
 673         }
 674         return NULL;
 675 }
 676
 677 static void snbep_pcu_put_constraint(struct intel_uncore_box *box, struct perf_event *event)
 678 {
 679         struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
 680         struct intel_uncore_extra_reg *er = &box->shared_regs[0];
 681
 682         if (uncore_box_is_fake(box) || !reg1->alloc)
 683                 return;
 684
 685         atomic_sub(1 << (reg1->idx * 8), &er->ref);
 686         reg1->alloc = 0;
 687 }
 688
 689 static int snbep_pcu_hw_config(struct intel_uncore_box *box, struct perf_event *event)
 690 {
 691         struct hw_perf_event *hwc = &event->hw;
 692         struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
 693         int ev_sel = hwc->config & SNBEP_PMON_CTL_EV_SEL_MASK;
 694
 695         if (ev_sel >= 0xb && ev_sel <= 0xe) {
 696                 reg1->reg = SNBEP_PCU_MSR_PMON_BOX_FILTER;
 697                 reg1->idx = ev_sel - 0xb;
 698                 reg1->config = event->attr.config1 & (0xff << reg1->idx);
 699         }
 700         return 0;
 701 }
 702
 703 static struct intel_uncore_ops snbep_uncore_pcu_ops = {
 704         SNBEP_UNCORE_MSR_OPS_COMMON_INIT(),
 705         .hw_config              = snbep_pcu_hw_config,
 706         .get_constraint         = snbep_pcu_get_constraint,
 707         .put_constraint         = snbep_pcu_put_constraint,
 708 };
 709
 710 static struct intel_uncore_type snbep_uncore_pcu = {
 711         .name                   = "pcu",
 712         .num_counters           = 4,
 713         .num_boxes              = 1,
 714         .perf_ctr_bits          = 48,
 715         .perf_ctr               = SNBEP_PCU_MSR_PMON_CTR0,
 716         .event_ctl              = SNBEP_PCU_MSR_PMON_CTL0,
 717         .event_mask             = SNBEP_PCU_MSR_PMON_RAW_EVENT_MASK,
 718         .box_ctl                = SNBEP_PCU_MSR_PMON_BOX_CTL,
 719         .num_shared_regs        = 1,
 720         .ops                    = &snbep_uncore_pcu_ops,
 721         .format_group           = &snbep_uncore_pcu_format_group,
 722 };
 723
 724 static struct intel_uncore_type *snbep_msr_uncores[] = {
 725         &snbep_uncore_ubox,
 726         &snbep_uncore_cbox,
 727         &snbep_uncore_pcu,
 728         NULL,
 729 };
 730
 731 #define SNBEP_UNCORE_PCI_COMMON_INIT()                          \
 732         .perf_ctr       = SNBEP_PCI_PMON_CTR0,                  \
 733         .event_ctl      = SNBEP_PCI_PMON_CTL0,                  \
 734         .event_mask     = SNBEP_PMON_RAW_EVENT_MASK,            \
 735         .box_ctl        = SNBEP_PCI_PMON_BOX_CTL,               \
 736         .ops            = &snbep_uncore_pci_ops,                \
 737         .format_group   = &snbep_uncore_format_group
 738
 739 static struct intel_uncore_type snbep_uncore_ha = {
 740         .name           = "ha",
 741         .num_counters   = 4,
 742         .num_boxes      = 1,
 743         .perf_ctr_bits  = 48,
 744         SNBEP_UNCORE_PCI_COMMON_INIT(),
 745 };
 746
 747 static struct intel_uncore_type snbep_uncore_imc = {
 748         .name           = "imc",
 749         .num_counters   = 4,
 750         .num_boxes      = 4,
 751         .perf_ctr_bits  = 48,
 752         .fixed_ctr_bits = 48,
 753         .fixed_ctr      = SNBEP_MC_CHy_PCI_PMON_FIXED_CTR,
 754         .fixed_ctl      = SNBEP_MC_CHy_PCI_PMON_FIXED_CTL,
 755         .event_descs    = snbep_uncore_imc_events,
 756         SNBEP_UNCORE_PCI_COMMON_INIT(),
 757 };
 758
 759 static struct intel_uncore_type snbep_uncore_qpi = {
 760         .name           = "qpi",
 761         .num_counters   = 4,
 762         .num_boxes      = 2,
 763         .perf_ctr_bits  = 48,
 764         .perf_ctr       = SNBEP_PCI_PMON_CTR0,
 765         .event_ctl      = SNBEP_PCI_PMON_CTL0,
 766         .event_mask     = SNBEP_QPI_PCI_PMON_RAW_EVENT_MASK,
 767         .box_ctl        = SNBEP_PCI_PMON_BOX_CTL,
 768         .ops            = &snbep_uncore_pci_ops,
 769         .event_descs    = snbep_uncore_qpi_events,
 770         .format_group   = &snbep_uncore_qpi_format_group,
 771 };
 772
 773
 774 static struct intel_uncore_type snbep_uncore_r2pcie = {
 775         .name           = "r2pcie",
 776         .num_counters   = 4,
 777         .num_boxes      = 1,
 778         .perf_ctr_bits  = 44,
 779         .constraints    = snbep_uncore_r2pcie_constraints,
 780         SNBEP_UNCORE_PCI_COMMON_INIT(),
 781 };
 782
 783 static struct intel_uncore_type snbep_uncore_r3qpi = {
 784         .name           = "r3qpi",
 785         .num_counters   = 3,
 786         .num_boxes      = 2,
 787         .perf_ctr_bits  = 44,
 788         .constraints    = snbep_uncore_r3qpi_constraints,
 789         SNBEP_UNCORE_PCI_COMMON_INIT(),
 790 };
 791
 792 enum {
 793         SNBEP_PCI_UNCORE_HA,
 794         SNBEP_PCI_UNCORE_IMC,
 795         SNBEP_PCI_UNCORE_QPI,
 796         SNBEP_PCI_UNCORE_R2PCIE,
 797         SNBEP_PCI_UNCORE_R3QPI,
 798 };
 799
 800 static struct intel_uncore_type *snbep_pci_uncores[] = {
 801         [SNBEP_PCI_UNCORE_HA]           = &snbep_uncore_ha,
 802         [SNBEP_PCI_UNCORE_IMC]          = &snbep_uncore_imc,
 803         [SNBEP_PCI_UNCORE_QPI]          = &snbep_uncore_qpi,
 804         [SNBEP_PCI_UNCORE_R2PCIE]       = &snbep_uncore_r2pcie,
 805         [SNBEP_PCI_UNCORE_R3QPI]        = &snbep_uncore_r3qpi,
 806         NULL,
 807 };
 808
 809 static DEFINE_PCI_DEVICE_TABLE(snbep_uncore_pci_ids) = {
 810         { /* Home Agent */
 811                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_HA),
 812                 .driver_data = UNCORE_PCI_DEV_DATA(SNBEP_PCI_UNCORE_HA, 0),
 813         },
 814         { /* MC Channel 0 */
 815                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_IMC0),
 816                 .driver_data = UNCORE_PCI_DEV_DATA(SNBEP_PCI_UNCORE_IMC, 0),
 817         },
 818         { /* MC Channel 1 */
 819                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_IMC1),
 820                 .driver_data = UNCORE_PCI_DEV_DATA(SNBEP_PCI_UNCORE_IMC, 1),
 821         },
 822         { /* MC Channel 2 */
 823                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_IMC2),
 824                 .driver_data = UNCORE_PCI_DEV_DATA(SNBEP_PCI_UNCORE_IMC, 2),
 825         },
 826         { /* MC Channel 3 */
 827                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_IMC3),
 828                 .driver_data = UNCORE_PCI_DEV_DATA(SNBEP_PCI_UNCORE_IMC, 3),
 829         },
 830         { /* QPI Port 0 */
 831                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_QPI0),
 832                 .driver_data = UNCORE_PCI_DEV_DATA(SNBEP_PCI_UNCORE_QPI, 0),
 833         },
 834         { /* QPI Port 1 */
 835                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_QPI1),
 836                 .driver_data = UNCORE_PCI_DEV_DATA(SNBEP_PCI_UNCORE_QPI, 1),
 837         },
 838         { /* R2PCIe */
 839                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_R2PCIE),
 840                 .driver_data = UNCORE_PCI_DEV_DATA(SNBEP_PCI_UNCORE_R2PCIE, 0),
 841         },
 842         { /* R3QPI Link 0 */
 843                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_R3QPI0),
 844                 .driver_data = UNCORE_PCI_DEV_DATA(SNBEP_PCI_UNCORE_R3QPI, 0),
 845         },
 846         { /* R3QPI Link 1 */
 847                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_R3QPI1),
 848                 .driver_data = UNCORE_PCI_DEV_DATA(SNBEP_PCI_UNCORE_R3QPI, 1),
 849         },
 850         { /* end: all zeroes */ }
 851 };
 852
 853 static struct pci_driver snbep_uncore_pci_driver = {
 854         .name           = "snbep_uncore",
 855         .id_table       = snbep_uncore_pci_ids,
 856 };
 857
 858 /*
 859  * build pci bus to socket mapping
 860  */
 861 static int snbep_pci2phy_map_init(int devid)
 862 {
 863         struct pci_dev *ubox_dev = NULL;
 864         int i, bus, nodeid;
 865         int err = 0;
 866         u32 config = 0;
 867
 868         while (1) {
 869                 /* find the UBOX device */
 870                 ubox_dev = pci_get_device(PCI_VENDOR_ID_INTEL, devid, ubox_dev);
 871                 if (!ubox_dev)
 872                         break;
 873                 bus = ubox_dev->bus->number;
 874                 /* get the Node ID of the local register */
 875                 err = pci_read_config_dword(ubox_dev, 0x40, &config);
 876                 if (err)
 877                         break;
 878                 nodeid = config;
 879                 /* get the Node ID mapping */
 880                 err = pci_read_config_dword(ubox_dev, 0x54, &config);
 881                 if (err)
 882                         break;
 883                 /*
 884                  * every three bits in the Node ID mapping register maps
 885                  * to a particular node.
 886                  */
 887                 for (i = 0; i < 8; i++) {
 888                         if (nodeid == ((config >> (3 * i)) & 0x7)) {
 889                                 pcibus_to_physid[bus] = i;
 890                                 break;
 891                         }
 892                 }
 893         }
 894
 895         if (ubox_dev)
 896                 pci_dev_put(ubox_dev);
 897
 898         return err ? pcibios_err_to_errno(err) : 0;
 899 }
 900 /* end of Sandy Bridge-EP uncore support */
 901
 902 /* IvyTown uncore support */
 903 static void ivt_uncore_msr_init_box(struct intel_uncore_box *box)
 904 {
 905         unsigned msr = uncore_msr_box_ctl(box);
 906         if (msr)
 907                 wrmsrl(msr, IVT_PMON_BOX_CTL_INT);
 908 }
 909
 910 static void ivt_uncore_pci_init_box(struct intel_uncore_box *box)
 911 {
 912         struct pci_dev *pdev = box->pci_dev;
 913
 914         pci_write_config_dword(pdev, SNBEP_PCI_PMON_BOX_CTL, IVT_PMON_BOX_CTL_INT);
 915 }
 916
 917 #define IVT_UNCORE_MSR_OPS_COMMON_INIT()                        \
 918         .init_box       = ivt_uncore_msr_init_box,              \
 919         .disable_box    = snbep_uncore_msr_disable_box,         \
 920         .enable_box     = snbep_uncore_msr_enable_box,          \
 921         .disable_event  = snbep_uncore_msr_disable_event,       \
 922         .enable_event   = snbep_uncore_msr_enable_event,        \
 923         .read_counter   = uncore_msr_read_counter
 924
 925 static struct intel_uncore_ops ivt_uncore_msr_ops = {
 926         IVT_UNCORE_MSR_OPS_COMMON_INIT(),
 927 };
 928
 929 static struct intel_uncore_ops ivt_uncore_pci_ops = {
 930         .init_box       = ivt_uncore_pci_init_box,
 931         .disable_box    = snbep_uncore_pci_disable_box,
 932         .enable_box     = snbep_uncore_pci_enable_box,
 933         .disable_event  = snbep_uncore_pci_disable_event,
 934         .enable_event   = snbep_uncore_pci_enable_event,
 935         .read_counter   = snbep_uncore_pci_read_counter,
 936 };
 937
 938 #define IVT_UNCORE_PCI_COMMON_INIT()                            \
 939         .perf_ctr       = SNBEP_PCI_PMON_CTR0,                  \
 940         .event_ctl      = SNBEP_PCI_PMON_CTL0,                  \
 941         .event_mask     = IVT_PMON_RAW_EVENT_MASK,              \
 942         .box_ctl        = SNBEP_PCI_PMON_BOX_CTL,               \
 943         .ops            = &ivt_uncore_pci_ops,                  \
 944         .format_group   = &ivt_uncore_format_group
 945
 946 static struct attribute *ivt_uncore_formats_attr[] = {
 947         &format_attr_event.attr,
 948         &format_attr_umask.attr,
 949         &format_attr_edge.attr,
 950         &format_attr_inv.attr,
 951         &format_attr_thresh8.attr,
 952         NULL,
 953 };
 954
 955 static struct attribute *ivt_uncore_ubox_formats_attr[] = {
 956         &format_attr_event.attr,
 957         &format_attr_umask.attr,
 958         &format_attr_edge.attr,
 959         &format_attr_inv.attr,
 960         &format_attr_thresh5.attr,
 961         NULL,
 962 };
 963
 964 static struct attribute *ivt_uncore_cbox_formats_attr[] = {
 965         &format_attr_event.attr,
 966         &format_attr_umask.attr,
 967         &format_attr_edge.attr,
 968         &format_attr_tid_en.attr,
 969         &format_attr_thresh8.attr,
 970         &format_attr_filter_tid.attr,
 971         &format_attr_filter_link.attr,
 972         &format_attr_filter_state2.attr,
 973         &format_attr_filter_nid2.attr,
 974         &format_attr_filter_opc2.attr,
 975         NULL,
 976 };
 977
 978 static struct attribute *ivt_uncore_pcu_formats_attr[] = {
 979         &format_attr_event_ext.attr,
 980         &format_attr_occ_sel.attr,
 981         &format_attr_edge.attr,
 982         &format_attr_thresh5.attr,
 983         &format_attr_occ_invert.attr,
 984         &format_attr_occ_edge.attr,
 985         &format_attr_filter_band0.attr,
 986         &format_attr_filter_band1.attr,
 987         &format_attr_filter_band2.attr,
 988         &format_attr_filter_band3.attr,
 989         NULL,
 990 };
 991
 992 static struct attribute *ivt_uncore_qpi_formats_attr[] = {
 993         &format_attr_event_ext.attr,
 994         &format_attr_umask.attr,
 995         &format_attr_edge.attr,
 996         &format_attr_thresh8.attr,
 997         NULL,
 998 };
 999
1000 static struct attribute_group ivt_uncore_format_group = {
1001         .name = "format",
1002         .attrs = ivt_uncore_formats_attr,
1003 };
1004
1005 static struct attribute_group ivt_uncore_ubox_format_group = {
1006         .name = "format",
1007         .attrs = ivt_uncore_ubox_formats_attr,
1008 };
1009
1010 static struct attribute_group ivt_uncore_cbox_format_group = {
1011         .name = "format",
1012         .attrs = ivt_uncore_cbox_formats_attr,
1013 };
1014
1015 static struct attribute_group ivt_uncore_pcu_format_group = {
1016         .name = "format",
1017         .attrs = ivt_uncore_pcu_formats_attr,
1018 };
1019
1020 static struct attribute_group ivt_uncore_qpi_format_group = {
1021         .name = "format",
1022         .attrs = ivt_uncore_qpi_formats_attr,
1023 };
1024
1025 static struct intel_uncore_type ivt_uncore_ubox = {
1026         .name           = "ubox",
1027         .num_counters   = 2,
1028         .num_boxes      = 1,
1029         .perf_ctr_bits  = 44,
1030         .fixed_ctr_bits = 48,
1031         .perf_ctr       = SNBEP_U_MSR_PMON_CTR0,
1032         .event_ctl      = SNBEP_U_MSR_PMON_CTL0,
1033         .event_mask     = IVT_U_MSR_PMON_RAW_EVENT_MASK,
1034         .fixed_ctr      = SNBEP_U_MSR_PMON_UCLK_FIXED_CTR,
1035         .fixed_ctl      = SNBEP_U_MSR_PMON_UCLK_FIXED_CTL,
1036         .ops            = &ivt_uncore_msr_ops,
1037         .format_group   = &ivt_uncore_ubox_format_group,
1038 };
1039
1040 static struct extra_reg ivt_uncore_cbox_extra_regs[] = {
1041         SNBEP_CBO_EVENT_EXTRA_REG(SNBEP_CBO_PMON_CTL_TID_EN,
1042                                   SNBEP_CBO_PMON_CTL_TID_EN, 0x1),
1043         SNBEP_CBO_EVENT_EXTRA_REG(0x1031, 0x10ff, 0x2),
1044         SNBEP_CBO_EVENT_EXTRA_REG(0x0334, 0xffff, 0x4),
1045         SNBEP_CBO_EVENT_EXTRA_REG(0x0534, 0xffff, 0x4),
1046         SNBEP_CBO_EVENT_EXTRA_REG(0x0934, 0xffff, 0x4),
1047         SNBEP_CBO_EVENT_EXTRA_REG(0x4134, 0xffff, 0xc),
1048         SNBEP_CBO_EVENT_EXTRA_REG(0x0135, 0xffff, 0x10),
1049         SNBEP_CBO_EVENT_EXTRA_REG(0x0335, 0xffff, 0x10),
1050         SNBEP_CBO_EVENT_EXTRA_REG(0x2135, 0xffff, 0x10),
1051         SNBEP_CBO_EVENT_EXTRA_REG(0x2335, 0xffff, 0x10),
1052         SNBEP_CBO_EVENT_EXTRA_REG(0x4135, 0xffff, 0x18),
1053         SNBEP_CBO_EVENT_EXTRA_REG(0x4335, 0xffff, 0x18),
1054         SNBEP_CBO_EVENT_EXTRA_REG(0x4435, 0xffff, 0x8),
1055         SNBEP_CBO_EVENT_EXTRA_REG(0x4835, 0xffff, 0x8),
1056         SNBEP_CBO_EVENT_EXTRA_REG(0x4a35, 0xffff, 0x8),
1057         SNBEP_CBO_EVENT_EXTRA_REG(0x5035, 0xffff, 0x8),
1058         SNBEP_CBO_EVENT_EXTRA_REG(0x8135, 0xffff, 0x10),
1059         SNBEP_CBO_EVENT_EXTRA_REG(0x8335, 0xffff, 0x10),
1060         SNBEP_CBO_EVENT_EXTRA_REG(0x0136, 0xffff, 0x10),
1061         SNBEP_CBO_EVENT_EXTRA_REG(0x0336, 0xffff, 0x10),
1062         SNBEP_CBO_EVENT_EXTRA_REG(0x2336, 0xffff, 0x10),
1063         SNBEP_CBO_EVENT_EXTRA_REG(0x2336, 0xffff, 0x10),
1064         SNBEP_CBO_EVENT_EXTRA_REG(0x4136, 0xffff, 0x18),
1065         SNBEP_CBO_EVENT_EXTRA_REG(0x4336, 0xffff, 0x18),
1066         SNBEP_CBO_EVENT_EXTRA_REG(0x4436, 0xffff, 0x8),
1067         SNBEP_CBO_EVENT_EXTRA_REG(0x4836, 0xffff, 0x8),
1068         SNBEP_CBO_EVENT_EXTRA_REG(0x4a36, 0xffff, 0x8),
1069         SNBEP_CBO_EVENT_EXTRA_REG(0x5036, 0xffff, 0x8),
1070         SNBEP_CBO_EVENT_EXTRA_REG(0x8136, 0xffff, 0x10),
1071         SNBEP_CBO_EVENT_EXTRA_REG(0x8336, 0xffff, 0x10),
1072         SNBEP_CBO_EVENT_EXTRA_REG(0x4037, 0x40ff, 0x8),
1073         EVENT_EXTRA_END
1074 };
1075
1076 static u64 ivt_cbox_filter_mask(int fields)
1077 {
1078         u64 mask = 0;
1079
1080         if (fields & 0x1)
1081                 mask |= IVT_CB0_MSR_PMON_BOX_FILTER_TID;
1082         if (fields & 0x2)
1083                 mask |= IVT_CB0_MSR_PMON_BOX_FILTER_LINK;
1084         if (fields & 0x4)
1085                 mask |= IVT_CB0_MSR_PMON_BOX_FILTER_STATE;
1086         if (fields & 0x8)
1087                 mask |= IVT_CB0_MSR_PMON_BOX_FILTER_NID;
1088         if (fields & 0x10)
1089                 mask |= IVT_CB0_MSR_PMON_BOX_FILTER_OPC;
1090
1091         return mask;
1092 }
1093
1094 static struct event_constraint *
1095 ivt_cbox_get_constraint(struct intel_uncore_box *box, struct perf_event *event)
1096 {
1097         return __snbep_cbox_get_constraint(box, event, ivt_cbox_filter_mask);
1098 }
1099
1100 static int ivt_cbox_hw_config(struct intel_uncore_box *box, struct perf_event *event)
1101 {
1102         struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
1103         struct extra_reg *er;
1104         int idx = 0;
1105
1106         for (er = ivt_uncore_cbox_extra_regs; er->msr; er++) {
1107                 if (er->event != (event->hw.config & er->config_mask))
1108                         continue;
1109                 idx |= er->idx;
1110         }
1111
1112         if (idx) {
1113                 reg1->reg = SNBEP_C0_MSR_PMON_BOX_FILTER +
1114                         SNBEP_CBO_MSR_OFFSET * box->pmu->pmu_idx;
1115                 reg1->config = event->attr.config1 & ivt_cbox_filter_mask(idx);
1116                 reg1->idx = idx;
1117         }
1118         return 0;
1119 }
1120
1121 static void ivt_cbox_enable_event(struct intel_uncore_box *box, struct perf_event *event)
1122 {
1123         struct hw_perf_event *hwc = &event->hw;
1124         struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
1125
1126         if (reg1->idx != EXTRA_REG_NONE) {
1127                 u64 filter = uncore_shared_reg_config(box, 0);
1128                 wrmsrl(reg1->reg, filter & 0xffffffff);
1129                 wrmsrl(reg1->reg + 6, filter >> 32);
1130         }
1131
1132         wrmsrl(hwc->config_base, hwc->config | SNBEP_PMON_CTL_EN);
1133 }
1134
1135 static struct intel_uncore_ops ivt_uncore_cbox_ops = {
1136         .init_box               = ivt_uncore_msr_init_box,
1137         .disable_box            = snbep_uncore_msr_disable_box,
1138         .enable_box             = snbep_uncore_msr_enable_box,
1139         .disable_event          = snbep_uncore_msr_disable_event,
1140         .enable_event           = ivt_cbox_enable_event,
1141         .read_counter           = uncore_msr_read_counter,
1142         .hw_config              = ivt_cbox_hw_config,
1143         .get_constraint         = ivt_cbox_get_constraint,
1144         .put_constraint         = snbep_cbox_put_constraint,
1145 };
1146
1147 static struct intel_uncore_type ivt_uncore_cbox = {
1148         .name                   = "cbox",
1149         .num_counters           = 4,
1150         .num_boxes              = 15,
1151         .perf_ctr_bits          = 44,
1152         .event_ctl              = SNBEP_C0_MSR_PMON_CTL0,
1153         .perf_ctr               = SNBEP_C0_MSR_PMON_CTR0,
1154         .event_mask             = IVT_CBO_MSR_PMON_RAW_EVENT_MASK,
1155         .box_ctl                = SNBEP_C0_MSR_PMON_BOX_CTL,
1156         .msr_offset             = SNBEP_CBO_MSR_OFFSET,
1157         .num_shared_regs        = 1,
1158         .constraints            = snbep_uncore_cbox_constraints,
1159         .ops                    = &ivt_uncore_cbox_ops,
1160         .format_group           = &ivt_uncore_cbox_format_group,
1161 };
1162
1163 static struct intel_uncore_ops ivt_uncore_pcu_ops = {
1164         IVT_UNCORE_MSR_OPS_COMMON_INIT(),
1165         .hw_config              = snbep_pcu_hw_config,
1166         .get_constraint         = snbep_pcu_get_constraint,
1167         .put_constraint         = snbep_pcu_put_constraint,
1168 };
1169
1170 static struct intel_uncore_type ivt_uncore_pcu = {
1171         .name                   = "pcu",
1172         .num_counters           = 4,
1173         .num_boxes              = 1,
1174         .perf_ctr_bits          = 48,
1175         .perf_ctr               = SNBEP_PCU_MSR_PMON_CTR0,
1176         .event_ctl              = SNBEP_PCU_MSR_PMON_CTL0,
1177         .event_mask             = IVT_PCU_MSR_PMON_RAW_EVENT_MASK,
1178         .box_ctl                = SNBEP_PCU_MSR_PMON_BOX_CTL,
1179         .num_shared_regs        = 1,
1180         .ops                    = &ivt_uncore_pcu_ops,
1181         .format_group           = &ivt_uncore_pcu_format_group,
1182 };
1183
1184 static struct intel_uncore_type *ivt_msr_uncores[] = {
1185         &ivt_uncore_ubox,
1186         &ivt_uncore_cbox,
1187         &ivt_uncore_pcu,
1188         NULL,
1189 };
1190
1191 static struct intel_uncore_type ivt_uncore_ha = {
1192         .name           = "ha",
1193         .num_counters   = 4,
1194         .num_boxes      = 2,
1195         .perf_ctr_bits  = 48,
1196         IVT_UNCORE_PCI_COMMON_INIT(),
1197 };
1198
1199 static struct intel_uncore_type ivt_uncore_imc = {
1200         .name           = "imc",
1201         .num_counters   = 4,
1202         .num_boxes      = 8,
1203         .perf_ctr_bits  = 48,
1204         .fixed_ctr_bits = 48,
1205         .fixed_ctr      = SNBEP_MC_CHy_PCI_PMON_FIXED_CTR,
1206         .fixed_ctl      = SNBEP_MC_CHy_PCI_PMON_FIXED_CTL,
1207         IVT_UNCORE_PCI_COMMON_INIT(),
1208 };
1209
1210 static struct intel_uncore_type ivt_uncore_qpi = {
1211         .name           = "qpi",
1212         .num_counters   = 4,
1213         .num_boxes      = 3,
1214         .perf_ctr_bits  = 48,
1215         .perf_ctr       = SNBEP_PCI_PMON_CTR0,
1216         .event_ctl      = SNBEP_PCI_PMON_CTL0,
1217         .event_mask     = IVT_QPI_PCI_PMON_RAW_EVENT_MASK,
1218         .box_ctl        = SNBEP_PCI_PMON_BOX_CTL,
1219         .ops            = &ivt_uncore_pci_ops,
1220         .format_group   = &ivt_uncore_qpi_format_group,
1221 };
1222
1223 static struct intel_uncore_type ivt_uncore_r2pcie = {
1224         .name           = "r2pcie",
1225         .num_counters   = 4,
1226         .num_boxes      = 1,
1227         .perf_ctr_bits  = 44,
1228         .constraints    = snbep_uncore_r2pcie_constraints,
1229         IVT_UNCORE_PCI_COMMON_INIT(),
1230 };
1231
1232 static struct intel_uncore_type ivt_uncore_r3qpi = {
1233         .name           = "r3qpi",
1234         .num_counters   = 3,
1235         .num_boxes      = 2,
1236         .perf_ctr_bits  = 44,
1237         .constraints    = snbep_uncore_r3qpi_constraints,
1238         IVT_UNCORE_PCI_COMMON_INIT(),
1239 };
1240
1241 enum {
1242         IVT_PCI_UNCORE_HA,
1243         IVT_PCI_UNCORE_IMC,
1244         IVT_PCI_UNCORE_QPI,
1245         IVT_PCI_UNCORE_R2PCIE,
1246         IVT_PCI_UNCORE_R3QPI,
1247 };
1248
1249 static struct intel_uncore_type *ivt_pci_uncores[] = {
1250         [IVT_PCI_UNCORE_HA]     = &ivt_uncore_ha,
1251         [IVT_PCI_UNCORE_IMC]    = &ivt_uncore_imc,
1252         [IVT_PCI_UNCORE_QPI]    = &ivt_uncore_qpi,
1253         [IVT_PCI_UNCORE_R2PCIE] = &ivt_uncore_r2pcie,
1254         [IVT_PCI_UNCORE_R3QPI]  = &ivt_uncore_r3qpi,
1255         NULL,
1256 };
1257
1258 static DEFINE_PCI_DEVICE_TABLE(ivt_uncore_pci_ids) = {
1259         { /* Home Agent 0 */
1260                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe30),
1261                 .driver_data = UNCORE_PCI_DEV_DATA(IVT_PCI_UNCORE_HA, 0),
1262         },
1263         { /* Home Agent 1 */
1264                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe38),
1265                 .driver_data = UNCORE_PCI_DEV_DATA(IVT_PCI_UNCORE_HA, 1),
1266         },
1267         { /* MC0 Channel 0 */
1268                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xeb4),
1269                 .driver_data = UNCORE_PCI_DEV_DATA(IVT_PCI_UNCORE_IMC, 0),
1270         },
1271         { /* MC0 Channel 1 */
1272                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xeb5),
1273                 .driver_data = UNCORE_PCI_DEV_DATA(IVT_PCI_UNCORE_IMC, 1),
1274         },
1275         { /* MC0 Channel 3 */
1276                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xeb0),
1277                 .driver_data = UNCORE_PCI_DEV_DATA(IVT_PCI_UNCORE_IMC, 2),
1278         },
1279         { /* MC0 Channel 4 */
1280                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xeb1),
1281                 .driver_data = UNCORE_PCI_DEV_DATA(IVT_PCI_UNCORE_IMC, 3),
1282         },
1283         { /* MC1 Channel 0 */
1284                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xef4),
1285                 .driver_data = UNCORE_PCI_DEV_DATA(IVT_PCI_UNCORE_IMC, 4),
1286         },
1287         { /* MC1 Channel 1 */
1288                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xef5),
1289                 .driver_data = UNCORE_PCI_DEV_DATA(IVT_PCI_UNCORE_IMC, 5),
1290         },
1291         { /* MC1 Channel 3 */
1292                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xef0),
1293                 .driver_data = UNCORE_PCI_DEV_DATA(IVT_PCI_UNCORE_IMC, 6),
1294         },
1295         { /* MC1 Channel 4 */
1296                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xef1),
1297                 .driver_data = UNCORE_PCI_DEV_DATA(IVT_PCI_UNCORE_IMC, 7),
1298         },
1299         { /* QPI0 Port 0 */
1300                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe32),
1301                 .driver_data = UNCORE_PCI_DEV_DATA(IVT_PCI_UNCORE_QPI, 0),
1302         },
1303         { /* QPI0 Port 1 */
1304                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe33),
1305                 .driver_data = UNCORE_PCI_DEV_DATA(IVT_PCI_UNCORE_QPI, 1),
1306         },
1307         { /* QPI1 Port 2 */
1308                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe3a),
1309                 .driver_data = UNCORE_PCI_DEV_DATA(IVT_PCI_UNCORE_QPI, 2),
1310         },
1311         { /* R2PCIe */
1312                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe34),
1313                 .driver_data = UNCORE_PCI_DEV_DATA(IVT_PCI_UNCORE_R2PCIE, 0),
1314         },
1315         { /* R3QPI0 Link 0 */
1316                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe36),
1317                 .driver_data = UNCORE_PCI_DEV_DATA(IVT_PCI_UNCORE_R3QPI, 0),
1318         },
1319         { /* R3QPI0 Link 1 */
1320                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe37),
1321                 .driver_data = UNCORE_PCI_DEV_DATA(IVT_PCI_UNCORE_R3QPI, 1),
1322         },
1323         { /* R3QPI1 Link 2 */
1324                 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe3e),
1325                 .driver_data = UNCORE_PCI_DEV_DATA(IVT_PCI_UNCORE_R3QPI, 2),
1326         },
1327         { /* end: all zeroes */ }
1328 };
1329
1330 static struct pci_driver ivt_uncore_pci_driver = {
1331         .name           = "ivt_uncore",
1332         .id_table       = ivt_uncore_pci_ids,
1333 };
1334 /* end of IvyTown uncore support */
1335
1336 /* Sandy Bridge uncore support */
1337 static void snb_uncore_msr_enable_event(struct intel_uncore_box *box, struct perf_event *event)
1338 {
1339         struct hw_perf_event *hwc = &event->hw;
1340
1341         if (hwc->idx < UNCORE_PMC_IDX_FIXED)
1342                 wrmsrl(hwc->config_base, hwc->config | SNB_UNC_CTL_EN);
1343         else
1344                 wrmsrl(hwc->config_base, SNB_UNC_CTL_EN);
1345 }
1346
1347 static void snb_uncore_msr_disable_event(struct intel_uncore_box *box, struct perf_event *event)
1348 {
1349         wrmsrl(event->hw.config_base, 0);
1350 }
1351
1352 static void snb_uncore_msr_init_box(struct intel_uncore_box *box)
1353 {
1354         if (box->pmu->pmu_idx == 0) {
1355                 wrmsrl(SNB_UNC_PERF_GLOBAL_CTL,
1356                         SNB_UNC_GLOBAL_CTL_EN | SNB_UNC_GLOBAL_CTL_CORE_ALL);
1357         }
1358 }
1359
1360 static struct uncore_event_desc snb_uncore_events[] = {
1361         INTEL_UNCORE_EVENT_DESC(clockticks, "event=0xff,umask=0x00"),
1362         { /* end: all zeroes */ },
1363 };
1364
1365 static struct attribute *snb_uncore_formats_attr[] = {
1366         &format_attr_event.attr,
1367         &format_attr_umask.attr,
1368         &format_attr_edge.attr,
1369         &format_attr_inv.attr,
1370         &format_attr_cmask5.attr,
1371         NULL,
1372 };
1373
1374 static struct attribute_group snb_uncore_format_group = {
1375         .name           = "format",
1376         .attrs          = snb_uncore_formats_attr,
1377 };
1378
1379 static struct intel_uncore_ops snb_uncore_msr_ops = {
1380         .init_box       = snb_uncore_msr_init_box,
1381         .disable_event  = snb_uncore_msr_disable_event,
1382         .enable_event   = snb_uncore_msr_enable_event,
1383         .read_counter   = uncore_msr_read_counter,
1384 };
1385
1386 static struct event_constraint snb_uncore_cbox_constraints[] = {
1387         UNCORE_EVENT_CONSTRAINT(0x80, 0x1),
1388         UNCORE_EVENT_CONSTRAINT(0x83, 0x1),
1389         EVENT_CONSTRAINT_END
1390 };
1391
1392 static struct intel_uncore_type snb_uncore_cbox = {
1393         .name           = "cbox",
1394         .num_counters   = 2,
1395         .num_boxes      = 4,
1396         .perf_ctr_bits  = 44,
1397         .fixed_ctr_bits = 48,
1398         .perf_ctr       = SNB_UNC_CBO_0_PER_CTR0,
1399         .event_ctl      = SNB_UNC_CBO_0_PERFEVTSEL0,
1400         .fixed_ctr      = SNB_UNC_FIXED_CTR,
1401         .fixed_ctl      = SNB_UNC_FIXED_CTR_CTRL,
1402         .single_fixed   = 1,
1403         .event_mask     = SNB_UNC_RAW_EVENT_MASK,
1404         .msr_offset     = SNB_UNC_CBO_MSR_OFFSET,
1405         .constraints    = snb_uncore_cbox_constraints,
1406         .ops            = &snb_uncore_msr_ops,
1407         .format_group   = &snb_uncore_format_group,
1408         .event_descs    = snb_uncore_events,
1409 };
1410
1411 static struct intel_uncore_type *snb_msr_uncores[] = {
1412         &snb_uncore_cbox,
1413         NULL,
1414 };
1415 /* end of Sandy Bridge uncore support */
1416
1417 /* Nehalem uncore support */
1418 static void nhm_uncore_msr_disable_box(struct intel_uncore_box *box)
1419 {
1420         wrmsrl(NHM_UNC_PERF_GLOBAL_CTL, 0);
1421 }
1422
1423 static void nhm_uncore_msr_enable_box(struct intel_uncore_box *box)
1424 {
1425         wrmsrl(NHM_UNC_PERF_GLOBAL_CTL, NHM_UNC_GLOBAL_CTL_EN_PC_ALL | NHM_UNC_GLOBAL_CTL_EN_FC);
1426 }
1427
1428 static void nhm_uncore_msr_enable_event(struct intel_uncore_box *box, struct perf_event *event)
1429 {
1430         struct hw_perf_event *hwc = &event->hw;
1431
1432         if (hwc->idx < UNCORE_PMC_IDX_FIXED)
1433                 wrmsrl(hwc->config_base, hwc->config | SNB_UNC_CTL_EN);
1434         else
1435                 wrmsrl(hwc->config_base, NHM_UNC_FIXED_CTR_CTL_EN);
1436 }
1437
1438 static struct attribute *nhm_uncore_formats_attr[] = {
1439         &format_attr_event.attr,
1440         &format_attr_umask.attr,
1441         &format_attr_edge.attr,
1442         &format_attr_inv.attr,
1443         &format_attr_cmask8.attr,
1444         NULL,
1445 };
1446
1447 static struct attribute_group nhm_uncore_format_group = {
1448         .name = "format",
1449         .attrs = nhm_uncore_formats_attr,
1450 };
1451
1452 static struct uncore_event_desc nhm_uncore_events[] = {
1453         INTEL_UNCORE_EVENT_DESC(clockticks,                "event=0xff,umask=0x00"),
1454         INTEL_UNCORE_EVENT_DESC(qmc_writes_full_any,       "event=0x2f,umask=0x0f"),
1455         INTEL_UNCORE_EVENT_DESC(qmc_normal_reads_any,      "event=0x2c,umask=0x0f"),
1456         INTEL_UNCORE_EVENT_DESC(qhl_request_ioh_reads,     "event=0x20,umask=0x01"),
1457         INTEL_UNCORE_EVENT_DESC(qhl_request_ioh_writes,    "event=0x20,umask=0x02"),
1458         INTEL_UNCORE_EVENT_DESC(qhl_request_remote_reads,  "event=0x20,umask=0x04"),
1459         INTEL_UNCORE_EVENT_DESC(qhl_request_remote_writes, "event=0x20,umask=0x08"),
1460         INTEL_UNCORE_EVENT_DESC(qhl_request_local_reads,   "event=0x20,umask=0x10"),
1461         INTEL_UNCORE_EVENT_DESC(qhl_request_local_writes,  "event=0x20,umask=0x20"),
1462         { /* end: all zeroes */ },
1463 };
1464
1465 static struct intel_uncore_ops nhm_uncore_msr_ops = {
1466         .disable_box    = nhm_uncore_msr_disable_box,
1467         .enable_box     = nhm_uncore_msr_enable_box,
1468         .disable_event  = snb_uncore_msr_disable_event,
1469         .enable_event   = nhm_uncore_msr_enable_event,
1470         .read_counter   = uncore_msr_read_counter,
1471 };
1472
1473 static struct intel_uncore_type nhm_uncore = {
1474         .name           = "",
1475         .num_counters   = 8,
1476         .num_boxes      = 1,
1477         .perf_ctr_bits  = 48,
1478         .fixed_ctr_bits = 48,
1479         .event_ctl      = NHM_UNC_PERFEVTSEL0,
1480         .perf_ctr       = NHM_UNC_UNCORE_PMC0,
1481         .fixed_ctr      = NHM_UNC_FIXED_CTR,
1482         .fixed_ctl      = NHM_UNC_FIXED_CTR_CTRL,
1483         .event_mask     = NHM_UNC_RAW_EVENT_MASK,
1484         .event_descs    = nhm_uncore_events,
1485         .ops            = &nhm_uncore_msr_ops,
1486         .format_group   = &nhm_uncore_format_group,
1487 };
1488
1489 static struct intel_uncore_type *nhm_msr_uncores[] = {
1490         &nhm_uncore,
1491         NULL,
1492 };
1493 /* end of Nehalem uncore support */
1494
1495 /* Nehalem-EX uncore support */
1496 DEFINE_UNCORE_FORMAT_ATTR(event5, event, "config:1-5");
1497 DEFINE_UNCORE_FORMAT_ATTR(counter, counter, "config:6-7");
1498 DEFINE_UNCORE_FORMAT_ATTR(match, match, "config1:0-63");
1499 DEFINE_UNCORE_FORMAT_ATTR(mask, mask, "config2:0-63");
1500
1501 static void nhmex_uncore_msr_init_box(struct intel_uncore_box *box)
1502 {
1503         wrmsrl(NHMEX_U_MSR_PMON_GLOBAL_CTL, NHMEX_U_PMON_GLOBAL_EN_ALL);
1504 }
1505
1506 static void nhmex_uncore_msr_disable_box(struct intel_uncore_box *box)
1507 {
1508         unsigned msr = uncore_msr_box_ctl(box);
1509         u64 config;
1510
1511         if (msr) {
1512                 rdmsrl(msr, config);
1513                 config &= ~((1ULL << uncore_num_counters(box)) - 1);
1514                 /* WBox has a fixed counter */
1515                 if (uncore_msr_fixed_ctl(box))
1516                         config &= ~NHMEX_W_PMON_GLOBAL_FIXED_EN;
1517                 wrmsrl(msr, config);
1518         }
1519 }
1520
1521 static void nhmex_uncore_msr_enable_box(struct intel_uncore_box *box)
1522 {
1523         unsigned msr = uncore_msr_box_ctl(box);
1524         u64 config;
1525
1526         if (msr) {
1527                 rdmsrl(msr, config);
1528                 config |= (1ULL << uncore_num_counters(box)) - 1;
1529                 /* WBox has a fixed counter */
1530                 if (uncore_msr_fixed_ctl(box))
1531                         config |= NHMEX_W_PMON_GLOBAL_FIXED_EN;
1532                 wrmsrl(msr, config);
1533         }
1534 }
1535
1536 static void nhmex_uncore_msr_disable_event(struct intel_uncore_box *box, struct perf_event *event)
1537 {
1538         wrmsrl(event->hw.config_base, 0);
1539 }
1540
1541 static void nhmex_uncore_msr_enable_event(struct intel_uncore_box *box, struct perf_event *event)
1542 {
1543         struct hw_perf_event *hwc = &event->hw;
1544
1545         if (hwc->idx >= UNCORE_PMC_IDX_FIXED)
1546                 wrmsrl(hwc->config_base, NHMEX_PMON_CTL_EN_BIT0);
1547         else if (box->pmu->type->event_mask & NHMEX_PMON_CTL_EN_BIT0)
1548                 wrmsrl(hwc->config_base, hwc->config | NHMEX_PMON_CTL_EN_BIT22);
1549         else
1550                 wrmsrl(hwc->config_base, hwc->config | NHMEX_PMON_CTL_EN_BIT0);
1551 }
1552
1553 #define NHMEX_UNCORE_OPS_COMMON_INIT()                          \
1554         .init_box       = nhmex_uncore_msr_init_box,            \
1555         .disable_box    = nhmex_uncore_msr_disable_box,         \
1556         .enable_box     = nhmex_uncore_msr_enable_box,          \
1557         .disable_event  = nhmex_uncore_msr_disable_event,       \
1558         .read_counter   = uncore_msr_read_counter
1559
1560 static struct intel_uncore_ops nhmex_uncore_ops = {
1561         NHMEX_UNCORE_OPS_COMMON_INIT(),
1562         .enable_event   = nhmex_uncore_msr_enable_event,
1563 };
1564
1565 static struct attribute *nhmex_uncore_ubox_formats_attr[] = {
1566         &format_attr_event.attr,
1567         &format_attr_edge.attr,
1568         NULL,
1569 };
1570
1571 static struct attribute_group nhmex_uncore_ubox_format_group = {
1572         .name           = "format",
1573         .attrs          = nhmex_uncore_ubox_formats_attr,
1574 };
1575
1576 static struct intel_uncore_type nhmex_uncore_ubox = {
1577         .name           = "ubox",
1578         .num_counters   = 1,
1579         .num_boxes      = 1,
1580         .perf_ctr_bits  = 48,
1581         .event_ctl      = NHMEX_U_MSR_PMON_EV_SEL,
1582         .perf_ctr       = NHMEX_U_MSR_PMON_CTR,
1583         .event_mask     = NHMEX_U_PMON_RAW_EVENT_MASK,
1584         .box_ctl        = NHMEX_U_MSR_PMON_GLOBAL_CTL,
1585         .ops            = &nhmex_uncore_ops,
1586         .format_group   = &nhmex_uncore_ubox_format_group
1587 };
1588
1589 static struct attribute *nhmex_uncore_cbox_formats_attr[] = {
1590         &format_attr_event.attr,
1591         &format_attr_umask.attr,
1592         &format_attr_edge.attr,
1593         &format_attr_inv.attr,
1594         &format_attr_thresh8.attr,
1595         NULL,
1596 };
1597
1598 static struct attribute_group nhmex_uncore_cbox_format_group = {
1599         .name = "format",
1600         .attrs = nhmex_uncore_cbox_formats_attr,
1601 };
1602
1603 /* msr offset for each instance of cbox */
1604 static unsigned nhmex_cbox_msr_offsets[] = {
1605         0x0, 0x80, 0x40, 0xc0, 0x20, 0xa0, 0x60, 0xe0, 0x240, 0x2c0,
1606 };
1607
1608 static struct intel_uncore_type nhmex_uncore_cbox = {
1609         .name                   = "cbox",
1610         .num_counters           = 6,
1611         .num_boxes              = 10,
1612         .perf_ctr_bits          = 48,
1613         .event_ctl              = NHMEX_C0_MSR_PMON_EV_SEL0,
1614         .perf_ctr               = NHMEX_C0_MSR_PMON_CTR0,
1615         .event_mask             = NHMEX_PMON_RAW_EVENT_MASK,
1616         .box_ctl                = NHMEX_C0_MSR_PMON_GLOBAL_CTL,
1617         .msr_offsets            = nhmex_cbox_msr_offsets,
1618         .pair_ctr_ctl           = 1,
1619         .ops                    = &nhmex_uncore_ops,
1620         .format_group           = &nhmex_uncore_cbox_format_group
1621 };
1622
1623 static struct uncore_event_desc nhmex_uncore_wbox_events[] = {
1624         INTEL_UNCORE_EVENT_DESC(clockticks, "event=0xff,umask=0"),
1625         { /* end: all zeroes */ },
1626 };
1627
1628 static struct intel_uncore_type nhmex_uncore_wbox = {
1629         .name                   = "wbox",
1630         .num_counters           = 4,
1631         .num_boxes              = 1,
1632         .perf_ctr_bits          = 48,
1633         .event_ctl              = NHMEX_W_MSR_PMON_CNT0,
1634         .perf_ctr               = NHMEX_W_MSR_PMON_EVT_SEL0,
1635         .fixed_ctr              = NHMEX_W_MSR_PMON_FIXED_CTR,
1636         .fixed_ctl              = NHMEX_W_MSR_PMON_FIXED_CTL,
1637         .event_mask             = NHMEX_PMON_RAW_EVENT_MASK,
1638         .box_ctl                = NHMEX_W_MSR_GLOBAL_CTL,
1639         .pair_ctr_ctl           = 1,
1640         .event_descs            = nhmex_uncore_wbox_events,
1641         .ops                    = &nhmex_uncore_ops,
1642         .format_group           = &nhmex_uncore_cbox_format_group
1643 };
1644
1645 static int nhmex_bbox_hw_config(struct intel_uncore_box *box, struct perf_event *event)
1646 {
1647         struct hw_perf_event *hwc = &event->hw;
1648         struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
1649         struct hw_perf_event_extra *reg2 = &hwc->branch_reg;
1650         int ctr, ev_sel;
1651
1652         ctr = (hwc->config & NHMEX_B_PMON_CTR_MASK) >>
1653                 NHMEX_B_PMON_CTR_SHIFT;
1654         ev_sel = (hwc->config & NHMEX_B_PMON_CTL_EV_SEL_MASK) >>
1655                   NHMEX_B_PMON_CTL_EV_SEL_SHIFT;
1656
1657         /* events that do not use the match/mask registers */
1658         if ((ctr == 0 && ev_sel > 0x3) || (ctr == 1 && ev_sel > 0x6) ||
1659             (ctr == 2 && ev_sel != 0x4) || ctr == 3)
1660                 return 0;
1661
1662         if (box->pmu->pmu_idx == 0)
1663                 reg1->reg = NHMEX_B0_MSR_MATCH;
1664         else
1665                 reg1->reg = NHMEX_B1_MSR_MATCH;
1666         reg1->idx = 0;
1667         reg1->config = event->attr.config1;
1668         reg2->config = event->attr.config2;
1669         return 0;
1670 }
1671
1672 static void nhmex_bbox_msr_enable_event(struct intel_uncore_box *box, struct perf_event *event)
1673 {
1674         struct hw_perf_event *hwc = &event->hw;
1675         struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
1676         struct hw_perf_event_extra *reg2 = &hwc->branch_reg;
1677
1678         if (reg1->idx != EXTRA_REG_NONE) {
1679                 wrmsrl(reg1->reg, reg1->config);
1680                 wrmsrl(reg1->reg + 1, reg2->config);
1681         }
1682         wrmsrl(hwc->config_base, NHMEX_PMON_CTL_EN_BIT0 |
1683                 (hwc->config & NHMEX_B_PMON_CTL_EV_SEL_MASK));
1684 }
1685
1686 /*
1687  * The Bbox has 4 counters, but each counter monitors different events.
1688  * Use bits 6-7 in the event config to select counter.
1689  */
1690 static struct event_constraint nhmex_uncore_bbox_constraints[] = {
1691         EVENT_CONSTRAINT(0 , 1, 0xc0),
1692         EVENT_CONSTRAINT(0x40, 2, 0xc0),
1693         EVENT_CONSTRAINT(0x80, 4, 0xc0),
1694         EVENT_CONSTRAINT(0xc0, 8, 0xc0),
1695         EVENT_CONSTRAINT_END,
1696 };
1697
1698 static struct attribute *nhmex_uncore_bbox_formats_attr[] = {
1699         &format_attr_event5.attr,
1700         &format_attr_counter.attr,
1701         &format_attr_match.attr,
1702         &format_attr_mask.attr,
1703         NULL,
1704 };
1705
1706 static struct attribute_group nhmex_uncore_bbox_format_group = {
1707         .name = "format",
1708         .attrs = nhmex_uncore_bbox_formats_attr,
1709 };
1710
1711 static struct intel_uncore_ops nhmex_uncore_bbox_ops = {
1712         NHMEX_UNCORE_OPS_COMMON_INIT(),
1713         .enable_event           = nhmex_bbox_msr_enable_event,
1714         .hw_config              = nhmex_bbox_hw_config,
1715         .get_constraint         = uncore_get_constraint,
1716         .put_constraint         = uncore_put_constraint,
1717 };
1718
1719 static struct intel_uncore_type nhmex_uncore_bbox = {
1720         .name                   = "bbox",
1721         .num_counters           = 4,
1722         .num_boxes              = 2,
1723         .perf_ctr_bits          = 48,
1724         .event_ctl              = NHMEX_B0_MSR_PMON_CTL0,
1725         .perf_ctr               = NHMEX_B0_MSR_PMON_CTR0,
1726         .event_mask             = NHMEX_B_PMON_RAW_EVENT_MASK,
1727         .box_ctl                = NHMEX_B0_MSR_PMON_GLOBAL_CTL,
1728         .msr_offset             = NHMEX_B_MSR_OFFSET,
1729         .pair_ctr_ctl           = 1,
1730         .num_shared_regs        = 1,
1731         .constraints            = nhmex_uncore_bbox_constraints,
1732         .ops                    = &nhmex_uncore_bbox_ops,
1733         .format_group           = &nhmex_uncore_bbox_format_group
1734 };
1735
1736 static int nhmex_sbox_hw_config(struct intel_uncore_box *box, struct perf_event *event)
1737 {
1738         struct hw_perf_event *hwc = &event->hw;
1739         struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
1740         struct hw_perf_event_extra *reg2 = &hwc->branch_reg;
1741
1742         /* only TO_R_PROG_EV event uses the match/mask register */
1743         if ((hwc->config & NHMEX_PMON_CTL_EV_SEL_MASK) !=
1744             NHMEX_S_EVENT_TO_R_PROG_EV)
1745                 return 0;
1746
1747         if (box->pmu->pmu_idx == 0)
1748                 reg1->reg = NHMEX_S0_MSR_MM_CFG;
1749         else
1750                 reg1->reg = NHMEX_S1_MSR_MM_CFG;
1751         reg1->idx = 0;
1752         reg1->config = event->attr.config1;
1753         reg2->config = event->attr.config2;
1754         return 0;
1755 }
1756
1757 static void nhmex_sbox_msr_enable_event(struct intel_uncore_box *box, struct perf_event *event)
1758 {
1759         struct hw_perf_event *hwc = &event->hw;
1760         struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
1761         struct hw_perf_event_extra *reg2 = &hwc->branch_reg;
1762
1763         if (reg1->idx != EXTRA_REG_NONE) {
1764                 wrmsrl(reg1->reg, 0);
1765                 wrmsrl(reg1->reg + 1, reg1->config);
1766                 wrmsrl(reg1->reg + 2, reg2->config);
1767                 wrmsrl(reg1->reg, NHMEX_S_PMON_MM_CFG_EN);
1768         }
1769         wrmsrl(hwc->config_base, hwc->config | NHMEX_PMON_CTL_EN_BIT22);
1770 }
1771
1772 static struct attribute *nhmex_uncore_sbox_formats_attr[] = {
1773         &format_attr_event.attr,
1774         &format_attr_umask.attr,
1775         &format_attr_edge.attr,
1776         &format_attr_inv.attr,
1777         &format_attr_thresh8.attr,
1778         &format_attr_match.attr,
1779         &format_attr_mask.attr,
1780         NULL,
1781 };
1782
1783 static struct attribute_group nhmex_uncore_sbox_format_group = {
1784         .name                   = "format",
1785         .attrs                  = nhmex_uncore_sbox_formats_attr,
1786 };
1787
1788 static struct intel_uncore_ops nhmex_uncore_sbox_ops = {
1789         NHMEX_UNCORE_OPS_COMMON_INIT(),
1790         .enable_event           = nhmex_sbox_msr_enable_event,
1791         .hw_config              = nhmex_sbox_hw_config,
1792         .get_constraint         = uncore_get_constraint,
1793         .put_constraint         = uncore_put_constraint,
1794 };
1795
1796 static struct intel_uncore_type nhmex_uncore_sbox = {
1797         .name                   = "sbox",
1798         .num_counters           = 4,
1799         .num_boxes              = 2,
1800         .perf_ctr_bits          = 48,
1801         .event_ctl              = NHMEX_S0_MSR_PMON_CTL0,
1802         .perf_ctr               = NHMEX_S0_MSR_PMON_CTR0,
1803         .event_mask             = NHMEX_PMON_RAW_EVENT_MASK,
1804         .box_ctl                = NHMEX_S0_MSR_PMON_GLOBAL_CTL,
1805         .msr_offset             = NHMEX_S_MSR_OFFSET,
1806         .pair_ctr_ctl           = 1,
1807         .num_shared_regs        = 1,
1808         .ops                    = &nhmex_uncore_sbox_ops,
1809         .format_group           = &nhmex_uncore_sbox_format_group
1810 };
1811
1812 enum {
1813         EXTRA_REG_NHMEX_M_FILTER,
1814         EXTRA_REG_NHMEX_M_DSP,
1815         EXTRA_REG_NHMEX_M_ISS,
1816         EXTRA_REG_NHMEX_M_MAP,
1817         EXTRA_REG_NHMEX_M_MSC_THR,
1818         EXTRA_REG_NHMEX_M_PGT,
1819         EXTRA_REG_NHMEX_M_PLD,
1820         EXTRA_REG_NHMEX_M_ZDP_CTL_FVC,
1821 };
1822
1823 static struct extra_reg nhmex_uncore_mbox_extra_regs[] = {
1824         MBOX_INC_SEL_EXTAR_REG(0x0, DSP),
1825         MBOX_INC_SEL_EXTAR_REG(0x4, MSC_THR),
1826         MBOX_INC_SEL_EXTAR_REG(0x5, MSC_THR),
1827         MBOX_INC_SEL_EXTAR_REG(0x9, ISS),
1828         /* event 0xa uses two extra registers */
1829         MBOX_INC_SEL_EXTAR_REG(0xa, ISS),
1830         MBOX_INC_SEL_EXTAR_REG(0xa, PLD),
1831         MBOX_INC_SEL_EXTAR_REG(0xb, PLD),
1832         /* events 0xd ~ 0x10 use the same extra register */
1833         MBOX_INC_SEL_EXTAR_REG(0xd, ZDP_CTL_FVC),
1834         MBOX_INC_SEL_EXTAR_REG(0xe, ZDP_CTL_FVC),
1835         MBOX_INC_SEL_EXTAR_REG(0xf, ZDP_CTL_FVC),
1836         MBOX_INC_SEL_EXTAR_REG(0x10, ZDP_CTL_FVC),
1837         MBOX_INC_SEL_EXTAR_REG(0x16, PGT),
1838         MBOX_SET_FLAG_SEL_EXTRA_REG(0x0, DSP),
1839         MBOX_SET_FLAG_SEL_EXTRA_REG(0x1, ISS),
1840         MBOX_SET_FLAG_SEL_EXTRA_REG(0x5, PGT),
1841         MBOX_SET_FLAG_SEL_EXTRA_REG(0x6, MAP),
1842         EVENT_EXTRA_END
1843 };
1844
1845 /* Nehalem-EX or Westmere-EX ? */
1846 static bool uncore_nhmex;
1847
1848 static bool nhmex_mbox_get_shared_reg(struct intel_uncore_box *box, int idx, u64 config)
1849 {
1850         struct intel_uncore_extra_reg *er;
1851         unsigned long flags;
1852         bool ret = false;
1853         u64 mask;
1854
1855         if (idx < EXTRA_REG_NHMEX_M_ZDP_CTL_FVC) {
1856                 er = &box->shared_regs[idx];
1857                 raw_spin_lock_irqsave(&er->lock, flags);
1858                 if (!atomic_read(&er->ref) || er->config == config) {
1859                         atomic_inc(&er->ref);
1860                         er->config = config;
1861                         ret = true;
1862                 }
1863                 raw_spin_unlock_irqrestore(&er->lock, flags);
1864
1865                 return ret;
1866         }
1867         /*
1868          * The ZDP_CTL_FVC MSR has 4 fields which are used to control
1869          * events 0xd ~ 0x10. Besides these 4 fields, there are additional
1870          * fields which are shared.
1871          */
1872         idx -= EXTRA_REG_NHMEX_M_ZDP_CTL_FVC;
1873         if (WARN_ON_ONCE(idx >= 4))
1874                 return false;
1875
1876         /* mask of the shared fields */
1877         if (uncore_nhmex)
1878                 mask = NHMEX_M_PMON_ZDP_CTL_FVC_MASK;
1879         else
1880                 mask = WSMEX_M_PMON_ZDP_CTL_FVC_MASK;
1881         er = &box->shared_regs[EXTRA_REG_NHMEX_M_ZDP_CTL_FVC];
1882
1883         raw_spin_lock_irqsave(&er->lock, flags);
1884         /* add mask of the non-shared field if it's in use */
1885         if (__BITS_VALUE(atomic_read(&er->ref), idx, 8)) {
1886                 if (uncore_nhmex)
1887                         mask |= NHMEX_M_PMON_ZDP_CTL_FVC_EVENT_MASK(idx);
1888                 else
1889                         mask |= WSMEX_M_PMON_ZDP_CTL_FVC_EVENT_MASK(idx);
1890         }
1891
1892         if (!atomic_read(&er->ref) || !((er->config ^ config) & mask)) {
1893                 atomic_add(1 << (idx * 8), &er->ref);
1894                 if (uncore_nhmex)
1895                         mask = NHMEX_M_PMON_ZDP_CTL_FVC_MASK |
1896                                 NHMEX_M_PMON_ZDP_CTL_FVC_EVENT_MASK(idx);
1897                 else
1898                         mask = WSMEX_M_PMON_ZDP_CTL_FVC_MASK |
1899                                 WSMEX_M_PMON_ZDP_CTL_FVC_EVENT_MASK(idx);
1900                 er->config &= ~mask;
1901                 er->config |= (config & mask);
1902                 ret = true;
1903         }
1904         raw_spin_unlock_irqrestore(&er->lock, flags);
1905
1906         return ret;
1907 }
1908
1909 static void nhmex_mbox_put_shared_reg(struct intel_uncore_box *box, int idx)
1910 {
1911         struct intel_uncore_extra_reg *er;
1912
1913         if (idx < EXTRA_REG_NHMEX_M_ZDP_CTL_FVC) {
1914                 er = &box->shared_regs[idx];
1915                 atomic_dec(&er->ref);
1916                 return;
1917         }
1918
1919         idx -= EXTRA_REG_NHMEX_M_ZDP_CTL_FVC;
1920         er = &box->shared_regs[EXTRA_REG_NHMEX_M_ZDP_CTL_FVC];
1921         atomic_sub(1 << (idx * 8), &er->ref);
1922 }
1923
1924 static u64 nhmex_mbox_alter_er(struct perf_event *event, int new_idx, bool modify)
1925 {
1926         struct hw_perf_event *hwc = &event->hw;
1927         struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
1928         u64 idx, orig_idx = __BITS_VALUE(reg1->idx, 0, 8);
1929         u64 config = reg1->config;
1930
1931         /* get the non-shared control bits and shift them */
1932         idx = orig_idx - EXTRA_REG_NHMEX_M_ZDP_CTL_FVC;
1933         if (uncore_nhmex)
1934                 config &= NHMEX_M_PMON_ZDP_CTL_FVC_EVENT_MASK(idx);
1935         else
1936                 config &= WSMEX_M_PMON_ZDP_CTL_FVC_EVENT_MASK(idx);
1937         if (new_idx > orig_idx) {
1938                 idx = new_idx - orig_idx;
1939                 config <<= 3 * idx;
1940         } else {
1941                 idx = orig_idx - new_idx;
1942                 config >>= 3 * idx;
1943         }
1944
1945         /* add the shared control bits back */
1946         if (uncore_nhmex)
1947                 config |= NHMEX_M_PMON_ZDP_CTL_FVC_MASK & reg1->config;
1948         else
1949                 config |= WSMEX_M_PMON_ZDP_CTL_FVC_MASK & reg1->config;
1950         config |= NHMEX_M_PMON_ZDP_CTL_FVC_MASK & reg1->config;
1951         if (modify) {
1952                 /* adjust the main event selector */
1953                 if (new_idx > orig_idx)
1954                         hwc->config += idx << NHMEX_M_PMON_CTL_INC_SEL_SHIFT;
1955                 else
1956                         hwc->config -= idx << NHMEX_M_PMON_CTL_INC_SEL_SHIFT;
1957                 reg1->config = config;
1958                 reg1->idx = ~0xff | new_idx;
1959         }
1960         return config;
1961 }
1962
1963 static struct event_constraint *
1964 nhmex_mbox_get_constraint(struct intel_uncore_box *box, struct perf_event *event)
1965 {
1966         struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
1967         struct hw_perf_event_extra *reg2 = &event->hw.branch_reg;
1968         int i, idx[2], alloc = 0;
1969         u64 config1 = reg1->config;
1970
1971         idx[0] = __BITS_VALUE(reg1->idx, 0, 8);
1972         idx[1] = __BITS_VALUE(reg1->idx, 1, 8);
1973 again:
1974         for (i = 0; i < 2; i++) {
1975                 if (!uncore_box_is_fake(box) && (reg1->alloc & (0x1 << i)))
1976                         idx[i] = 0xff;
1977
1978                 if (idx[i] == 0xff)
1979                         continue;
1980
1981                 if (!nhmex_mbox_get_shared_reg(box, idx[i],
1982                                 __BITS_VALUE(config1, i, 32)))
1983                         goto fail;
1984                 alloc |= (0x1 << i);
1985         }
1986
1987         /* for the match/mask registers */
1988         if (reg2->idx != EXTRA_REG_NONE &&
1989             (uncore_box_is_fake(box) || !reg2->alloc) &&
1990             !nhmex_mbox_get_shared_reg(box, reg2->idx, reg2->config))
1991                 goto fail;
1992
1993         /*
1994          * If it's a fake box -- as per validate_{group,event}() we
1995          * shouldn't touch event state and we can avoid doing so
1996          * since both will only call get_event_constraints() once
1997          * on each event, this avoids the need for reg->alloc.
1998          */
1999         if (!uncore_box_is_fake(box)) {
2000                 if (idx[0] != 0xff && idx[0] != __BITS_VALUE(reg1->idx, 0, 8))
2001                         nhmex_mbox_alter_er(event, idx[0], true);
2002                 reg1->alloc |= alloc;
2003                 if (reg2->idx != EXTRA_REG_NONE)
2004                         reg2->alloc = 1;
2005         }
2006         return NULL;
2007 fail:
2008         if (idx[0] != 0xff && !(alloc & 0x1) &&
2009             idx[0] >= EXTRA_REG_NHMEX_M_ZDP_CTL_FVC) {
2010                 /*
2011                  * events 0xd ~ 0x10 are functional identical, but are
2012                  * controlled by different fields in the ZDP_CTL_FVC
2013                  * register. If we failed to take one field, try the
2014                  * rest 3 choices.
2015                  */
2016                 BUG_ON(__BITS_VALUE(reg1->idx, 1, 8) != 0xff);
2017                 idx[0] -= EXTRA_REG_NHMEX_M_ZDP_CTL_FVC;
2018                 idx[0] = (idx[0] + 1) % 4;
2019                 idx[0] += EXTRA_REG_NHMEX_M_ZDP_CTL_FVC;
2020                 if (idx[0] != __BITS_VALUE(reg1->idx, 0, 8)) {
2021                         config1 = nhmex_mbox_alter_er(event, idx[0], false);
2022                         goto again;
2023                 }
2024         }
2025
2026         if (alloc & 0x1)
2027                 nhmex_mbox_put_shared_reg(box, idx[0]);
2028         if (alloc & 0x2)
2029                 nhmex_mbox_put_shared_reg(box, idx[1]);
2030         return &constraint_empty;
2031 }
2032
2033 static void nhmex_mbox_put_constraint(struct intel_uncore_box *box, struct perf_event *event)
2034 {
2035         struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
2036         struct hw_perf_event_extra *reg2 = &event->hw.branch_reg;
2037
2038         if (uncore_box_is_fake(box))
2039                 return;
2040
2041         if (reg1->alloc & 0x1)
2042                 nhmex_mbox_put_shared_reg(box, __BITS_VALUE(reg1->idx, 0, 8));
2043         if (reg1->alloc & 0x2)
2044                 nhmex_mbox_put_shared_reg(box, __BITS_VALUE(reg1->idx, 1, 8));
2045         reg1->alloc = 0;
2046
2047         if (reg2->alloc) {
2048                 nhmex_mbox_put_shared_reg(box, reg2->idx);
2049                 reg2->alloc = 0;
2050         }
2051 }
2052
2053 static int nhmex_mbox_extra_reg_idx(struct extra_reg *er)
2054 {
2055         if (er->idx < EXTRA_REG_NHMEX_M_ZDP_CTL_FVC)
2056                 return er->idx;
2057         return er->idx + (er->event >> NHMEX_M_PMON_CTL_INC_SEL_SHIFT) - 0xd;
2058 }
2059
2060 static int nhmex_mbox_hw_config(struct intel_uncore_box *box, struct perf_event *event)
2061 {
2062         struct intel_uncore_type *type = box->pmu->type;
2063         struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
2064         struct hw_perf_event_extra *reg2 = &event->hw.branch_reg;
2065         struct extra_reg *er;
2066         unsigned msr;
2067         int reg_idx = 0;
2068         /*
2069          * The mbox events may require 2 extra MSRs at the most. But only
2070          * the lower 32 bits in these MSRs are significant, so we can use
2071          * config1 to pass two MSRs' config.
2072          */
2073         for (er = nhmex_uncore_mbox_extra_regs; er->msr; er++) {
2074                 if (er->event != (event->hw.config & er->config_mask))
2075                         continue;
2076                 if (event->attr.config1 & ~er->valid_mask)
2077                         return -EINVAL;
2078
2079                 msr = er->msr + type->msr_offset * box->pmu->pmu_idx;
2080                 if (WARN_ON_ONCE(msr >= 0xffff || er->idx >= 0xff))
2081                         return -EINVAL;
2082
2083                 /* always use the 32~63 bits to pass the PLD config */
2084                 if (er->idx == EXTRA_REG_NHMEX_M_PLD)
2085                         reg_idx = 1;
2086                 else if (WARN_ON_ONCE(reg_idx > 0))
2087                         return -EINVAL;
2088
2089                 reg1->idx &= ~(0xff << (reg_idx * 8));
2090                 reg1->reg &= ~(0xffff << (reg_idx * 16));
2091                 reg1->idx |= nhmex_mbox_extra_reg_idx(er) << (reg_idx * 8);
2092                 reg1->reg |= msr << (reg_idx * 16);
2093                 reg1->config = event->attr.config1;
2094                 reg_idx++;
2095         }
2096         /*
2097          * The mbox only provides ability to perform address matching
2098          * for the PLD events.
2099          */
2100         if (reg_idx == 2) {
2101                 reg2->idx = EXTRA_REG_NHMEX_M_FILTER;
2102                 if (event->attr.config2 & NHMEX_M_PMON_MM_CFG_EN)
2103                         reg2->config = event->attr.config2;
2104                 else
2105                         reg2->config = ~0ULL;
2106                 if (box->pmu->pmu_idx == 0)
2107                         reg2->reg = NHMEX_M0_MSR_PMU_MM_CFG;
2108                 else
2109                         reg2->reg = NHMEX_M1_MSR_PMU_MM_CFG;
2110         }
2111         return 0;
2112 }
2113
2114 static u64 nhmex_mbox_shared_reg_config(struct intel_uncore_box *box, int idx)
2115 {
2116         struct intel_uncore_extra_reg *er;
2117         unsigned long flags;
2118         u64 config;
2119
2120         if (idx < EXTRA_REG_NHMEX_M_ZDP_CTL_FVC)
2121                 return box->shared_regs[idx].config;
2122
2123         er = &box->shared_regs[EXTRA_REG_NHMEX_M_ZDP_CTL_FVC];
2124         raw_spin_lock_irqsave(&er->lock, flags);
2125         config = er->config;
2126         raw_spin_unlock_irqrestore(&er->lock, flags);
2127         return config;
2128 }
2129
2130 static void nhmex_mbox_msr_enable_event(struct intel_uncore_box *box, struct perf_event *event)
2131 {
2132         struct hw_perf_event *hwc = &event->hw;
2133         struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
2134         struct hw_perf_event_extra *reg2 = &hwc->branch_reg;
2135         int idx;
2136
2137         idx = __BITS_VALUE(reg1->idx, 0, 8);
2138         if (idx != 0xff)
2139                 wrmsrl(__BITS_VALUE(reg1->reg, 0, 16),
2140                         nhmex_mbox_shared_reg_config(box, idx));
2141         idx = __BITS_VALUE(reg1->idx, 1, 8);
2142         if (idx != 0xff)
2143                 wrmsrl(__BITS_VALUE(reg1->reg, 1, 16),
2144                         nhmex_mbox_shared_reg_config(box, idx));
2145
2146         if (reg2->idx != EXTRA_REG_NONE) {
2147                 wrmsrl(reg2->reg, 0);
2148                 if (reg2->config != ~0ULL) {
2149                         wrmsrl(reg2->reg + 1,
2150                                 reg2->config & NHMEX_M_PMON_ADDR_MATCH_MASK);
2151                         wrmsrl(reg2->reg + 2, NHMEX_M_PMON_ADDR_MASK_MASK &
2152                                 (reg2->config >> NHMEX_M_PMON_ADDR_MASK_SHIFT));
2153                         wrmsrl(reg2->reg, NHMEX_M_PMON_MM_CFG_EN);
2154                 }
2155         }
2156
2157         wrmsrl(hwc->config_base, hwc->config | NHMEX_PMON_CTL_EN_BIT0);
2158 }
2159
2160 DEFINE_UNCORE_FORMAT_ATTR(count_mode,           count_mode,     "config:2-3");
2161 DEFINE_UNCORE_FORMAT_ATTR(storage_mode,         storage_mode,   "config:4-5");
2162 DEFINE_UNCORE_FORMAT_ATTR(wrap_mode,            wrap_mode,      "config:6");
2163 DEFINE_UNCORE_FORMAT_ATTR(flag_mode,            flag_mode,      "config:7");
2164 DEFINE_UNCORE_FORMAT_ATTR(inc_sel,              inc_sel,        "config:9-13");
2165 DEFINE_UNCORE_FORMAT_ATTR(set_flag_sel,         set_flag_sel,   "config:19-21");
2166 DEFINE_UNCORE_FORMAT_ATTR(filter_cfg_en,        filter_cfg_en,  "config2:63");
2167 DEFINE_UNCORE_FORMAT_ATTR(filter_match,         filter_match,   "config2:0-33");
2168 DEFINE_UNCORE_FORMAT_ATTR(filter_mask,          filter_mask,    "config2:34-61");
2169 DEFINE_UNCORE_FORMAT_ATTR(dsp,                  dsp,            "config1:0-31");
2170 DEFINE_UNCORE_FORMAT_ATTR(thr,                  thr,            "config1:0-31");
2171 DEFINE_UNCORE_FORMAT_ATTR(fvc,                  fvc,            "config1:0-31");
2172 DEFINE_UNCORE_FORMAT_ATTR(pgt,                  pgt,            "config1:0-31");
2173 DEFINE_UNCORE_FORMAT_ATTR(map,                  map,            "config1:0-31");
2174 DEFINE_UNCORE_FORMAT_ATTR(iss,                  iss,            "config1:0-31");
2175 DEFINE_UNCORE_FORMAT_ATTR(pld,                  pld,            "config1:32-63");
2176
2177 static struct attribute *nhmex_uncore_mbox_formats_attr[] = {
2178         &format_attr_count_mode.attr,
2179         &format_attr_storage_mode.attr,
2180         &format_attr_wrap_mode.attr,
2181         &format_attr_flag_mode.attr,
2182         &format_attr_inc_sel.attr,
2183         &format_attr_set_flag_sel.attr,
2184         &format_attr_filter_cfg_en.attr,
2185         &format_attr_filter_match.attr,
2186         &format_attr_filter_mask.attr,
2187         &format_attr_dsp.attr,
2188         &format_attr_thr.attr,
2189         &format_attr_fvc.attr,
2190         &format_attr_pgt.attr,
2191         &format_attr_map.attr,
2192         &format_attr_iss.attr,
2193         &format_attr_pld.attr,
2194         NULL,
2195 };
2196
2197 static struct attribute_group nhmex_uncore_mbox_format_group = {
2198         .name           = "format",
2199         .attrs          = nhmex_uncore_mbox_formats_attr,
2200 };
2201
2202 static struct uncore_event_desc nhmex_uncore_mbox_events[] = {
2203         INTEL_UNCORE_EVENT_DESC(bbox_cmds_read, "inc_sel=0xd,fvc=0x2800"),
2204         INTEL_UNCORE_EVENT_DESC(bbox_cmds_write, "inc_sel=0xd,fvc=0x2820"),
2205         { /* end: all zeroes */ },
2206 };
2207
2208 static struct uncore_event_desc wsmex_uncore_mbox_events[] = {
2209         INTEL_UNCORE_EVENT_DESC(bbox_cmds_read, "inc_sel=0xd,fvc=0x5000"),
2210         INTEL_UNCORE_EVENT_DESC(bbox_cmds_write, "inc_sel=0xd,fvc=0x5040"),
2211         { /* end: all zeroes */ },
2212 };
2213
2214 static struct intel_uncore_ops nhmex_uncore_mbox_ops = {
2215         NHMEX_UNCORE_OPS_COMMON_INIT(),
2216         .enable_event   = nhmex_mbox_msr_enable_event,
2217         .hw_config      = nhmex_mbox_hw_config,
2218         .get_constraint = nhmex_mbox_get_constraint,
2219         .put_constraint = nhmex_mbox_put_constraint,
2220 };
2221
2222 static struct intel_uncore_type nhmex_uncore_mbox = {
2223         .name                   = "mbox",
2224         .num_counters           = 6,
2225         .num_boxes              = 2,
2226         .perf_ctr_bits          = 48,
2227         .event_ctl              = NHMEX_M0_MSR_PMU_CTL0,
2228         .perf_ctr               = NHMEX_M0_MSR_PMU_CNT0,
2229         .event_mask             = NHMEX_M_PMON_RAW_EVENT_MASK,
2230         .box_ctl                = NHMEX_M0_MSR_GLOBAL_CTL,
2231         .msr_offset             = NHMEX_M_MSR_OFFSET,
2232         .pair_ctr_ctl           = 1,
2233         .num_shared_regs        = 8,
2234         .event_descs            = nhmex_uncore_mbox_events,
2235         .ops                    = &nhmex_uncore_mbox_ops,
2236         .format_group           = &nhmex_uncore_mbox_format_group,
2237 };
2238
2239 static void nhmex_rbox_alter_er(struct intel_uncore_box *box, struct perf_event *event)
2240 {
2241         struct hw_perf_event *hwc = &event->hw;
2242         struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
2243
2244         /* adjust the main event selector and extra register index */
2245         if (reg1->idx % 2) {
2246                 reg1->idx--;
2247                 hwc->config -= 1 << NHMEX_R_PMON_CTL_EV_SEL_SHIFT;
2248         } else {
2249                 reg1->idx++;
2250                 hwc->config += 1 << NHMEX_R_PMON_CTL_EV_SEL_SHIFT;
2251         }
2252
2253         /* adjust extra register config */
2254         switch (reg1->idx % 6) {
2255         case 2:
2256                 /* shift the 8~15 bits to the 0~7 bits */
2257                 reg1->config >>= 8;
2258                 break;
2259         case 3:
2260                 /* shift the 0~7 bits to the 8~15 bits */
2261                 reg1->config <<= 8;
2262                 break;
2263         };
2264 }
2265
2266 /*
2267  * Each rbox has 4 event set which monitor PQI port 0~3 or 4~7.
2268  * An event set consists of 6 events, the 3rd and 4th events in
2269  * an event set use the same extra register. So an event set uses
2270  * 5 extra registers.
2271  */
2272 static struct event_constraint *
2273 nhmex_rbox_get_constraint(struct intel_uncore_box *box, struct perf_event *event)
2274 {
2275         struct hw_perf_event *hwc = &event->hw;
2276         struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
2277         struct hw_perf_event_extra *reg2 = &hwc->branch_reg;
2278         struct intel_uncore_extra_reg *er;
2279         unsigned long flags;
2280         int idx, er_idx;
2281         u64 config1;
2282         bool ok = false;
2283
2284         if (!uncore_box_is_fake(box) && reg1->alloc)
2285                 return NULL;
2286
2287         idx = reg1->idx % 6;
2288         config1 = reg1->config;
2289 again:
2290         er_idx = idx;
2291         /* the 3rd and 4th events use the same extra register */
2292         if (er_idx > 2)
2293                 er_idx--;
2294         er_idx += (reg1->idx / 6) * 5;
2295
2296         er = &box->shared_regs[er_idx];
2297         raw_spin_lock_irqsave(&er->lock, flags);
2298         if (idx < 2) {
2299                 if (!atomic_read(&er->ref) || er->config == reg1->config) {
2300                         atomic_inc(&er->ref);
2301                         er->config = reg1->config;
2302                         ok = true;
2303                 }
2304         } else if (idx == 2 || idx == 3) {
2305                 /*
2306                  * these two events use different fields in a extra register,
2307                  * the 0~7 bits and the 8~15 bits respectively.
2308                  */
2309                 u64 mask = 0xff << ((idx - 2) * 8);
2310                 if (!__BITS_VALUE(atomic_read(&er->ref), idx - 2, 8) ||
2311                                 !((er->config ^ config1) & mask)) {
2312                         atomic_add(1 << ((idx - 2) * 8), &er->ref);
2313                         er->config &= ~mask;
2314                         er->config |= config1 & mask;
2315                         ok = true;
2316                 }
2317         } else {
2318                 if (!atomic_read(&er->ref) ||
2319                                 (er->config == (hwc->config >> 32) &&
2320                                  er->config1 == reg1->config &&
2321                                  er->config2 == reg2->config)) {
2322                         atomic_inc(&er->ref);
2323                         er->config = (hwc->config >> 32);
2324                         er->config1 = reg1->config;
2325                         er->config2 = reg2->config;
2326                         ok = true;
2327                 }
2328         }
2329         raw_spin_unlock_irqrestore(&er->lock, flags);
2330
2331         if (!ok) {
2332                 /*
2333                  * The Rbox events are always in pairs. The paired
2334                  * events are functional identical, but use different
2335                  * extra registers. If we failed to take an extra
2336                  * register, try the alternative.
2337                  */
2338                 if (idx % 2)
2339                         idx--;
2340                 else
2341                         idx++;
2342                 if (idx != reg1->idx % 6) {
2343                         if (idx == 2)
2344                                 config1 >>= 8;
2345                         else if (idx == 3)
2346                                 config1 <<= 8;
2347                         goto again;
2348                 }
2349         } else {
2350                 if (!uncore_box_is_fake(box)) {
2351                         if (idx != reg1->idx % 6)
2352                                 nhmex_rbox_alter_er(box, event);
2353                         reg1->alloc = 1;
2354                 }
2355                 return NULL;
2356         }
2357         return &constraint_empty;
2358 }
2359
2360 static void nhmex_rbox_put_constraint(struct intel_uncore_box *box, struct perf_event *event)
2361 {
2362         struct intel_uncore_extra_reg *er;
2363         struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
2364         int idx, er_idx;
2365
2366         if (uncore_box_is_fake(box) || !reg1->alloc)
2367                 return;
2368
2369         idx = reg1->idx % 6;
2370         er_idx = idx;
2371         if (er_idx > 2)
2372                 er_idx--;
2373         er_idx += (reg1->idx / 6) * 5;
2374
2375         er = &box->shared_regs[er_idx];
2376         if (idx == 2 || idx == 3)
2377                 atomic_sub(1 << ((idx - 2) * 8), &er->ref);
2378         else
2379                 atomic_dec(&er->ref);
2380
2381         reg1->alloc = 0;
2382 }
2383
2384 static int nhmex_rbox_hw_config(struct intel_uncore_box *box, struct perf_event *event)
2385 {
2386         struct hw_perf_event *hwc = &event->hw;
2387         struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
2388         struct hw_perf_event_extra *reg2 = &event->hw.branch_reg;
2389         int idx;
2390
2391         idx = (event->hw.config & NHMEX_R_PMON_CTL_EV_SEL_MASK) >>
2392                 NHMEX_R_PMON_CTL_EV_SEL_SHIFT;
2393         if (idx >= 0x18)
2394                 return -EINVAL;
2395
2396         reg1->idx = idx;
2397         reg1->config = event->attr.config1;
2398
2399         switch (idx % 6) {
2400         case 4:
2401         case 5:
2402                 hwc->config |= event->attr.config & (~0ULL << 32);
2403                 reg2->config = event->attr.config2;
2404                 break;
2405         };
2406         return 0;
2407 }
2408
2409 static void nhmex_rbox_msr_enable_event(struct intel_uncore_box *box, struct perf_event *event)
2410 {
2411         struct hw_perf_event *hwc = &event->hw;
2412         struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
2413         struct hw_perf_event_extra *reg2 = &hwc->branch_reg;
2414         int idx, port;
2415
2416         idx = reg1->idx;
2417         port = idx / 6 + box->pmu->pmu_idx * 4;
2418
2419         switch (idx % 6) {
2420         case 0:
2421                 wrmsrl(NHMEX_R_MSR_PORTN_IPERF_CFG0(port), reg1->config);
2422                 break;
2423         case 1:
2424                 wrmsrl(NHMEX_R_MSR_PORTN_IPERF_CFG1(port), reg1->config);
2425                 break;
2426         case 2:
2427         case 3:
2428                 wrmsrl(NHMEX_R_MSR_PORTN_QLX_CFG(port),
2429                         uncore_shared_reg_config(box, 2 + (idx / 6) * 5));
2430                 break;
2431         case 4:
2432                 wrmsrl(NHMEX_R_MSR_PORTN_XBR_SET1_MM_CFG(port),
2433                         hwc->config >> 32);
2434                 wrmsrl(NHMEX_R_MSR_PORTN_XBR_SET1_MATCH(port), reg1->config);
2435                 wrmsrl(NHMEX_R_MSR_PORTN_XBR_SET1_MASK(port), reg2->config);
2436                 break;
2437         case 5:
2438                 wrmsrl(NHMEX_R_MSR_PORTN_XBR_SET2_MM_CFG(port),
2439                         hwc->config >> 32);
2440                 wrmsrl(NHMEX_R_MSR_PORTN_XBR_SET2_MATCH(port), reg1->config);
2441                 wrmsrl(NHMEX_R_MSR_PORTN_XBR_SET2_MASK(port), reg2->config);
2442                 break;
2443         };
2444
2445         wrmsrl(hwc->config_base, NHMEX_PMON_CTL_EN_BIT0 |
2446                 (hwc->config & NHMEX_R_PMON_CTL_EV_SEL_MASK));
2447 }
2448
2449 DEFINE_UNCORE_FORMAT_ATTR(xbr_mm_cfg, xbr_mm_cfg, "config:32-63");
2450 DEFINE_UNCORE_FORMAT_ATTR(xbr_match, xbr_match, "config1:0-63");
2451 DEFINE_UNCORE_FORMAT_ATTR(xbr_mask, xbr_mask, "config2:0-63");
2452 DEFINE_UNCORE_FORMAT_ATTR(qlx_cfg, qlx_cfg, "config1:0-15");
2453 DEFINE_UNCORE_FORMAT_ATTR(iperf_cfg, iperf_cfg, "config1:0-31");
2454
2455 static struct attribute *nhmex_uncore_rbox_formats_attr[] = {
2456         &format_attr_event5.attr,
2457         &format_attr_xbr_mm_cfg.attr,
2458         &format_attr_xbr_match.attr,
2459         &format_attr_xbr_mask.attr,
2460         &format_attr_qlx_cfg.attr,
2461         &format_attr_iperf_cfg.attr,
2462         NULL,
2463 };
2464
2465 static struct attribute_group nhmex_uncore_rbox_format_group = {
2466         .name = "format",
2467         .attrs = nhmex_uncore_rbox_formats_attr,
2468 };
2469
2470 static struct uncore_event_desc nhmex_uncore_rbox_events[] = {
2471         INTEL_UNCORE_EVENT_DESC(qpi0_flit_send,         "event=0x0,iperf_cfg=0x80000000"),
2472         INTEL_UNCORE_EVENT_DESC(qpi1_filt_send,         "event=0x6,iperf_cfg=0x80000000"),
2473         INTEL_UNCORE_EVENT_DESC(qpi0_idle_filt,         "event=0x0,iperf_cfg=0x40000000"),
2474         INTEL_UNCORE_EVENT_DESC(qpi1_idle_filt,         "event=0x6,iperf_cfg=0x40000000"),
2475         INTEL_UNCORE_EVENT_DESC(qpi0_date_response,     "event=0x0,iperf_cfg=0xc4"),
2476         INTEL_UNCORE_EVENT_DESC(qpi1_date_response,     "event=0x6,iperf_cfg=0xc4"),
2477         { /* end: all zeroes */ },
2478 };
2479
2480 static struct intel_uncore_ops nhmex_uncore_rbox_ops = {
2481         NHMEX_UNCORE_OPS_COMMON_INIT(),
2482         .enable_event           = nhmex_rbox_msr_enable_event,
2483         .hw_config              = nhmex_rbox_hw_config,
2484         .get_constraint         = nhmex_rbox_get_constraint,
2485         .put_constraint         = nhmex_rbox_put_constraint,
2486 };
2487
2488 static struct intel_uncore_type nhmex_uncore_rbox = {
2489         .name                   = "rbox",
2490         .num_counters           = 8,
2491         .num_boxes              = 2,
2492         .perf_ctr_bits          = 48,
2493         .event_ctl              = NHMEX_R_MSR_PMON_CTL0,
2494         .perf_ctr               = NHMEX_R_MSR_PMON_CNT0,
2495         .event_mask             = NHMEX_R_PMON_RAW_EVENT_MASK,
2496         .box_ctl                = NHMEX_R_MSR_GLOBAL_CTL,
2497         .msr_offset             = NHMEX_R_MSR_OFFSET,
2498         .pair_ctr_ctl           = 1,
2499         .num_shared_regs        = 20,
2500         .event_descs            = nhmex_uncore_rbox_events,
2501         .ops                    = &nhmex_uncore_rbox_ops,
2502         .format_group           = &nhmex_uncore_rbox_format_group
2503 };
2504
2505 static struct intel_uncore_type *nhmex_msr_uncores[] = {
2506         &nhmex_uncore_ubox,
2507         &nhmex_uncore_cbox,
2508         &nhmex_uncore_bbox,
2509         &nhmex_uncore_sbox,
2510         &nhmex_uncore_mbox,
2511         &nhmex_uncore_rbox,
2512         &nhmex_uncore_wbox,
2513         NULL,
2514 };
2515 /* end of Nehalem-EX uncore support */
2516
2517 static void uncore_assign_hw_event(struct intel_uncore_box *box, struct perf_event *event, int idx)
2518 {
2519         struct hw_perf_event *hwc = &event->hw;
2520
2521         hwc->idx = idx;
2522         hwc->last_tag = ++box->tags[idx];
2523
2524         if (hwc->idx == UNCORE_PMC_IDX_FIXED) {
2525                 hwc->event_base = uncore_fixed_ctr(box);
2526                 hwc->config_base = uncore_fixed_ctl(box);
2527                 return;
2528         }
2529
2530         hwc->config_base = uncore_event_ctl(box, hwc->idx);
2531         hwc->event_base  = uncore_perf_ctr(box, hwc->idx);
2532 }
2533
2534 static void uncore_perf_event_update(struct intel_uncore_box *box, struct perf_event *event)
2535 {
2536         u64 prev_count, new_count, delta;
2537         int shift;
2538
2539         if (event->hw.idx >= UNCORE_PMC_IDX_FIXED)
2540                 shift = 64 - uncore_fixed_ctr_bits(box);
2541         else
2542                 shift = 64 - uncore_perf_ctr_bits(box);
2543
2544         /* the hrtimer might modify the previous event value */
2545 again:
2546         prev_count = local64_read(&event->hw.prev_count);
2547         new_count = uncore_read_counter(box, event);
2548         if (local64_xchg(&event->hw.prev_count, new_count) != prev_count)
2549                 goto again;
2550
2551         delta = (new_count << shift) - (prev_count << shift);
2552         delta >>= shift;
2553
2554         local64_add(delta, &event->count);
2555 }
2556
2557 /*
2558  * The overflow interrupt is unavailable for SandyBridge-EP, is broken
2559  * for SandyBridge. So we use hrtimer to periodically poll the counter
2560  * to avoid overflow.
2561  */
2562 static enum hrtimer_restart uncore_pmu_hrtimer(struct hrtimer *hrtimer)
2563 {
2564         struct intel_uncore_box *box;
2565         unsigned long flags;
2566         int bit;
2567
2568         box = container_of(hrtimer, struct intel_uncore_box, hrtimer);
2569         if (!box->n_active || box->cpu != smp_processor_id())
2570                 return HRTIMER_NORESTART;
2571         /*
2572          * disable local interrupt to prevent uncore_pmu_event_start/stop
2573          * to interrupt the update process
2574          */
2575         local_irq_save(flags);
2576
2577         for_each_set_bit(bit, box->active_mask, UNCORE_PMC_IDX_MAX)
2578                 uncore_perf_event_update(box, box->events[bit]);
2579
2580         local_irq_restore(flags);
2581
2582         hrtimer_forward_now(hrtimer, ns_to_ktime(UNCORE_PMU_HRTIMER_INTERVAL));
2583         return HRTIMER_RESTART;
2584 }
2585
2586 static void uncore_pmu_start_hrtimer(struct intel_uncore_box *box)
2587 {
2588         __hrtimer_start_range_ns(&box->hrtimer,
2589                         ns_to_ktime(UNCORE_PMU_HRTIMER_INTERVAL), 0,
2590                         HRTIMER_MODE_REL_PINNED, 0);
2591 }
2592
2593 static void uncore_pmu_cancel_hrtimer(struct intel_uncore_box *box)
2594 {
2595         hrtimer_cancel(&box->hrtimer);
2596 }
2597
2598 static void uncore_pmu_init_hrtimer(struct intel_uncore_box *box)
2599 {
2600         hrtimer_init(&box->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
2601         box->hrtimer.function = uncore_pmu_hrtimer;
2602 }
2603
2604 struct intel_uncore_box *uncore_alloc_box(struct intel_uncore_type *type, int cpu)
2605 {
2606         struct intel_uncore_box *box;
2607         int i, size;
2608
2609         size = sizeof(*box) + type->num_shared_regs * sizeof(struct intel_uncore_extra_reg);
2610
2611         box = kmalloc_node(size, GFP_KERNEL | __GFP_ZERO, cpu_to_node(cpu));
2612         if (!box)
2613                 return NULL;
2614
2615         for (i = 0; i < type->num_shared_regs; i++)
2616                 raw_spin_lock_init(&box->shared_regs[i].lock);
2617
2618         uncore_pmu_init_hrtimer(box);
2619         atomic_set(&box->refcnt, 1);
2620         box->cpu = -1;
2621         box->phys_id = -1;
2622
2623         return box;
2624 }
2625
2626 static struct intel_uncore_box *
2627 uncore_pmu_to_box(struct intel_uncore_pmu *pmu, int cpu)
2628 {
2629         struct intel_uncore_box *box;
2630
2631         box = *per_cpu_ptr(pmu->box, cpu);
2632         if (box)
2633                 return box;
2634
2635         raw_spin_lock(&uncore_box_lock);
2636         list_for_each_entry(box, &pmu->box_list, list) {
2637                 if (box->phys_id == topology_physical_package_id(cpu)) {
2638                         atomic_inc(&box->refcnt);
2639                         *per_cpu_ptr(pmu->box, cpu) = box;
2640                         break;
2641                 }
2642         }
2643         raw_spin_unlock(&uncore_box_lock);
2644
2645         return *per_cpu_ptr(pmu->box, cpu);
2646 }
2647
2648 static struct intel_uncore_pmu *uncore_event_to_pmu(struct perf_event *event)
2649 {
2650         return container_of(event->pmu, struct intel_uncore_pmu, pmu);
2651 }
2652
2653 static struct intel_uncore_box *uncore_event_to_box(struct perf_event *event)
2654 {
2655         /*
2656          * perf core schedules event on the basis of cpu, uncore events are
2657          * collected by one of the cpus inside a physical package.
2658          */
2659         return uncore_pmu_to_box(uncore_event_to_pmu(event), smp_processor_id());
2660 }
2661
2662 static int
2663 uncore_collect_events(struct intel_uncore_box *box, struct perf_event *leader, bool dogrp)
2664 {
2665         struct perf_event *event;
2666         int n, max_count;
2667
2668         max_count = box->pmu->type->num_counters;
2669         if (box->pmu->type->fixed_ctl)
2670                 max_count++;
2671
2672         if (box->n_events >= max_count)
2673                 return -EINVAL;
2674
2675         n = box->n_events;
2676         box->event_list[n] = leader;
2677         n++;
2678         if (!dogrp)
2679                 return n;
2680
2681         list_for_each_entry(event, &leader->sibling_list, group_entry) {
2682                 if (event->state <= PERF_EVENT_STATE_OFF)
2683                         continue;
2684
2685                 if (n >= max_count)
2686                         return -EINVAL;
2687
2688                 box->event_list[n] = event;
2689                 n++;
2690         }
2691         return n;
2692 }
2693
2694 static struct event_constraint *
2695 uncore_get_event_constraint(struct intel_uncore_box *box, struct perf_event *event)
2696 {
2697         struct intel_uncore_type *type = box->pmu->type;
2698         struct event_constraint *c;
2699
2700         if (type->ops->get_constraint) {
2701                 c = type->ops->get_constraint(box, event);
2702                 if (c)
2703                         return c;
2704         }
2705
2706         if (event->hw.config == ~0ULL)
2707                 return &constraint_fixed;
2708
2709         if (type->constraints) {
2710                 for_each_event_constraint(c, type->constraints) {
2711                         if ((event->hw.config & c->cmask) == c->code)
2712                                 return c;
2713                 }
2714         }
2715
2716         return &type->unconstrainted;
2717 }
2718
2719 static void uncore_put_event_constraint(struct intel_uncore_box *box, struct perf_event *event)
2720 {
2721         if (box->pmu->type->ops->put_constraint)
2722                 box->pmu->type->ops->put_constraint(box, event);
2723 }
2724
2725 static int uncore_assign_events(struct intel_uncore_box *box, int assign[], int n)
2726 {
2727         unsigned long used_mask[BITS_TO_LONGS(UNCORE_PMC_IDX_MAX)];
2728         struct event_constraint *c;
2729         int i, wmin, wmax, ret = 0;
2730         struct hw_perf_event *hwc;
2731
2732         bitmap_zero(used_mask, UNCORE_PMC_IDX_MAX);
2733
2734         for (i = 0, wmin = UNCORE_PMC_IDX_MAX, wmax = 0; i < n; i++) {
2735                 hwc = &box->event_list[i]->hw;
2736                 c = uncore_get_event_constraint(box, box->event_list[i]);
2737                 hwc->constraint = c;
2738                 wmin = min(wmin, c->weight);
2739                 wmax = max(wmax, c->weight);
2740         }
2741
2742         /* fastpath, try to reuse previous register */
2743         for (i = 0; i < n; i++) {
2744                 hwc = &box->event_list[i]->hw;
2745                 c = hwc->constraint;
2746
2747                 /* never assigned */
2748                 if (hwc->idx == -1)
2749                         break;
2750
2751                 /* constraint still honored */
2752                 if (!test_bit(hwc->idx, c->idxmsk))
2753                         break;
2754
2755                 /* not already used */
2756                 if (test_bit(hwc->idx, used_mask))
2757                         break;
2758
2759                 __set_bit(hwc->idx, used_mask);
2760                 if (assign)
2761                         assign[i] = hwc->idx;
2762         }
2763         /* slow path */
2764         if (i != n)
2765                 ret = perf_assign_events(box->event_list, n,
2766                                          wmin, wmax, assign);
2767
2768         if (!assign || ret) {
2769                 for (i = 0; i < n; i++)
2770                         uncore_put_event_constraint(box, box->event_list[i]);
2771         }
2772         return ret ? -EINVAL : 0;
2773 }
2774
2775 static void uncore_pmu_event_start(struct perf_event *event, int flags)
2776 {
2777         struct intel_uncore_box *box = uncore_event_to_box(event);
2778         int idx = event->hw.idx;
2779
2780         if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED)))
2781                 return;
2782
2783         if (WARN_ON_ONCE(idx == -1 || idx >= UNCORE_PMC_IDX_MAX))
2784                 return;
2785
2786         event->hw.state = 0;
2787         box->events[idx] = event;
2788         box->n_active++;
2789         __set_bit(idx, box->active_mask);
2790
2791         local64_set(&event->hw.prev_count, uncore_read_counter(box, event));
2792         uncore_enable_event(box, event);
2793
2794         if (box->n_active == 1) {
2795                 uncore_enable_box(box);
2796                 uncore_pmu_start_hrtimer(box);
2797         }
2798 }
2799
2800 static void uncore_pmu_event_stop(struct perf_event *event, int flags)
2801 {
2802         struct intel_uncore_box *box = uncore_event_to_box(event);
2803         struct hw_perf_event *hwc = &event->hw;
2804
2805         if (__test_and_clear_bit(hwc->idx, box->active_mask)) {
2806                 uncore_disable_event(box, event);
2807                 box->n_active--;
2808                 box->events[hwc->idx] = NULL;
2809                 WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
2810                 hwc->state |= PERF_HES_STOPPED;
2811
2812                 if (box->n_active == 0) {
2813                         uncore_disable_box(box);
2814                         uncore_pmu_cancel_hrtimer(box);
2815                 }
2816         }
2817
2818         if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) {
2819                 /*
2820                  * Drain the remaining delta count out of a event
2821                  * that we are disabling:
2822                  */
2823                 uncore_perf_event_update(box, event);
2824                 hwc->state |= PERF_HES_UPTODATE;
2825         }
2826 }
2827
2828 static int uncore_pmu_event_add(struct perf_event *event, int flags)
2829 {
2830         struct intel_uncore_box *box = uncore_event_to_box(event);
2831         struct hw_perf_event *hwc = &event->hw;
2832         int assign[UNCORE_PMC_IDX_MAX];
2833         int i, n, ret;
2834
2835         if (!box)
2836                 return -ENODEV;
2837
2838         ret = n = uncore_collect_events(box, event, false);
2839         if (ret < 0)
2840                 return ret;
2841
2842         hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
2843         if (!(flags & PERF_EF_START))
2844                 hwc->state |= PERF_HES_ARCH;
2845
2846         ret = uncore_assign_events(box, assign, n);
2847         if (ret)
2848                 return ret;
2849
2850         /* save events moving to new counters */
2851         for (i = 0; i < box->n_events; i++) {
2852                 event = box->event_list[i];
2853                 hwc = &event->hw;
2854
2855                 if (hwc->idx == assign[i] &&
2856                         hwc->last_tag == box->tags[assign[i]])
2857                         continue;
2858                 /*
2859                  * Ensure we don't accidentally enable a stopped
2860                  * counter simply because we rescheduled.
2861                  */
2862                 if (hwc->state & PERF_HES_STOPPED)
2863                         hwc->state |= PERF_HES_ARCH;
2864
2865                 uncore_pmu_event_stop(event, PERF_EF_UPDATE);
2866         }
2867
2868         /* reprogram moved events into new counters */
2869         for (i = 0; i < n; i++) {
2870                 event = box->event_list[i];
2871                 hwc = &event->hw;
2872
2873                 if (hwc->idx != assign[i] ||
2874                         hwc->last_tag != box->tags[assign[i]])
2875                         uncore_assign_hw_event(box, event, assign[i]);
2876                 else if (i < box->n_events)
2877                         continue;
2878
2879                 if (hwc->state & PERF_HES_ARCH)
2880                         continue;
2881
2882                 uncore_pmu_event_start(event, 0);
2883         }
2884         box->n_events = n;
2885
2886         return 0;
2887 }
2888
2889 static void uncore_pmu_event_del(struct perf_event *event, int flags)
2890 {
2891         struct intel_uncore_box *box = uncore_event_to_box(event);
2892         int i;
2893
2894         uncore_pmu_event_stop(event, PERF_EF_UPDATE);
2895
2896         for (i = 0; i < box->n_events; i++) {
2897                 if (event == box->event_list[i]) {
2898                         uncore_put_event_constraint(box, event);
2899
2900                         while (++i < box->n_events)
2901                                 box->event_list[i - 1] = box->event_list[i];
2902
2903                         --box->n_events;
2904                         break;
2905                 }
2906         }
2907
2908         event->hw.idx = -1;
2909         event->hw.last_tag = ~0ULL;
2910 }
2911
2912 static void uncore_pmu_event_read(struct perf_event *event)
2913 {
2914         struct intel_uncore_box *box = uncore_event_to_box(event);
2915         uncore_perf_event_update(box, event);
2916 }
2917
2918 /*
2919  * validation ensures the group can be loaded onto the
2920  * PMU if it was the only group available.
2921  */
2922 static int uncore_validate_group(struct intel_uncore_pmu *pmu,
2923                                 struct perf_event *event)
2924 {
2925         struct perf_event *leader = event->group_leader;
2926         struct intel_uncore_box *fake_box;
2927         int ret = -EINVAL, n;
2928
2929         fake_box = uncore_alloc_box(pmu->type, smp_processor_id());
2930         if (!fake_box)
2931                 return -ENOMEM;
2932
2933         fake_box->pmu = pmu;
2934         /*
2935          * the event is not yet connected with its
2936          * siblings therefore we must first collect
2937          * existing siblings, then add the new event
2938          * before we can simulate the scheduling
2939          */
2940         n = uncore_collect_events(fake_box, leader, true);
2941         if (n < 0)
2942                 goto out;
2943
2944         fake_box->n_events = n;
2945         n = uncore_collect_events(fake_box, event, false);
2946         if (n < 0)
2947                 goto out;
2948
2949         fake_box->n_events = n;
2950
2951         ret = uncore_assign_events(fake_box, NULL, n);
2952 out:
2953         kfree(fake_box);
2954         return ret;
2955 }
2956
2957 static int uncore_pmu_event_init(struct perf_event *event)
2958 {
2959         struct intel_uncore_pmu *pmu;
2960         struct intel_uncore_box *box;
2961         struct hw_perf_event *hwc = &event->hw;
2962         int ret;
2963
2964         if (event->attr.type != event->pmu->type)
2965                 return -ENOENT;
2966
2967         pmu = uncore_event_to_pmu(event);
2968         /* no device found for this pmu */
2969         if (pmu->func_id < 0)
2970                 return -ENOENT;
2971
2972         /*
2973          * Uncore PMU does measure at all privilege level all the time.
2974          * So it doesn't make sense to specify any exclude bits.
2975          */
2976         if (event->attr.exclude_user || event->attr.exclude_kernel ||
2977                         event->attr.exclude_hv || event->attr.exclude_idle)
2978                 return -EINVAL;
2979
2980         /* Sampling not supported yet */
2981         if (hwc->sample_period)
2982                 return -EINVAL;
2983
2984         /*
2985          * Place all uncore events for a particular physical package
2986          * onto a single cpu
2987          */
2988         if (event->cpu < 0)
2989                 return -EINVAL;
2990         box = uncore_pmu_to_box(pmu, event->cpu);
2991         if (!box || box->cpu < 0)
2992                 return -EINVAL;
2993         event->cpu = box->cpu;
2994
2995         event->hw.idx = -1;
2996         event->hw.last_tag = ~0ULL;
2997         event->hw.extra_reg.idx = EXTRA_REG_NONE;
2998         event->hw.branch_reg.idx = EXTRA_REG_NONE;
2999
3000         if (event->attr.config == UNCORE_FIXED_EVENT) {
3001                 /* no fixed counter */
3002                 if (!pmu->type->fixed_ctl)
3003                         return -EINVAL;
3004                 /*
3005                  * if there is only one fixed counter, only the first pmu
3006                  * can access the fixed counter
3007                  */
3008                 if (pmu->type->single_fixed && pmu->pmu_idx > 0)
3009                         return -EINVAL;
3010                 hwc->config = ~0ULL;
3011         } else {
3012                 hwc->config = event->attr.config & pmu->type->event_mask;
3013                 if (pmu->type->ops->hw_config) {
3014                         ret = pmu->type->ops->hw_config(box, event);
3015                         if (ret)
3016                                 return ret;
3017                 }
3018         }
3019
3020         if (event->group_leader != event)
3021                 ret = uncore_validate_group(pmu, event);
3022         else
3023                 ret = 0;
3024
3025         return ret;
3026 }
3027
3028 static ssize_t uncore_get_attr_cpumask(struct device *dev,
3029                                 struct device_attribute *attr, char *buf)
3030 {
3031         int n = cpulist_scnprintf(buf, PAGE_SIZE - 2, &uncore_cpu_mask);
3032
3033         buf[n++] = '\n';
3034         buf[n] = '\0';
3035         return n;
3036 }
3037
3038 static DEVICE_ATTR(cpumask, S_IRUGO, uncore_get_attr_cpumask, NULL);
3039
3040 static struct attribute *uncore_pmu_attrs[] = {
3041         &dev_attr_cpumask.attr,
3042         NULL,
3043 };
3044
3045 static struct attribute_group uncore_pmu_attr_group = {
3046         .attrs = uncore_pmu_attrs,
3047 };
3048
3049 static int __init uncore_pmu_register(struct intel_uncore_pmu *pmu)
3050 {
3051         int ret;
3052
3053         pmu->pmu = (struct pmu) {
3054                 .attr_groups    = pmu->type->attr_groups,
3055                 .task_ctx_nr    = perf_invalid_context,
3056                 .event_init     = uncore_pmu_event_init,
3057                 .add            = uncore_pmu_event_add,
3058                 .del            = uncore_pmu_event_del,
3059                 .start          = uncore_pmu_event_start,
3060                 .stop           = uncore_pmu_event_stop,
3061                 .read           = uncore_pmu_event_read,
3062         };
3063
3064         if (pmu->type->num_boxes == 1) {
3065                 if (strlen(pmu->type->name) > 0)
3066                         sprintf(pmu->name, "uncore_%s", pmu->type->name);
3067                 else
3068                         sprintf(pmu->name, "uncore");
3069         } else {
3070                 sprintf(pmu->name, "uncore_%s_%d", pmu->type->name,
3071                         pmu->pmu_idx);
3072         }
3073
3074         ret = perf_pmu_register(&pmu->pmu, pmu->name, -1);
3075         return ret;
3076 }
3077
3078 static void __init uncore_type_exit(struct intel_uncore_type *type)
3079 {
3080         int i;
3081
3082         for (i = 0; i < type->num_boxes; i++)
3083                 free_percpu(type->pmus[i].box);
3084         kfree(type->pmus);
3085         type->pmus = NULL;
3086         kfree(type->events_group);
3087         type->events_group = NULL;
3088 }
3089
3090 static void __init uncore_types_exit(struct intel_uncore_type **types)
3091 {
3092         int i;
3093         for (i = 0; types[i]; i++)
3094                 uncore_type_exit(types[i]);
3095 }
3096
3097 static int __init uncore_type_init(struct intel_uncore_type *type)
3098 {
3099         struct intel_uncore_pmu *pmus;
3100         struct attribute_group *attr_group;
3101         struct attribute **attrs;
3102         int i, j;
3103
3104         pmus = kzalloc(sizeof(*pmus) * type->num_boxes, GFP_KERNEL);
3105         if (!pmus)
3106                 return -ENOMEM;
3107
3108         type->unconstrainted = (struct event_constraint)
3109                 __EVENT_CONSTRAINT(0, (1ULL << type->num_counters) - 1,
3110                                 0, type->num_counters, 0, 0);
3111
3112         for (i = 0; i < type->num_boxes; i++) {
3113                 pmus[i].func_id = -1;
3114                 pmus[i].pmu_idx = i;
3115                 pmus[i].type = type;
3116                 INIT_LIST_HEAD(&pmus[i].box_list);
3117                 pmus[i].box = alloc_percpu(struct intel_uncore_box *);
3118                 if (!pmus[i].box)
3119                         goto fail;
3120         }
3121
3122         if (type->event_descs) {
3123                 i = 0;
3124                 while (type->event_descs[i].attr.attr.name)
3125                         i++;
3126
3127                 attr_group = kzalloc(sizeof(struct attribute *) * (i + 1) +
3128                                         sizeof(*attr_group), GFP_KERNEL);
3129                 if (!attr_group)
3130                         goto fail;
3131
3132                 attrs = (struct attribute **)(attr_group + 1);
3133                 attr_group->name = "events";
3134                 attr_group->attrs = attrs;
3135
3136                 for (j = 0; j < i; j++)
3137                         attrs[j] = &type->event_descs[j].attr.attr;
3138
3139                 type->events_group = attr_group;
3140         }
3141
3142         type->pmu_group = &uncore_pmu_attr_group;
3143         type->pmus = pmus;
3144         return 0;
3145 fail:
3146         uncore_type_exit(type);
3147         return -ENOMEM;
3148 }
3149
3150 static int __init uncore_types_init(struct intel_uncore_type **types)
3151 {
3152         int i, ret;
3153
3154         for (i = 0; types[i]; i++) {
3155                 ret = uncore_type_init(types[i]);
3156                 if (ret)
3157                         goto fail;
3158         }
3159         return 0;
3160 fail:
3161         while (--i >= 0)
3162                 uncore_type_exit(types[i]);
3163         return ret;
3164 }
3165
3166 static struct pci_driver *uncore_pci_driver;
3167 static bool pcidrv_registered;
3168
3169 /*
3170  * add a pci uncore device
3171  */
3172 static int uncore_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
3173 {
3174         struct intel_uncore_pmu *pmu;
3175         struct intel_uncore_box *box;
3176         struct intel_uncore_type *type;
3177         int phys_id;
3178
3179         phys_id = pcibus_to_physid[pdev->bus->number];
3180         if (phys_id < 0)
3181                 return -ENODEV;
3182
3183         if (UNCORE_PCI_DEV_TYPE(id->driver_data) == UNCORE_EXTRA_PCI_DEV) {
3184                 extra_pci_dev[phys_id][UNCORE_PCI_DEV_IDX(id->driver_data)] = pdev;
3185                 pci_set_drvdata(pdev, NULL);
3186                 return 0;
3187         }
3188
3189         type = pci_uncores[UNCORE_PCI_DEV_TYPE(id->driver_data)];
3190         box = uncore_alloc_box(type, 0);
3191         if (!box)
3192                 return -ENOMEM;
3193
3194         /*
3195          * for performance monitoring unit with multiple boxes,
3196          * each box has a different function id.
3197          */
3198         pmu = &type->pmus[UNCORE_PCI_DEV_IDX(id->driver_data)];
3199         if (pmu->func_id < 0)
3200                 pmu->func_id = pdev->devfn;
3201         else
3202                 WARN_ON_ONCE(pmu->func_id != pdev->devfn);
3203
3204         box->phys_id = phys_id;
3205         box->pci_dev = pdev;
3206         box->pmu = pmu;
3207         uncore_box_init(box);
3208         pci_set_drvdata(pdev, box);
3209
3210         raw_spin_lock(&uncore_box_lock);
3211         list_add_tail(&box->list, &pmu->box_list);
3212         raw_spin_unlock(&uncore_box_lock);
3213
3214         return 0;
3215 }
3216
3217 static void uncore_pci_remove(struct pci_dev *pdev)
3218 {
3219         struct intel_uncore_box *box = pci_get_drvdata(pdev);
3220         struct intel_uncore_pmu *pmu;
3221         int i, cpu, phys_id = pcibus_to_physid[pdev->bus->number];
3222
3223         box = pci_get_drvdata(pdev);
3224         if (!box) {
3225                 for (i = 0; i < UNCORE_EXTRA_PCI_DEV_MAX; i++) {
3226                         if (extra_pci_dev[phys_id][i] == pdev) {
3227                                 extra_pci_dev[phys_id][i] = NULL;
3228                                 break;
3229                         }
3230                 }
3231                 WARN_ON_ONCE(i >= UNCORE_EXTRA_PCI_DEV_MAX);
3232                 return;
3233         }
3234
3235         pmu = box->pmu;
3236         if (WARN_ON_ONCE(phys_id != box->phys_id))
3237                 return;
3238
3239         pci_set_drvdata(pdev, NULL);
3240
3241         raw_spin_lock(&uncore_box_lock);
3242         list_del(&box->list);
3243         raw_spin_unlock(&uncore_box_lock);
3244
3245         for_each_possible_cpu(cpu) {
3246                 if (*per_cpu_ptr(pmu->box, cpu) == box) {
3247                         *per_cpu_ptr(pmu->box, cpu) = NULL;
3248                         atomic_dec(&box->refcnt);
3249                 }
3250         }
3251
3252         WARN_ON_ONCE(atomic_read(&box->refcnt) != 1);
3253         kfree(box);
3254 }
3255
3256 static int __init uncore_pci_init(void)
3257 {
3258         int ret;
3259
3260         switch (boot_cpu_data.x86_model) {
3261         case 45: /* Sandy Bridge-EP */
3262                 ret = snbep_pci2phy_map_init(0x3ce0);
3263                 if (ret)
3264                         return ret;
3265                 pci_uncores = snbep_pci_uncores;
3266                 uncore_pci_driver = &snbep_uncore_pci_driver;
3267                 break;
3268         case 62: /* IvyTown */
3269                 ret = snbep_pci2phy_map_init(0x0e1e);
3270                 if (ret)
3271                         return ret;
3272                 pci_uncores = ivt_pci_uncores;
3273                 uncore_pci_driver = &ivt_uncore_pci_driver;
3274                 break;
3275         default:
3276                 return 0;
3277         }
3278
3279         ret = uncore_types_init(pci_uncores);
3280         if (ret)
3281                 return ret;
3282
3283         uncore_pci_driver->probe = uncore_pci_probe;
3284         uncore_pci_driver->remove = uncore_pci_remove;
3285
3286         ret = pci_register_driver(uncore_pci_driver);
3287         if (ret == 0)
3288                 pcidrv_registered = true;
3289         else
3290                 uncore_types_exit(pci_uncores);
3291
3292         return ret;
3293 }
3294
3295 static void __init uncore_pci_exit(void)
3296 {
3297         if (pcidrv_registered) {
3298                 pcidrv_registered = false;
3299                 pci_unregister_driver(uncore_pci_driver);
3300                 uncore_types_exit(pci_uncores);
3301         }
3302 }
3303
3304 /* CPU hot plug/unplug are serialized by cpu_add_remove_lock mutex */
3305 static LIST_HEAD(boxes_to_free);
3306
3307 static void uncore_kfree_boxes(void)
3308 {
3309         struct intel_uncore_box *box;
3310
3311         while (!list_empty(&boxes_to_free)) {
3312                 box = list_entry(boxes_to_free.next,
3313                                  struct intel_uncore_box, list);
3314                 list_del(&box->list);
3315                 kfree(box);
3316         }
3317 }
3318
3319 static void uncore_cpu_dying(int cpu)
3320 {
3321         struct intel_uncore_type *type;
3322         struct intel_uncore_pmu *pmu;
3323         struct intel_uncore_box *box;
3324         int i, j;
3325
3326         for (i = 0; msr_uncores[i]; i++) {
3327                 type = msr_uncores[i];
3328                 for (j = 0; j < type->num_boxes; j++) {
3329                         pmu = &type->pmus[j];
3330                         box = *per_cpu_ptr(pmu->box, cpu);
3331                         *per_cpu_ptr(pmu->box, cpu) = NULL;
3332                         if (box && atomic_dec_and_test(&box->refcnt))
3333                                 list_add(&box->list, &boxes_to_free);
3334                 }
3335         }
3336 }
3337
3338 static int uncore_cpu_starting(int cpu)
3339 {
3340         struct intel_uncore_type *type;
3341         struct intel_uncore_pmu *pmu;
3342         struct intel_uncore_box *box, *exist;
3343         int i, j, k, phys_id;
3344
3345         phys_id = topology_physical_package_id(cpu);
3346
3347         for (i = 0; msr_uncores[i]; i++) {
3348                 type = msr_uncores[i];
3349                 for (j = 0; j < type->num_boxes; j++) {
3350                         pmu = &type->pmus[j];
3351                         box = *per_cpu_ptr(pmu->box, cpu);
3352                         /* called by uncore_cpu_init? */
3353                         if (box && box->phys_id >= 0) {
3354                                 uncore_box_init(box);
3355                                 continue;
3356                         }
3357
3358                         for_each_online_cpu(k) {
3359                                 exist = *per_cpu_ptr(pmu->box, k);
3360                                 if (exist && exist->phys_id == phys_id) {
3361                                         atomic_inc(&exist->refcnt);
3362                                         *per_cpu_ptr(pmu->box, cpu) = exist;
3363                                         if (box) {
3364                                                 list_add(&box->list,
3365                                                          &boxes_to_free);
3366                                                 box = NULL;
3367                                         }
3368                                         break;
3369                                 }
3370                         }
3371
3372                         if (box) {
3373                                 box->phys_id = phys_id;
3374                                 uncore_box_init(box);
3375                         }
3376                 }
3377         }
3378         return 0;
3379 }
3380
3381 static int uncore_cpu_prepare(int cpu, int phys_id)
3382 {
3383         struct intel_uncore_type *type;
3384         struct intel_uncore_pmu *pmu;
3385         struct intel_uncore_box *box;
3386         int i, j;
3387
3388         for (i = 0; msr_uncores[i]; i++) {
3389                 type = msr_uncores[i];
3390                 for (j = 0; j < type->num_boxes; j++) {
3391                         pmu = &type->pmus[j];
3392                         if (pmu->func_id < 0)
3393                                 pmu->func_id = j;
3394
3395                         box = uncore_alloc_box(type, cpu);
3396                         if (!box)
3397                                 return -ENOMEM;
3398
3399                         box->pmu = pmu;
3400                         box->phys_id = phys_id;
3401                         *per_cpu_ptr(pmu->box, cpu) = box;
3402                 }
3403         }
3404         return 0;
3405 }
3406
3407 static void
3408 uncore_change_context(struct intel_uncore_type **uncores, int old_cpu, int new_cpu)
3409 {
3410         struct intel_uncore_type *type;
3411         struct intel_uncore_pmu *pmu;
3412         struct intel_uncore_box *box;
3413         int i, j;
3414
3415         for (i = 0; uncores[i]; i++) {
3416                 type = uncores[i];
3417                 for (j = 0; j < type->num_boxes; j++) {
3418                         pmu = &type->pmus[j];
3419                         if (old_cpu < 0)
3420                                 box = uncore_pmu_to_box(pmu, new_cpu);
3421                         else
3422                                 box = uncore_pmu_to_box(pmu, old_cpu);
3423                         if (!box)
3424                                 continue;
3425
3426                         if (old_cpu < 0) {
3427                                 WARN_ON_ONCE(box->cpu != -1);
3428                                 box->cpu = new_cpu;
3429                                 continue;
3430                         }
3431
3432                         WARN_ON_ONCE(box->cpu != old_cpu);
3433                         if (new_cpu >= 0) {
3434                                 uncore_pmu_cancel_hrtimer(box);
3435                                 perf_pmu_migrate_context(&pmu->pmu,
3436                                                 old_cpu, new_cpu);
3437                                 box->cpu = new_cpu;
3438                         } else {
3439                                 box->cpu = -1;
3440                         }
3441                 }
3442         }
3443 }
3444
3445 static void uncore_event_exit_cpu(int cpu)
3446 {
3447         int i, phys_id, target;
3448
3449         /* if exiting cpu is used for collecting uncore events */
3450         if (!cpumask_test_and_clear_cpu(cpu, &uncore_cpu_mask))
3451                 return;
3452
3453         /* find a new cpu to collect uncore events */
3454         phys_id = topology_physical_package_id(cpu);
3455         target = -1;
3456         for_each_online_cpu(i) {
3457                 if (i == cpu)
3458                         continue;
3459                 if (phys_id == topology_physical_package_id(i)) {
3460                         target = i;
3461                         break;
3462                 }
3463         }
3464
3465         /* migrate uncore events to the new cpu */
3466         if (target >= 0)
3467                 cpumask_set_cpu(target, &uncore_cpu_mask);
3468
3469         uncore_change_context(msr_uncores, cpu, target);
3470         uncore_change_context(pci_uncores, cpu, target);
3471 }
3472
3473 static void uncore_event_init_cpu(int cpu)
3474 {
3475         int i, phys_id;
3476
3477         phys_id = topology_physical_package_id(cpu);
3478         for_each_cpu(i, &uncore_cpu_mask) {
3479                 if (phys_id == topology_physical_package_id(i))
3480                         return;
3481         }
3482
3483         cpumask_set_cpu(cpu, &uncore_cpu_mask);
3484
3485         uncore_change_context(msr_uncores, -1, cpu);
3486         uncore_change_context(pci_uncores, -1, cpu);
3487 }
3488
3489 static int uncore_cpu_notifier(struct notifier_block *self,
3490                                unsigned long action, void *hcpu)
3491 {
3492         unsigned int cpu = (long)hcpu;
3493
3494         /* allocate/free data structure for uncore box */
3495         switch (action & ~CPU_TASKS_FROZEN) {
3496         case CPU_UP_PREPARE:
3497                 uncore_cpu_prepare(cpu, -1);
3498                 break;
3499         case CPU_STARTING:
3500                 uncore_cpu_starting(cpu);
3501                 break;
3502         case CPU_UP_CANCELED:
3503         case CPU_DYING:
3504                 uncore_cpu_dying(cpu);
3505                 break;
3506         case CPU_ONLINE:
3507         case CPU_DEAD:
3508                 uncore_kfree_boxes();
3509                 break;
3510         default:
3511                 break;
3512         }
3513
3514         /* select the cpu that collects uncore events */
3515         switch (action & ~CPU_TASKS_FROZEN) {
3516         case CPU_DOWN_FAILED:
3517         case CPU_STARTING:
3518                 uncore_event_init_cpu(cpu);
3519                 break;
3520         case CPU_DOWN_PREPARE:
3521                 uncore_event_exit_cpu(cpu);
3522                 break;
3523         default:
3524                 break;
3525         }
3526
3527         return NOTIFY_OK;
3528 }
3529
3530 static struct notifier_block uncore_cpu_nb = {
3531         .notifier_call  = uncore_cpu_notifier,
3532         /*
3533          * to migrate uncore events, our notifier should be executed
3534          * before perf core's notifier.
3535          */
3536         .priority       = CPU_PRI_PERF + 1,
3537 };
3538
3539 static void __init uncore_cpu_setup(void *dummy)
3540 {
3541         uncore_cpu_starting(smp_processor_id());
3542 }
3543
3544 static int __init uncore_cpu_init(void)
3545 {
3546         int ret, cpu, max_cores;
3547
3548         max_cores = boot_cpu_data.x86_max_cores;
3549         switch (boot_cpu_data.x86_model) {
3550         case 26: /* Nehalem */
3551         case 30:
3552         case 37: /* Westmere */
3553         case 44:
3554                 msr_uncores = nhm_msr_uncores;
3555                 break;
3556         case 42: /* Sandy Bridge */
3557         case 58: /* Ivy Bridge */
3558                 if (snb_uncore_cbox.num_boxes > max_cores)
3559                         snb_uncore_cbox.num_boxes = max_cores;
3560                 msr_uncores = snb_msr_uncores;
3561                 break;
3562         case 45: /* Sandy Bridge-EP */
3563                 if (snbep_uncore_cbox.num_boxes > max_cores)
3564                         snbep_uncore_cbox.num_boxes = max_cores;
3565                 msr_uncores = snbep_msr_uncores;
3566                 break;
3567         case 46: /* Nehalem-EX */
3568                 uncore_nhmex = true;
3569         case 47: /* Westmere-EX aka. Xeon E7 */
3570                 if (!uncore_nhmex)
3571                         nhmex_uncore_mbox.event_descs = wsmex_uncore_mbox_events;
3572                 if (nhmex_uncore_cbox.num_boxes > max_cores)
3573                         nhmex_uncore_cbox.num_boxes = max_cores;
3574                 msr_uncores = nhmex_msr_uncores;
3575                 break;
3576         case 62: /* IvyTown */
3577                 if (ivt_uncore_cbox.num_boxes > max_cores)
3578                         ivt_uncore_cbox.num_boxes = max_cores;
3579                 msr_uncores = ivt_msr_uncores;
3580                 break;
3581
3582         default:
3583                 return 0;
3584         }
3585
3586         ret = uncore_types_init(msr_uncores);
3587         if (ret)
3588                 return ret;
3589
3590         get_online_cpus();
3591
3592         for_each_online_cpu(cpu) {
3593                 int i, phys_id = topology_physical_package_id(cpu);
3594
3595                 for_each_cpu(i, &uncore_cpu_mask) {
3596                         if (phys_id == topology_physical_package_id(i)) {
3597                                 phys_id = -1;
3598                                 break;
3599                         }
3600                 }
3601                 if (phys_id < 0)
3602                         continue;
3603
3604                 uncore_cpu_prepare(cpu, phys_id);
3605                 uncore_event_init_cpu(cpu);
3606         }
3607         on_each_cpu(uncore_cpu_setup, NULL, 1);
3608
3609         register_cpu_notifier(&uncore_cpu_nb);
3610
3611         put_online_cpus();
3612
3613         return 0;
3614 }
3615
3616 static int __init uncore_pmus_register(void)
3617 {
3618         struct intel_uncore_pmu *pmu;
3619         struct intel_uncore_type *type;
3620         int i, j;
3621
3622         for (i = 0; msr_uncores[i]; i++) {
3623                 type = msr_uncores[i];
3624                 for (j = 0; j < type->num_boxes; j++) {
3625                         pmu = &type->pmus[j];
3626                         uncore_pmu_register(pmu);
3627                 }
3628         }
3629
3630         for (i = 0; pci_uncores[i]; i++) {
3631                 type = pci_uncores[i];
3632                 for (j = 0; j < type->num_boxes; j++) {
3633                         pmu = &type->pmus[j];
3634                         uncore_pmu_register(pmu);
3635                 }
3636         }
3637
3638         return 0;
3639 }
3640
3641 static int __init intel_uncore_init(void)
3642 {
3643         int ret;
3644
3645         if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
3646                 return -ENODEV;
3647
3648         if (cpu_has_hypervisor)
3649                 return -ENODEV;
3650
3651         ret = uncore_pci_init();
3652         if (ret)
3653                 goto fail;
3654         ret = uncore_cpu_init();
3655         if (ret) {
3656                 uncore_pci_exit();
3657                 goto fail;
3658         }
3659
3660         uncore_pmus_register();
3661         return 0;
3662 fail:
3663         return ret;
3664 }
3665 device_initcall(intel_uncore_init);