drivers/gpu/drm/i915/i915_pmu.c

   1 /*
   2  * Copyright © 2017 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  */
  24
  25 #include <linux/perf_event.h>
  26 #include <linux/pm_runtime.h>
  27
  28 #include "i915_drv.h"
  29 #include "i915_pmu.h"
  30 #include "intel_ringbuffer.h"
  31
  32 /* Frequency for the sampling timer for events which need it. */
  33 #define FREQUENCY 200
  34 #define PERIOD max_t(u64, 10000, NSEC_PER_SEC / FREQUENCY)
  35
  36 #define ENGINE_SAMPLE_MASK \
  37         (BIT(I915_SAMPLE_BUSY) | \
  38          BIT(I915_SAMPLE_WAIT) | \
  39          BIT(I915_SAMPLE_SEMA))
  40
  41 #define ENGINE_SAMPLE_BITS (1 << I915_PMU_SAMPLE_BITS)
  42
  43 static cpumask_t i915_pmu_cpumask;
  44
  45 static u8 engine_config_sample(u64 config)
  46 {
  47         return config & I915_PMU_SAMPLE_MASK;
  48 }
  49
  50 static u8 engine_event_sample(struct perf_event *event)
  51 {
  52         return engine_config_sample(event->attr.config);
  53 }
  54
  55 static u8 engine_event_class(struct perf_event *event)
  56 {
  57         return (event->attr.config >> I915_PMU_CLASS_SHIFT) & 0xff;
  58 }
  59
  60 static u8 engine_event_instance(struct perf_event *event)
  61 {
  62         return (event->attr.config >> I915_PMU_SAMPLE_BITS) & 0xff;
  63 }
  64
  65 static bool is_engine_config(u64 config)
  66 {
  67         return config < __I915_PMU_OTHER(0);
  68 }
  69
  70 static unsigned int config_enabled_bit(u64 config)
  71 {
  72         if (is_engine_config(config))
  73                 return engine_config_sample(config);
  74         else
  75                 return ENGINE_SAMPLE_BITS + (config - __I915_PMU_OTHER(0));
  76 }
  77
  78 static u64 config_enabled_mask(u64 config)
  79 {
  80         return BIT_ULL(config_enabled_bit(config));
  81 }
  82
  83 static bool is_engine_event(struct perf_event *event)
  84 {
  85         return is_engine_config(event->attr.config);
  86 }
  87
  88 static unsigned int event_enabled_bit(struct perf_event *event)
  89 {
  90         return config_enabled_bit(event->attr.config);
  91 }
  92
  93 static bool pmu_needs_timer(struct drm_i915_private *i915, bool gpu_active)
  94 {
  95         u64 enable;
  96
  97         /*
  98          * Only some counters need the sampling timer.
  99          *
 100          * We start with a bitmask of all currently enabled events.
 101          */
 102         enable = i915->pmu.enable;
 103
 104         /*
 105          * Mask out all the ones which do not need the timer, or in
 106          * other words keep all the ones that could need the timer.
 107          */
 108         enable &= config_enabled_mask(I915_PMU_ACTUAL_FREQUENCY) |
 109                   config_enabled_mask(I915_PMU_REQUESTED_FREQUENCY) |
 110                   ENGINE_SAMPLE_MASK;
 111
 112         /*
 113          * When the GPU is idle per-engine counters do not need to be
 114          * running so clear those bits out.
 115          */
 116         if (!gpu_active)
 117                 enable &= ~ENGINE_SAMPLE_MASK;
 118         /*
 119          * Also there is software busyness tracking available we do not
 120          * need the timer for I915_SAMPLE_BUSY counter.
 121          *
 122          * Use RCS as proxy for all engines.
 123          */
 124         else if (intel_engine_supports_stats(i915->engine[RCS]))
 125                 enable &= ~BIT(I915_SAMPLE_BUSY);
 126
 127         /*
 128          * If some bits remain it means we need the sampling timer running.
 129          */
 130         return enable;
 131 }
 132
 133 void i915_pmu_gt_parked(struct drm_i915_private *i915)
 134 {
 135         if (!i915->pmu.base.event_init)
 136                 return;
 137
 138         spin_lock_irq(&i915->pmu.lock);
 139         /*
 140          * Signal sampling timer to stop if only engine events are enabled and
 141          * GPU went idle.
 142          */
 143         i915->pmu.timer_enabled = pmu_needs_timer(i915, false);
 144         spin_unlock_irq(&i915->pmu.lock);
 145 }
 146
 147 static void __i915_pmu_maybe_start_timer(struct drm_i915_private *i915)
 148 {
 149         if (!i915->pmu.timer_enabled && pmu_needs_timer(i915, true)) {
 150                 i915->pmu.timer_enabled = true;
 151                 hrtimer_start_range_ns(&i915->pmu.timer,
 152                                        ns_to_ktime(PERIOD), 0,
 153                                        HRTIMER_MODE_REL_PINNED);
 154         }
 155 }
 156
 157 void i915_pmu_gt_unparked(struct drm_i915_private *i915)
 158 {
 159         if (!i915->pmu.base.event_init)
 160                 return;
 161
 162         spin_lock_irq(&i915->pmu.lock);
 163         /*
 164          * Re-enable sampling timer when GPU goes active.
 165          */
 166         __i915_pmu_maybe_start_timer(i915);
 167         spin_unlock_irq(&i915->pmu.lock);
 168 }
 169
 170 static bool grab_forcewake(struct drm_i915_private *i915, bool fw)
 171 {
 172         if (!fw)
 173                 intel_uncore_forcewake_get(i915, FORCEWAKE_ALL);
 174
 175         return true;
 176 }
 177
 178 static void
 179 update_sample(struct i915_pmu_sample *sample, u32 unit, u32 val)
 180 {
 181         sample->cur += mul_u32_u32(val, unit);
 182 }
 183
 184 static void engines_sample(struct drm_i915_private *dev_priv)
 185 {
 186         struct intel_engine_cs *engine;
 187         enum intel_engine_id id;
 188         bool fw = false;
 189
 190         if ((dev_priv->pmu.enable & ENGINE_SAMPLE_MASK) == 0)
 191                 return;
 192
 193         if (!dev_priv->gt.awake)
 194                 return;
 195
 196         if (!intel_runtime_pm_get_if_in_use(dev_priv))
 197                 return;
 198
 199         for_each_engine(engine, dev_priv, id) {
 200                 u32 current_seqno = intel_engine_get_seqno(engine);
 201                 u32 last_seqno = intel_engine_last_submit(engine);
 202                 u32 val;
 203
 204                 val = !i915_seqno_passed(current_seqno, last_seqno);
 205
 206                 update_sample(&engine->pmu.sample[I915_SAMPLE_BUSY],
 207                               PERIOD, val);
 208
 209                 if (val && (engine->pmu.enable &
 210                     (BIT(I915_SAMPLE_WAIT) | BIT(I915_SAMPLE_SEMA)))) {
 211                         fw = grab_forcewake(dev_priv, fw);
 212
 213                         val = I915_READ_FW(RING_CTL(engine->mmio_base));
 214                 } else {
 215                         val = 0;
 216                 }
 217
 218                 update_sample(&engine->pmu.sample[I915_SAMPLE_WAIT],
 219                               PERIOD, !!(val & RING_WAIT));
 220
 221                 update_sample(&engine->pmu.sample[I915_SAMPLE_SEMA],
 222                               PERIOD, !!(val & RING_WAIT_SEMAPHORE));
 223         }
 224
 225         if (fw)
 226                 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
 227
 228         intel_runtime_pm_put(dev_priv);
 229 }
 230
 231 static void frequency_sample(struct drm_i915_private *dev_priv)
 232 {
 233         if (dev_priv->pmu.enable &
 234             config_enabled_mask(I915_PMU_ACTUAL_FREQUENCY)) {
 235                 u32 val;
 236
 237                 val = dev_priv->gt_pm.rps.cur_freq;
 238                 if (dev_priv->gt.awake &&
 239                     intel_runtime_pm_get_if_in_use(dev_priv)) {
 240                         val = intel_get_cagf(dev_priv,
 241                                              I915_READ_NOTRACE(GEN6_RPSTAT1));
 242                         intel_runtime_pm_put(dev_priv);
 243                 }
 244
 245                 update_sample(&dev_priv->pmu.sample[__I915_SAMPLE_FREQ_ACT],
 246                               1, intel_gpu_freq(dev_priv, val));
 247         }
 248
 249         if (dev_priv->pmu.enable &
 250             config_enabled_mask(I915_PMU_REQUESTED_FREQUENCY)) {
 251                 update_sample(&dev_priv->pmu.sample[__I915_SAMPLE_FREQ_REQ], 1,
 252                               intel_gpu_freq(dev_priv,
 253                                              dev_priv->gt_pm.rps.cur_freq));
 254         }
 255 }
 256
 257 static enum hrtimer_restart i915_sample(struct hrtimer *hrtimer)
 258 {
 259         struct drm_i915_private *i915 =
 260                 container_of(hrtimer, struct drm_i915_private, pmu.timer);
 261
 262         if (!READ_ONCE(i915->pmu.timer_enabled))
 263                 return HRTIMER_NORESTART;
 264
 265         engines_sample(i915);
 266         frequency_sample(i915);
 267
 268         hrtimer_forward_now(hrtimer, ns_to_ktime(PERIOD));
 269         return HRTIMER_RESTART;
 270 }
 271
 272 static u64 count_interrupts(struct drm_i915_private *i915)
 273 {
 274         /* open-coded kstat_irqs() */
 275         struct irq_desc *desc = irq_to_desc(i915->drm.pdev->irq);
 276         u64 sum = 0;
 277         int cpu;
 278
 279         if (!desc || !desc->kstat_irqs)
 280                 return 0;
 281
 282         for_each_possible_cpu(cpu)
 283                 sum += *per_cpu_ptr(desc->kstat_irqs, cpu);
 284
 285         return sum;
 286 }
 287
 288 static void engine_event_destroy(struct perf_event *event)
 289 {
 290         struct drm_i915_private *i915 =
 291                 container_of(event->pmu, typeof(*i915), pmu.base);
 292         struct intel_engine_cs *engine;
 293
 294         engine = intel_engine_lookup_user(i915,
 295                                           engine_event_class(event),
 296                                           engine_event_instance(event));
 297         if (WARN_ON_ONCE(!engine))
 298                 return;
 299
 300         if (engine_event_sample(event) == I915_SAMPLE_BUSY &&
 301             intel_engine_supports_stats(engine))
 302                 intel_disable_engine_stats(engine);
 303 }
 304
 305 static void i915_pmu_event_destroy(struct perf_event *event)
 306 {
 307         WARN_ON(event->parent);
 308
 309         if (is_engine_event(event))
 310                 engine_event_destroy(event);
 311 }
 312
 313 static int
 314 engine_event_status(struct intel_engine_cs *engine,
 315                     enum drm_i915_pmu_engine_sample sample)
 316 {
 317         switch (sample) {
 318         case I915_SAMPLE_BUSY:
 319         case I915_SAMPLE_WAIT:
 320                 break;
 321         case I915_SAMPLE_SEMA:
 322                 if (INTEL_GEN(engine->i915) < 6)
 323                         return -ENODEV;
 324                 break;
 325         default:
 326                 return -ENOENT;
 327         }
 328
 329         return 0;
 330 }
 331
 332 static int engine_event_init(struct perf_event *event)
 333 {
 334         struct drm_i915_private *i915 =
 335                 container_of(event->pmu, typeof(*i915), pmu.base);
 336         struct intel_engine_cs *engine;
 337         u8 sample;
 338         int ret;
 339
 340         engine = intel_engine_lookup_user(i915, engine_event_class(event),
 341                                           engine_event_instance(event));
 342         if (!engine)
 343                 return -ENODEV;
 344
 345         sample = engine_event_sample(event);
 346         ret = engine_event_status(engine, sample);
 347         if (ret)
 348                 return ret;
 349
 350         if (sample == I915_SAMPLE_BUSY && intel_engine_supports_stats(engine))
 351                 ret = intel_enable_engine_stats(engine);
 352
 353         return ret;
 354 }
 355
 356 static int i915_pmu_event_init(struct perf_event *event)
 357 {
 358         struct drm_i915_private *i915 =
 359                 container_of(event->pmu, typeof(*i915), pmu.base);
 360         int ret;
 361
 362         if (event->attr.type != event->pmu->type)
 363                 return -ENOENT;
 364
 365         /* unsupported modes and filters */
 366         if (event->attr.sample_period) /* no sampling */
 367                 return -EINVAL;
 368
 369         if (has_branch_stack(event))
 370                 return -EOPNOTSUPP;
 371
 372         if (event->cpu < 0)
 373                 return -EINVAL;
 374
 375         /* only allow running on one cpu at a time */
 376         if (!cpumask_test_cpu(event->cpu, &i915_pmu_cpumask))
 377                 return -EINVAL;
 378
 379         if (is_engine_event(event)) {
 380                 ret = engine_event_init(event);
 381         } else {
 382                 ret = 0;
 383                 switch (event->attr.config) {
 384                 case I915_PMU_ACTUAL_FREQUENCY:
 385                         if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915))
 386                                  /* Requires a mutex for sampling! */
 387                                 ret = -ENODEV;
 388                 case I915_PMU_REQUESTED_FREQUENCY:
 389                         if (INTEL_GEN(i915) < 6)
 390                                 ret = -ENODEV;
 391                         break;
 392                 case I915_PMU_INTERRUPTS:
 393                         break;
 394                 case I915_PMU_RC6_RESIDENCY:
 395                         if (!HAS_RC6(i915))
 396                                 ret = -ENODEV;
 397                         break;
 398                 default:
 399                         ret = -ENOENT;
 400                         break;
 401                 }
 402         }
 403         if (ret)
 404                 return ret;
 405
 406         if (!event->parent)
 407                 event->destroy = i915_pmu_event_destroy;
 408
 409         return 0;
 410 }
 411
 412 static u64 __i915_pmu_event_read(struct perf_event *event)
 413 {
 414         struct drm_i915_private *i915 =
 415                 container_of(event->pmu, typeof(*i915), pmu.base);
 416         u64 val = 0;
 417
 418         if (is_engine_event(event)) {
 419                 u8 sample = engine_event_sample(event);
 420                 struct intel_engine_cs *engine;
 421
 422                 engine = intel_engine_lookup_user(i915,
 423                                                   engine_event_class(event),
 424                                                   engine_event_instance(event));
 425
 426                 if (WARN_ON_ONCE(!engine)) {
 427                         /* Do nothing */
 428                 } else if (sample == I915_SAMPLE_BUSY &&
 429                            intel_engine_supports_stats(engine)) {
 430                         val = ktime_to_ns(intel_engine_get_busy_time(engine));
 431                 } else {
 432                         val = engine->pmu.sample[sample].cur;
 433                 }
 434         } else {
 435                 switch (event->attr.config) {
 436                 case I915_PMU_ACTUAL_FREQUENCY:
 437                         val =
 438                            div_u64(i915->pmu.sample[__I915_SAMPLE_FREQ_ACT].cur,
 439                                    FREQUENCY);
 440                         break;
 441                 case I915_PMU_REQUESTED_FREQUENCY:
 442                         val =
 443                            div_u64(i915->pmu.sample[__I915_SAMPLE_FREQ_REQ].cur,
 444                                    FREQUENCY);
 445                         break;
 446                 case I915_PMU_INTERRUPTS:
 447                         val = count_interrupts(i915);
 448                         break;
 449                 case I915_PMU_RC6_RESIDENCY:
 450                         intel_runtime_pm_get(i915);
 451                         val = intel_rc6_residency_ns(i915,
 452                                                      IS_VALLEYVIEW(i915) ?
 453                                                      VLV_GT_RENDER_RC6 :
 454                                                      GEN6_GT_GFX_RC6);
 455                         if (HAS_RC6p(i915))
 456                                 val += intel_rc6_residency_ns(i915,
 457                                                               GEN6_GT_GFX_RC6p);
 458                         if (HAS_RC6pp(i915))
 459                                 val += intel_rc6_residency_ns(i915,
 460                                                               GEN6_GT_GFX_RC6pp);
 461                         intel_runtime_pm_put(i915);
 462                         break;
 463                 }
 464         }
 465
 466         return val;
 467 }
 468
 469 static void i915_pmu_event_read(struct perf_event *event)
 470 {
 471         struct hw_perf_event *hwc = &event->hw;
 472         u64 prev, new;
 473
 474 again:
 475         prev = local64_read(&hwc->prev_count);
 476         new = __i915_pmu_event_read(event);
 477
 478         if (local64_cmpxchg(&hwc->prev_count, prev, new) != prev)
 479                 goto again;
 480
 481         local64_add(new - prev, &event->count);
 482 }
 483
 484 static void i915_pmu_enable(struct perf_event *event)
 485 {
 486         struct drm_i915_private *i915 =
 487                 container_of(event->pmu, typeof(*i915), pmu.base);
 488         unsigned int bit = event_enabled_bit(event);
 489         unsigned long flags;
 490
 491         spin_lock_irqsave(&i915->pmu.lock, flags);
 492
 493         /*
 494          * Update the bitmask of enabled events and increment
 495          * the event reference counter.
 496          */
 497         GEM_BUG_ON(bit >= I915_PMU_MASK_BITS);
 498         GEM_BUG_ON(i915->pmu.enable_count[bit] == ~0);
 499         i915->pmu.enable |= BIT_ULL(bit);
 500         i915->pmu.enable_count[bit]++;
 501
 502         /*
 503          * Start the sampling timer if needed and not already enabled.
 504          */
 505         __i915_pmu_maybe_start_timer(i915);
 506
 507         /*
 508          * For per-engine events the bitmask and reference counting
 509          * is stored per engine.
 510          */
 511         if (is_engine_event(event)) {
 512                 u8 sample = engine_event_sample(event);
 513                 struct intel_engine_cs *engine;
 514
 515                 engine = intel_engine_lookup_user(i915,
 516                                                   engine_event_class(event),
 517                                                   engine_event_instance(event));
 518                 GEM_BUG_ON(!engine);
 519                 engine->pmu.enable |= BIT(sample);
 520
 521                 GEM_BUG_ON(sample >= I915_PMU_SAMPLE_BITS);
 522                 GEM_BUG_ON(engine->pmu.enable_count[sample] == ~0);
 523                 engine->pmu.enable_count[sample]++;
 524         }
 525
 526         /*
 527          * Store the current counter value so we can report the correct delta
 528          * for all listeners. Even when the event was already enabled and has
 529          * an existing non-zero value.
 530          */
 531         local64_set(&event->hw.prev_count, __i915_pmu_event_read(event));
 532
 533         spin_unlock_irqrestore(&i915->pmu.lock, flags);
 534 }
 535
 536 static void i915_pmu_disable(struct perf_event *event)
 537 {
 538         struct drm_i915_private *i915 =
 539                 container_of(event->pmu, typeof(*i915), pmu.base);
 540         unsigned int bit = event_enabled_bit(event);
 541         unsigned long flags;
 542
 543         spin_lock_irqsave(&i915->pmu.lock, flags);
 544
 545         if (is_engine_event(event)) {
 546                 u8 sample = engine_event_sample(event);
 547                 struct intel_engine_cs *engine;
 548
 549                 engine = intel_engine_lookup_user(i915,
 550                                                   engine_event_class(event),
 551                                                   engine_event_instance(event));
 552                 GEM_BUG_ON(!engine);
 553                 GEM_BUG_ON(sample >= I915_PMU_SAMPLE_BITS);
 554                 GEM_BUG_ON(engine->pmu.enable_count[sample] == 0);
 555                 /*
 556                  * Decrement the reference count and clear the enabled
 557                  * bitmask when the last listener on an event goes away.
 558                  */
 559                 if (--engine->pmu.enable_count[sample] == 0)
 560                         engine->pmu.enable &= ~BIT(sample);
 561         }
 562
 563         GEM_BUG_ON(bit >= I915_PMU_MASK_BITS);
 564         GEM_BUG_ON(i915->pmu.enable_count[bit] == 0);
 565         /*
 566          * Decrement the reference count and clear the enabled
 567          * bitmask when the last listener on an event goes away.
 568          */
 569         if (--i915->pmu.enable_count[bit] == 0) {
 570                 i915->pmu.enable &= ~BIT_ULL(bit);
 571                 i915->pmu.timer_enabled &= pmu_needs_timer(i915, true);
 572         }
 573
 574         spin_unlock_irqrestore(&i915->pmu.lock, flags);
 575 }
 576
 577 static void i915_pmu_event_start(struct perf_event *event, int flags)
 578 {
 579         i915_pmu_enable(event);
 580         event->hw.state = 0;
 581 }
 582
 583 static void i915_pmu_event_stop(struct perf_event *event, int flags)
 584 {
 585         if (flags & PERF_EF_UPDATE)
 586                 i915_pmu_event_read(event);
 587         i915_pmu_disable(event);
 588         event->hw.state = PERF_HES_STOPPED;
 589 }
 590
 591 static int i915_pmu_event_add(struct perf_event *event, int flags)
 592 {
 593         if (flags & PERF_EF_START)
 594                 i915_pmu_event_start(event, flags);
 595
 596         return 0;
 597 }
 598
 599 static void i915_pmu_event_del(struct perf_event *event, int flags)
 600 {
 601         i915_pmu_event_stop(event, PERF_EF_UPDATE);
 602 }
 603
 604 static int i915_pmu_event_event_idx(struct perf_event *event)
 605 {
 606         return 0;
 607 }
 608
 609 struct i915_str_attribute {
 610         struct device_attribute attr;
 611         const char *str;
 612 };
 613
 614 static ssize_t i915_pmu_format_show(struct device *dev,
 615                                     struct device_attribute *attr, char *buf)
 616 {
 617         struct i915_str_attribute *eattr;
 618
 619         eattr = container_of(attr, struct i915_str_attribute, attr);
 620         return sprintf(buf, "%s\n", eattr->str);
 621 }
 622
 623 #define I915_PMU_FORMAT_ATTR(_name, _config) \
 624         (&((struct i915_str_attribute[]) { \
 625                 { .attr = __ATTR(_name, 0444, i915_pmu_format_show, NULL), \
 626                   .str = _config, } \
 627         })[0].attr.attr)
 628
 629 static struct attribute *i915_pmu_format_attrs[] = {
 630         I915_PMU_FORMAT_ATTR(i915_eventid, "config:0-20"),
 631         NULL,
 632 };
 633
 634 static const struct attribute_group i915_pmu_format_attr_group = {
 635         .name = "format",
 636         .attrs = i915_pmu_format_attrs,
 637 };
 638
 639 struct i915_ext_attribute {
 640         struct device_attribute attr;
 641         unsigned long val;
 642 };
 643
 644 static ssize_t i915_pmu_event_show(struct device *dev,
 645                                    struct device_attribute *attr, char *buf)
 646 {
 647         struct i915_ext_attribute *eattr;
 648
 649         eattr = container_of(attr, struct i915_ext_attribute, attr);
 650         return sprintf(buf, "config=0x%lx\n", eattr->val);
 651 }
 652
 653 #define I915_EVENT_ATTR(_name, _config) \
 654         (&((struct i915_ext_attribute[]) { \
 655                 { .attr = __ATTR(_name, 0444, i915_pmu_event_show, NULL), \
 656                   .val = _config, } \
 657         })[0].attr.attr)
 658
 659 #define I915_EVENT_STR(_name, _str) \
 660         (&((struct perf_pmu_events_attr[]) { \
 661                 { .attr      = __ATTR(_name, 0444, perf_event_sysfs_show, NULL), \
 662                   .id        = 0, \
 663                   .event_str = _str, } \
 664         })[0].attr.attr)
 665
 666 #define I915_EVENT(_name, _config, _unit) \
 667         I915_EVENT_ATTR(_name, _config), \
 668         I915_EVENT_STR(_name.unit, _unit)
 669
 670 #define I915_ENGINE_EVENT(_name, _class, _instance, _sample) \
 671         I915_EVENT_ATTR(_name, __I915_PMU_ENGINE(_class, _instance, _sample)), \
 672         I915_EVENT_STR(_name.unit, "ns")
 673
 674 #define I915_ENGINE_EVENTS(_name, _class, _instance) \
 675         I915_ENGINE_EVENT(_name##_instance-busy, _class, _instance, I915_SAMPLE_BUSY), \
 676         I915_ENGINE_EVENT(_name##_instance-sema, _class, _instance, I915_SAMPLE_SEMA), \
 677         I915_ENGINE_EVENT(_name##_instance-wait, _class, _instance, I915_SAMPLE_WAIT)
 678
 679 static struct attribute *i915_pmu_events_attrs[] = {
 680         I915_ENGINE_EVENTS(rcs, I915_ENGINE_CLASS_RENDER, 0),
 681         I915_ENGINE_EVENTS(bcs, I915_ENGINE_CLASS_COPY, 0),
 682         I915_ENGINE_EVENTS(vcs, I915_ENGINE_CLASS_VIDEO, 0),
 683         I915_ENGINE_EVENTS(vcs, I915_ENGINE_CLASS_VIDEO, 1),
 684         I915_ENGINE_EVENTS(vecs, I915_ENGINE_CLASS_VIDEO_ENHANCE, 0),
 685
 686         I915_EVENT(actual-frequency,    I915_PMU_ACTUAL_FREQUENCY,    "MHz"),
 687         I915_EVENT(requested-frequency, I915_PMU_REQUESTED_FREQUENCY, "MHz"),
 688
 689         I915_EVENT_ATTR(interrupts, I915_PMU_INTERRUPTS),
 690
 691         I915_EVENT(rc6-residency,   I915_PMU_RC6_RESIDENCY,   "ns"),
 692
 693         NULL,
 694 };
 695
 696 static const struct attribute_group i915_pmu_events_attr_group = {
 697         .name = "events",
 698         .attrs = i915_pmu_events_attrs,
 699 };
 700
 701 static ssize_t
 702 i915_pmu_get_attr_cpumask(struct device *dev,
 703                           struct device_attribute *attr,
 704                           char *buf)
 705 {
 706         return cpumap_print_to_pagebuf(true, buf, &i915_pmu_cpumask);
 707 }
 708
 709 static DEVICE_ATTR(cpumask, 0444, i915_pmu_get_attr_cpumask, NULL);
 710
 711 static struct attribute *i915_cpumask_attrs[] = {
 712         &dev_attr_cpumask.attr,
 713         NULL,
 714 };
 715
 716 static struct attribute_group i915_pmu_cpumask_attr_group = {
 717         .attrs = i915_cpumask_attrs,
 718 };
 719
 720 static const struct attribute_group *i915_pmu_attr_groups[] = {
 721         &i915_pmu_format_attr_group,
 722         &i915_pmu_events_attr_group,
 723         &i915_pmu_cpumask_attr_group,
 724         NULL
 725 };
 726
 727 static int i915_pmu_cpu_online(unsigned int cpu, struct hlist_node *node)
 728 {
 729         struct i915_pmu *pmu = hlist_entry_safe(node, typeof(*pmu), node);
 730
 731         GEM_BUG_ON(!pmu->base.event_init);
 732
 733         /* Select the first online CPU as a designated reader. */
 734         if (!cpumask_weight(&i915_pmu_cpumask))
 735                 cpumask_set_cpu(cpu, &i915_pmu_cpumask);
 736
 737         return 0;
 738 }
 739
 740 static int i915_pmu_cpu_offline(unsigned int cpu, struct hlist_node *node)
 741 {
 742         struct i915_pmu *pmu = hlist_entry_safe(node, typeof(*pmu), node);
 743         unsigned int target;
 744
 745         GEM_BUG_ON(!pmu->base.event_init);
 746
 747         if (cpumask_test_and_clear_cpu(cpu, &i915_pmu_cpumask)) {
 748                 target = cpumask_any_but(topology_sibling_cpumask(cpu), cpu);
 749                 /* Migrate events if there is a valid target */
 750                 if (target < nr_cpu_ids) {
 751                         cpumask_set_cpu(target, &i915_pmu_cpumask);
 752                         perf_pmu_migrate_context(&pmu->base, cpu, target);
 753                 }
 754         }
 755
 756         return 0;
 757 }
 758
 759 static enum cpuhp_state cpuhp_slot = CPUHP_INVALID;
 760
 761 static int i915_pmu_register_cpuhp_state(struct drm_i915_private *i915)
 762 {
 763         enum cpuhp_state slot;
 764         int ret;
 765
 766         ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
 767                                       "perf/x86/intel/i915:online",
 768                                       i915_pmu_cpu_online,
 769                                       i915_pmu_cpu_offline);
 770         if (ret < 0)
 771                 return ret;
 772
 773         slot = ret;
 774         ret = cpuhp_state_add_instance(slot, &i915->pmu.node);
 775         if (ret) {
 776                 cpuhp_remove_multi_state(slot);
 777                 return ret;
 778         }
 779
 780         cpuhp_slot = slot;
 781         return 0;
 782 }
 783
 784 static void i915_pmu_unregister_cpuhp_state(struct drm_i915_private *i915)
 785 {
 786         WARN_ON(cpuhp_slot == CPUHP_INVALID);
 787         WARN_ON(cpuhp_state_remove_instance(cpuhp_slot, &i915->pmu.node));
 788         cpuhp_remove_multi_state(cpuhp_slot);
 789 }
 790
 791 void i915_pmu_register(struct drm_i915_private *i915)
 792 {
 793         int ret;
 794
 795         if (INTEL_GEN(i915) <= 2) {
 796                 DRM_INFO("PMU not supported for this GPU.");
 797                 return;
 798         }
 799
 800         i915->pmu.base.attr_groups      = i915_pmu_attr_groups;
 801         i915->pmu.base.task_ctx_nr      = perf_invalid_context;
 802         i915->pmu.base.event_init       = i915_pmu_event_init;
 803         i915->pmu.base.add              = i915_pmu_event_add;
 804         i915->pmu.base.del              = i915_pmu_event_del;
 805         i915->pmu.base.start            = i915_pmu_event_start;
 806         i915->pmu.base.stop             = i915_pmu_event_stop;
 807         i915->pmu.base.read             = i915_pmu_event_read;
 808         i915->pmu.base.event_idx        = i915_pmu_event_event_idx;
 809
 810         spin_lock_init(&i915->pmu.lock);
 811         hrtimer_init(&i915->pmu.timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
 812         i915->pmu.timer.function = i915_sample;
 813
 814         ret = perf_pmu_register(&i915->pmu.base, "i915", -1);
 815         if (ret)
 816                 goto err;
 817
 818         ret = i915_pmu_register_cpuhp_state(i915);
 819         if (ret)
 820                 goto err_unreg;
 821
 822         return;
 823
 824 err_unreg:
 825         perf_pmu_unregister(&i915->pmu.base);
 826 err:
 827         i915->pmu.base.event_init = NULL;
 828         DRM_NOTE("Failed to register PMU! (err=%d)\n", ret);
 829 }
 830
 831 void i915_pmu_unregister(struct drm_i915_private *i915)
 832 {
 833         if (!i915->pmu.base.event_init)
 834                 return;
 835
 836         WARN_ON(i915->pmu.enable);
 837
 838         hrtimer_cancel(&i915->pmu.timer);
 839
 840         i915_pmu_unregister_cpuhp_state(i915);
 841
 842         perf_pmu_unregister(&i915->pmu.base);
 843         i915->pmu.base.event_init = NULL;
 844 }