1 /* 2 * Copyright © 2017 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 */ 24 25 #include <linux/perf_event.h> 26 #include <linux/pm_runtime.h> 27 28 #include "i915_drv.h" 29 #include "i915_pmu.h" 30 #include "intel_ringbuffer.h" 31 32 /* Frequency for the sampling timer for events which need it. */ 33 #define FREQUENCY 200 34 #define PERIOD max_t(u64, 10000, NSEC_PER_SEC / FREQUENCY) 35 36 #define ENGINE_SAMPLE_MASK \ 37 (BIT(I915_SAMPLE_BUSY) | \ 38 BIT(I915_SAMPLE_WAIT) | \ 39 BIT(I915_SAMPLE_SEMA)) 40 41 #define ENGINE_SAMPLE_BITS (1 << I915_PMU_SAMPLE_BITS) 42 43 static cpumask_t i915_pmu_cpumask; 44 45 static u8 engine_config_sample(u64 config) 46 { 47 return config & I915_PMU_SAMPLE_MASK; 48 } 49 50 static u8 engine_event_sample(struct perf_event *event) 51 { 52 return engine_config_sample(event->attr.config); 53 } 54 55 static u8 engine_event_class(struct perf_event *event) 56 { 57 return (event->attr.config >> I915_PMU_CLASS_SHIFT) & 0xff; 58 } 59 60 static u8 engine_event_instance(struct perf_event *event) 61 { 62 return (event->attr.config >> I915_PMU_SAMPLE_BITS) & 0xff; 63 } 64 65 static bool is_engine_config(u64 config) 66 { 67 return config < __I915_PMU_OTHER(0); 68 } 69 70 static unsigned int config_enabled_bit(u64 config) 71 { 72 if (is_engine_config(config)) 73 return engine_config_sample(config); 74 else 75 return ENGINE_SAMPLE_BITS + (config - __I915_PMU_OTHER(0)); 76 } 77 78 static u64 config_enabled_mask(u64 config) 79 { 80 return BIT_ULL(config_enabled_bit(config)); 81 } 82 83 static bool is_engine_event(struct perf_event *event) 84 { 85 return is_engine_config(event->attr.config); 86 } 87 88 static unsigned int event_enabled_bit(struct perf_event *event) 89 { 90 return config_enabled_bit(event->attr.config); 91 } 92 93 static bool pmu_needs_timer(struct drm_i915_private *i915, bool gpu_active) 94 { 95 u64 enable; 96 97 /* 98 * Only some counters need the sampling timer. 99 * 100 * We start with a bitmask of all currently enabled events. 101 */ 102 enable = i915->pmu.enable; 103 104 /* 105 * Mask out all the ones which do not need the timer, or in 106 * other words keep all the ones that could need the timer. 107 */ 108 enable &= config_enabled_mask(I915_PMU_ACTUAL_FREQUENCY) | 109 config_enabled_mask(I915_PMU_REQUESTED_FREQUENCY) | 110 ENGINE_SAMPLE_MASK; 111 112 /* 113 * When the GPU is idle per-engine counters do not need to be 114 * running so clear those bits out. 115 */ 116 if (!gpu_active) 117 enable &= ~ENGINE_SAMPLE_MASK; 118 /* 119 * Also there is software busyness tracking available we do not 120 * need the timer for I915_SAMPLE_BUSY counter. 121 * 122 * Use RCS as proxy for all engines. 123 */ 124 else if (intel_engine_supports_stats(i915->engine[RCS])) 125 enable &= ~BIT(I915_SAMPLE_BUSY); 126 127 /* 128 * If some bits remain it means we need the sampling timer running. 129 */ 130 return enable; 131 } 132 133 void i915_pmu_gt_parked(struct drm_i915_private *i915) 134 { 135 if (!i915->pmu.base.event_init) 136 return; 137 138 spin_lock_irq(&i915->pmu.lock); 139 /* 140 * Signal sampling timer to stop if only engine events are enabled and 141 * GPU went idle. 142 */ 143 i915->pmu.timer_enabled = pmu_needs_timer(i915, false); 144 spin_unlock_irq(&i915->pmu.lock); 145 } 146 147 static void __i915_pmu_maybe_start_timer(struct drm_i915_private *i915) 148 { 149 if (!i915->pmu.timer_enabled && pmu_needs_timer(i915, true)) { 150 i915->pmu.timer_enabled = true; 151 hrtimer_start_range_ns(&i915->pmu.timer, 152 ns_to_ktime(PERIOD), 0, 153 HRTIMER_MODE_REL_PINNED); 154 } 155 } 156 157 void i915_pmu_gt_unparked(struct drm_i915_private *i915) 158 { 159 if (!i915->pmu.base.event_init) 160 return; 161 162 spin_lock_irq(&i915->pmu.lock); 163 /* 164 * Re-enable sampling timer when GPU goes active. 165 */ 166 __i915_pmu_maybe_start_timer(i915); 167 spin_unlock_irq(&i915->pmu.lock); 168 } 169 170 static bool grab_forcewake(struct drm_i915_private *i915, bool fw) 171 { 172 if (!fw) 173 intel_uncore_forcewake_get(i915, FORCEWAKE_ALL); 174 175 return true; 176 } 177 178 static void 179 update_sample(struct i915_pmu_sample *sample, u32 unit, u32 val) 180 { 181 sample->cur += mul_u32_u32(val, unit); 182 } 183 184 static void engines_sample(struct drm_i915_private *dev_priv) 185 { 186 struct intel_engine_cs *engine; 187 enum intel_engine_id id; 188 bool fw = false; 189 190 if ((dev_priv->pmu.enable & ENGINE_SAMPLE_MASK) == 0) 191 return; 192 193 if (!dev_priv->gt.awake) 194 return; 195 196 if (!intel_runtime_pm_get_if_in_use(dev_priv)) 197 return; 198 199 for_each_engine(engine, dev_priv, id) { 200 u32 current_seqno = intel_engine_get_seqno(engine); 201 u32 last_seqno = intel_engine_last_submit(engine); 202 u32 val; 203 204 val = !i915_seqno_passed(current_seqno, last_seqno); 205 206 update_sample(&engine->pmu.sample[I915_SAMPLE_BUSY], 207 PERIOD, val); 208 209 if (val && (engine->pmu.enable & 210 (BIT(I915_SAMPLE_WAIT) | BIT(I915_SAMPLE_SEMA)))) { 211 fw = grab_forcewake(dev_priv, fw); 212 213 val = I915_READ_FW(RING_CTL(engine->mmio_base)); 214 } else { 215 val = 0; 216 } 217 218 update_sample(&engine->pmu.sample[I915_SAMPLE_WAIT], 219 PERIOD, !!(val & RING_WAIT)); 220 221 update_sample(&engine->pmu.sample[I915_SAMPLE_SEMA], 222 PERIOD, !!(val & RING_WAIT_SEMAPHORE)); 223 } 224 225 if (fw) 226 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); 227 228 intel_runtime_pm_put(dev_priv); 229 } 230 231 static void frequency_sample(struct drm_i915_private *dev_priv) 232 { 233 if (dev_priv->pmu.enable & 234 config_enabled_mask(I915_PMU_ACTUAL_FREQUENCY)) { 235 u32 val; 236 237 val = dev_priv->gt_pm.rps.cur_freq; 238 if (dev_priv->gt.awake && 239 intel_runtime_pm_get_if_in_use(dev_priv)) { 240 val = intel_get_cagf(dev_priv, 241 I915_READ_NOTRACE(GEN6_RPSTAT1)); 242 intel_runtime_pm_put(dev_priv); 243 } 244 245 update_sample(&dev_priv->pmu.sample[__I915_SAMPLE_FREQ_ACT], 246 1, intel_gpu_freq(dev_priv, val)); 247 } 248 249 if (dev_priv->pmu.enable & 250 config_enabled_mask(I915_PMU_REQUESTED_FREQUENCY)) { 251 update_sample(&dev_priv->pmu.sample[__I915_SAMPLE_FREQ_REQ], 1, 252 intel_gpu_freq(dev_priv, 253 dev_priv->gt_pm.rps.cur_freq)); 254 } 255 } 256 257 static enum hrtimer_restart i915_sample(struct hrtimer *hrtimer) 258 { 259 struct drm_i915_private *i915 = 260 container_of(hrtimer, struct drm_i915_private, pmu.timer); 261 262 if (!READ_ONCE(i915->pmu.timer_enabled)) 263 return HRTIMER_NORESTART; 264 265 engines_sample(i915); 266 frequency_sample(i915); 267 268 hrtimer_forward_now(hrtimer, ns_to_ktime(PERIOD)); 269 return HRTIMER_RESTART; 270 } 271 272 static u64 count_interrupts(struct drm_i915_private *i915) 273 { 274 /* open-coded kstat_irqs() */ 275 struct irq_desc *desc = irq_to_desc(i915->drm.pdev->irq); 276 u64 sum = 0; 277 int cpu; 278 279 if (!desc || !desc->kstat_irqs) 280 return 0; 281 282 for_each_possible_cpu(cpu) 283 sum += *per_cpu_ptr(desc->kstat_irqs, cpu); 284 285 return sum; 286 } 287 288 static void engine_event_destroy(struct perf_event *event) 289 { 290 struct drm_i915_private *i915 = 291 container_of(event->pmu, typeof(*i915), pmu.base); 292 struct intel_engine_cs *engine; 293 294 engine = intel_engine_lookup_user(i915, 295 engine_event_class(event), 296 engine_event_instance(event)); 297 if (WARN_ON_ONCE(!engine)) 298 return; 299 300 if (engine_event_sample(event) == I915_SAMPLE_BUSY && 301 intel_engine_supports_stats(engine)) 302 intel_disable_engine_stats(engine); 303 } 304 305 static void i915_pmu_event_destroy(struct perf_event *event) 306 { 307 WARN_ON(event->parent); 308 309 if (is_engine_event(event)) 310 engine_event_destroy(event); 311 } 312 313 static int 314 engine_event_status(struct intel_engine_cs *engine, 315 enum drm_i915_pmu_engine_sample sample) 316 { 317 switch (sample) { 318 case I915_SAMPLE_BUSY: 319 case I915_SAMPLE_WAIT: 320 break; 321 case I915_SAMPLE_SEMA: 322 if (INTEL_GEN(engine->i915) < 6) 323 return -ENODEV; 324 break; 325 default: 326 return -ENOENT; 327 } 328 329 return 0; 330 } 331 332 static int 333 config_status(struct drm_i915_private *i915, u64 config) 334 { 335 switch (config) { 336 case I915_PMU_ACTUAL_FREQUENCY: 337 if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) 338 /* Requires a mutex for sampling! */ 339 return -ENODEV; 340 /* Fall-through. */ 341 case I915_PMU_REQUESTED_FREQUENCY: 342 if (INTEL_GEN(i915) < 6) 343 return -ENODEV; 344 break; 345 case I915_PMU_INTERRUPTS: 346 break; 347 case I915_PMU_RC6_RESIDENCY: 348 if (!HAS_RC6(i915)) 349 return -ENODEV; 350 break; 351 default: 352 return -ENOENT; 353 } 354 355 return 0; 356 } 357 358 static int engine_event_init(struct perf_event *event) 359 { 360 struct drm_i915_private *i915 = 361 container_of(event->pmu, typeof(*i915), pmu.base); 362 struct intel_engine_cs *engine; 363 u8 sample; 364 int ret; 365 366 engine = intel_engine_lookup_user(i915, engine_event_class(event), 367 engine_event_instance(event)); 368 if (!engine) 369 return -ENODEV; 370 371 sample = engine_event_sample(event); 372 ret = engine_event_status(engine, sample); 373 if (ret) 374 return ret; 375 376 if (sample == I915_SAMPLE_BUSY && intel_engine_supports_stats(engine)) 377 ret = intel_enable_engine_stats(engine); 378 379 return ret; 380 } 381 382 static int i915_pmu_event_init(struct perf_event *event) 383 { 384 struct drm_i915_private *i915 = 385 container_of(event->pmu, typeof(*i915), pmu.base); 386 int ret; 387 388 if (event->attr.type != event->pmu->type) 389 return -ENOENT; 390 391 /* unsupported modes and filters */ 392 if (event->attr.sample_period) /* no sampling */ 393 return -EINVAL; 394 395 if (has_branch_stack(event)) 396 return -EOPNOTSUPP; 397 398 if (event->cpu < 0) 399 return -EINVAL; 400 401 /* only allow running on one cpu at a time */ 402 if (!cpumask_test_cpu(event->cpu, &i915_pmu_cpumask)) 403 return -EINVAL; 404 405 if (is_engine_event(event)) 406 ret = engine_event_init(event); 407 else 408 ret = config_status(i915, event->attr.config); 409 if (ret) 410 return ret; 411 412 if (!event->parent) 413 event->destroy = i915_pmu_event_destroy; 414 415 return 0; 416 } 417 418 static u64 __get_rc6(struct drm_i915_private *i915) 419 { 420 u64 val; 421 422 val = intel_rc6_residency_ns(i915, 423 IS_VALLEYVIEW(i915) ? 424 VLV_GT_RENDER_RC6 : 425 GEN6_GT_GFX_RC6); 426 427 if (HAS_RC6p(i915)) 428 val += intel_rc6_residency_ns(i915, GEN6_GT_GFX_RC6p); 429 430 if (HAS_RC6pp(i915)) 431 val += intel_rc6_residency_ns(i915, GEN6_GT_GFX_RC6pp); 432 433 return val; 434 } 435 436 static u64 get_rc6(struct drm_i915_private *i915, bool locked) 437 { 438 #if IS_ENABLED(CONFIG_PM) 439 unsigned long flags; 440 u64 val; 441 442 if (intel_runtime_pm_get_if_in_use(i915)) { 443 val = __get_rc6(i915); 444 intel_runtime_pm_put(i915); 445 446 /* 447 * If we are coming back from being runtime suspended we must 448 * be careful not to report a larger value than returned 449 * previously. 450 */ 451 452 if (!locked) 453 spin_lock_irqsave(&i915->pmu.lock, flags); 454 455 if (val >= i915->pmu.sample[__I915_SAMPLE_RC6_ESTIMATED].cur) { 456 i915->pmu.sample[__I915_SAMPLE_RC6_ESTIMATED].cur = 0; 457 i915->pmu.sample[__I915_SAMPLE_RC6].cur = val; 458 } else { 459 val = i915->pmu.sample[__I915_SAMPLE_RC6_ESTIMATED].cur; 460 } 461 462 if (!locked) 463 spin_unlock_irqrestore(&i915->pmu.lock, flags); 464 } else { 465 struct pci_dev *pdev = i915->drm.pdev; 466 struct device *kdev = &pdev->dev; 467 unsigned long flags2; 468 469 /* 470 * We are runtime suspended. 471 * 472 * Report the delta from when the device was suspended to now, 473 * on top of the last known real value, as the approximated RC6 474 * counter value. 475 */ 476 if (!locked) 477 spin_lock_irqsave(&i915->pmu.lock, flags); 478 479 spin_lock_irqsave(&kdev->power.lock, flags2); 480 481 if (!i915->pmu.sample[__I915_SAMPLE_RC6_ESTIMATED].cur) 482 i915->pmu.suspended_jiffies_last = 483 kdev->power.suspended_jiffies; 484 485 val = kdev->power.suspended_jiffies - 486 i915->pmu.suspended_jiffies_last; 487 val += jiffies - kdev->power.accounting_timestamp; 488 489 spin_unlock_irqrestore(&kdev->power.lock, flags2); 490 491 val = jiffies_to_nsecs(val); 492 val += i915->pmu.sample[__I915_SAMPLE_RC6].cur; 493 i915->pmu.sample[__I915_SAMPLE_RC6_ESTIMATED].cur = val; 494 495 if (!locked) 496 spin_unlock_irqrestore(&i915->pmu.lock, flags); 497 } 498 499 return val; 500 #else 501 return __get_rc6(i915); 502 #endif 503 } 504 505 static u64 __i915_pmu_event_read(struct perf_event *event, bool locked) 506 { 507 struct drm_i915_private *i915 = 508 container_of(event->pmu, typeof(*i915), pmu.base); 509 u64 val = 0; 510 511 if (is_engine_event(event)) { 512 u8 sample = engine_event_sample(event); 513 struct intel_engine_cs *engine; 514 515 engine = intel_engine_lookup_user(i915, 516 engine_event_class(event), 517 engine_event_instance(event)); 518 519 if (WARN_ON_ONCE(!engine)) { 520 /* Do nothing */ 521 } else if (sample == I915_SAMPLE_BUSY && 522 intel_engine_supports_stats(engine)) { 523 val = ktime_to_ns(intel_engine_get_busy_time(engine)); 524 } else { 525 val = engine->pmu.sample[sample].cur; 526 } 527 } else { 528 switch (event->attr.config) { 529 case I915_PMU_ACTUAL_FREQUENCY: 530 val = 531 div_u64(i915->pmu.sample[__I915_SAMPLE_FREQ_ACT].cur, 532 FREQUENCY); 533 break; 534 case I915_PMU_REQUESTED_FREQUENCY: 535 val = 536 div_u64(i915->pmu.sample[__I915_SAMPLE_FREQ_REQ].cur, 537 FREQUENCY); 538 break; 539 case I915_PMU_INTERRUPTS: 540 val = count_interrupts(i915); 541 break; 542 case I915_PMU_RC6_RESIDENCY: 543 val = get_rc6(i915, locked); 544 break; 545 } 546 } 547 548 return val; 549 } 550 551 static void i915_pmu_event_read(struct perf_event *event) 552 { 553 struct hw_perf_event *hwc = &event->hw; 554 u64 prev, new; 555 556 again: 557 prev = local64_read(&hwc->prev_count); 558 new = __i915_pmu_event_read(event, false); 559 560 if (local64_cmpxchg(&hwc->prev_count, prev, new) != prev) 561 goto again; 562 563 local64_add(new - prev, &event->count); 564 } 565 566 static void i915_pmu_enable(struct perf_event *event) 567 { 568 struct drm_i915_private *i915 = 569 container_of(event->pmu, typeof(*i915), pmu.base); 570 unsigned int bit = event_enabled_bit(event); 571 unsigned long flags; 572 573 spin_lock_irqsave(&i915->pmu.lock, flags); 574 575 /* 576 * Update the bitmask of enabled events and increment 577 * the event reference counter. 578 */ 579 GEM_BUG_ON(bit >= I915_PMU_MASK_BITS); 580 GEM_BUG_ON(i915->pmu.enable_count[bit] == ~0); 581 i915->pmu.enable |= BIT_ULL(bit); 582 i915->pmu.enable_count[bit]++; 583 584 /* 585 * Start the sampling timer if needed and not already enabled. 586 */ 587 __i915_pmu_maybe_start_timer(i915); 588 589 /* 590 * For per-engine events the bitmask and reference counting 591 * is stored per engine. 592 */ 593 if (is_engine_event(event)) { 594 u8 sample = engine_event_sample(event); 595 struct intel_engine_cs *engine; 596 597 engine = intel_engine_lookup_user(i915, 598 engine_event_class(event), 599 engine_event_instance(event)); 600 GEM_BUG_ON(!engine); 601 engine->pmu.enable |= BIT(sample); 602 603 GEM_BUG_ON(sample >= I915_PMU_SAMPLE_BITS); 604 GEM_BUG_ON(engine->pmu.enable_count[sample] == ~0); 605 engine->pmu.enable_count[sample]++; 606 } 607 608 /* 609 * Store the current counter value so we can report the correct delta 610 * for all listeners. Even when the event was already enabled and has 611 * an existing non-zero value. 612 */ 613 local64_set(&event->hw.prev_count, __i915_pmu_event_read(event, true)); 614 615 spin_unlock_irqrestore(&i915->pmu.lock, flags); 616 } 617 618 static void i915_pmu_disable(struct perf_event *event) 619 { 620 struct drm_i915_private *i915 = 621 container_of(event->pmu, typeof(*i915), pmu.base); 622 unsigned int bit = event_enabled_bit(event); 623 unsigned long flags; 624 625 spin_lock_irqsave(&i915->pmu.lock, flags); 626 627 if (is_engine_event(event)) { 628 u8 sample = engine_event_sample(event); 629 struct intel_engine_cs *engine; 630 631 engine = intel_engine_lookup_user(i915, 632 engine_event_class(event), 633 engine_event_instance(event)); 634 GEM_BUG_ON(!engine); 635 GEM_BUG_ON(sample >= I915_PMU_SAMPLE_BITS); 636 GEM_BUG_ON(engine->pmu.enable_count[sample] == 0); 637 /* 638 * Decrement the reference count and clear the enabled 639 * bitmask when the last listener on an event goes away. 640 */ 641 if (--engine->pmu.enable_count[sample] == 0) 642 engine->pmu.enable &= ~BIT(sample); 643 } 644 645 GEM_BUG_ON(bit >= I915_PMU_MASK_BITS); 646 GEM_BUG_ON(i915->pmu.enable_count[bit] == 0); 647 /* 648 * Decrement the reference count and clear the enabled 649 * bitmask when the last listener on an event goes away. 650 */ 651 if (--i915->pmu.enable_count[bit] == 0) { 652 i915->pmu.enable &= ~BIT_ULL(bit); 653 i915->pmu.timer_enabled &= pmu_needs_timer(i915, true); 654 } 655 656 spin_unlock_irqrestore(&i915->pmu.lock, flags); 657 } 658 659 static void i915_pmu_event_start(struct perf_event *event, int flags) 660 { 661 i915_pmu_enable(event); 662 event->hw.state = 0; 663 } 664 665 static void i915_pmu_event_stop(struct perf_event *event, int flags) 666 { 667 if (flags & PERF_EF_UPDATE) 668 i915_pmu_event_read(event); 669 i915_pmu_disable(event); 670 event->hw.state = PERF_HES_STOPPED; 671 } 672 673 static int i915_pmu_event_add(struct perf_event *event, int flags) 674 { 675 if (flags & PERF_EF_START) 676 i915_pmu_event_start(event, flags); 677 678 return 0; 679 } 680 681 static void i915_pmu_event_del(struct perf_event *event, int flags) 682 { 683 i915_pmu_event_stop(event, PERF_EF_UPDATE); 684 } 685 686 static int i915_pmu_event_event_idx(struct perf_event *event) 687 { 688 return 0; 689 } 690 691 struct i915_str_attribute { 692 struct device_attribute attr; 693 const char *str; 694 }; 695 696 static ssize_t i915_pmu_format_show(struct device *dev, 697 struct device_attribute *attr, char *buf) 698 { 699 struct i915_str_attribute *eattr; 700 701 eattr = container_of(attr, struct i915_str_attribute, attr); 702 return sprintf(buf, "%s\n", eattr->str); 703 } 704 705 #define I915_PMU_FORMAT_ATTR(_name, _config) \ 706 (&((struct i915_str_attribute[]) { \ 707 { .attr = __ATTR(_name, 0444, i915_pmu_format_show, NULL), \ 708 .str = _config, } \ 709 })[0].attr.attr) 710 711 static struct attribute *i915_pmu_format_attrs[] = { 712 I915_PMU_FORMAT_ATTR(i915_eventid, "config:0-20"), 713 NULL, 714 }; 715 716 static const struct attribute_group i915_pmu_format_attr_group = { 717 .name = "format", 718 .attrs = i915_pmu_format_attrs, 719 }; 720 721 struct i915_ext_attribute { 722 struct device_attribute attr; 723 unsigned long val; 724 }; 725 726 static ssize_t i915_pmu_event_show(struct device *dev, 727 struct device_attribute *attr, char *buf) 728 { 729 struct i915_ext_attribute *eattr; 730 731 eattr = container_of(attr, struct i915_ext_attribute, attr); 732 return sprintf(buf, "config=0x%lx\n", eattr->val); 733 } 734 735 static struct attribute_group i915_pmu_events_attr_group = { 736 .name = "events", 737 /* Patch in attrs at runtime. */ 738 }; 739 740 static ssize_t 741 i915_pmu_get_attr_cpumask(struct device *dev, 742 struct device_attribute *attr, 743 char *buf) 744 { 745 return cpumap_print_to_pagebuf(true, buf, &i915_pmu_cpumask); 746 } 747 748 static DEVICE_ATTR(cpumask, 0444, i915_pmu_get_attr_cpumask, NULL); 749 750 static struct attribute *i915_cpumask_attrs[] = { 751 &dev_attr_cpumask.attr, 752 NULL, 753 }; 754 755 static const struct attribute_group i915_pmu_cpumask_attr_group = { 756 .attrs = i915_cpumask_attrs, 757 }; 758 759 static const struct attribute_group *i915_pmu_attr_groups[] = { 760 &i915_pmu_format_attr_group, 761 &i915_pmu_events_attr_group, 762 &i915_pmu_cpumask_attr_group, 763 NULL 764 }; 765 766 #define __event(__config, __name, __unit) \ 767 { \ 768 .config = (__config), \ 769 .name = (__name), \ 770 .unit = (__unit), \ 771 } 772 773 #define __engine_event(__sample, __name) \ 774 { \ 775 .sample = (__sample), \ 776 .name = (__name), \ 777 } 778 779 static struct i915_ext_attribute * 780 add_i915_attr(struct i915_ext_attribute *attr, const char *name, u64 config) 781 { 782 sysfs_attr_init(&attr->attr.attr); 783 attr->attr.attr.name = name; 784 attr->attr.attr.mode = 0444; 785 attr->attr.show = i915_pmu_event_show; 786 attr->val = config; 787 788 return ++attr; 789 } 790 791 static struct perf_pmu_events_attr * 792 add_pmu_attr(struct perf_pmu_events_attr *attr, const char *name, 793 const char *str) 794 { 795 sysfs_attr_init(&attr->attr.attr); 796 attr->attr.attr.name = name; 797 attr->attr.attr.mode = 0444; 798 attr->attr.show = perf_event_sysfs_show; 799 attr->event_str = str; 800 801 return ++attr; 802 } 803 804 static struct attribute ** 805 create_event_attributes(struct drm_i915_private *i915) 806 { 807 static const struct { 808 u64 config; 809 const char *name; 810 const char *unit; 811 } events[] = { 812 __event(I915_PMU_ACTUAL_FREQUENCY, "actual-frequency", "MHz"), 813 __event(I915_PMU_REQUESTED_FREQUENCY, "requested-frequency", "MHz"), 814 __event(I915_PMU_INTERRUPTS, "interrupts", NULL), 815 __event(I915_PMU_RC6_RESIDENCY, "rc6-residency", "ns"), 816 }; 817 static const struct { 818 enum drm_i915_pmu_engine_sample sample; 819 char *name; 820 } engine_events[] = { 821 __engine_event(I915_SAMPLE_BUSY, "busy"), 822 __engine_event(I915_SAMPLE_SEMA, "sema"), 823 __engine_event(I915_SAMPLE_WAIT, "wait"), 824 }; 825 unsigned int count = 0; 826 struct perf_pmu_events_attr *pmu_attr = NULL, *pmu_iter; 827 struct i915_ext_attribute *i915_attr = NULL, *i915_iter; 828 struct attribute **attr = NULL, **attr_iter; 829 struct intel_engine_cs *engine; 830 enum intel_engine_id id; 831 unsigned int i; 832 833 /* Count how many counters we will be exposing. */ 834 for (i = 0; i < ARRAY_SIZE(events); i++) { 835 if (!config_status(i915, events[i].config)) 836 count++; 837 } 838 839 for_each_engine(engine, i915, id) { 840 for (i = 0; i < ARRAY_SIZE(engine_events); i++) { 841 if (!engine_event_status(engine, 842 engine_events[i].sample)) 843 count++; 844 } 845 } 846 847 /* Allocate attribute objects and table. */ 848 i915_attr = kcalloc(count, sizeof(*i915_attr), GFP_KERNEL); 849 if (!i915_attr) 850 goto err_alloc; 851 852 pmu_attr = kcalloc(count, sizeof(*pmu_attr), GFP_KERNEL); 853 if (!pmu_attr) 854 goto err_alloc; 855 856 /* Max one pointer of each attribute type plus a termination entry. */ 857 attr = kcalloc(count * 2 + 1, sizeof(*attr), GFP_KERNEL); 858 if (!attr) 859 goto err_alloc; 860 861 i915_iter = i915_attr; 862 pmu_iter = pmu_attr; 863 attr_iter = attr; 864 865 /* Initialize supported non-engine counters. */ 866 for (i = 0; i < ARRAY_SIZE(events); i++) { 867 char *str; 868 869 if (config_status(i915, events[i].config)) 870 continue; 871 872 str = kstrdup(events[i].name, GFP_KERNEL); 873 if (!str) 874 goto err; 875 876 *attr_iter++ = &i915_iter->attr.attr; 877 i915_iter = add_i915_attr(i915_iter, str, events[i].config); 878 879 if (events[i].unit) { 880 str = kasprintf(GFP_KERNEL, "%s.unit", events[i].name); 881 if (!str) 882 goto err; 883 884 *attr_iter++ = &pmu_iter->attr.attr; 885 pmu_iter = add_pmu_attr(pmu_iter, str, events[i].unit); 886 } 887 } 888 889 /* Initialize supported engine counters. */ 890 for_each_engine(engine, i915, id) { 891 for (i = 0; i < ARRAY_SIZE(engine_events); i++) { 892 char *str; 893 894 if (engine_event_status(engine, 895 engine_events[i].sample)) 896 continue; 897 898 str = kasprintf(GFP_KERNEL, "%s-%s", 899 engine->name, engine_events[i].name); 900 if (!str) 901 goto err; 902 903 *attr_iter++ = &i915_iter->attr.attr; 904 i915_iter = 905 add_i915_attr(i915_iter, str, 906 __I915_PMU_ENGINE(engine->uabi_class, 907 engine->instance, 908 engine_events[i].sample)); 909 910 str = kasprintf(GFP_KERNEL, "%s-%s.unit", 911 engine->name, engine_events[i].name); 912 if (!str) 913 goto err; 914 915 *attr_iter++ = &pmu_iter->attr.attr; 916 pmu_iter = add_pmu_attr(pmu_iter, str, "ns"); 917 } 918 } 919 920 i915->pmu.i915_attr = i915_attr; 921 i915->pmu.pmu_attr = pmu_attr; 922 923 return attr; 924 925 err:; 926 for (attr_iter = attr; *attr_iter; attr_iter++) 927 kfree((*attr_iter)->name); 928 929 err_alloc: 930 kfree(attr); 931 kfree(i915_attr); 932 kfree(pmu_attr); 933 934 return NULL; 935 } 936 937 static void free_event_attributes(struct drm_i915_private *i915) 938 { 939 struct attribute **attr_iter = i915_pmu_events_attr_group.attrs; 940 941 for (; *attr_iter; attr_iter++) 942 kfree((*attr_iter)->name); 943 944 kfree(i915_pmu_events_attr_group.attrs); 945 kfree(i915->pmu.i915_attr); 946 kfree(i915->pmu.pmu_attr); 947 948 i915_pmu_events_attr_group.attrs = NULL; 949 i915->pmu.i915_attr = NULL; 950 i915->pmu.pmu_attr = NULL; 951 } 952 953 static int i915_pmu_cpu_online(unsigned int cpu, struct hlist_node *node) 954 { 955 struct i915_pmu *pmu = hlist_entry_safe(node, typeof(*pmu), node); 956 957 GEM_BUG_ON(!pmu->base.event_init); 958 959 /* Select the first online CPU as a designated reader. */ 960 if (!cpumask_weight(&i915_pmu_cpumask)) 961 cpumask_set_cpu(cpu, &i915_pmu_cpumask); 962 963 return 0; 964 } 965 966 static int i915_pmu_cpu_offline(unsigned int cpu, struct hlist_node *node) 967 { 968 struct i915_pmu *pmu = hlist_entry_safe(node, typeof(*pmu), node); 969 unsigned int target; 970 971 GEM_BUG_ON(!pmu->base.event_init); 972 973 if (cpumask_test_and_clear_cpu(cpu, &i915_pmu_cpumask)) { 974 target = cpumask_any_but(topology_sibling_cpumask(cpu), cpu); 975 /* Migrate events if there is a valid target */ 976 if (target < nr_cpu_ids) { 977 cpumask_set_cpu(target, &i915_pmu_cpumask); 978 perf_pmu_migrate_context(&pmu->base, cpu, target); 979 } 980 } 981 982 return 0; 983 } 984 985 static enum cpuhp_state cpuhp_slot = CPUHP_INVALID; 986 987 static int i915_pmu_register_cpuhp_state(struct drm_i915_private *i915) 988 { 989 enum cpuhp_state slot; 990 int ret; 991 992 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, 993 "perf/x86/intel/i915:online", 994 i915_pmu_cpu_online, 995 i915_pmu_cpu_offline); 996 if (ret < 0) 997 return ret; 998 999 slot = ret; 1000 ret = cpuhp_state_add_instance(slot, &i915->pmu.node); 1001 if (ret) { 1002 cpuhp_remove_multi_state(slot); 1003 return ret; 1004 } 1005 1006 cpuhp_slot = slot; 1007 return 0; 1008 } 1009 1010 static void i915_pmu_unregister_cpuhp_state(struct drm_i915_private *i915) 1011 { 1012 WARN_ON(cpuhp_slot == CPUHP_INVALID); 1013 WARN_ON(cpuhp_state_remove_instance(cpuhp_slot, &i915->pmu.node)); 1014 cpuhp_remove_multi_state(cpuhp_slot); 1015 } 1016 1017 void i915_pmu_register(struct drm_i915_private *i915) 1018 { 1019 int ret; 1020 1021 if (INTEL_GEN(i915) <= 2) { 1022 DRM_INFO("PMU not supported for this GPU."); 1023 return; 1024 } 1025 1026 i915_pmu_events_attr_group.attrs = create_event_attributes(i915); 1027 if (!i915_pmu_events_attr_group.attrs) { 1028 ret = -ENOMEM; 1029 goto err; 1030 } 1031 1032 i915->pmu.base.attr_groups = i915_pmu_attr_groups; 1033 i915->pmu.base.task_ctx_nr = perf_invalid_context; 1034 i915->pmu.base.event_init = i915_pmu_event_init; 1035 i915->pmu.base.add = i915_pmu_event_add; 1036 i915->pmu.base.del = i915_pmu_event_del; 1037 i915->pmu.base.start = i915_pmu_event_start; 1038 i915->pmu.base.stop = i915_pmu_event_stop; 1039 i915->pmu.base.read = i915_pmu_event_read; 1040 i915->pmu.base.event_idx = i915_pmu_event_event_idx; 1041 1042 spin_lock_init(&i915->pmu.lock); 1043 hrtimer_init(&i915->pmu.timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 1044 i915->pmu.timer.function = i915_sample; 1045 1046 ret = perf_pmu_register(&i915->pmu.base, "i915", -1); 1047 if (ret) 1048 goto err; 1049 1050 ret = i915_pmu_register_cpuhp_state(i915); 1051 if (ret) 1052 goto err_unreg; 1053 1054 return; 1055 1056 err_unreg: 1057 perf_pmu_unregister(&i915->pmu.base); 1058 err: 1059 i915->pmu.base.event_init = NULL; 1060 free_event_attributes(i915); 1061 DRM_NOTE("Failed to register PMU! (err=%d)\n", ret); 1062 } 1063 1064 void i915_pmu_unregister(struct drm_i915_private *i915) 1065 { 1066 if (!i915->pmu.base.event_init) 1067 return; 1068 1069 WARN_ON(i915->pmu.enable); 1070 1071 hrtimer_cancel(&i915->pmu.timer); 1072 1073 i915_pmu_unregister_cpuhp_state(i915); 1074 1075 perf_pmu_unregister(&i915->pmu.base); 1076 i915->pmu.base.event_init = NULL; 1077 free_event_attributes(i915); 1078 } 1079