1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2017-2018 Intel Corporation 5 */ 6 7 #include <linux/pm_runtime.h> 8 9 #include "gt/intel_engine.h" 10 #include "gt/intel_engine_pm.h" 11 #include "gt/intel_engine_user.h" 12 #include "gt/intel_gt_pm.h" 13 #include "gt/intel_rc6.h" 14 #include "gt/intel_rps.h" 15 16 #include "i915_drv.h" 17 #include "i915_pmu.h" 18 #include "intel_pm.h" 19 20 /* Frequency for the sampling timer for events which need it. */ 21 #define FREQUENCY 200 22 #define PERIOD max_t(u64, 10000, NSEC_PER_SEC / FREQUENCY) 23 24 #define ENGINE_SAMPLE_MASK \ 25 (BIT(I915_SAMPLE_BUSY) | \ 26 BIT(I915_SAMPLE_WAIT) | \ 27 BIT(I915_SAMPLE_SEMA)) 28 29 #define ENGINE_SAMPLE_BITS (1 << I915_PMU_SAMPLE_BITS) 30 31 static cpumask_t i915_pmu_cpumask; 32 static unsigned int i915_pmu_target_cpu = -1; 33 34 static u8 engine_config_sample(u64 config) 35 { 36 return config & I915_PMU_SAMPLE_MASK; 37 } 38 39 static u8 engine_event_sample(struct perf_event *event) 40 { 41 return engine_config_sample(event->attr.config); 42 } 43 44 static u8 engine_event_class(struct perf_event *event) 45 { 46 return (event->attr.config >> I915_PMU_CLASS_SHIFT) & 0xff; 47 } 48 49 static u8 engine_event_instance(struct perf_event *event) 50 { 51 return (event->attr.config >> I915_PMU_SAMPLE_BITS) & 0xff; 52 } 53 54 static bool is_engine_config(u64 config) 55 { 56 return config < __I915_PMU_OTHER(0); 57 } 58 59 static unsigned int config_enabled_bit(u64 config) 60 { 61 if (is_engine_config(config)) 62 return engine_config_sample(config); 63 else 64 return ENGINE_SAMPLE_BITS + (config - __I915_PMU_OTHER(0)); 65 } 66 67 static u64 config_enabled_mask(u64 config) 68 { 69 return BIT_ULL(config_enabled_bit(config)); 70 } 71 72 static bool is_engine_event(struct perf_event *event) 73 { 74 return is_engine_config(event->attr.config); 75 } 76 77 static unsigned int event_enabled_bit(struct perf_event *event) 78 { 79 return config_enabled_bit(event->attr.config); 80 } 81 82 static bool pmu_needs_timer(struct i915_pmu *pmu, bool gpu_active) 83 { 84 struct drm_i915_private *i915 = container_of(pmu, typeof(*i915), pmu); 85 u64 enable; 86 87 /* 88 * Only some counters need the sampling timer. 89 * 90 * We start with a bitmask of all currently enabled events. 91 */ 92 enable = pmu->enable; 93 94 /* 95 * Mask out all the ones which do not need the timer, or in 96 * other words keep all the ones that could need the timer. 97 */ 98 enable &= config_enabled_mask(I915_PMU_ACTUAL_FREQUENCY) | 99 config_enabled_mask(I915_PMU_REQUESTED_FREQUENCY) | 100 ENGINE_SAMPLE_MASK; 101 102 /* 103 * When the GPU is idle per-engine counters do not need to be 104 * running so clear those bits out. 105 */ 106 if (!gpu_active) 107 enable &= ~ENGINE_SAMPLE_MASK; 108 /* 109 * Also there is software busyness tracking available we do not 110 * need the timer for I915_SAMPLE_BUSY counter. 111 */ 112 else if (i915->caps.scheduler & I915_SCHEDULER_CAP_ENGINE_BUSY_STATS) 113 enable &= ~BIT(I915_SAMPLE_BUSY); 114 115 /* 116 * If some bits remain it means we need the sampling timer running. 117 */ 118 return enable; 119 } 120 121 static u64 __get_rc6(struct intel_gt *gt) 122 { 123 struct drm_i915_private *i915 = gt->i915; 124 u64 val; 125 126 val = intel_rc6_residency_ns(>->rc6, 127 IS_VALLEYVIEW(i915) ? 128 VLV_GT_RENDER_RC6 : 129 GEN6_GT_GFX_RC6); 130 131 if (HAS_RC6p(i915)) 132 val += intel_rc6_residency_ns(>->rc6, GEN6_GT_GFX_RC6p); 133 134 if (HAS_RC6pp(i915)) 135 val += intel_rc6_residency_ns(>->rc6, GEN6_GT_GFX_RC6pp); 136 137 return val; 138 } 139 140 #if IS_ENABLED(CONFIG_PM) 141 142 static inline s64 ktime_since(const ktime_t kt) 143 { 144 return ktime_to_ns(ktime_sub(ktime_get(), kt)); 145 } 146 147 static u64 get_rc6(struct intel_gt *gt) 148 { 149 struct drm_i915_private *i915 = gt->i915; 150 struct i915_pmu *pmu = &i915->pmu; 151 unsigned long flags; 152 bool awake = false; 153 u64 val; 154 155 if (intel_gt_pm_get_if_awake(gt)) { 156 val = __get_rc6(gt); 157 intel_gt_pm_put_async(gt); 158 awake = true; 159 } 160 161 spin_lock_irqsave(&pmu->lock, flags); 162 163 if (awake) { 164 pmu->sample[__I915_SAMPLE_RC6].cur = val; 165 } else { 166 /* 167 * We think we are runtime suspended. 168 * 169 * Report the delta from when the device was suspended to now, 170 * on top of the last known real value, as the approximated RC6 171 * counter value. 172 */ 173 val = ktime_since(pmu->sleep_last); 174 val += pmu->sample[__I915_SAMPLE_RC6].cur; 175 } 176 177 if (val < pmu->sample[__I915_SAMPLE_RC6_LAST_REPORTED].cur) 178 val = pmu->sample[__I915_SAMPLE_RC6_LAST_REPORTED].cur; 179 else 180 pmu->sample[__I915_SAMPLE_RC6_LAST_REPORTED].cur = val; 181 182 spin_unlock_irqrestore(&pmu->lock, flags); 183 184 return val; 185 } 186 187 static void init_rc6(struct i915_pmu *pmu) 188 { 189 struct drm_i915_private *i915 = container_of(pmu, typeof(*i915), pmu); 190 intel_wakeref_t wakeref; 191 192 with_intel_runtime_pm(i915->gt.uncore->rpm, wakeref) { 193 pmu->sample[__I915_SAMPLE_RC6].cur = __get_rc6(&i915->gt); 194 pmu->sample[__I915_SAMPLE_RC6_LAST_REPORTED].cur = 195 pmu->sample[__I915_SAMPLE_RC6].cur; 196 pmu->sleep_last = ktime_get(); 197 } 198 } 199 200 static void park_rc6(struct drm_i915_private *i915) 201 { 202 struct i915_pmu *pmu = &i915->pmu; 203 204 pmu->sample[__I915_SAMPLE_RC6].cur = __get_rc6(&i915->gt); 205 pmu->sleep_last = ktime_get(); 206 } 207 208 #else 209 210 static u64 get_rc6(struct intel_gt *gt) 211 { 212 return __get_rc6(gt); 213 } 214 215 static void init_rc6(struct i915_pmu *pmu) { } 216 static void park_rc6(struct drm_i915_private *i915) {} 217 218 #endif 219 220 static void __i915_pmu_maybe_start_timer(struct i915_pmu *pmu) 221 { 222 if (!pmu->timer_enabled && pmu_needs_timer(pmu, true)) { 223 pmu->timer_enabled = true; 224 pmu->timer_last = ktime_get(); 225 hrtimer_start_range_ns(&pmu->timer, 226 ns_to_ktime(PERIOD), 0, 227 HRTIMER_MODE_REL_PINNED); 228 } 229 } 230 231 void i915_pmu_gt_parked(struct drm_i915_private *i915) 232 { 233 struct i915_pmu *pmu = &i915->pmu; 234 235 if (!pmu->base.event_init) 236 return; 237 238 spin_lock_irq(&pmu->lock); 239 240 park_rc6(i915); 241 242 /* 243 * Signal sampling timer to stop if only engine events are enabled and 244 * GPU went idle. 245 */ 246 pmu->timer_enabled = pmu_needs_timer(pmu, false); 247 248 spin_unlock_irq(&pmu->lock); 249 } 250 251 void i915_pmu_gt_unparked(struct drm_i915_private *i915) 252 { 253 struct i915_pmu *pmu = &i915->pmu; 254 255 if (!pmu->base.event_init) 256 return; 257 258 spin_lock_irq(&pmu->lock); 259 260 /* 261 * Re-enable sampling timer when GPU goes active. 262 */ 263 __i915_pmu_maybe_start_timer(pmu); 264 265 spin_unlock_irq(&pmu->lock); 266 } 267 268 static void 269 add_sample(struct i915_pmu_sample *sample, u32 val) 270 { 271 sample->cur += val; 272 } 273 274 static bool exclusive_mmio_access(const struct drm_i915_private *i915) 275 { 276 /* 277 * We have to avoid concurrent mmio cache line access on gen7 or 278 * risk a machine hang. For a fun history lesson dig out the old 279 * userspace intel_gpu_top and run it on Ivybridge or Haswell! 280 */ 281 return IS_GEN(i915, 7); 282 } 283 284 static void engine_sample(struct intel_engine_cs *engine, unsigned int period_ns) 285 { 286 struct intel_engine_pmu *pmu = &engine->pmu; 287 bool busy; 288 u32 val; 289 290 val = ENGINE_READ_FW(engine, RING_CTL); 291 if (val == 0) /* powerwell off => engine idle */ 292 return; 293 294 if (val & RING_WAIT) 295 add_sample(&pmu->sample[I915_SAMPLE_WAIT], period_ns); 296 if (val & RING_WAIT_SEMAPHORE) 297 add_sample(&pmu->sample[I915_SAMPLE_SEMA], period_ns); 298 299 /* No need to sample when busy stats are supported. */ 300 if (intel_engine_supports_stats(engine)) 301 return; 302 303 /* 304 * While waiting on a semaphore or event, MI_MODE reports the 305 * ring as idle. However, previously using the seqno, and with 306 * execlists sampling, we account for the ring waiting as the 307 * engine being busy. Therefore, we record the sample as being 308 * busy if either waiting or !idle. 309 */ 310 busy = val & (RING_WAIT_SEMAPHORE | RING_WAIT); 311 if (!busy) { 312 val = ENGINE_READ_FW(engine, RING_MI_MODE); 313 busy = !(val & MODE_IDLE); 314 } 315 if (busy) 316 add_sample(&pmu->sample[I915_SAMPLE_BUSY], period_ns); 317 } 318 319 static void 320 engines_sample(struct intel_gt *gt, unsigned int period_ns) 321 { 322 struct drm_i915_private *i915 = gt->i915; 323 struct intel_engine_cs *engine; 324 enum intel_engine_id id; 325 unsigned long flags; 326 327 if ((i915->pmu.enable & ENGINE_SAMPLE_MASK) == 0) 328 return; 329 330 if (!intel_gt_pm_is_awake(gt)) 331 return; 332 333 for_each_engine(engine, gt, id) { 334 if (!intel_engine_pm_get_if_awake(engine)) 335 continue; 336 337 if (exclusive_mmio_access(i915)) { 338 spin_lock_irqsave(&engine->uncore->lock, flags); 339 engine_sample(engine, period_ns); 340 spin_unlock_irqrestore(&engine->uncore->lock, flags); 341 } else { 342 engine_sample(engine, period_ns); 343 } 344 345 intel_engine_pm_put_async(engine); 346 } 347 } 348 349 static void 350 add_sample_mult(struct i915_pmu_sample *sample, u32 val, u32 mul) 351 { 352 sample->cur += mul_u32_u32(val, mul); 353 } 354 355 static bool frequency_sampling_enabled(struct i915_pmu *pmu) 356 { 357 return pmu->enable & 358 (config_enabled_mask(I915_PMU_ACTUAL_FREQUENCY) | 359 config_enabled_mask(I915_PMU_REQUESTED_FREQUENCY)); 360 } 361 362 static void 363 frequency_sample(struct intel_gt *gt, unsigned int period_ns) 364 { 365 struct drm_i915_private *i915 = gt->i915; 366 struct intel_uncore *uncore = gt->uncore; 367 struct i915_pmu *pmu = &i915->pmu; 368 struct intel_rps *rps = >->rps; 369 370 if (!frequency_sampling_enabled(pmu)) 371 return; 372 373 /* Report 0/0 (actual/requested) frequency while parked. */ 374 if (!intel_gt_pm_get_if_awake(gt)) 375 return; 376 377 if (pmu->enable & config_enabled_mask(I915_PMU_ACTUAL_FREQUENCY)) { 378 u32 val; 379 380 /* 381 * We take a quick peek here without using forcewake 382 * so that we don't perturb the system under observation 383 * (forcewake => !rc6 => increased power use). We expect 384 * that if the read fails because it is outside of the 385 * mmio power well, then it will return 0 -- in which 386 * case we assume the system is running at the intended 387 * frequency. Fortunately, the read should rarely fail! 388 */ 389 val = intel_uncore_read_fw(uncore, GEN6_RPSTAT1); 390 if (val) 391 val = intel_rps_get_cagf(rps, val); 392 else 393 val = rps->cur_freq; 394 395 add_sample_mult(&pmu->sample[__I915_SAMPLE_FREQ_ACT], 396 intel_gpu_freq(rps, val), period_ns / 1000); 397 } 398 399 if (pmu->enable & config_enabled_mask(I915_PMU_REQUESTED_FREQUENCY)) { 400 add_sample_mult(&pmu->sample[__I915_SAMPLE_FREQ_REQ], 401 intel_gpu_freq(rps, rps->cur_freq), 402 period_ns / 1000); 403 } 404 405 intel_gt_pm_put_async(gt); 406 } 407 408 static enum hrtimer_restart i915_sample(struct hrtimer *hrtimer) 409 { 410 struct drm_i915_private *i915 = 411 container_of(hrtimer, struct drm_i915_private, pmu.timer); 412 struct i915_pmu *pmu = &i915->pmu; 413 struct intel_gt *gt = &i915->gt; 414 unsigned int period_ns; 415 ktime_t now; 416 417 if (!READ_ONCE(pmu->timer_enabled)) 418 return HRTIMER_NORESTART; 419 420 now = ktime_get(); 421 period_ns = ktime_to_ns(ktime_sub(now, pmu->timer_last)); 422 pmu->timer_last = now; 423 424 /* 425 * Strictly speaking the passed in period may not be 100% accurate for 426 * all internal calculation, since some amount of time can be spent on 427 * grabbing the forcewake. However the potential error from timer call- 428 * back delay greatly dominates this so we keep it simple. 429 */ 430 engines_sample(gt, period_ns); 431 frequency_sample(gt, period_ns); 432 433 hrtimer_forward(hrtimer, now, ns_to_ktime(PERIOD)); 434 435 return HRTIMER_RESTART; 436 } 437 438 static void i915_pmu_event_destroy(struct perf_event *event) 439 { 440 struct drm_i915_private *i915 = 441 container_of(event->pmu, typeof(*i915), pmu.base); 442 443 drm_WARN_ON(&i915->drm, event->parent); 444 445 drm_dev_put(&i915->drm); 446 } 447 448 static int 449 engine_event_status(struct intel_engine_cs *engine, 450 enum drm_i915_pmu_engine_sample sample) 451 { 452 switch (sample) { 453 case I915_SAMPLE_BUSY: 454 case I915_SAMPLE_WAIT: 455 break; 456 case I915_SAMPLE_SEMA: 457 if (INTEL_GEN(engine->i915) < 6) 458 return -ENODEV; 459 break; 460 default: 461 return -ENOENT; 462 } 463 464 return 0; 465 } 466 467 static int 468 config_status(struct drm_i915_private *i915, u64 config) 469 { 470 switch (config) { 471 case I915_PMU_ACTUAL_FREQUENCY: 472 if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) 473 /* Requires a mutex for sampling! */ 474 return -ENODEV; 475 fallthrough; 476 case I915_PMU_REQUESTED_FREQUENCY: 477 if (INTEL_GEN(i915) < 6) 478 return -ENODEV; 479 break; 480 case I915_PMU_INTERRUPTS: 481 break; 482 case I915_PMU_RC6_RESIDENCY: 483 if (!HAS_RC6(i915)) 484 return -ENODEV; 485 break; 486 default: 487 return -ENOENT; 488 } 489 490 return 0; 491 } 492 493 static int engine_event_init(struct perf_event *event) 494 { 495 struct drm_i915_private *i915 = 496 container_of(event->pmu, typeof(*i915), pmu.base); 497 struct intel_engine_cs *engine; 498 499 engine = intel_engine_lookup_user(i915, engine_event_class(event), 500 engine_event_instance(event)); 501 if (!engine) 502 return -ENODEV; 503 504 return engine_event_status(engine, engine_event_sample(event)); 505 } 506 507 static int i915_pmu_event_init(struct perf_event *event) 508 { 509 struct drm_i915_private *i915 = 510 container_of(event->pmu, typeof(*i915), pmu.base); 511 struct i915_pmu *pmu = &i915->pmu; 512 int ret; 513 514 if (pmu->closed) 515 return -ENODEV; 516 517 if (event->attr.type != event->pmu->type) 518 return -ENOENT; 519 520 /* unsupported modes and filters */ 521 if (event->attr.sample_period) /* no sampling */ 522 return -EINVAL; 523 524 if (has_branch_stack(event)) 525 return -EOPNOTSUPP; 526 527 if (event->cpu < 0) 528 return -EINVAL; 529 530 /* only allow running on one cpu at a time */ 531 if (!cpumask_test_cpu(event->cpu, &i915_pmu_cpumask)) 532 return -EINVAL; 533 534 if (is_engine_event(event)) 535 ret = engine_event_init(event); 536 else 537 ret = config_status(i915, event->attr.config); 538 if (ret) 539 return ret; 540 541 if (!event->parent) { 542 drm_dev_get(&i915->drm); 543 event->destroy = i915_pmu_event_destroy; 544 } 545 546 return 0; 547 } 548 549 static u64 __i915_pmu_event_read(struct perf_event *event) 550 { 551 struct drm_i915_private *i915 = 552 container_of(event->pmu, typeof(*i915), pmu.base); 553 struct i915_pmu *pmu = &i915->pmu; 554 u64 val = 0; 555 556 if (is_engine_event(event)) { 557 u8 sample = engine_event_sample(event); 558 struct intel_engine_cs *engine; 559 560 engine = intel_engine_lookup_user(i915, 561 engine_event_class(event), 562 engine_event_instance(event)); 563 564 if (drm_WARN_ON_ONCE(&i915->drm, !engine)) { 565 /* Do nothing */ 566 } else if (sample == I915_SAMPLE_BUSY && 567 intel_engine_supports_stats(engine)) { 568 ktime_t unused; 569 570 val = ktime_to_ns(intel_engine_get_busy_time(engine, 571 &unused)); 572 } else { 573 val = engine->pmu.sample[sample].cur; 574 } 575 } else { 576 switch (event->attr.config) { 577 case I915_PMU_ACTUAL_FREQUENCY: 578 val = 579 div_u64(pmu->sample[__I915_SAMPLE_FREQ_ACT].cur, 580 USEC_PER_SEC /* to MHz */); 581 break; 582 case I915_PMU_REQUESTED_FREQUENCY: 583 val = 584 div_u64(pmu->sample[__I915_SAMPLE_FREQ_REQ].cur, 585 USEC_PER_SEC /* to MHz */); 586 break; 587 case I915_PMU_INTERRUPTS: 588 val = READ_ONCE(pmu->irq_count); 589 break; 590 case I915_PMU_RC6_RESIDENCY: 591 val = get_rc6(&i915->gt); 592 break; 593 } 594 } 595 596 return val; 597 } 598 599 static void i915_pmu_event_read(struct perf_event *event) 600 { 601 struct drm_i915_private *i915 = 602 container_of(event->pmu, typeof(*i915), pmu.base); 603 struct hw_perf_event *hwc = &event->hw; 604 struct i915_pmu *pmu = &i915->pmu; 605 u64 prev, new; 606 607 if (pmu->closed) { 608 event->hw.state = PERF_HES_STOPPED; 609 return; 610 } 611 again: 612 prev = local64_read(&hwc->prev_count); 613 new = __i915_pmu_event_read(event); 614 615 if (local64_cmpxchg(&hwc->prev_count, prev, new) != prev) 616 goto again; 617 618 local64_add(new - prev, &event->count); 619 } 620 621 static void i915_pmu_enable(struct perf_event *event) 622 { 623 struct drm_i915_private *i915 = 624 container_of(event->pmu, typeof(*i915), pmu.base); 625 unsigned int bit = event_enabled_bit(event); 626 struct i915_pmu *pmu = &i915->pmu; 627 unsigned long flags; 628 629 spin_lock_irqsave(&pmu->lock, flags); 630 631 /* 632 * Update the bitmask of enabled events and increment 633 * the event reference counter. 634 */ 635 BUILD_BUG_ON(ARRAY_SIZE(pmu->enable_count) != I915_PMU_MASK_BITS); 636 GEM_BUG_ON(bit >= ARRAY_SIZE(pmu->enable_count)); 637 GEM_BUG_ON(pmu->enable_count[bit] == ~0); 638 639 pmu->enable |= BIT_ULL(bit); 640 pmu->enable_count[bit]++; 641 642 /* 643 * Start the sampling timer if needed and not already enabled. 644 */ 645 __i915_pmu_maybe_start_timer(pmu); 646 647 /* 648 * For per-engine events the bitmask and reference counting 649 * is stored per engine. 650 */ 651 if (is_engine_event(event)) { 652 u8 sample = engine_event_sample(event); 653 struct intel_engine_cs *engine; 654 655 engine = intel_engine_lookup_user(i915, 656 engine_event_class(event), 657 engine_event_instance(event)); 658 659 BUILD_BUG_ON(ARRAY_SIZE(engine->pmu.enable_count) != 660 I915_ENGINE_SAMPLE_COUNT); 661 BUILD_BUG_ON(ARRAY_SIZE(engine->pmu.sample) != 662 I915_ENGINE_SAMPLE_COUNT); 663 GEM_BUG_ON(sample >= ARRAY_SIZE(engine->pmu.enable_count)); 664 GEM_BUG_ON(sample >= ARRAY_SIZE(engine->pmu.sample)); 665 GEM_BUG_ON(engine->pmu.enable_count[sample] == ~0); 666 667 engine->pmu.enable |= BIT(sample); 668 engine->pmu.enable_count[sample]++; 669 } 670 671 spin_unlock_irqrestore(&pmu->lock, flags); 672 673 /* 674 * Store the current counter value so we can report the correct delta 675 * for all listeners. Even when the event was already enabled and has 676 * an existing non-zero value. 677 */ 678 local64_set(&event->hw.prev_count, __i915_pmu_event_read(event)); 679 } 680 681 static void i915_pmu_disable(struct perf_event *event) 682 { 683 struct drm_i915_private *i915 = 684 container_of(event->pmu, typeof(*i915), pmu.base); 685 unsigned int bit = event_enabled_bit(event); 686 struct i915_pmu *pmu = &i915->pmu; 687 unsigned long flags; 688 689 spin_lock_irqsave(&pmu->lock, flags); 690 691 if (is_engine_event(event)) { 692 u8 sample = engine_event_sample(event); 693 struct intel_engine_cs *engine; 694 695 engine = intel_engine_lookup_user(i915, 696 engine_event_class(event), 697 engine_event_instance(event)); 698 699 GEM_BUG_ON(sample >= ARRAY_SIZE(engine->pmu.enable_count)); 700 GEM_BUG_ON(sample >= ARRAY_SIZE(engine->pmu.sample)); 701 GEM_BUG_ON(engine->pmu.enable_count[sample] == 0); 702 703 /* 704 * Decrement the reference count and clear the enabled 705 * bitmask when the last listener on an event goes away. 706 */ 707 if (--engine->pmu.enable_count[sample] == 0) 708 engine->pmu.enable &= ~BIT(sample); 709 } 710 711 GEM_BUG_ON(bit >= ARRAY_SIZE(pmu->enable_count)); 712 GEM_BUG_ON(pmu->enable_count[bit] == 0); 713 /* 714 * Decrement the reference count and clear the enabled 715 * bitmask when the last listener on an event goes away. 716 */ 717 if (--pmu->enable_count[bit] == 0) { 718 pmu->enable &= ~BIT_ULL(bit); 719 pmu->timer_enabled &= pmu_needs_timer(pmu, true); 720 } 721 722 spin_unlock_irqrestore(&pmu->lock, flags); 723 } 724 725 static void i915_pmu_event_start(struct perf_event *event, int flags) 726 { 727 struct drm_i915_private *i915 = 728 container_of(event->pmu, typeof(*i915), pmu.base); 729 struct i915_pmu *pmu = &i915->pmu; 730 731 if (pmu->closed) 732 return; 733 734 i915_pmu_enable(event); 735 event->hw.state = 0; 736 } 737 738 static void i915_pmu_event_stop(struct perf_event *event, int flags) 739 { 740 if (flags & PERF_EF_UPDATE) 741 i915_pmu_event_read(event); 742 i915_pmu_disable(event); 743 event->hw.state = PERF_HES_STOPPED; 744 } 745 746 static int i915_pmu_event_add(struct perf_event *event, int flags) 747 { 748 struct drm_i915_private *i915 = 749 container_of(event->pmu, typeof(*i915), pmu.base); 750 struct i915_pmu *pmu = &i915->pmu; 751 752 if (pmu->closed) 753 return -ENODEV; 754 755 if (flags & PERF_EF_START) 756 i915_pmu_event_start(event, flags); 757 758 return 0; 759 } 760 761 static void i915_pmu_event_del(struct perf_event *event, int flags) 762 { 763 i915_pmu_event_stop(event, PERF_EF_UPDATE); 764 } 765 766 static int i915_pmu_event_event_idx(struct perf_event *event) 767 { 768 return 0; 769 } 770 771 struct i915_str_attribute { 772 struct device_attribute attr; 773 const char *str; 774 }; 775 776 static ssize_t i915_pmu_format_show(struct device *dev, 777 struct device_attribute *attr, char *buf) 778 { 779 struct i915_str_attribute *eattr; 780 781 eattr = container_of(attr, struct i915_str_attribute, attr); 782 return sprintf(buf, "%s\n", eattr->str); 783 } 784 785 #define I915_PMU_FORMAT_ATTR(_name, _config) \ 786 (&((struct i915_str_attribute[]) { \ 787 { .attr = __ATTR(_name, 0444, i915_pmu_format_show, NULL), \ 788 .str = _config, } \ 789 })[0].attr.attr) 790 791 static struct attribute *i915_pmu_format_attrs[] = { 792 I915_PMU_FORMAT_ATTR(i915_eventid, "config:0-20"), 793 NULL, 794 }; 795 796 static const struct attribute_group i915_pmu_format_attr_group = { 797 .name = "format", 798 .attrs = i915_pmu_format_attrs, 799 }; 800 801 struct i915_ext_attribute { 802 struct device_attribute attr; 803 unsigned long val; 804 }; 805 806 static ssize_t i915_pmu_event_show(struct device *dev, 807 struct device_attribute *attr, char *buf) 808 { 809 struct i915_ext_attribute *eattr; 810 811 eattr = container_of(attr, struct i915_ext_attribute, attr); 812 return sprintf(buf, "config=0x%lx\n", eattr->val); 813 } 814 815 static ssize_t 816 i915_pmu_get_attr_cpumask(struct device *dev, 817 struct device_attribute *attr, 818 char *buf) 819 { 820 return cpumap_print_to_pagebuf(true, buf, &i915_pmu_cpumask); 821 } 822 823 static DEVICE_ATTR(cpumask, 0444, i915_pmu_get_attr_cpumask, NULL); 824 825 static struct attribute *i915_cpumask_attrs[] = { 826 &dev_attr_cpumask.attr, 827 NULL, 828 }; 829 830 static const struct attribute_group i915_pmu_cpumask_attr_group = { 831 .attrs = i915_cpumask_attrs, 832 }; 833 834 #define __event(__config, __name, __unit) \ 835 { \ 836 .config = (__config), \ 837 .name = (__name), \ 838 .unit = (__unit), \ 839 } 840 841 #define __engine_event(__sample, __name) \ 842 { \ 843 .sample = (__sample), \ 844 .name = (__name), \ 845 } 846 847 static struct i915_ext_attribute * 848 add_i915_attr(struct i915_ext_attribute *attr, const char *name, u64 config) 849 { 850 sysfs_attr_init(&attr->attr.attr); 851 attr->attr.attr.name = name; 852 attr->attr.attr.mode = 0444; 853 attr->attr.show = i915_pmu_event_show; 854 attr->val = config; 855 856 return ++attr; 857 } 858 859 static struct perf_pmu_events_attr * 860 add_pmu_attr(struct perf_pmu_events_attr *attr, const char *name, 861 const char *str) 862 { 863 sysfs_attr_init(&attr->attr.attr); 864 attr->attr.attr.name = name; 865 attr->attr.attr.mode = 0444; 866 attr->attr.show = perf_event_sysfs_show; 867 attr->event_str = str; 868 869 return ++attr; 870 } 871 872 static struct attribute ** 873 create_event_attributes(struct i915_pmu *pmu) 874 { 875 struct drm_i915_private *i915 = container_of(pmu, typeof(*i915), pmu); 876 static const struct { 877 u64 config; 878 const char *name; 879 const char *unit; 880 } events[] = { 881 __event(I915_PMU_ACTUAL_FREQUENCY, "actual-frequency", "M"), 882 __event(I915_PMU_REQUESTED_FREQUENCY, "requested-frequency", "M"), 883 __event(I915_PMU_INTERRUPTS, "interrupts", NULL), 884 __event(I915_PMU_RC6_RESIDENCY, "rc6-residency", "ns"), 885 }; 886 static const struct { 887 enum drm_i915_pmu_engine_sample sample; 888 char *name; 889 } engine_events[] = { 890 __engine_event(I915_SAMPLE_BUSY, "busy"), 891 __engine_event(I915_SAMPLE_SEMA, "sema"), 892 __engine_event(I915_SAMPLE_WAIT, "wait"), 893 }; 894 unsigned int count = 0; 895 struct perf_pmu_events_attr *pmu_attr = NULL, *pmu_iter; 896 struct i915_ext_attribute *i915_attr = NULL, *i915_iter; 897 struct attribute **attr = NULL, **attr_iter; 898 struct intel_engine_cs *engine; 899 unsigned int i; 900 901 /* Count how many counters we will be exposing. */ 902 for (i = 0; i < ARRAY_SIZE(events); i++) { 903 if (!config_status(i915, events[i].config)) 904 count++; 905 } 906 907 for_each_uabi_engine(engine, i915) { 908 for (i = 0; i < ARRAY_SIZE(engine_events); i++) { 909 if (!engine_event_status(engine, 910 engine_events[i].sample)) 911 count++; 912 } 913 } 914 915 /* Allocate attribute objects and table. */ 916 i915_attr = kcalloc(count, sizeof(*i915_attr), GFP_KERNEL); 917 if (!i915_attr) 918 goto err_alloc; 919 920 pmu_attr = kcalloc(count, sizeof(*pmu_attr), GFP_KERNEL); 921 if (!pmu_attr) 922 goto err_alloc; 923 924 /* Max one pointer of each attribute type plus a termination entry. */ 925 attr = kcalloc(count * 2 + 1, sizeof(*attr), GFP_KERNEL); 926 if (!attr) 927 goto err_alloc; 928 929 i915_iter = i915_attr; 930 pmu_iter = pmu_attr; 931 attr_iter = attr; 932 933 /* Initialize supported non-engine counters. */ 934 for (i = 0; i < ARRAY_SIZE(events); i++) { 935 char *str; 936 937 if (config_status(i915, events[i].config)) 938 continue; 939 940 str = kstrdup(events[i].name, GFP_KERNEL); 941 if (!str) 942 goto err; 943 944 *attr_iter++ = &i915_iter->attr.attr; 945 i915_iter = add_i915_attr(i915_iter, str, events[i].config); 946 947 if (events[i].unit) { 948 str = kasprintf(GFP_KERNEL, "%s.unit", events[i].name); 949 if (!str) 950 goto err; 951 952 *attr_iter++ = &pmu_iter->attr.attr; 953 pmu_iter = add_pmu_attr(pmu_iter, str, events[i].unit); 954 } 955 } 956 957 /* Initialize supported engine counters. */ 958 for_each_uabi_engine(engine, i915) { 959 for (i = 0; i < ARRAY_SIZE(engine_events); i++) { 960 char *str; 961 962 if (engine_event_status(engine, 963 engine_events[i].sample)) 964 continue; 965 966 str = kasprintf(GFP_KERNEL, "%s-%s", 967 engine->name, engine_events[i].name); 968 if (!str) 969 goto err; 970 971 *attr_iter++ = &i915_iter->attr.attr; 972 i915_iter = 973 add_i915_attr(i915_iter, str, 974 __I915_PMU_ENGINE(engine->uabi_class, 975 engine->uabi_instance, 976 engine_events[i].sample)); 977 978 str = kasprintf(GFP_KERNEL, "%s-%s.unit", 979 engine->name, engine_events[i].name); 980 if (!str) 981 goto err; 982 983 *attr_iter++ = &pmu_iter->attr.attr; 984 pmu_iter = add_pmu_attr(pmu_iter, str, "ns"); 985 } 986 } 987 988 pmu->i915_attr = i915_attr; 989 pmu->pmu_attr = pmu_attr; 990 991 return attr; 992 993 err:; 994 for (attr_iter = attr; *attr_iter; attr_iter++) 995 kfree((*attr_iter)->name); 996 997 err_alloc: 998 kfree(attr); 999 kfree(i915_attr); 1000 kfree(pmu_attr); 1001 1002 return NULL; 1003 } 1004 1005 static void free_event_attributes(struct i915_pmu *pmu) 1006 { 1007 struct attribute **attr_iter = pmu->events_attr_group.attrs; 1008 1009 for (; *attr_iter; attr_iter++) 1010 kfree((*attr_iter)->name); 1011 1012 kfree(pmu->events_attr_group.attrs); 1013 kfree(pmu->i915_attr); 1014 kfree(pmu->pmu_attr); 1015 1016 pmu->events_attr_group.attrs = NULL; 1017 pmu->i915_attr = NULL; 1018 pmu->pmu_attr = NULL; 1019 } 1020 1021 static int i915_pmu_cpu_online(unsigned int cpu, struct hlist_node *node) 1022 { 1023 struct i915_pmu *pmu = hlist_entry_safe(node, typeof(*pmu), cpuhp.node); 1024 1025 GEM_BUG_ON(!pmu->base.event_init); 1026 1027 /* Select the first online CPU as a designated reader. */ 1028 if (!cpumask_weight(&i915_pmu_cpumask)) 1029 cpumask_set_cpu(cpu, &i915_pmu_cpumask); 1030 1031 return 0; 1032 } 1033 1034 static int i915_pmu_cpu_offline(unsigned int cpu, struct hlist_node *node) 1035 { 1036 struct i915_pmu *pmu = hlist_entry_safe(node, typeof(*pmu), cpuhp.node); 1037 unsigned int target = i915_pmu_target_cpu; 1038 1039 GEM_BUG_ON(!pmu->base.event_init); 1040 1041 /* 1042 * Unregistering an instance generates a CPU offline event which we must 1043 * ignore to avoid incorrectly modifying the shared i915_pmu_cpumask. 1044 */ 1045 if (pmu->closed) 1046 return 0; 1047 1048 if (cpumask_test_and_clear_cpu(cpu, &i915_pmu_cpumask)) { 1049 target = cpumask_any_but(topology_sibling_cpumask(cpu), cpu); 1050 1051 /* Migrate events if there is a valid target */ 1052 if (target < nr_cpu_ids) { 1053 cpumask_set_cpu(target, &i915_pmu_cpumask); 1054 i915_pmu_target_cpu = target; 1055 } 1056 } 1057 1058 if (target < nr_cpu_ids && target != pmu->cpuhp.cpu) { 1059 perf_pmu_migrate_context(&pmu->base, cpu, target); 1060 pmu->cpuhp.cpu = target; 1061 } 1062 1063 return 0; 1064 } 1065 1066 static enum cpuhp_state cpuhp_slot = CPUHP_INVALID; 1067 1068 void i915_pmu_init(void) 1069 { 1070 int ret; 1071 1072 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, 1073 "perf/x86/intel/i915:online", 1074 i915_pmu_cpu_online, 1075 i915_pmu_cpu_offline); 1076 if (ret < 0) 1077 pr_notice("Failed to setup cpuhp state for i915 PMU! (%d)\n", 1078 ret); 1079 else 1080 cpuhp_slot = ret; 1081 } 1082 1083 void i915_pmu_exit(void) 1084 { 1085 if (cpuhp_slot != CPUHP_INVALID) 1086 cpuhp_remove_multi_state(cpuhp_slot); 1087 } 1088 1089 static int i915_pmu_register_cpuhp_state(struct i915_pmu *pmu) 1090 { 1091 if (cpuhp_slot == CPUHP_INVALID) 1092 return -EINVAL; 1093 1094 return cpuhp_state_add_instance(cpuhp_slot, &pmu->cpuhp.node); 1095 } 1096 1097 static void i915_pmu_unregister_cpuhp_state(struct i915_pmu *pmu) 1098 { 1099 cpuhp_state_remove_instance(cpuhp_slot, &pmu->cpuhp.node); 1100 } 1101 1102 static bool is_igp(struct drm_i915_private *i915) 1103 { 1104 struct pci_dev *pdev = i915->drm.pdev; 1105 1106 /* IGP is 0000:00:02.0 */ 1107 return pci_domain_nr(pdev->bus) == 0 && 1108 pdev->bus->number == 0 && 1109 PCI_SLOT(pdev->devfn) == 2 && 1110 PCI_FUNC(pdev->devfn) == 0; 1111 } 1112 1113 void i915_pmu_register(struct drm_i915_private *i915) 1114 { 1115 struct i915_pmu *pmu = &i915->pmu; 1116 const struct attribute_group *attr_groups[] = { 1117 &i915_pmu_format_attr_group, 1118 &pmu->events_attr_group, 1119 &i915_pmu_cpumask_attr_group, 1120 NULL 1121 }; 1122 1123 int ret = -ENOMEM; 1124 1125 if (INTEL_GEN(i915) <= 2) { 1126 drm_info(&i915->drm, "PMU not supported for this GPU."); 1127 return; 1128 } 1129 1130 spin_lock_init(&pmu->lock); 1131 hrtimer_init(&pmu->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 1132 pmu->timer.function = i915_sample; 1133 pmu->cpuhp.cpu = -1; 1134 init_rc6(pmu); 1135 1136 if (!is_igp(i915)) { 1137 pmu->name = kasprintf(GFP_KERNEL, 1138 "i915_%s", 1139 dev_name(i915->drm.dev)); 1140 if (pmu->name) { 1141 /* tools/perf reserves colons as special. */ 1142 strreplace((char *)pmu->name, ':', '_'); 1143 } 1144 } else { 1145 pmu->name = "i915"; 1146 } 1147 if (!pmu->name) 1148 goto err; 1149 1150 pmu->events_attr_group.name = "events"; 1151 pmu->events_attr_group.attrs = create_event_attributes(pmu); 1152 if (!pmu->events_attr_group.attrs) 1153 goto err_name; 1154 1155 pmu->base.attr_groups = kmemdup(attr_groups, sizeof(attr_groups), 1156 GFP_KERNEL); 1157 if (!pmu->base.attr_groups) 1158 goto err_attr; 1159 1160 pmu->base.module = THIS_MODULE; 1161 pmu->base.task_ctx_nr = perf_invalid_context; 1162 pmu->base.event_init = i915_pmu_event_init; 1163 pmu->base.add = i915_pmu_event_add; 1164 pmu->base.del = i915_pmu_event_del; 1165 pmu->base.start = i915_pmu_event_start; 1166 pmu->base.stop = i915_pmu_event_stop; 1167 pmu->base.read = i915_pmu_event_read; 1168 pmu->base.event_idx = i915_pmu_event_event_idx; 1169 1170 ret = perf_pmu_register(&pmu->base, pmu->name, -1); 1171 if (ret) 1172 goto err_groups; 1173 1174 ret = i915_pmu_register_cpuhp_state(pmu); 1175 if (ret) 1176 goto err_unreg; 1177 1178 return; 1179 1180 err_unreg: 1181 perf_pmu_unregister(&pmu->base); 1182 err_groups: 1183 kfree(pmu->base.attr_groups); 1184 err_attr: 1185 pmu->base.event_init = NULL; 1186 free_event_attributes(pmu); 1187 err_name: 1188 if (!is_igp(i915)) 1189 kfree(pmu->name); 1190 err: 1191 drm_notice(&i915->drm, "Failed to register PMU!\n"); 1192 } 1193 1194 void i915_pmu_unregister(struct drm_i915_private *i915) 1195 { 1196 struct i915_pmu *pmu = &i915->pmu; 1197 1198 if (!pmu->base.event_init) 1199 return; 1200 1201 /* 1202 * "Disconnect" the PMU callbacks - since all are atomic synchronize_rcu 1203 * ensures all currently executing ones will have exited before we 1204 * proceed with unregistration. 1205 */ 1206 pmu->closed = true; 1207 synchronize_rcu(); 1208 1209 hrtimer_cancel(&pmu->timer); 1210 1211 i915_pmu_unregister_cpuhp_state(pmu); 1212 1213 perf_pmu_unregister(&pmu->base); 1214 pmu->base.event_init = NULL; 1215 kfree(pmu->base.attr_groups); 1216 if (!is_igp(i915)) 1217 kfree(pmu->name); 1218 free_event_attributes(pmu); 1219 } 1220