Lines Matching +full:re +full:- +full:sampling
2 * SPDX-License-Identifier: MIT
4 * Copyright © 2017-2018 Intel Corporation
22 /* Frequency for the sampling timer for events which need it. */
33 return container_of(event->pmu, struct i915_pmu, base); in event_to_pmu()
48 return engine_config_sample(event->attr.config); in engine_event_sample()
53 return (event->attr.config >> I915_PMU_CLASS_SHIFT) & 0xff; in engine_event_class()
58 return (event->attr.config >> I915_PMU_SAMPLE_BITS) & 0xff; in engine_event_instance()
92 * Events that do not require sampling, or tracking state in other_bit()
95 return -1; in other_bit()
118 enable)) - 1); in config_mask()
122 enable)) - 1); in config_mask()
129 return is_engine_config(event->attr.config); in is_engine_event()
134 return config_bit(event->attr.config); in event_bit()
155 * Only some counters need the sampling timer. in pmu_needs_timer()
159 enable = pmu->enable; in pmu_needs_timer()
171 if (i915->caps.scheduler & I915_SCHEDULER_CAP_ENGINE_BUSY_STATS) in pmu_needs_timer()
175 * If some bits remain it means we need the sampling timer running. in pmu_needs_timer()
182 struct drm_i915_private *i915 = gt->i915; in __get_rc6()
185 val = intel_rc6_residency_ns(>->rc6, INTEL_RC6_RES_RC6); in __get_rc6()
188 val += intel_rc6_residency_ns(>->rc6, INTEL_RC6_RES_RC6p); in __get_rc6()
191 val += intel_rc6_residency_ns(>->rc6, INTEL_RC6_RES_RC6pp); in __get_rc6()
203 return pmu->sample[gt_id][sample].cur; in read_sample()
209 pmu->sample[gt_id][sample].cur = val; in store_sample()
215 pmu->sample[gt_id][sample].cur += mul_u32_u32(val, mul); in add_sample_mult()
220 struct drm_i915_private *i915 = gt->i915; in get_rc6()
221 const unsigned int gt_id = gt->info.id; in get_rc6()
222 struct i915_pmu *pmu = &i915->pmu; in get_rc6()
233 spin_lock_irqsave(&pmu->lock, flags); in get_rc6()
245 val = ktime_since_raw(pmu->sleep_last[gt_id]); in get_rc6()
254 spin_unlock_irqrestore(&pmu->lock, flags); in get_rc6()
268 with_intel_runtime_pm(gt->uncore->rpm, wakeref) { in init_rc6()
274 pmu->sleep_last[i] = ktime_get_raw(); in init_rc6()
281 struct i915_pmu *pmu = >->i915->pmu; in park_rc6()
283 store_sample(pmu, gt->info.id, __I915_SAMPLE_RC6, __get_rc6(gt)); in park_rc6()
284 pmu->sleep_last[gt->info.id] = ktime_get_raw(); in park_rc6()
289 if (!pmu->timer_enabled && pmu_needs_timer(pmu)) { in __i915_pmu_maybe_start_timer()
290 pmu->timer_enabled = true; in __i915_pmu_maybe_start_timer()
291 pmu->timer_last = ktime_get(); in __i915_pmu_maybe_start_timer()
292 hrtimer_start_range_ns(&pmu->timer, in __i915_pmu_maybe_start_timer()
300 struct i915_pmu *pmu = >->i915->pmu; in i915_pmu_gt_parked()
302 if (!pmu->registered) in i915_pmu_gt_parked()
305 spin_lock_irq(&pmu->lock); in i915_pmu_gt_parked()
310 * Signal sampling timer to stop if only engine events are enabled and in i915_pmu_gt_parked()
313 pmu->unparked &= ~BIT(gt->info.id); in i915_pmu_gt_parked()
314 if (pmu->unparked == 0) in i915_pmu_gt_parked()
315 pmu->timer_enabled = false; in i915_pmu_gt_parked()
317 spin_unlock_irq(&pmu->lock); in i915_pmu_gt_parked()
322 struct i915_pmu *pmu = >->i915->pmu; in i915_pmu_gt_unparked()
324 if (!pmu->registered) in i915_pmu_gt_unparked()
327 spin_lock_irq(&pmu->lock); in i915_pmu_gt_unparked()
330 * Re-enable sampling timer when GPU goes active. in i915_pmu_gt_unparked()
332 if (pmu->unparked == 0) in i915_pmu_gt_unparked()
335 pmu->unparked |= BIT(gt->info.id); in i915_pmu_gt_unparked()
337 spin_unlock_irq(&pmu->lock); in i915_pmu_gt_unparked()
343 sample->cur += val; in add_sample()
358 struct intel_engine_pmu *pmu = &engine->pmu; in gen3_engine_sample()
367 add_sample(&pmu->sample[I915_SAMPLE_WAIT], period_ns); in gen3_engine_sample()
369 add_sample(&pmu->sample[I915_SAMPLE_SEMA], period_ns); in gen3_engine_sample()
378 * execlists sampling, we account for the ring waiting as the in gen3_engine_sample()
388 add_sample(&pmu->sample[I915_SAMPLE_BUSY], period_ns); in gen3_engine_sample()
393 struct intel_engine_pmu *pmu = &engine->pmu; in gen2_engine_sample()
401 add_sample(&pmu->sample[I915_SAMPLE_WAIT], period_ns); in gen2_engine_sample()
405 add_sample(&pmu->sample[I915_SAMPLE_BUSY], period_ns); in gen2_engine_sample()
410 if (GRAPHICS_VER(engine->i915) >= 3) in engine_sample()
419 struct drm_i915_private *i915 = gt->i915; in engines_sample()
424 if ((i915->pmu.enable & ENGINE_SAMPLE_MASK) == 0) in engines_sample()
431 if (!engine->pmu.enable) in engines_sample()
438 spin_lock_irqsave(&engine->uncore->lock, flags); in engines_sample()
440 spin_unlock_irqrestore(&engine->uncore->lock, flags); in engines_sample()
452 return pmu->enable & in frequency_sampling_enabled()
460 struct drm_i915_private *i915 = gt->i915; in frequency_sample()
461 const unsigned int gt_id = gt->info.id; in frequency_sample()
462 struct i915_pmu *pmu = &i915->pmu; in frequency_sample()
463 struct intel_rps *rps = >->rps; in frequency_sample()
474 if (pmu->enable & config_mask(__I915_PMU_ACTUAL_FREQUENCY(gt_id))) { in frequency_sample()
482 * mmio power well, then it will return 0 -- in which in frequency_sample()
488 val = intel_gpu_freq(rps, rps->cur_freq); in frequency_sample()
494 if (pmu->enable & config_mask(__I915_PMU_REQUESTED_FREQUENCY(gt_id))) { in frequency_sample()
512 if (!READ_ONCE(pmu->timer_enabled)) in i915_sample()
516 period_ns = ktime_to_ns(ktime_sub(now, pmu->timer_last)); in i915_sample()
517 pmu->timer_last = now; in i915_sample()
522 * grabbing the forcewake. However the potential error from timer call- in i915_sample()
527 if (!(pmu->unparked & BIT(i))) in i915_sample()
544 drm_WARN_ON(&i915->drm, event->parent); in i915_pmu_event_destroy()
546 drm_dev_put(&i915->drm); in i915_pmu_event_destroy()
558 if (GRAPHICS_VER(engine->i915) < 6) in engine_event_status()
559 return -ENODEV; in engine_event_status()
562 return -ENOENT; in engine_event_status()
577 return -ENOENT; in config_status()
582 /* Requires a mutex for sampling! */ in config_status()
583 return -ENODEV; in config_status()
587 return -ENODEV; in config_status()
591 return -ENOENT; in config_status()
594 if (!gt->rc6.supported) in config_status()
595 return -ENODEV; in config_status()
600 return -ENOENT; in config_status()
615 return -ENODEV; in engine_event_init()
626 if (!pmu->registered) in i915_pmu_event_init()
627 return -ENODEV; in i915_pmu_event_init()
629 if (event->attr.type != event->pmu->type) in i915_pmu_event_init()
630 return -ENOENT; in i915_pmu_event_init()
633 if (event->attr.sample_period) /* no sampling */ in i915_pmu_event_init()
634 return -EINVAL; in i915_pmu_event_init()
637 return -EOPNOTSUPP; in i915_pmu_event_init()
639 if (event->cpu < 0) in i915_pmu_event_init()
640 return -EINVAL; in i915_pmu_event_init()
645 ret = config_status(i915, event->attr.config); in i915_pmu_event_init()
649 if (!event->parent) { in i915_pmu_event_init()
650 drm_dev_get(&i915->drm); in i915_pmu_event_init()
651 event->destroy = i915_pmu_event_destroy; in i915_pmu_event_init()
671 if (drm_WARN_ON_ONCE(&i915->drm, !engine)) { in __i915_pmu_event_read()
680 val = engine->pmu.sample[sample].cur; in __i915_pmu_event_read()
683 const unsigned int gt_id = config_gt_id(event->attr.config); in __i915_pmu_event_read()
684 const u64 config = config_counter(event->attr.config); in __i915_pmu_event_read()
700 val = READ_ONCE(pmu->irq_count); in __i915_pmu_event_read()
703 val = get_rc6(i915->gt[gt_id]); in __i915_pmu_event_read()
717 struct hw_perf_event *hwc = &event->hw; in i915_pmu_event_read()
720 if (!pmu->registered) { in i915_pmu_event_read()
721 event->hw.state = PERF_HES_STOPPED; in i915_pmu_event_read()
725 prev = local64_read(&hwc->prev_count); in i915_pmu_event_read()
728 } while (!local64_try_cmpxchg(&hwc->prev_count, &prev, new)); in i915_pmu_event_read()
730 local64_add(new - prev, &event->count); in i915_pmu_event_read()
740 if (bit == -1) in i915_pmu_enable()
743 spin_lock_irqsave(&pmu->lock, flags); in i915_pmu_enable()
749 BUILD_BUG_ON(ARRAY_SIZE(pmu->enable_count) != I915_PMU_MASK_BITS); in i915_pmu_enable()
750 GEM_BUG_ON(bit >= ARRAY_SIZE(pmu->enable_count)); in i915_pmu_enable()
751 GEM_BUG_ON(pmu->enable_count[bit] == ~0); in i915_pmu_enable()
753 pmu->enable |= BIT(bit); in i915_pmu_enable()
754 pmu->enable_count[bit]++; in i915_pmu_enable()
757 * Start the sampling timer if needed and not already enabled. in i915_pmu_enable()
762 * For per-engine events the bitmask and reference counting in i915_pmu_enable()
773 BUILD_BUG_ON(ARRAY_SIZE(engine->pmu.enable_count) != in i915_pmu_enable()
775 BUILD_BUG_ON(ARRAY_SIZE(engine->pmu.sample) != in i915_pmu_enable()
777 GEM_BUG_ON(sample >= ARRAY_SIZE(engine->pmu.enable_count)); in i915_pmu_enable()
778 GEM_BUG_ON(sample >= ARRAY_SIZE(engine->pmu.sample)); in i915_pmu_enable()
779 GEM_BUG_ON(engine->pmu.enable_count[sample] == ~0); in i915_pmu_enable()
781 engine->pmu.enable |= BIT(sample); in i915_pmu_enable()
782 engine->pmu.enable_count[sample]++; in i915_pmu_enable()
785 spin_unlock_irqrestore(&pmu->lock, flags); in i915_pmu_enable()
791 * an existing non-zero value. in i915_pmu_enable()
793 local64_set(&event->hw.prev_count, __i915_pmu_event_read(event)); in i915_pmu_enable()
803 if (bit == -1) in i915_pmu_disable()
806 spin_lock_irqsave(&pmu->lock, flags); in i915_pmu_disable()
816 GEM_BUG_ON(sample >= ARRAY_SIZE(engine->pmu.enable_count)); in i915_pmu_disable()
817 GEM_BUG_ON(sample >= ARRAY_SIZE(engine->pmu.sample)); in i915_pmu_disable()
818 GEM_BUG_ON(engine->pmu.enable_count[sample] == 0); in i915_pmu_disable()
824 if (--engine->pmu.enable_count[sample] == 0) in i915_pmu_disable()
825 engine->pmu.enable &= ~BIT(sample); in i915_pmu_disable()
828 GEM_BUG_ON(bit >= ARRAY_SIZE(pmu->enable_count)); in i915_pmu_disable()
829 GEM_BUG_ON(pmu->enable_count[bit] == 0); in i915_pmu_disable()
834 if (--pmu->enable_count[bit] == 0) { in i915_pmu_disable()
835 pmu->enable &= ~BIT(bit); in i915_pmu_disable()
836 pmu->timer_enabled &= pmu_needs_timer(pmu); in i915_pmu_disable()
839 spin_unlock_irqrestore(&pmu->lock, flags); in i915_pmu_disable()
846 if (!pmu->registered) in i915_pmu_event_start()
850 event->hw.state = 0; in i915_pmu_event_start()
857 if (!pmu->registered) in i915_pmu_event_stop()
866 event->hw.state = PERF_HES_STOPPED; in i915_pmu_event_stop()
873 if (!pmu->registered) in i915_pmu_event_add()
874 return -ENODEV; in i915_pmu_event_add()
898 return sprintf(buf, "%s\n", eattr->str); in i915_pmu_format_show()
908 I915_PMU_FORMAT_ATTR(i915_eventid, "config:0-20"),
928 return sprintf(buf, "config=0x%lx\n", eattr->val); in i915_pmu_event_show()
956 sysfs_attr_init(&attr->attr.attr); in add_i915_attr()
957 attr->attr.attr.name = name; in add_i915_attr()
958 attr->attr.attr.mode = 0444; in add_i915_attr()
959 attr->attr.show = i915_pmu_event_show; in add_i915_attr()
960 attr->val = config; in add_i915_attr()
969 sysfs_attr_init(&attr->attr.attr); in add_pmu_attr()
970 attr->attr.attr.name = name; in add_pmu_attr()
971 attr->attr.attr.mode = 0444; in add_pmu_attr()
972 attr->attr.show = perf_event_sysfs_show; in add_pmu_attr()
973 attr->event_str = str; in add_pmu_attr()
988 __event(0, "actual-frequency", "M"), in create_event_attributes()
989 __event(1, "requested-frequency", "M"), in create_event_attributes()
991 __event(3, "rc6-residency", "ns"), in create_event_attributes()
992 __event(4, "software-gt-awake-time", "ns"), in create_event_attributes()
1046 /* Initialize supported non-engine counters. */ in create_event_attributes()
1058 str = kasprintf(GFP_KERNEL, "%s-gt%u", in create_event_attributes()
1063 *attr_iter++ = &i915_iter->attr.attr; in create_event_attributes()
1071 str = kasprintf(GFP_KERNEL, "%s-gt%u.unit", in create_event_attributes()
1076 *attr_iter++ = &pmu_iter->attr.attr; in create_event_attributes()
1092 str = kasprintf(GFP_KERNEL, "%s-%s", in create_event_attributes()
1093 engine->name, engine_events[i].name); in create_event_attributes()
1097 *attr_iter++ = &i915_iter->attr.attr; in create_event_attributes()
1100 __I915_PMU_ENGINE(engine->uabi_class, in create_event_attributes()
1101 engine->uabi_instance, in create_event_attributes()
1104 str = kasprintf(GFP_KERNEL, "%s-%s.unit", in create_event_attributes()
1105 engine->name, engine_events[i].name); in create_event_attributes()
1109 *attr_iter++ = &pmu_iter->attr.attr; in create_event_attributes()
1114 pmu->i915_attr = i915_attr; in create_event_attributes()
1115 pmu->pmu_attr = pmu_attr; in create_event_attributes()
1121 kfree((*attr_iter)->name); in create_event_attributes()
1133 struct attribute **attr_iter = pmu->events_attr_group.attrs; in free_event_attributes()
1136 kfree((*attr_iter)->name); in free_event_attributes()
1138 kfree(pmu->events_attr_group.attrs); in free_event_attributes()
1139 kfree(pmu->i915_attr); in free_event_attributes()
1140 kfree(pmu->pmu_attr); in free_event_attributes()
1142 pmu->events_attr_group.attrs = NULL; in free_event_attributes()
1143 pmu->i915_attr = NULL; in free_event_attributes()
1144 pmu->pmu_attr = NULL; in free_event_attributes()
1149 struct i915_pmu *pmu = &i915->pmu; in i915_pmu_register()
1152 &pmu->events_attr_group, in i915_pmu_register()
1155 int ret = -ENOMEM; in i915_pmu_register()
1157 spin_lock_init(&pmu->lock); in i915_pmu_register()
1158 hrtimer_setup(&pmu->timer, i915_sample, CLOCK_MONOTONIC, HRTIMER_MODE_REL); in i915_pmu_register()
1162 pmu->name = kasprintf(GFP_KERNEL, in i915_pmu_register()
1164 dev_name(i915->drm.dev)); in i915_pmu_register()
1165 if (pmu->name) { in i915_pmu_register()
1167 strreplace((char *)pmu->name, ':', '_'); in i915_pmu_register()
1170 pmu->name = "i915"; in i915_pmu_register()
1172 if (!pmu->name) in i915_pmu_register()
1175 pmu->events_attr_group.name = "events"; in i915_pmu_register()
1176 pmu->events_attr_group.attrs = create_event_attributes(pmu); in i915_pmu_register()
1177 if (!pmu->events_attr_group.attrs) in i915_pmu_register()
1180 pmu->base.attr_groups = kmemdup(attr_groups, sizeof(attr_groups), in i915_pmu_register()
1182 if (!pmu->base.attr_groups) in i915_pmu_register()
1185 pmu->base.module = THIS_MODULE; in i915_pmu_register()
1186 pmu->base.task_ctx_nr = perf_invalid_context; in i915_pmu_register()
1187 pmu->base.scope = PERF_PMU_SCOPE_SYS_WIDE; in i915_pmu_register()
1188 pmu->base.event_init = i915_pmu_event_init; in i915_pmu_register()
1189 pmu->base.add = i915_pmu_event_add; in i915_pmu_register()
1190 pmu->base.del = i915_pmu_event_del; in i915_pmu_register()
1191 pmu->base.start = i915_pmu_event_start; in i915_pmu_register()
1192 pmu->base.stop = i915_pmu_event_stop; in i915_pmu_register()
1193 pmu->base.read = i915_pmu_event_read; in i915_pmu_register()
1195 ret = perf_pmu_register(&pmu->base, pmu->name, -1); in i915_pmu_register()
1199 pmu->registered = true; in i915_pmu_register()
1204 kfree(pmu->base.attr_groups); in i915_pmu_register()
1209 kfree(pmu->name); in i915_pmu_register()
1211 drm_notice(&i915->drm, "Failed to register PMU!\n"); in i915_pmu_register()
1216 struct i915_pmu *pmu = &i915->pmu; in i915_pmu_unregister()
1218 if (!pmu->registered) in i915_pmu_unregister()
1222 pmu->registered = false; in i915_pmu_unregister()
1224 hrtimer_cancel(&pmu->timer); in i915_pmu_unregister()
1226 perf_pmu_unregister(&pmu->base); in i915_pmu_unregister()
1227 kfree(pmu->base.attr_groups); in i915_pmu_unregister()
1229 kfree(pmu->name); in i915_pmu_unregister()