1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2023 Intel Corporation
4 */
5
6 #include <linux/time64.h>
7
8 #include <drm/drm_managed.h>
9
10 #include <generated/xe_wa_oob.h>
11 #include "xe_force_wake.h"
12 #include "xe_device.h"
13 #include "xe_gt.h"
14 #include "xe_gt_idle.h"
15 #include "xe_gt_sysfs.h"
16 #include "xe_guc_pc.h"
17 #include "regs/xe_gt_regs.h"
18 #include "xe_mmio.h"
19 #include "xe_pm.h"
20 #include "xe_sriov.h"
21 #include "xe_wa.h"
22
23 /**
24 * DOC: Xe GT Idle
25 *
26 * Contains functions that init GT idle features like C6
27 *
28 * device/gt#/gtidle/name - name of the state
29 * device/gt#/gtidle/idle_residency_ms - Provides residency of the idle state in ms
30 * device/gt#/gtidle/idle_status - Provides current idle state
31 */
32
dev_to_gtidle(struct device * dev)33 static struct xe_gt_idle *dev_to_gtidle(struct device *dev)
34 {
35 struct kobject *kobj = &dev->kobj;
36
37 return &kobj_to_gt(kobj->parent)->gtidle;
38 }
39
gtidle_to_gt(struct xe_gt_idle * gtidle)40 static struct xe_gt *gtidle_to_gt(struct xe_gt_idle *gtidle)
41 {
42 return container_of(gtidle, struct xe_gt, gtidle);
43 }
44
gtidle_to_pc(struct xe_gt_idle * gtidle)45 static struct xe_guc_pc *gtidle_to_pc(struct xe_gt_idle *gtidle)
46 {
47 return >idle_to_gt(gtidle)->uc.guc.pc;
48 }
49
50 static struct xe_device *
pc_to_xe(struct xe_guc_pc * pc)51 pc_to_xe(struct xe_guc_pc *pc)
52 {
53 struct xe_guc *guc = container_of(pc, struct xe_guc, pc);
54 struct xe_gt *gt = container_of(guc, struct xe_gt, uc.guc);
55
56 return gt_to_xe(gt);
57 }
58
gt_idle_state_to_string(enum xe_gt_idle_state state)59 static const char *gt_idle_state_to_string(enum xe_gt_idle_state state)
60 {
61 switch (state) {
62 case GT_IDLE_C0:
63 return "gt-c0";
64 case GT_IDLE_C6:
65 return "gt-c6";
66 default:
67 return "unknown";
68 }
69 }
70
get_residency_ms(struct xe_gt_idle * gtidle,u64 cur_residency)71 static u64 get_residency_ms(struct xe_gt_idle *gtidle, u64 cur_residency)
72 {
73 u64 delta, overflow_residency, prev_residency;
74
75 lockdep_assert_held(>idle->lock);
76
77 overflow_residency = BIT_ULL(32);
78
79 /*
80 * Counter wrap handling
81 * Store previous hw counter values for counter wrap-around handling
82 * Relying on sufficient frequency of queries otherwise counters can still wrap.
83 */
84 prev_residency = gtidle->prev_residency;
85 gtidle->prev_residency = cur_residency;
86
87 /* delta */
88 if (cur_residency >= prev_residency)
89 delta = cur_residency - prev_residency;
90 else
91 delta = cur_residency + (overflow_residency - prev_residency);
92
93 /* Add delta to extended raw driver copy of idle residency */
94 cur_residency = gtidle->cur_residency + delta;
95 gtidle->cur_residency = cur_residency;
96
97 /* residency multiplier in ns, convert to ms */
98 cur_residency = mul_u64_u32_div(cur_residency, gtidle->residency_multiplier, NSEC_PER_MSEC);
99
100 return cur_residency;
101 }
102
xe_gt_idle_enable_pg(struct xe_gt * gt)103 void xe_gt_idle_enable_pg(struct xe_gt *gt)
104 {
105 struct xe_device *xe = gt_to_xe(gt);
106 struct xe_gt_idle *gtidle = >->gtidle;
107 struct xe_mmio *mmio = >->mmio;
108 u32 vcs_mask, vecs_mask;
109 int i, j;
110
111 if (IS_SRIOV_VF(xe))
112 return;
113
114 /* Disable CPG for PVC */
115 if (xe->info.platform == XE_PVC)
116 return;
117
118 xe_device_assert_mem_access(gt_to_xe(gt));
119
120 vcs_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_VIDEO_DECODE);
121 vecs_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_VIDEO_ENHANCE);
122
123 if (vcs_mask || vecs_mask)
124 gtidle->powergate_enable = MEDIA_POWERGATE_ENABLE;
125
126 if (xe_gt_is_main_type(gt))
127 gtidle->powergate_enable |= RENDER_POWERGATE_ENABLE;
128
129 if (MEDIA_VERx100(xe) >= 1100 && MEDIA_VERx100(xe) < 1255)
130 gtidle->powergate_enable |= MEDIA_SAMPLERS_POWERGATE_ENABLE;
131
132 if (xe->info.platform != XE_DG1) {
133 for (i = XE_HW_ENGINE_VCS0, j = 0; i <= XE_HW_ENGINE_VCS7; ++i, ++j) {
134 if ((gt->info.engine_mask & BIT(i)))
135 gtidle->powergate_enable |= (VDN_HCP_POWERGATE_ENABLE(j) |
136 VDN_MFXVDENC_POWERGATE_ENABLE(j));
137 }
138 }
139
140 CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
141 if (xe->info.skip_guc_pc) {
142 /*
143 * GuC sets the hysteresis value when GuC PC is enabled
144 * else set it to 25 (25 * 1.28us)
145 */
146 xe_mmio_write32(mmio, MEDIA_POWERGATE_IDLE_HYSTERESIS, 25);
147 xe_mmio_write32(mmio, RENDER_POWERGATE_IDLE_HYSTERESIS, 25);
148 }
149
150 if (XE_GT_WA(gt, 14020316580))
151 gtidle->powergate_enable &= ~(VDN_HCP_POWERGATE_ENABLE(0) |
152 VDN_MFXVDENC_POWERGATE_ENABLE(0) |
153 VDN_HCP_POWERGATE_ENABLE(2) |
154 VDN_MFXVDENC_POWERGATE_ENABLE(2));
155
156 xe_mmio_write32(mmio, POWERGATE_ENABLE, gtidle->powergate_enable);
157 }
158
xe_gt_idle_disable_pg(struct xe_gt * gt)159 void xe_gt_idle_disable_pg(struct xe_gt *gt)
160 {
161 struct xe_gt_idle *gtidle = >->gtidle;
162
163 if (IS_SRIOV_VF(gt_to_xe(gt)))
164 return;
165
166 xe_device_assert_mem_access(gt_to_xe(gt));
167 gtidle->powergate_enable = 0;
168
169 CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
170 xe_mmio_write32(>->mmio, POWERGATE_ENABLE, gtidle->powergate_enable);
171 }
172
force_wake_domains_show(struct xe_gt * gt,struct drm_printer * p)173 static void force_wake_domains_show(struct xe_gt *gt, struct drm_printer *p)
174 {
175 struct xe_force_wake_domain *domain;
176 struct xe_force_wake *fw = gt_to_fw(gt);
177 unsigned int tmp;
178 unsigned long flags;
179
180 spin_lock_irqsave(&fw->lock, flags);
181 for_each_fw_domain(domain, fw, tmp) {
182 drm_printf(p, "%s.ref_count=%u, %s.fwake=0x%x\n",
183 xe_force_wake_domain_to_str(domain->id),
184 READ_ONCE(domain->ref),
185 xe_force_wake_domain_to_str(domain->id),
186 xe_mmio_read32(>->mmio, domain->reg_ctl));
187 }
188 spin_unlock_irqrestore(&fw->lock, flags);
189 }
190
191 /**
192 * xe_gt_idle_pg_print - Xe powergating info
193 * @gt: GT object
194 * @p: drm_printer.
195 *
196 * This function prints the powergating information
197 *
198 * Return: 0 on success, negative error code otherwise
199 */
xe_gt_idle_pg_print(struct xe_gt * gt,struct drm_printer * p)200 int xe_gt_idle_pg_print(struct xe_gt *gt, struct drm_printer *p)
201 {
202 struct xe_gt_idle *gtidle = >->gtidle;
203 struct xe_device *xe = gt_to_xe(gt);
204 enum xe_gt_idle_state state;
205 u32 pg_enabled, pg_status = 0;
206 u32 vcs_mask, vecs_mask;
207 int n;
208 /*
209 * Media Slices
210 *
211 * Slice 0: VCS0, VCS1, VECS0
212 * Slice 1: VCS2, VCS3, VECS1
213 * Slice 2: VCS4, VCS5, VECS2
214 * Slice 3: VCS6, VCS7, VECS3
215 */
216 static const struct {
217 u64 engines;
218 u32 status_bit;
219 } media_slices[] = {
220 {(BIT(XE_HW_ENGINE_VCS0) | BIT(XE_HW_ENGINE_VCS1) |
221 BIT(XE_HW_ENGINE_VECS0)), MEDIA_SLICE0_AWAKE_STATUS},
222
223 {(BIT(XE_HW_ENGINE_VCS2) | BIT(XE_HW_ENGINE_VCS3) |
224 BIT(XE_HW_ENGINE_VECS1)), MEDIA_SLICE1_AWAKE_STATUS},
225
226 {(BIT(XE_HW_ENGINE_VCS4) | BIT(XE_HW_ENGINE_VCS5) |
227 BIT(XE_HW_ENGINE_VECS2)), MEDIA_SLICE2_AWAKE_STATUS},
228
229 {(BIT(XE_HW_ENGINE_VCS6) | BIT(XE_HW_ENGINE_VCS7) |
230 BIT(XE_HW_ENGINE_VECS3)), MEDIA_SLICE3_AWAKE_STATUS},
231 };
232
233 if (xe->info.platform == XE_PVC) {
234 drm_printf(p, "Power Gating not supported\n");
235 return 0;
236 }
237
238 state = gtidle->idle_status(gtidle_to_pc(gtidle));
239 pg_enabled = gtidle->powergate_enable;
240
241 /* Do not wake the GT to read powergating status */
242 if (state != GT_IDLE_C6) {
243 CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
244 if (!fw_ref.domains)
245 return -ETIMEDOUT;
246
247 pg_enabled = xe_mmio_read32(>->mmio, POWERGATE_ENABLE);
248 pg_status = xe_mmio_read32(>->mmio, POWERGATE_DOMAIN_STATUS);
249 }
250
251 if (gt->info.engine_mask &
252 (XE_HW_ENGINE_RCS_MASK | XE_HW_ENGINE_CCS_MASK)) {
253 drm_printf(p, "Render Power Gating Enabled: %s\n",
254 str_yes_no(pg_enabled & RENDER_POWERGATE_ENABLE));
255
256 drm_printf(p, "Render Power Gate Status: %s\n",
257 str_up_down(pg_status & RENDER_AWAKE_STATUS));
258 }
259
260 vcs_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_VIDEO_DECODE);
261 vecs_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_VIDEO_ENHANCE);
262
263 /* Print media CPG status only if media is present */
264 if (vcs_mask || vecs_mask) {
265 drm_printf(p, "Media Power Gating Enabled: %s\n",
266 str_yes_no(pg_enabled & MEDIA_POWERGATE_ENABLE));
267
268 for (n = 0; n < ARRAY_SIZE(media_slices); n++)
269 if (gt->info.engine_mask & media_slices[n].engines)
270 drm_printf(p, "Media Slice%d Power Gate Status: %s\n", n,
271 str_up_down(pg_status & media_slices[n].status_bit));
272 }
273
274 if (MEDIA_VERx100(xe) >= 1100 && MEDIA_VERx100(xe) < 1255)
275 drm_printf(p, "Media Samplers Power Gating Enabled: %s\n",
276 str_yes_no(pg_enabled & MEDIA_SAMPLERS_POWERGATE_ENABLE));
277
278 if (gt->info.engine_mask & BIT(XE_HW_ENGINE_GSCCS0)) {
279 drm_printf(p, "GSC Power Gate Status: %s\n",
280 str_up_down(pg_status & GSC_AWAKE_STATUS));
281 }
282
283 force_wake_domains_show(gt, p);
284
285 return 0;
286 }
287
name_show(struct kobject * kobj,struct kobj_attribute * attr,char * buff)288 static ssize_t name_show(struct kobject *kobj,
289 struct kobj_attribute *attr, char *buff)
290 {
291 struct device *dev = kobj_to_dev(kobj);
292 struct xe_gt_idle *gtidle = dev_to_gtidle(dev);
293 struct xe_guc_pc *pc = gtidle_to_pc(gtidle);
294
295 guard(xe_pm_runtime)(pc_to_xe(pc));
296 return sysfs_emit(buff, "%s\n", gtidle->name);
297 }
298 static struct kobj_attribute name_attr = __ATTR_RO(name);
299
idle_status_show(struct kobject * kobj,struct kobj_attribute * attr,char * buff)300 static ssize_t idle_status_show(struct kobject *kobj,
301 struct kobj_attribute *attr, char *buff)
302 {
303 struct device *dev = kobj_to_dev(kobj);
304 struct xe_gt_idle *gtidle = dev_to_gtidle(dev);
305 struct xe_guc_pc *pc = gtidle_to_pc(gtidle);
306 enum xe_gt_idle_state state;
307
308 scoped_guard(xe_pm_runtime, pc_to_xe(pc))
309 state = gtidle->idle_status(pc);
310
311 return sysfs_emit(buff, "%s\n", gt_idle_state_to_string(state));
312 }
313 static struct kobj_attribute idle_status_attr = __ATTR_RO(idle_status);
314
xe_gt_idle_residency_msec(struct xe_gt_idle * gtidle)315 u64 xe_gt_idle_residency_msec(struct xe_gt_idle *gtidle)
316 {
317 struct xe_guc_pc *pc = gtidle_to_pc(gtidle);
318 u64 residency;
319 unsigned long flags;
320
321 raw_spin_lock_irqsave(>idle->lock, flags);
322 residency = get_residency_ms(gtidle, gtidle->idle_residency(pc));
323 raw_spin_unlock_irqrestore(>idle->lock, flags);
324
325 return residency;
326 }
327
328
idle_residency_ms_show(struct kobject * kobj,struct kobj_attribute * attr,char * buff)329 static ssize_t idle_residency_ms_show(struct kobject *kobj,
330 struct kobj_attribute *attr, char *buff)
331 {
332 struct device *dev = kobj_to_dev(kobj);
333 struct xe_gt_idle *gtidle = dev_to_gtidle(dev);
334 struct xe_guc_pc *pc = gtidle_to_pc(gtidle);
335 u64 residency;
336
337 scoped_guard(xe_pm_runtime, pc_to_xe(pc))
338 residency = xe_gt_idle_residency_msec(gtidle);
339
340 return sysfs_emit(buff, "%llu\n", residency);
341 }
342 static struct kobj_attribute idle_residency_attr = __ATTR_RO(idle_residency_ms);
343
344 static const struct attribute *gt_idle_attrs[] = {
345 &name_attr.attr,
346 &idle_status_attr.attr,
347 &idle_residency_attr.attr,
348 NULL,
349 };
350
gt_idle_fini(void * arg)351 static void gt_idle_fini(void *arg)
352 {
353 struct kobject *kobj = arg;
354 struct xe_gt *gt = kobj_to_gt(kobj->parent);
355
356 xe_gt_idle_disable_pg(gt);
357
358 if (gt_to_xe(gt)->info.skip_guc_pc)
359 xe_gt_idle_disable_c6(gt);
360
361 sysfs_remove_files(kobj, gt_idle_attrs);
362 kobject_put(kobj);
363 }
364
xe_gt_idle_init(struct xe_gt_idle * gtidle)365 int xe_gt_idle_init(struct xe_gt_idle *gtidle)
366 {
367 struct xe_gt *gt = gtidle_to_gt(gtidle);
368 struct xe_device *xe = gt_to_xe(gt);
369 struct kobject *kobj;
370 int err;
371
372 if (IS_SRIOV_VF(xe))
373 return 0;
374
375 kobj = kobject_create_and_add("gtidle", gt->sysfs);
376 if (!kobj)
377 return -ENOMEM;
378
379 raw_spin_lock_init(>idle->lock);
380
381 if (xe_gt_is_media_type(gt)) {
382 snprintf(gtidle->name, sizeof(gtidle->name), "gt%d-mc", gt->info.id);
383 gtidle->idle_residency = xe_guc_pc_mc6_residency;
384 } else {
385 snprintf(gtidle->name, sizeof(gtidle->name), "gt%d-rc", gt->info.id);
386 gtidle->idle_residency = xe_guc_pc_rc6_residency;
387 }
388
389 /* Multiplier for Residency counter in units of 1.28us */
390 gtidle->residency_multiplier = 1280;
391 gtidle->idle_status = xe_guc_pc_c_status;
392
393 err = sysfs_create_files(kobj, gt_idle_attrs);
394 if (err) {
395 kobject_put(kobj);
396 return err;
397 }
398
399 xe_gt_idle_enable_pg(gt);
400
401 return devm_add_action_or_reset(xe->drm.dev, gt_idle_fini, kobj);
402 }
403
xe_gt_idle_enable_c6(struct xe_gt * gt)404 void xe_gt_idle_enable_c6(struct xe_gt *gt)
405 {
406 xe_device_assert_mem_access(gt_to_xe(gt));
407 xe_force_wake_assert_held(gt_to_fw(gt), XE_FW_GT);
408
409 if (IS_SRIOV_VF(gt_to_xe(gt)))
410 return;
411
412 /* Units of 1280 ns for a total of 5s */
413 xe_mmio_write32(>->mmio, RC_IDLE_HYSTERSIS, 0x3B9ACA);
414 /* Enable RC6 */
415 xe_mmio_write32(>->mmio, RC_CONTROL,
416 RC_CTL_HW_ENABLE | RC_CTL_TO_MODE | RC_CTL_RC6_ENABLE);
417 }
418
xe_gt_idle_disable_c6(struct xe_gt * gt)419 int xe_gt_idle_disable_c6(struct xe_gt *gt)
420 {
421 xe_device_assert_mem_access(gt_to_xe(gt));
422
423 if (IS_SRIOV_VF(gt_to_xe(gt)))
424 return 0;
425
426 CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
427 if (!fw_ref.domains)
428 return -ETIMEDOUT;
429
430 xe_mmio_write32(>->mmio, RC_CONTROL, 0);
431 xe_mmio_write32(>->mmio, RC_STATE, 0);
432
433 return 0;
434 }
435