1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2023 Intel Corporation
4 */
5
6 #include <drm/drm_managed.h>
7
8 #include <generated/xe_wa_oob.h>
9 #include "xe_force_wake.h"
10 #include "xe_device.h"
11 #include "xe_gt.h"
12 #include "xe_gt_idle.h"
13 #include "xe_gt_sysfs.h"
14 #include "xe_guc_pc.h"
15 #include "regs/xe_gt_regs.h"
16 #include "xe_macros.h"
17 #include "xe_mmio.h"
18 #include "xe_pm.h"
19 #include "xe_sriov.h"
20 #include "xe_wa.h"
21
22 /**
23 * DOC: Xe GT Idle
24 *
25 * Contains functions that init GT idle features like C6
26 *
27 * device/gt#/gtidle/name - name of the state
28 * device/gt#/gtidle/idle_residency_ms - Provides residency of the idle state in ms
29 * device/gt#/gtidle/idle_status - Provides current idle state
30 */
31
dev_to_gtidle(struct device * dev)32 static struct xe_gt_idle *dev_to_gtidle(struct device *dev)
33 {
34 struct kobject *kobj = &dev->kobj;
35
36 return &kobj_to_gt(kobj->parent)->gtidle;
37 }
38
gtidle_to_gt(struct xe_gt_idle * gtidle)39 static struct xe_gt *gtidle_to_gt(struct xe_gt_idle *gtidle)
40 {
41 return container_of(gtidle, struct xe_gt, gtidle);
42 }
43
gtidle_to_pc(struct xe_gt_idle * gtidle)44 static struct xe_guc_pc *gtidle_to_pc(struct xe_gt_idle *gtidle)
45 {
46 return >idle_to_gt(gtidle)->uc.guc.pc;
47 }
48
49 static struct xe_device *
pc_to_xe(struct xe_guc_pc * pc)50 pc_to_xe(struct xe_guc_pc *pc)
51 {
52 struct xe_guc *guc = container_of(pc, struct xe_guc, pc);
53 struct xe_gt *gt = container_of(guc, struct xe_gt, uc.guc);
54
55 return gt_to_xe(gt);
56 }
57
gt_idle_state_to_string(enum xe_gt_idle_state state)58 static const char *gt_idle_state_to_string(enum xe_gt_idle_state state)
59 {
60 switch (state) {
61 case GT_IDLE_C0:
62 return "gt-c0";
63 case GT_IDLE_C6:
64 return "gt-c6";
65 default:
66 return "unknown";
67 }
68 }
69
get_residency_ms(struct xe_gt_idle * gtidle,u64 cur_residency)70 static u64 get_residency_ms(struct xe_gt_idle *gtidle, u64 cur_residency)
71 {
72 u64 delta, overflow_residency, prev_residency;
73
74 lockdep_assert_held(>idle->lock);
75
76 overflow_residency = BIT_ULL(32);
77
78 /*
79 * Counter wrap handling
80 * Store previous hw counter values for counter wrap-around handling
81 * Relying on sufficient frequency of queries otherwise counters can still wrap.
82 */
83 prev_residency = gtidle->prev_residency;
84 gtidle->prev_residency = cur_residency;
85
86 /* delta */
87 if (cur_residency >= prev_residency)
88 delta = cur_residency - prev_residency;
89 else
90 delta = cur_residency + (overflow_residency - prev_residency);
91
92 /* Add delta to extended raw driver copy of idle residency */
93 cur_residency = gtidle->cur_residency + delta;
94 gtidle->cur_residency = cur_residency;
95
96 /* residency multiplier in ns, convert to ms */
97 cur_residency = mul_u64_u32_div(cur_residency, gtidle->residency_multiplier, 1e6);
98
99 return cur_residency;
100 }
101
xe_gt_idle_enable_pg(struct xe_gt * gt)102 void xe_gt_idle_enable_pg(struct xe_gt *gt)
103 {
104 struct xe_device *xe = gt_to_xe(gt);
105 struct xe_gt_idle *gtidle = >->gtidle;
106 struct xe_mmio *mmio = >->mmio;
107 u32 vcs_mask, vecs_mask;
108 unsigned int fw_ref;
109 int i, j;
110
111 if (IS_SRIOV_VF(xe))
112 return;
113
114 /* Disable CPG for PVC */
115 if (xe->info.platform == XE_PVC)
116 return;
117
118 xe_device_assert_mem_access(gt_to_xe(gt));
119
120 vcs_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_VIDEO_DECODE);
121 vecs_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_VIDEO_ENHANCE);
122
123 if (vcs_mask || vecs_mask)
124 gtidle->powergate_enable = MEDIA_POWERGATE_ENABLE;
125
126 if (xe_gt_is_main_type(gt))
127 gtidle->powergate_enable |= RENDER_POWERGATE_ENABLE;
128
129 if (MEDIA_VERx100(xe) >= 1100 && MEDIA_VERx100(xe) < 1255)
130 gtidle->powergate_enable |= MEDIA_SAMPLERS_POWERGATE_ENABLE;
131
132 if (xe->info.platform != XE_DG1) {
133 for (i = XE_HW_ENGINE_VCS0, j = 0; i <= XE_HW_ENGINE_VCS7; ++i, ++j) {
134 if ((gt->info.engine_mask & BIT(i)))
135 gtidle->powergate_enable |= (VDN_HCP_POWERGATE_ENABLE(j) |
136 VDN_MFXVDENC_POWERGATE_ENABLE(j));
137 }
138 }
139
140 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
141 if (xe->info.skip_guc_pc) {
142 /*
143 * GuC sets the hysteresis value when GuC PC is enabled
144 * else set it to 25 (25 * 1.28us)
145 */
146 xe_mmio_write32(mmio, MEDIA_POWERGATE_IDLE_HYSTERESIS, 25);
147 xe_mmio_write32(mmio, RENDER_POWERGATE_IDLE_HYSTERESIS, 25);
148 }
149
150 if (XE_GT_WA(gt, 14020316580))
151 gtidle->powergate_enable &= ~(VDN_HCP_POWERGATE_ENABLE(0) |
152 VDN_MFXVDENC_POWERGATE_ENABLE(0) |
153 VDN_HCP_POWERGATE_ENABLE(2) |
154 VDN_MFXVDENC_POWERGATE_ENABLE(2));
155
156 xe_mmio_write32(mmio, POWERGATE_ENABLE, gtidle->powergate_enable);
157 xe_force_wake_put(gt_to_fw(gt), fw_ref);
158 }
159
xe_gt_idle_disable_pg(struct xe_gt * gt)160 void xe_gt_idle_disable_pg(struct xe_gt *gt)
161 {
162 struct xe_gt_idle *gtidle = >->gtidle;
163 unsigned int fw_ref;
164
165 if (IS_SRIOV_VF(gt_to_xe(gt)))
166 return;
167
168 xe_device_assert_mem_access(gt_to_xe(gt));
169 gtidle->powergate_enable = 0;
170
171 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
172 xe_mmio_write32(>->mmio, POWERGATE_ENABLE, gtidle->powergate_enable);
173 xe_force_wake_put(gt_to_fw(gt), fw_ref);
174 }
175
176 /**
177 * xe_gt_idle_pg_print - Xe powergating info
178 * @gt: GT object
179 * @p: drm_printer.
180 *
181 * This function prints the powergating information
182 *
183 * Return: 0 on success, negative error code otherwise
184 */
xe_gt_idle_pg_print(struct xe_gt * gt,struct drm_printer * p)185 int xe_gt_idle_pg_print(struct xe_gt *gt, struct drm_printer *p)
186 {
187 struct xe_gt_idle *gtidle = >->gtidle;
188 struct xe_device *xe = gt_to_xe(gt);
189 enum xe_gt_idle_state state;
190 u32 pg_enabled, pg_status = 0;
191 u32 vcs_mask, vecs_mask;
192 unsigned int fw_ref;
193 int n;
194 /*
195 * Media Slices
196 *
197 * Slice 0: VCS0, VCS1, VECS0
198 * Slice 1: VCS2, VCS3, VECS1
199 * Slice 2: VCS4, VCS5, VECS2
200 * Slice 3: VCS6, VCS7, VECS3
201 */
202 static const struct {
203 u64 engines;
204 u32 status_bit;
205 } media_slices[] = {
206 {(BIT(XE_HW_ENGINE_VCS0) | BIT(XE_HW_ENGINE_VCS1) |
207 BIT(XE_HW_ENGINE_VECS0)), MEDIA_SLICE0_AWAKE_STATUS},
208
209 {(BIT(XE_HW_ENGINE_VCS2) | BIT(XE_HW_ENGINE_VCS3) |
210 BIT(XE_HW_ENGINE_VECS1)), MEDIA_SLICE1_AWAKE_STATUS},
211
212 {(BIT(XE_HW_ENGINE_VCS4) | BIT(XE_HW_ENGINE_VCS5) |
213 BIT(XE_HW_ENGINE_VECS2)), MEDIA_SLICE2_AWAKE_STATUS},
214
215 {(BIT(XE_HW_ENGINE_VCS6) | BIT(XE_HW_ENGINE_VCS7) |
216 BIT(XE_HW_ENGINE_VECS3)), MEDIA_SLICE3_AWAKE_STATUS},
217 };
218
219 if (xe->info.platform == XE_PVC) {
220 drm_printf(p, "Power Gating not supported\n");
221 return 0;
222 }
223
224 state = gtidle->idle_status(gtidle_to_pc(gtidle));
225 pg_enabled = gtidle->powergate_enable;
226
227 /* Do not wake the GT to read powergating status */
228 if (state != GT_IDLE_C6) {
229 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
230 if (!fw_ref)
231 return -ETIMEDOUT;
232
233 pg_enabled = xe_mmio_read32(>->mmio, POWERGATE_ENABLE);
234 pg_status = xe_mmio_read32(>->mmio, POWERGATE_DOMAIN_STATUS);
235
236 xe_force_wake_put(gt_to_fw(gt), fw_ref);
237 }
238
239 if (gt->info.engine_mask & XE_HW_ENGINE_RCS_MASK) {
240 drm_printf(p, "Render Power Gating Enabled: %s\n",
241 str_yes_no(pg_enabled & RENDER_POWERGATE_ENABLE));
242
243 drm_printf(p, "Render Power Gate Status: %s\n",
244 str_up_down(pg_status & RENDER_AWAKE_STATUS));
245 }
246
247 vcs_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_VIDEO_DECODE);
248 vecs_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_VIDEO_ENHANCE);
249
250 /* Print media CPG status only if media is present */
251 if (vcs_mask || vecs_mask) {
252 drm_printf(p, "Media Power Gating Enabled: %s\n",
253 str_yes_no(pg_enabled & MEDIA_POWERGATE_ENABLE));
254
255 for (n = 0; n < ARRAY_SIZE(media_slices); n++)
256 if (gt->info.engine_mask & media_slices[n].engines)
257 drm_printf(p, "Media Slice%d Power Gate Status: %s\n", n,
258 str_up_down(pg_status & media_slices[n].status_bit));
259 }
260
261 if (MEDIA_VERx100(xe) >= 1100 && MEDIA_VERx100(xe) < 1255)
262 drm_printf(p, "Media Samplers Power Gating Enabled: %s\n",
263 str_yes_no(pg_enabled & MEDIA_SAMPLERS_POWERGATE_ENABLE));
264
265 return 0;
266 }
267
name_show(struct kobject * kobj,struct kobj_attribute * attr,char * buff)268 static ssize_t name_show(struct kobject *kobj,
269 struct kobj_attribute *attr, char *buff)
270 {
271 struct device *dev = kobj_to_dev(kobj);
272 struct xe_gt_idle *gtidle = dev_to_gtidle(dev);
273 struct xe_guc_pc *pc = gtidle_to_pc(gtidle);
274 ssize_t ret;
275
276 xe_pm_runtime_get(pc_to_xe(pc));
277 ret = sysfs_emit(buff, "%s\n", gtidle->name);
278 xe_pm_runtime_put(pc_to_xe(pc));
279
280 return ret;
281 }
282 static struct kobj_attribute name_attr = __ATTR_RO(name);
283
idle_status_show(struct kobject * kobj,struct kobj_attribute * attr,char * buff)284 static ssize_t idle_status_show(struct kobject *kobj,
285 struct kobj_attribute *attr, char *buff)
286 {
287 struct device *dev = kobj_to_dev(kobj);
288 struct xe_gt_idle *gtidle = dev_to_gtidle(dev);
289 struct xe_guc_pc *pc = gtidle_to_pc(gtidle);
290 enum xe_gt_idle_state state;
291
292 xe_pm_runtime_get(pc_to_xe(pc));
293 state = gtidle->idle_status(pc);
294 xe_pm_runtime_put(pc_to_xe(pc));
295
296 return sysfs_emit(buff, "%s\n", gt_idle_state_to_string(state));
297 }
298 static struct kobj_attribute idle_status_attr = __ATTR_RO(idle_status);
299
xe_gt_idle_residency_msec(struct xe_gt_idle * gtidle)300 u64 xe_gt_idle_residency_msec(struct xe_gt_idle *gtidle)
301 {
302 struct xe_guc_pc *pc = gtidle_to_pc(gtidle);
303 u64 residency;
304 unsigned long flags;
305
306 raw_spin_lock_irqsave(>idle->lock, flags);
307 residency = get_residency_ms(gtidle, gtidle->idle_residency(pc));
308 raw_spin_unlock_irqrestore(>idle->lock, flags);
309
310 return residency;
311 }
312
313
idle_residency_ms_show(struct kobject * kobj,struct kobj_attribute * attr,char * buff)314 static ssize_t idle_residency_ms_show(struct kobject *kobj,
315 struct kobj_attribute *attr, char *buff)
316 {
317 struct device *dev = kobj_to_dev(kobj);
318 struct xe_gt_idle *gtidle = dev_to_gtidle(dev);
319 struct xe_guc_pc *pc = gtidle_to_pc(gtidle);
320 u64 residency;
321
322 xe_pm_runtime_get(pc_to_xe(pc));
323 residency = xe_gt_idle_residency_msec(gtidle);
324 xe_pm_runtime_put(pc_to_xe(pc));
325
326 return sysfs_emit(buff, "%llu\n", residency);
327 }
328 static struct kobj_attribute idle_residency_attr = __ATTR_RO(idle_residency_ms);
329
330 static const struct attribute *gt_idle_attrs[] = {
331 &name_attr.attr,
332 &idle_status_attr.attr,
333 &idle_residency_attr.attr,
334 NULL,
335 };
336
gt_idle_fini(void * arg)337 static void gt_idle_fini(void *arg)
338 {
339 struct kobject *kobj = arg;
340 struct xe_gt *gt = kobj_to_gt(kobj->parent);
341
342 xe_gt_idle_disable_pg(gt);
343
344 if (gt_to_xe(gt)->info.skip_guc_pc)
345 xe_gt_idle_disable_c6(gt);
346
347 sysfs_remove_files(kobj, gt_idle_attrs);
348 kobject_put(kobj);
349 }
350
xe_gt_idle_init(struct xe_gt_idle * gtidle)351 int xe_gt_idle_init(struct xe_gt_idle *gtidle)
352 {
353 struct xe_gt *gt = gtidle_to_gt(gtidle);
354 struct xe_device *xe = gt_to_xe(gt);
355 struct kobject *kobj;
356 int err;
357
358 if (IS_SRIOV_VF(xe))
359 return 0;
360
361 kobj = kobject_create_and_add("gtidle", gt->sysfs);
362 if (!kobj)
363 return -ENOMEM;
364
365 raw_spin_lock_init(>idle->lock);
366
367 if (xe_gt_is_media_type(gt)) {
368 snprintf(gtidle->name, sizeof(gtidle->name), "gt%d-mc", gt->info.id);
369 gtidle->idle_residency = xe_guc_pc_mc6_residency;
370 } else {
371 snprintf(gtidle->name, sizeof(gtidle->name), "gt%d-rc", gt->info.id);
372 gtidle->idle_residency = xe_guc_pc_rc6_residency;
373 }
374
375 /* Multiplier for Residency counter in units of 1.28us */
376 gtidle->residency_multiplier = 1280;
377 gtidle->idle_status = xe_guc_pc_c_status;
378
379 err = sysfs_create_files(kobj, gt_idle_attrs);
380 if (err) {
381 kobject_put(kobj);
382 return err;
383 }
384
385 xe_gt_idle_enable_pg(gt);
386
387 return devm_add_action_or_reset(xe->drm.dev, gt_idle_fini, kobj);
388 }
389
xe_gt_idle_enable_c6(struct xe_gt * gt)390 void xe_gt_idle_enable_c6(struct xe_gt *gt)
391 {
392 xe_device_assert_mem_access(gt_to_xe(gt));
393 xe_force_wake_assert_held(gt_to_fw(gt), XE_FW_GT);
394
395 if (IS_SRIOV_VF(gt_to_xe(gt)))
396 return;
397
398 /* Units of 1280 ns for a total of 5s */
399 xe_mmio_write32(>->mmio, RC_IDLE_HYSTERSIS, 0x3B9ACA);
400 /* Enable RC6 */
401 xe_mmio_write32(>->mmio, RC_CONTROL,
402 RC_CTL_HW_ENABLE | RC_CTL_TO_MODE | RC_CTL_RC6_ENABLE);
403 }
404
xe_gt_idle_disable_c6(struct xe_gt * gt)405 int xe_gt_idle_disable_c6(struct xe_gt *gt)
406 {
407 unsigned int fw_ref;
408
409 xe_device_assert_mem_access(gt_to_xe(gt));
410
411 if (IS_SRIOV_VF(gt_to_xe(gt)))
412 return 0;
413
414 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
415 if (!fw_ref)
416 return -ETIMEDOUT;
417
418 xe_mmio_write32(>->mmio, RC_CONTROL, 0);
419 xe_mmio_write32(>->mmio, RC_STATE, 0);
420
421 xe_force_wake_put(gt_to_fw(gt), fw_ref);
422
423 return 0;
424 }
425