1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2023 Intel Corporation
4 */
5
6 #include <drm/drm_managed.h>
7
8 #include <generated/xe_wa_oob.h>
9 #include "xe_force_wake.h"
10 #include "xe_device.h"
11 #include "xe_gt.h"
12 #include "xe_gt_idle.h"
13 #include "xe_gt_sysfs.h"
14 #include "xe_guc_pc.h"
15 #include "regs/xe_gt_regs.h"
16 #include "xe_mmio.h"
17 #include "xe_pm.h"
18 #include "xe_sriov.h"
19 #include "xe_wa.h"
20
21 /**
22 * DOC: Xe GT Idle
23 *
24 * Contains functions that init GT idle features like C6
25 *
26 * device/gt#/gtidle/name - name of the state
27 * device/gt#/gtidle/idle_residency_ms - Provides residency of the idle state in ms
28 * device/gt#/gtidle/idle_status - Provides current idle state
29 */
30
dev_to_gtidle(struct device * dev)31 static struct xe_gt_idle *dev_to_gtidle(struct device *dev)
32 {
33 struct kobject *kobj = &dev->kobj;
34
35 return &kobj_to_gt(kobj->parent)->gtidle;
36 }
37
gtidle_to_gt(struct xe_gt_idle * gtidle)38 static struct xe_gt *gtidle_to_gt(struct xe_gt_idle *gtidle)
39 {
40 return container_of(gtidle, struct xe_gt, gtidle);
41 }
42
gtidle_to_pc(struct xe_gt_idle * gtidle)43 static struct xe_guc_pc *gtidle_to_pc(struct xe_gt_idle *gtidle)
44 {
45 return >idle_to_gt(gtidle)->uc.guc.pc;
46 }
47
48 static struct xe_device *
pc_to_xe(struct xe_guc_pc * pc)49 pc_to_xe(struct xe_guc_pc *pc)
50 {
51 struct xe_guc *guc = container_of(pc, struct xe_guc, pc);
52 struct xe_gt *gt = container_of(guc, struct xe_gt, uc.guc);
53
54 return gt_to_xe(gt);
55 }
56
gt_idle_state_to_string(enum xe_gt_idle_state state)57 static const char *gt_idle_state_to_string(enum xe_gt_idle_state state)
58 {
59 switch (state) {
60 case GT_IDLE_C0:
61 return "gt-c0";
62 case GT_IDLE_C6:
63 return "gt-c6";
64 default:
65 return "unknown";
66 }
67 }
68
get_residency_ms(struct xe_gt_idle * gtidle,u64 cur_residency)69 static u64 get_residency_ms(struct xe_gt_idle *gtidle, u64 cur_residency)
70 {
71 u64 delta, overflow_residency, prev_residency;
72
73 lockdep_assert_held(>idle->lock);
74
75 overflow_residency = BIT_ULL(32);
76
77 /*
78 * Counter wrap handling
79 * Store previous hw counter values for counter wrap-around handling
80 * Relying on sufficient frequency of queries otherwise counters can still wrap.
81 */
82 prev_residency = gtidle->prev_residency;
83 gtidle->prev_residency = cur_residency;
84
85 /* delta */
86 if (cur_residency >= prev_residency)
87 delta = cur_residency - prev_residency;
88 else
89 delta = cur_residency + (overflow_residency - prev_residency);
90
91 /* Add delta to extended raw driver copy of idle residency */
92 cur_residency = gtidle->cur_residency + delta;
93 gtidle->cur_residency = cur_residency;
94
95 /* residency multiplier in ns, convert to ms */
96 cur_residency = mul_u64_u32_div(cur_residency, gtidle->residency_multiplier, 1e6);
97
98 return cur_residency;
99 }
100
xe_gt_idle_enable_pg(struct xe_gt * gt)101 void xe_gt_idle_enable_pg(struct xe_gt *gt)
102 {
103 struct xe_device *xe = gt_to_xe(gt);
104 struct xe_gt_idle *gtidle = >->gtidle;
105 struct xe_mmio *mmio = >->mmio;
106 u32 vcs_mask, vecs_mask;
107 int i, j;
108
109 if (IS_SRIOV_VF(xe))
110 return;
111
112 /* Disable CPG for PVC */
113 if (xe->info.platform == XE_PVC)
114 return;
115
116 xe_device_assert_mem_access(gt_to_xe(gt));
117
118 vcs_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_VIDEO_DECODE);
119 vecs_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_VIDEO_ENHANCE);
120
121 if (vcs_mask || vecs_mask)
122 gtidle->powergate_enable = MEDIA_POWERGATE_ENABLE;
123
124 if (xe_gt_is_main_type(gt))
125 gtidle->powergate_enable |= RENDER_POWERGATE_ENABLE;
126
127 if (MEDIA_VERx100(xe) >= 1100 && MEDIA_VERx100(xe) < 1255)
128 gtidle->powergate_enable |= MEDIA_SAMPLERS_POWERGATE_ENABLE;
129
130 if (xe->info.platform != XE_DG1) {
131 for (i = XE_HW_ENGINE_VCS0, j = 0; i <= XE_HW_ENGINE_VCS7; ++i, ++j) {
132 if ((gt->info.engine_mask & BIT(i)))
133 gtidle->powergate_enable |= (VDN_HCP_POWERGATE_ENABLE(j) |
134 VDN_MFXVDENC_POWERGATE_ENABLE(j));
135 }
136 }
137
138 CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
139 if (xe->info.skip_guc_pc) {
140 /*
141 * GuC sets the hysteresis value when GuC PC is enabled
142 * else set it to 25 (25 * 1.28us)
143 */
144 xe_mmio_write32(mmio, MEDIA_POWERGATE_IDLE_HYSTERESIS, 25);
145 xe_mmio_write32(mmio, RENDER_POWERGATE_IDLE_HYSTERESIS, 25);
146 }
147
148 if (XE_GT_WA(gt, 14020316580))
149 gtidle->powergate_enable &= ~(VDN_HCP_POWERGATE_ENABLE(0) |
150 VDN_MFXVDENC_POWERGATE_ENABLE(0) |
151 VDN_HCP_POWERGATE_ENABLE(2) |
152 VDN_MFXVDENC_POWERGATE_ENABLE(2));
153
154 xe_mmio_write32(mmio, POWERGATE_ENABLE, gtidle->powergate_enable);
155 }
156
xe_gt_idle_disable_pg(struct xe_gt * gt)157 void xe_gt_idle_disable_pg(struct xe_gt *gt)
158 {
159 struct xe_gt_idle *gtidle = >->gtidle;
160
161 if (IS_SRIOV_VF(gt_to_xe(gt)))
162 return;
163
164 xe_device_assert_mem_access(gt_to_xe(gt));
165 gtidle->powergate_enable = 0;
166
167 CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
168 xe_mmio_write32(>->mmio, POWERGATE_ENABLE, gtidle->powergate_enable);
169 }
170
171 /**
172 * xe_gt_idle_pg_print - Xe powergating info
173 * @gt: GT object
174 * @p: drm_printer.
175 *
176 * This function prints the powergating information
177 *
178 * Return: 0 on success, negative error code otherwise
179 */
xe_gt_idle_pg_print(struct xe_gt * gt,struct drm_printer * p)180 int xe_gt_idle_pg_print(struct xe_gt *gt, struct drm_printer *p)
181 {
182 struct xe_gt_idle *gtidle = >->gtidle;
183 struct xe_device *xe = gt_to_xe(gt);
184 enum xe_gt_idle_state state;
185 u32 pg_enabled, pg_status = 0;
186 u32 vcs_mask, vecs_mask;
187 int n;
188 /*
189 * Media Slices
190 *
191 * Slice 0: VCS0, VCS1, VECS0
192 * Slice 1: VCS2, VCS3, VECS1
193 * Slice 2: VCS4, VCS5, VECS2
194 * Slice 3: VCS6, VCS7, VECS3
195 */
196 static const struct {
197 u64 engines;
198 u32 status_bit;
199 } media_slices[] = {
200 {(BIT(XE_HW_ENGINE_VCS0) | BIT(XE_HW_ENGINE_VCS1) |
201 BIT(XE_HW_ENGINE_VECS0)), MEDIA_SLICE0_AWAKE_STATUS},
202
203 {(BIT(XE_HW_ENGINE_VCS2) | BIT(XE_HW_ENGINE_VCS3) |
204 BIT(XE_HW_ENGINE_VECS1)), MEDIA_SLICE1_AWAKE_STATUS},
205
206 {(BIT(XE_HW_ENGINE_VCS4) | BIT(XE_HW_ENGINE_VCS5) |
207 BIT(XE_HW_ENGINE_VECS2)), MEDIA_SLICE2_AWAKE_STATUS},
208
209 {(BIT(XE_HW_ENGINE_VCS6) | BIT(XE_HW_ENGINE_VCS7) |
210 BIT(XE_HW_ENGINE_VECS3)), MEDIA_SLICE3_AWAKE_STATUS},
211 };
212
213 if (xe->info.platform == XE_PVC) {
214 drm_printf(p, "Power Gating not supported\n");
215 return 0;
216 }
217
218 state = gtidle->idle_status(gtidle_to_pc(gtidle));
219 pg_enabled = gtidle->powergate_enable;
220
221 /* Do not wake the GT to read powergating status */
222 if (state != GT_IDLE_C6) {
223 CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
224 if (!fw_ref.domains)
225 return -ETIMEDOUT;
226
227 pg_enabled = xe_mmio_read32(>->mmio, POWERGATE_ENABLE);
228 pg_status = xe_mmio_read32(>->mmio, POWERGATE_DOMAIN_STATUS);
229 }
230
231 if (gt->info.engine_mask & XE_HW_ENGINE_RCS_MASK) {
232 drm_printf(p, "Render Power Gating Enabled: %s\n",
233 str_yes_no(pg_enabled & RENDER_POWERGATE_ENABLE));
234
235 drm_printf(p, "Render Power Gate Status: %s\n",
236 str_up_down(pg_status & RENDER_AWAKE_STATUS));
237 }
238
239 vcs_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_VIDEO_DECODE);
240 vecs_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_VIDEO_ENHANCE);
241
242 /* Print media CPG status only if media is present */
243 if (vcs_mask || vecs_mask) {
244 drm_printf(p, "Media Power Gating Enabled: %s\n",
245 str_yes_no(pg_enabled & MEDIA_POWERGATE_ENABLE));
246
247 for (n = 0; n < ARRAY_SIZE(media_slices); n++)
248 if (gt->info.engine_mask & media_slices[n].engines)
249 drm_printf(p, "Media Slice%d Power Gate Status: %s\n", n,
250 str_up_down(pg_status & media_slices[n].status_bit));
251 }
252
253 if (MEDIA_VERx100(xe) >= 1100 && MEDIA_VERx100(xe) < 1255)
254 drm_printf(p, "Media Samplers Power Gating Enabled: %s\n",
255 str_yes_no(pg_enabled & MEDIA_SAMPLERS_POWERGATE_ENABLE));
256
257 return 0;
258 }
259
name_show(struct kobject * kobj,struct kobj_attribute * attr,char * buff)260 static ssize_t name_show(struct kobject *kobj,
261 struct kobj_attribute *attr, char *buff)
262 {
263 struct device *dev = kobj_to_dev(kobj);
264 struct xe_gt_idle *gtidle = dev_to_gtidle(dev);
265 struct xe_guc_pc *pc = gtidle_to_pc(gtidle);
266
267 guard(xe_pm_runtime)(pc_to_xe(pc));
268 return sysfs_emit(buff, "%s\n", gtidle->name);
269 }
270 static struct kobj_attribute name_attr = __ATTR_RO(name);
271
idle_status_show(struct kobject * kobj,struct kobj_attribute * attr,char * buff)272 static ssize_t idle_status_show(struct kobject *kobj,
273 struct kobj_attribute *attr, char *buff)
274 {
275 struct device *dev = kobj_to_dev(kobj);
276 struct xe_gt_idle *gtidle = dev_to_gtidle(dev);
277 struct xe_guc_pc *pc = gtidle_to_pc(gtidle);
278 enum xe_gt_idle_state state;
279
280 scoped_guard(xe_pm_runtime, pc_to_xe(pc))
281 state = gtidle->idle_status(pc);
282
283 return sysfs_emit(buff, "%s\n", gt_idle_state_to_string(state));
284 }
285 static struct kobj_attribute idle_status_attr = __ATTR_RO(idle_status);
286
xe_gt_idle_residency_msec(struct xe_gt_idle * gtidle)287 u64 xe_gt_idle_residency_msec(struct xe_gt_idle *gtidle)
288 {
289 struct xe_guc_pc *pc = gtidle_to_pc(gtidle);
290 u64 residency;
291 unsigned long flags;
292
293 raw_spin_lock_irqsave(>idle->lock, flags);
294 residency = get_residency_ms(gtidle, gtidle->idle_residency(pc));
295 raw_spin_unlock_irqrestore(>idle->lock, flags);
296
297 return residency;
298 }
299
300
idle_residency_ms_show(struct kobject * kobj,struct kobj_attribute * attr,char * buff)301 static ssize_t idle_residency_ms_show(struct kobject *kobj,
302 struct kobj_attribute *attr, char *buff)
303 {
304 struct device *dev = kobj_to_dev(kobj);
305 struct xe_gt_idle *gtidle = dev_to_gtidle(dev);
306 struct xe_guc_pc *pc = gtidle_to_pc(gtidle);
307 u64 residency;
308
309 scoped_guard(xe_pm_runtime, pc_to_xe(pc))
310 residency = xe_gt_idle_residency_msec(gtidle);
311
312 return sysfs_emit(buff, "%llu\n", residency);
313 }
314 static struct kobj_attribute idle_residency_attr = __ATTR_RO(idle_residency_ms);
315
316 static const struct attribute *gt_idle_attrs[] = {
317 &name_attr.attr,
318 &idle_status_attr.attr,
319 &idle_residency_attr.attr,
320 NULL,
321 };
322
gt_idle_fini(void * arg)323 static void gt_idle_fini(void *arg)
324 {
325 struct kobject *kobj = arg;
326 struct xe_gt *gt = kobj_to_gt(kobj->parent);
327
328 xe_gt_idle_disable_pg(gt);
329
330 if (gt_to_xe(gt)->info.skip_guc_pc)
331 xe_gt_idle_disable_c6(gt);
332
333 sysfs_remove_files(kobj, gt_idle_attrs);
334 kobject_put(kobj);
335 }
336
xe_gt_idle_init(struct xe_gt_idle * gtidle)337 int xe_gt_idle_init(struct xe_gt_idle *gtidle)
338 {
339 struct xe_gt *gt = gtidle_to_gt(gtidle);
340 struct xe_device *xe = gt_to_xe(gt);
341 struct kobject *kobj;
342 int err;
343
344 if (IS_SRIOV_VF(xe))
345 return 0;
346
347 kobj = kobject_create_and_add("gtidle", gt->sysfs);
348 if (!kobj)
349 return -ENOMEM;
350
351 raw_spin_lock_init(>idle->lock);
352
353 if (xe_gt_is_media_type(gt)) {
354 snprintf(gtidle->name, sizeof(gtidle->name), "gt%d-mc", gt->info.id);
355 gtidle->idle_residency = xe_guc_pc_mc6_residency;
356 } else {
357 snprintf(gtidle->name, sizeof(gtidle->name), "gt%d-rc", gt->info.id);
358 gtidle->idle_residency = xe_guc_pc_rc6_residency;
359 }
360
361 /* Multiplier for Residency counter in units of 1.28us */
362 gtidle->residency_multiplier = 1280;
363 gtidle->idle_status = xe_guc_pc_c_status;
364
365 err = sysfs_create_files(kobj, gt_idle_attrs);
366 if (err) {
367 kobject_put(kobj);
368 return err;
369 }
370
371 xe_gt_idle_enable_pg(gt);
372
373 return devm_add_action_or_reset(xe->drm.dev, gt_idle_fini, kobj);
374 }
375
xe_gt_idle_enable_c6(struct xe_gt * gt)376 void xe_gt_idle_enable_c6(struct xe_gt *gt)
377 {
378 xe_device_assert_mem_access(gt_to_xe(gt));
379 xe_force_wake_assert_held(gt_to_fw(gt), XE_FW_GT);
380
381 if (IS_SRIOV_VF(gt_to_xe(gt)))
382 return;
383
384 /* Units of 1280 ns for a total of 5s */
385 xe_mmio_write32(>->mmio, RC_IDLE_HYSTERSIS, 0x3B9ACA);
386 /* Enable RC6 */
387 xe_mmio_write32(>->mmio, RC_CONTROL,
388 RC_CTL_HW_ENABLE | RC_CTL_TO_MODE | RC_CTL_RC6_ENABLE);
389 }
390
xe_gt_idle_disable_c6(struct xe_gt * gt)391 int xe_gt_idle_disable_c6(struct xe_gt *gt)
392 {
393 xe_device_assert_mem_access(gt_to_xe(gt));
394
395 if (IS_SRIOV_VF(gt_to_xe(gt)))
396 return 0;
397
398 CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
399 if (!fw_ref.domains)
400 return -ETIMEDOUT;
401
402 xe_mmio_write32(>->mmio, RC_CONTROL, 0);
403 xe_mmio_write32(>->mmio, RC_STATE, 0);
404
405 return 0;
406 }
407