1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2023 Intel Corporation
4 */
5
6 #include <drm/drm_managed.h>
7
8 #include "xe_force_wake.h"
9 #include "xe_device.h"
10 #include "xe_gt.h"
11 #include "xe_gt_idle.h"
12 #include "xe_gt_sysfs.h"
13 #include "xe_guc_pc.h"
14 #include "regs/xe_gt_regs.h"
15 #include "xe_macros.h"
16 #include "xe_mmio.h"
17 #include "xe_pm.h"
18 #include "xe_sriov.h"
19
20 /**
21 * DOC: Xe GT Idle
22 *
23 * Contains functions that init GT idle features like C6
24 *
25 * device/gt#/gtidle/name - name of the state
26 * device/gt#/gtidle/idle_residency_ms - Provides residency of the idle state in ms
27 * device/gt#/gtidle/idle_status - Provides current idle state
28 */
29
dev_to_gtidle(struct device * dev)30 static struct xe_gt_idle *dev_to_gtidle(struct device *dev)
31 {
32 struct kobject *kobj = &dev->kobj;
33
34 return &kobj_to_gt(kobj->parent)->gtidle;
35 }
36
gtidle_to_gt(struct xe_gt_idle * gtidle)37 static struct xe_gt *gtidle_to_gt(struct xe_gt_idle *gtidle)
38 {
39 return container_of(gtidle, struct xe_gt, gtidle);
40 }
41
gtidle_to_pc(struct xe_gt_idle * gtidle)42 static struct xe_guc_pc *gtidle_to_pc(struct xe_gt_idle *gtidle)
43 {
44 return >idle_to_gt(gtidle)->uc.guc.pc;
45 }
46
47 static struct xe_device *
pc_to_xe(struct xe_guc_pc * pc)48 pc_to_xe(struct xe_guc_pc *pc)
49 {
50 struct xe_guc *guc = container_of(pc, struct xe_guc, pc);
51 struct xe_gt *gt = container_of(guc, struct xe_gt, uc.guc);
52
53 return gt_to_xe(gt);
54 }
55
gt_idle_state_to_string(enum xe_gt_idle_state state)56 static const char *gt_idle_state_to_string(enum xe_gt_idle_state state)
57 {
58 switch (state) {
59 case GT_IDLE_C0:
60 return "gt-c0";
61 case GT_IDLE_C6:
62 return "gt-c6";
63 default:
64 return "unknown";
65 }
66 }
67
get_residency_ms(struct xe_gt_idle * gtidle,u64 cur_residency)68 static u64 get_residency_ms(struct xe_gt_idle *gtidle, u64 cur_residency)
69 {
70 u64 delta, overflow_residency, prev_residency;
71
72 lockdep_assert_held(>idle->lock);
73
74 overflow_residency = BIT_ULL(32);
75
76 /*
77 * Counter wrap handling
78 * Store previous hw counter values for counter wrap-around handling
79 * Relying on sufficient frequency of queries otherwise counters can still wrap.
80 */
81 prev_residency = gtidle->prev_residency;
82 gtidle->prev_residency = cur_residency;
83
84 /* delta */
85 if (cur_residency >= prev_residency)
86 delta = cur_residency - prev_residency;
87 else
88 delta = cur_residency + (overflow_residency - prev_residency);
89
90 /* Add delta to extended raw driver copy of idle residency */
91 cur_residency = gtidle->cur_residency + delta;
92 gtidle->cur_residency = cur_residency;
93
94 /* residency multiplier in ns, convert to ms */
95 cur_residency = mul_u64_u32_div(cur_residency, gtidle->residency_multiplier, 1e6);
96
97 return cur_residency;
98 }
99
xe_gt_idle_enable_pg(struct xe_gt * gt)100 void xe_gt_idle_enable_pg(struct xe_gt *gt)
101 {
102 struct xe_device *xe = gt_to_xe(gt);
103 struct xe_gt_idle *gtidle = >->gtidle;
104 struct xe_mmio *mmio = >->mmio;
105 u32 vcs_mask, vecs_mask;
106 unsigned int fw_ref;
107 int i, j;
108
109 if (IS_SRIOV_VF(xe))
110 return;
111
112 /* Disable CPG for PVC */
113 if (xe->info.platform == XE_PVC)
114 return;
115
116 xe_device_assert_mem_access(gt_to_xe(gt));
117
118 vcs_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_VIDEO_DECODE);
119 vecs_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_VIDEO_ENHANCE);
120
121 if (vcs_mask || vecs_mask)
122 gtidle->powergate_enable = MEDIA_POWERGATE_ENABLE;
123
124 if (xe_gt_is_main_type(gt))
125 gtidle->powergate_enable |= RENDER_POWERGATE_ENABLE;
126
127 if (MEDIA_VERx100(xe) >= 1100 && MEDIA_VERx100(xe) < 1255)
128 gtidle->powergate_enable |= MEDIA_SAMPLERS_POWERGATE_ENABLE;
129
130 if (xe->info.platform != XE_DG1) {
131 for (i = XE_HW_ENGINE_VCS0, j = 0; i <= XE_HW_ENGINE_VCS7; ++i, ++j) {
132 if ((gt->info.engine_mask & BIT(i)))
133 gtidle->powergate_enable |= (VDN_HCP_POWERGATE_ENABLE(j) |
134 VDN_MFXVDENC_POWERGATE_ENABLE(j));
135 }
136 }
137
138 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
139 if (xe->info.skip_guc_pc) {
140 /*
141 * GuC sets the hysteresis value when GuC PC is enabled
142 * else set it to 25 (25 * 1.28us)
143 */
144 xe_mmio_write32(mmio, MEDIA_POWERGATE_IDLE_HYSTERESIS, 25);
145 xe_mmio_write32(mmio, RENDER_POWERGATE_IDLE_HYSTERESIS, 25);
146 }
147
148 xe_mmio_write32(mmio, POWERGATE_ENABLE, gtidle->powergate_enable);
149 xe_force_wake_put(gt_to_fw(gt), fw_ref);
150 }
151
xe_gt_idle_disable_pg(struct xe_gt * gt)152 void xe_gt_idle_disable_pg(struct xe_gt *gt)
153 {
154 struct xe_gt_idle *gtidle = >->gtidle;
155 unsigned int fw_ref;
156
157 if (IS_SRIOV_VF(gt_to_xe(gt)))
158 return;
159
160 xe_device_assert_mem_access(gt_to_xe(gt));
161 gtidle->powergate_enable = 0;
162
163 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
164 xe_mmio_write32(>->mmio, POWERGATE_ENABLE, gtidle->powergate_enable);
165 xe_force_wake_put(gt_to_fw(gt), fw_ref);
166 }
167
168 /**
169 * xe_gt_idle_pg_print - Xe powergating info
170 * @gt: GT object
171 * @p: drm_printer.
172 *
173 * This function prints the powergating information
174 *
175 * Return: 0 on success, negative error code otherwise
176 */
xe_gt_idle_pg_print(struct xe_gt * gt,struct drm_printer * p)177 int xe_gt_idle_pg_print(struct xe_gt *gt, struct drm_printer *p)
178 {
179 struct xe_gt_idle *gtidle = >->gtidle;
180 struct xe_device *xe = gt_to_xe(gt);
181 enum xe_gt_idle_state state;
182 u32 pg_enabled, pg_status = 0;
183 u32 vcs_mask, vecs_mask;
184 unsigned int fw_ref;
185 int n;
186 /*
187 * Media Slices
188 *
189 * Slice 0: VCS0, VCS1, VECS0
190 * Slice 1: VCS2, VCS3, VECS1
191 * Slice 2: VCS4, VCS5, VECS2
192 * Slice 3: VCS6, VCS7, VECS3
193 */
194 static const struct {
195 u64 engines;
196 u32 status_bit;
197 } media_slices[] = {
198 {(BIT(XE_HW_ENGINE_VCS0) | BIT(XE_HW_ENGINE_VCS1) |
199 BIT(XE_HW_ENGINE_VECS0)), MEDIA_SLICE0_AWAKE_STATUS},
200
201 {(BIT(XE_HW_ENGINE_VCS2) | BIT(XE_HW_ENGINE_VCS3) |
202 BIT(XE_HW_ENGINE_VECS1)), MEDIA_SLICE1_AWAKE_STATUS},
203
204 {(BIT(XE_HW_ENGINE_VCS4) | BIT(XE_HW_ENGINE_VCS5) |
205 BIT(XE_HW_ENGINE_VECS2)), MEDIA_SLICE2_AWAKE_STATUS},
206
207 {(BIT(XE_HW_ENGINE_VCS6) | BIT(XE_HW_ENGINE_VCS7) |
208 BIT(XE_HW_ENGINE_VECS3)), MEDIA_SLICE3_AWAKE_STATUS},
209 };
210
211 if (xe->info.platform == XE_PVC) {
212 drm_printf(p, "Power Gating not supported\n");
213 return 0;
214 }
215
216 state = gtidle->idle_status(gtidle_to_pc(gtidle));
217 pg_enabled = gtidle->powergate_enable;
218
219 /* Do not wake the GT to read powergating status */
220 if (state != GT_IDLE_C6) {
221 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
222 if (!fw_ref)
223 return -ETIMEDOUT;
224
225 pg_enabled = xe_mmio_read32(>->mmio, POWERGATE_ENABLE);
226 pg_status = xe_mmio_read32(>->mmio, POWERGATE_DOMAIN_STATUS);
227
228 xe_force_wake_put(gt_to_fw(gt), fw_ref);
229 }
230
231 if (gt->info.engine_mask & XE_HW_ENGINE_RCS_MASK) {
232 drm_printf(p, "Render Power Gating Enabled: %s\n",
233 str_yes_no(pg_enabled & RENDER_POWERGATE_ENABLE));
234
235 drm_printf(p, "Render Power Gate Status: %s\n",
236 str_up_down(pg_status & RENDER_AWAKE_STATUS));
237 }
238
239 vcs_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_VIDEO_DECODE);
240 vecs_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_VIDEO_ENHANCE);
241
242 /* Print media CPG status only if media is present */
243 if (vcs_mask || vecs_mask) {
244 drm_printf(p, "Media Power Gating Enabled: %s\n",
245 str_yes_no(pg_enabled & MEDIA_POWERGATE_ENABLE));
246
247 for (n = 0; n < ARRAY_SIZE(media_slices); n++)
248 if (gt->info.engine_mask & media_slices[n].engines)
249 drm_printf(p, "Media Slice%d Power Gate Status: %s\n", n,
250 str_up_down(pg_status & media_slices[n].status_bit));
251 }
252
253 if (MEDIA_VERx100(xe) >= 1100 && MEDIA_VERx100(xe) < 1255)
254 drm_printf(p, "Media Samplers Power Gating Enabled: %s\n",
255 str_yes_no(pg_enabled & MEDIA_SAMPLERS_POWERGATE_ENABLE));
256
257 return 0;
258 }
259
name_show(struct kobject * kobj,struct kobj_attribute * attr,char * buff)260 static ssize_t name_show(struct kobject *kobj,
261 struct kobj_attribute *attr, char *buff)
262 {
263 struct device *dev = kobj_to_dev(kobj);
264 struct xe_gt_idle *gtidle = dev_to_gtidle(dev);
265 struct xe_guc_pc *pc = gtidle_to_pc(gtidle);
266 ssize_t ret;
267
268 xe_pm_runtime_get(pc_to_xe(pc));
269 ret = sysfs_emit(buff, "%s\n", gtidle->name);
270 xe_pm_runtime_put(pc_to_xe(pc));
271
272 return ret;
273 }
274 static struct kobj_attribute name_attr = __ATTR_RO(name);
275
idle_status_show(struct kobject * kobj,struct kobj_attribute * attr,char * buff)276 static ssize_t idle_status_show(struct kobject *kobj,
277 struct kobj_attribute *attr, char *buff)
278 {
279 struct device *dev = kobj_to_dev(kobj);
280 struct xe_gt_idle *gtidle = dev_to_gtidle(dev);
281 struct xe_guc_pc *pc = gtidle_to_pc(gtidle);
282 enum xe_gt_idle_state state;
283
284 xe_pm_runtime_get(pc_to_xe(pc));
285 state = gtidle->idle_status(pc);
286 xe_pm_runtime_put(pc_to_xe(pc));
287
288 return sysfs_emit(buff, "%s\n", gt_idle_state_to_string(state));
289 }
290 static struct kobj_attribute idle_status_attr = __ATTR_RO(idle_status);
291
xe_gt_idle_residency_msec(struct xe_gt_idle * gtidle)292 u64 xe_gt_idle_residency_msec(struct xe_gt_idle *gtidle)
293 {
294 struct xe_guc_pc *pc = gtidle_to_pc(gtidle);
295 u64 residency;
296 unsigned long flags;
297
298 raw_spin_lock_irqsave(>idle->lock, flags);
299 residency = get_residency_ms(gtidle, gtidle->idle_residency(pc));
300 raw_spin_unlock_irqrestore(>idle->lock, flags);
301
302 return residency;
303 }
304
305
idle_residency_ms_show(struct kobject * kobj,struct kobj_attribute * attr,char * buff)306 static ssize_t idle_residency_ms_show(struct kobject *kobj,
307 struct kobj_attribute *attr, char *buff)
308 {
309 struct device *dev = kobj_to_dev(kobj);
310 struct xe_gt_idle *gtidle = dev_to_gtidle(dev);
311 struct xe_guc_pc *pc = gtidle_to_pc(gtidle);
312 u64 residency;
313
314 xe_pm_runtime_get(pc_to_xe(pc));
315 residency = xe_gt_idle_residency_msec(gtidle);
316 xe_pm_runtime_put(pc_to_xe(pc));
317
318 return sysfs_emit(buff, "%llu\n", residency);
319 }
320 static struct kobj_attribute idle_residency_attr = __ATTR_RO(idle_residency_ms);
321
322 static const struct attribute *gt_idle_attrs[] = {
323 &name_attr.attr,
324 &idle_status_attr.attr,
325 &idle_residency_attr.attr,
326 NULL,
327 };
328
gt_idle_fini(void * arg)329 static void gt_idle_fini(void *arg)
330 {
331 struct kobject *kobj = arg;
332 struct xe_gt *gt = kobj_to_gt(kobj->parent);
333
334 xe_gt_idle_disable_pg(gt);
335
336 if (gt_to_xe(gt)->info.skip_guc_pc)
337 xe_gt_idle_disable_c6(gt);
338
339 sysfs_remove_files(kobj, gt_idle_attrs);
340 kobject_put(kobj);
341 }
342
xe_gt_idle_init(struct xe_gt_idle * gtidle)343 int xe_gt_idle_init(struct xe_gt_idle *gtidle)
344 {
345 struct xe_gt *gt = gtidle_to_gt(gtidle);
346 struct xe_device *xe = gt_to_xe(gt);
347 struct kobject *kobj;
348 int err;
349
350 if (IS_SRIOV_VF(xe))
351 return 0;
352
353 kobj = kobject_create_and_add("gtidle", gt->sysfs);
354 if (!kobj)
355 return -ENOMEM;
356
357 raw_spin_lock_init(>idle->lock);
358
359 if (xe_gt_is_media_type(gt)) {
360 snprintf(gtidle->name, sizeof(gtidle->name), "gt%d-mc", gt->info.id);
361 gtidle->idle_residency = xe_guc_pc_mc6_residency;
362 } else {
363 snprintf(gtidle->name, sizeof(gtidle->name), "gt%d-rc", gt->info.id);
364 gtidle->idle_residency = xe_guc_pc_rc6_residency;
365 }
366
367 /* Multiplier for Residency counter in units of 1.28us */
368 gtidle->residency_multiplier = 1280;
369 gtidle->idle_status = xe_guc_pc_c_status;
370
371 err = sysfs_create_files(kobj, gt_idle_attrs);
372 if (err) {
373 kobject_put(kobj);
374 return err;
375 }
376
377 xe_gt_idle_enable_pg(gt);
378
379 return devm_add_action_or_reset(xe->drm.dev, gt_idle_fini, kobj);
380 }
381
xe_gt_idle_enable_c6(struct xe_gt * gt)382 void xe_gt_idle_enable_c6(struct xe_gt *gt)
383 {
384 xe_device_assert_mem_access(gt_to_xe(gt));
385 xe_force_wake_assert_held(gt_to_fw(gt), XE_FW_GT);
386
387 if (IS_SRIOV_VF(gt_to_xe(gt)))
388 return;
389
390 /* Units of 1280 ns for a total of 5s */
391 xe_mmio_write32(>->mmio, RC_IDLE_HYSTERSIS, 0x3B9ACA);
392 /* Enable RC6 */
393 xe_mmio_write32(>->mmio, RC_CONTROL,
394 RC_CTL_HW_ENABLE | RC_CTL_TO_MODE | RC_CTL_RC6_ENABLE);
395 }
396
xe_gt_idle_disable_c6(struct xe_gt * gt)397 int xe_gt_idle_disable_c6(struct xe_gt *gt)
398 {
399 unsigned int fw_ref;
400
401 xe_device_assert_mem_access(gt_to_xe(gt));
402
403 if (IS_SRIOV_VF(gt_to_xe(gt)))
404 return 0;
405
406 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
407 if (!fw_ref)
408 return -ETIMEDOUT;
409
410 xe_mmio_write32(>->mmio, RC_CONTROL, 0);
411 xe_mmio_write32(>->mmio, RC_STATE, 0);
412
413 xe_force_wake_put(gt_to_fw(gt), fw_ref);
414
415 return 0;
416 }
417