1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2022 Intel Corporation
4 */
5
6 #include "xe_guc_pc.h"
7
8 #include <linux/delay.h>
9 #include <linux/ktime.h>
10
11 #include <drm/drm_managed.h>
12 #include <generated/xe_wa_oob.h>
13
14 #include "abi/guc_actions_slpc_abi.h"
15 #include "regs/xe_gt_regs.h"
16 #include "regs/xe_regs.h"
17 #include "xe_bo.h"
18 #include "xe_device.h"
19 #include "xe_force_wake.h"
20 #include "xe_gt.h"
21 #include "xe_gt_idle.h"
22 #include "xe_gt_printk.h"
23 #include "xe_gt_throttle.h"
24 #include "xe_gt_types.h"
25 #include "xe_guc.h"
26 #include "xe_guc_ct.h"
27 #include "xe_map.h"
28 #include "xe_mmio.h"
29 #include "xe_pcode.h"
30 #include "xe_pm.h"
31 #include "xe_sriov.h"
32 #include "xe_wa.h"
33
34 #define MCHBAR_MIRROR_BASE_SNB 0x140000
35
36 #define RP_STATE_CAP XE_REG(MCHBAR_MIRROR_BASE_SNB + 0x5998)
37 #define RP0_MASK REG_GENMASK(7, 0)
38 #define RP1_MASK REG_GENMASK(15, 8)
39 #define RPN_MASK REG_GENMASK(23, 16)
40
41 #define FREQ_INFO_REC XE_REG(MCHBAR_MIRROR_BASE_SNB + 0x5ef0)
42 #define RPE_MASK REG_GENMASK(15, 8)
43 #define RPA_MASK REG_GENMASK(31, 16)
44
45 #define GT_PERF_STATUS XE_REG(0x1381b4)
46 #define CAGF_MASK REG_GENMASK(19, 11)
47
48 #define GT_FREQUENCY_MULTIPLIER 50
49 #define GT_FREQUENCY_SCALER 3
50
51 #define LNL_MERT_FREQ_CAP 800
52 #define BMG_MERT_FREQ_CAP 2133
53
54 #define SLPC_RESET_TIMEOUT_MS 5 /* roughly 5ms, but no need for precision */
55 #define SLPC_RESET_EXTENDED_TIMEOUT_MS 1000 /* To be used only at pc_start */
56
57 /**
58 * DOC: GuC Power Conservation (PC)
59 *
60 * GuC Power Conservation (PC) supports multiple features for the most
61 * efficient and performing use of the GT when GuC submission is enabled,
62 * including frequency management, Render-C states management, and various
63 * algorithms for power balancing.
64 *
65 * Single Loop Power Conservation (SLPC) is the name given to the suite of
66 * connected power conservation features in the GuC firmware. The firmware
67 * exposes a programming interface to the host for the control of SLPC.
68 *
69 * Frequency management:
70 * =====================
71 *
72 * Xe driver enables SLPC with all of its defaults features and frequency
73 * selection, which varies per platform.
74 *
75 * Render-C States:
76 * ================
77 *
78 * Render-C states is also a GuC PC feature that is now enabled in Xe for
79 * all platforms.
80 *
81 */
82
pc_to_guc(struct xe_guc_pc * pc)83 static struct xe_guc *pc_to_guc(struct xe_guc_pc *pc)
84 {
85 return container_of(pc, struct xe_guc, pc);
86 }
87
pc_to_ct(struct xe_guc_pc * pc)88 static struct xe_guc_ct *pc_to_ct(struct xe_guc_pc *pc)
89 {
90 return &pc_to_guc(pc)->ct;
91 }
92
pc_to_gt(struct xe_guc_pc * pc)93 static struct xe_gt *pc_to_gt(struct xe_guc_pc *pc)
94 {
95 return guc_to_gt(pc_to_guc(pc));
96 }
97
pc_to_xe(struct xe_guc_pc * pc)98 static struct xe_device *pc_to_xe(struct xe_guc_pc *pc)
99 {
100 return guc_to_xe(pc_to_guc(pc));
101 }
102
pc_to_maps(struct xe_guc_pc * pc)103 static struct iosys_map *pc_to_maps(struct xe_guc_pc *pc)
104 {
105 return &pc->bo->vmap;
106 }
107
108 #define slpc_shared_data_read(pc_, field_) \
109 xe_map_rd_field(pc_to_xe(pc_), pc_to_maps(pc_), 0, \
110 struct slpc_shared_data, field_)
111
112 #define slpc_shared_data_write(pc_, field_, val_) \
113 xe_map_wr_field(pc_to_xe(pc_), pc_to_maps(pc_), 0, \
114 struct slpc_shared_data, field_, val_)
115
116 #define SLPC_EVENT(id, count) \
117 (FIELD_PREP(HOST2GUC_PC_SLPC_REQUEST_MSG_1_EVENT_ID, id) | \
118 FIELD_PREP(HOST2GUC_PC_SLPC_REQUEST_MSG_1_EVENT_ARGC, count))
119
wait_for_pc_state(struct xe_guc_pc * pc,enum slpc_global_state state,int timeout_ms)120 static int wait_for_pc_state(struct xe_guc_pc *pc,
121 enum slpc_global_state state,
122 int timeout_ms)
123 {
124 int timeout_us = 1000 * timeout_ms;
125 int slept, wait = 10;
126
127 xe_device_assert_mem_access(pc_to_xe(pc));
128
129 for (slept = 0; slept < timeout_us;) {
130 if (slpc_shared_data_read(pc, header.global_state) == state)
131 return 0;
132
133 usleep_range(wait, wait << 1);
134 slept += wait;
135 wait <<= 1;
136 if (slept + wait > timeout_us)
137 wait = timeout_us - slept;
138 }
139
140 return -ETIMEDOUT;
141 }
142
pc_action_reset(struct xe_guc_pc * pc)143 static int pc_action_reset(struct xe_guc_pc *pc)
144 {
145 struct xe_guc_ct *ct = pc_to_ct(pc);
146 u32 action[] = {
147 GUC_ACTION_HOST2GUC_PC_SLPC_REQUEST,
148 SLPC_EVENT(SLPC_EVENT_RESET, 2),
149 xe_bo_ggtt_addr(pc->bo),
150 0,
151 };
152 int ret;
153
154 ret = xe_guc_ct_send(ct, action, ARRAY_SIZE(action), 0, 0);
155 if (ret)
156 xe_gt_err(pc_to_gt(pc), "GuC PC reset failed: %pe\n",
157 ERR_PTR(ret));
158
159 return ret;
160 }
161
pc_action_query_task_state(struct xe_guc_pc * pc)162 static int pc_action_query_task_state(struct xe_guc_pc *pc)
163 {
164 struct xe_guc_ct *ct = pc_to_ct(pc);
165 u32 action[] = {
166 GUC_ACTION_HOST2GUC_PC_SLPC_REQUEST,
167 SLPC_EVENT(SLPC_EVENT_QUERY_TASK_STATE, 2),
168 xe_bo_ggtt_addr(pc->bo),
169 0,
170 };
171 int ret;
172
173 if (wait_for_pc_state(pc, SLPC_GLOBAL_STATE_RUNNING,
174 SLPC_RESET_TIMEOUT_MS))
175 return -EAGAIN;
176
177 /* Blocking here to ensure the results are ready before reading them */
178 ret = xe_guc_ct_send_block(ct, action, ARRAY_SIZE(action));
179 if (ret)
180 xe_gt_err(pc_to_gt(pc), "GuC PC query task state failed: %pe\n",
181 ERR_PTR(ret));
182
183 return ret;
184 }
185
pc_action_set_param(struct xe_guc_pc * pc,u8 id,u32 value)186 static int pc_action_set_param(struct xe_guc_pc *pc, u8 id, u32 value)
187 {
188 struct xe_guc_ct *ct = pc_to_ct(pc);
189 u32 action[] = {
190 GUC_ACTION_HOST2GUC_PC_SLPC_REQUEST,
191 SLPC_EVENT(SLPC_EVENT_PARAMETER_SET, 2),
192 id,
193 value,
194 };
195 int ret;
196
197 if (wait_for_pc_state(pc, SLPC_GLOBAL_STATE_RUNNING,
198 SLPC_RESET_TIMEOUT_MS))
199 return -EAGAIN;
200
201 ret = xe_guc_ct_send(ct, action, ARRAY_SIZE(action), 0, 0);
202 if (ret)
203 xe_gt_err(pc_to_gt(pc), "GuC PC set param[%u]=%u failed: %pe\n",
204 id, value, ERR_PTR(ret));
205
206 return ret;
207 }
208
pc_action_unset_param(struct xe_guc_pc * pc,u8 id)209 static int pc_action_unset_param(struct xe_guc_pc *pc, u8 id)
210 {
211 u32 action[] = {
212 GUC_ACTION_HOST2GUC_PC_SLPC_REQUEST,
213 SLPC_EVENT(SLPC_EVENT_PARAMETER_UNSET, 1),
214 id,
215 };
216 struct xe_guc_ct *ct = &pc_to_guc(pc)->ct;
217 int ret;
218
219 if (wait_for_pc_state(pc, SLPC_GLOBAL_STATE_RUNNING,
220 SLPC_RESET_TIMEOUT_MS))
221 return -EAGAIN;
222
223 ret = xe_guc_ct_send(ct, action, ARRAY_SIZE(action), 0, 0);
224 if (ret)
225 xe_gt_err(pc_to_gt(pc), "GuC PC unset param failed: %pe",
226 ERR_PTR(ret));
227
228 return ret;
229 }
230
pc_action_setup_gucrc(struct xe_guc_pc * pc,u32 mode)231 static int pc_action_setup_gucrc(struct xe_guc_pc *pc, u32 mode)
232 {
233 struct xe_guc_ct *ct = pc_to_ct(pc);
234 u32 action[] = {
235 GUC_ACTION_HOST2GUC_SETUP_PC_GUCRC,
236 mode,
237 };
238 int ret;
239
240 ret = xe_guc_ct_send(ct, action, ARRAY_SIZE(action), 0, 0);
241 if (ret)
242 xe_gt_err(pc_to_gt(pc), "GuC RC enable mode=%u failed: %pe\n",
243 mode, ERR_PTR(ret));
244 return ret;
245 }
246
decode_freq(u32 raw)247 static u32 decode_freq(u32 raw)
248 {
249 return DIV_ROUND_CLOSEST(raw * GT_FREQUENCY_MULTIPLIER,
250 GT_FREQUENCY_SCALER);
251 }
252
encode_freq(u32 freq)253 static u32 encode_freq(u32 freq)
254 {
255 return DIV_ROUND_CLOSEST(freq * GT_FREQUENCY_SCALER,
256 GT_FREQUENCY_MULTIPLIER);
257 }
258
pc_get_min_freq(struct xe_guc_pc * pc)259 static u32 pc_get_min_freq(struct xe_guc_pc *pc)
260 {
261 u32 freq;
262
263 freq = FIELD_GET(SLPC_MIN_UNSLICE_FREQ_MASK,
264 slpc_shared_data_read(pc, task_state_data.freq));
265
266 return decode_freq(freq);
267 }
268
pc_set_manual_rp_ctrl(struct xe_guc_pc * pc,bool enable)269 static void pc_set_manual_rp_ctrl(struct xe_guc_pc *pc, bool enable)
270 {
271 struct xe_gt *gt = pc_to_gt(pc);
272 u32 state = enable ? RPSWCTL_ENABLE : RPSWCTL_DISABLE;
273
274 /* Allow/Disallow punit to process software freq requests */
275 xe_mmio_write32(>->mmio, RP_CONTROL, state);
276 }
277
pc_set_cur_freq(struct xe_guc_pc * pc,u32 freq)278 static void pc_set_cur_freq(struct xe_guc_pc *pc, u32 freq)
279 {
280 struct xe_gt *gt = pc_to_gt(pc);
281 u32 rpnswreq;
282
283 pc_set_manual_rp_ctrl(pc, true);
284
285 /* Req freq is in units of 16.66 Mhz */
286 rpnswreq = REG_FIELD_PREP(REQ_RATIO_MASK, encode_freq(freq));
287 xe_mmio_write32(>->mmio, RPNSWREQ, rpnswreq);
288
289 /* Sleep for a small time to allow pcode to respond */
290 usleep_range(100, 300);
291
292 pc_set_manual_rp_ctrl(pc, false);
293 }
294
pc_set_min_freq(struct xe_guc_pc * pc,u32 freq)295 static int pc_set_min_freq(struct xe_guc_pc *pc, u32 freq)
296 {
297 /*
298 * Let's only check for the rpn-rp0 range. If max < min,
299 * min becomes a fixed request.
300 */
301 if (freq < pc->rpn_freq || freq > pc->rp0_freq)
302 return -EINVAL;
303
304 /*
305 * GuC policy is to elevate minimum frequency to the efficient levels
306 * Our goal is to have the admin choices respected.
307 */
308 pc_action_set_param(pc, SLPC_PARAM_IGNORE_EFFICIENT_FREQUENCY,
309 freq < pc->rpe_freq);
310
311 return pc_action_set_param(pc,
312 SLPC_PARAM_GLOBAL_MIN_GT_UNSLICE_FREQ_MHZ,
313 freq);
314 }
315
pc_get_max_freq(struct xe_guc_pc * pc)316 static int pc_get_max_freq(struct xe_guc_pc *pc)
317 {
318 u32 freq;
319
320 freq = FIELD_GET(SLPC_MAX_UNSLICE_FREQ_MASK,
321 slpc_shared_data_read(pc, task_state_data.freq));
322
323 return decode_freq(freq);
324 }
325
pc_set_max_freq(struct xe_guc_pc * pc,u32 freq)326 static int pc_set_max_freq(struct xe_guc_pc *pc, u32 freq)
327 {
328 /*
329 * Let's only check for the rpn-rp0 range. If max < min,
330 * min becomes a fixed request.
331 * Also, overclocking is not supported.
332 */
333 if (freq < pc->rpn_freq || freq > pc->rp0_freq)
334 return -EINVAL;
335
336 return pc_action_set_param(pc,
337 SLPC_PARAM_GLOBAL_MAX_GT_UNSLICE_FREQ_MHZ,
338 freq);
339 }
340
mtl_update_rpa_value(struct xe_guc_pc * pc)341 static void mtl_update_rpa_value(struct xe_guc_pc *pc)
342 {
343 struct xe_gt *gt = pc_to_gt(pc);
344 u32 reg;
345
346 if (xe_gt_is_media_type(gt))
347 reg = xe_mmio_read32(>->mmio, MTL_MPA_FREQUENCY);
348 else
349 reg = xe_mmio_read32(>->mmio, MTL_GT_RPA_FREQUENCY);
350
351 pc->rpa_freq = decode_freq(REG_FIELD_GET(MTL_RPA_MASK, reg));
352 }
353
mtl_update_rpe_value(struct xe_guc_pc * pc)354 static void mtl_update_rpe_value(struct xe_guc_pc *pc)
355 {
356 struct xe_gt *gt = pc_to_gt(pc);
357 u32 reg;
358
359 if (xe_gt_is_media_type(gt))
360 reg = xe_mmio_read32(>->mmio, MTL_MPE_FREQUENCY);
361 else
362 reg = xe_mmio_read32(>->mmio, MTL_GT_RPE_FREQUENCY);
363
364 pc->rpe_freq = decode_freq(REG_FIELD_GET(MTL_RPE_MASK, reg));
365 }
366
tgl_update_rpa_value(struct xe_guc_pc * pc)367 static void tgl_update_rpa_value(struct xe_guc_pc *pc)
368 {
369 struct xe_gt *gt = pc_to_gt(pc);
370 struct xe_device *xe = gt_to_xe(gt);
371 u32 reg;
372
373 /*
374 * For PVC we still need to use fused RP1 as the approximation for RPe
375 * For other platforms than PVC we get the resolved RPe directly from
376 * PCODE at a different register
377 */
378 if (xe->info.platform == XE_PVC)
379 reg = xe_mmio_read32(>->mmio, PVC_RP_STATE_CAP);
380 else
381 reg = xe_mmio_read32(>->mmio, FREQ_INFO_REC);
382
383 pc->rpa_freq = REG_FIELD_GET(RPA_MASK, reg) * GT_FREQUENCY_MULTIPLIER;
384 }
385
tgl_update_rpe_value(struct xe_guc_pc * pc)386 static void tgl_update_rpe_value(struct xe_guc_pc *pc)
387 {
388 struct xe_gt *gt = pc_to_gt(pc);
389 struct xe_device *xe = gt_to_xe(gt);
390 u32 reg;
391
392 /*
393 * For PVC we still need to use fused RP1 as the approximation for RPe
394 * For other platforms than PVC we get the resolved RPe directly from
395 * PCODE at a different register
396 */
397 if (xe->info.platform == XE_PVC)
398 reg = xe_mmio_read32(>->mmio, PVC_RP_STATE_CAP);
399 else
400 reg = xe_mmio_read32(>->mmio, FREQ_INFO_REC);
401
402 pc->rpe_freq = REG_FIELD_GET(RPE_MASK, reg) * GT_FREQUENCY_MULTIPLIER;
403 }
404
pc_update_rp_values(struct xe_guc_pc * pc)405 static void pc_update_rp_values(struct xe_guc_pc *pc)
406 {
407 struct xe_gt *gt = pc_to_gt(pc);
408 struct xe_device *xe = gt_to_xe(gt);
409
410 if (GRAPHICS_VERx100(xe) >= 1270) {
411 mtl_update_rpa_value(pc);
412 mtl_update_rpe_value(pc);
413 } else {
414 tgl_update_rpa_value(pc);
415 tgl_update_rpe_value(pc);
416 }
417
418 /*
419 * RPe is decided at runtime by PCODE. In the rare case where that's
420 * smaller than the fused min, we will trust the PCODE and use that
421 * as our minimum one.
422 */
423 pc->rpn_freq = min(pc->rpn_freq, pc->rpe_freq);
424 }
425
426 /**
427 * xe_guc_pc_get_act_freq - Get Actual running frequency
428 * @pc: The GuC PC
429 *
430 * Returns: The Actual running frequency. Which might be 0 if GT is in Render-C sleep state (RC6).
431 */
xe_guc_pc_get_act_freq(struct xe_guc_pc * pc)432 u32 xe_guc_pc_get_act_freq(struct xe_guc_pc *pc)
433 {
434 struct xe_gt *gt = pc_to_gt(pc);
435 struct xe_device *xe = gt_to_xe(gt);
436 u32 freq;
437
438 /* When in RC6, actual frequency reported will be 0. */
439 if (GRAPHICS_VERx100(xe) >= 1270) {
440 freq = xe_mmio_read32(>->mmio, MTL_MIRROR_TARGET_WP1);
441 freq = REG_FIELD_GET(MTL_CAGF_MASK, freq);
442 } else {
443 freq = xe_mmio_read32(>->mmio, GT_PERF_STATUS);
444 freq = REG_FIELD_GET(CAGF_MASK, freq);
445 }
446
447 freq = decode_freq(freq);
448
449 return freq;
450 }
451
get_cur_freq(struct xe_gt * gt)452 static u32 get_cur_freq(struct xe_gt *gt)
453 {
454 u32 freq;
455
456 freq = xe_mmio_read32(>->mmio, RPNSWREQ);
457 freq = REG_FIELD_GET(REQ_RATIO_MASK, freq);
458 return decode_freq(freq);
459 }
460
461 /**
462 * xe_guc_pc_get_cur_freq - Get Current requested frequency
463 * @pc: The GuC PC
464 * @freq: A pointer to a u32 where the freq value will be returned
465 *
466 * Returns: 0 on success,
467 * -EAGAIN if GuC PC not ready (likely in middle of a reset).
468 */
xe_guc_pc_get_cur_freq(struct xe_guc_pc * pc,u32 * freq)469 int xe_guc_pc_get_cur_freq(struct xe_guc_pc *pc, u32 *freq)
470 {
471 struct xe_gt *gt = pc_to_gt(pc);
472 unsigned int fw_ref;
473
474 /*
475 * GuC SLPC plays with cur freq request when GuCRC is enabled
476 * Block RC6 for a more reliable read.
477 */
478 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
479 if (!xe_force_wake_ref_has_domain(fw_ref, XE_FW_GT)) {
480 xe_force_wake_put(gt_to_fw(gt), fw_ref);
481 return -ETIMEDOUT;
482 }
483
484 *freq = get_cur_freq(gt);
485
486 xe_force_wake_put(gt_to_fw(gt), fw_ref);
487 return 0;
488 }
489
490 /**
491 * xe_guc_pc_get_rp0_freq - Get the RP0 freq
492 * @pc: The GuC PC
493 *
494 * Returns: RP0 freq.
495 */
xe_guc_pc_get_rp0_freq(struct xe_guc_pc * pc)496 u32 xe_guc_pc_get_rp0_freq(struct xe_guc_pc *pc)
497 {
498 return pc->rp0_freq;
499 }
500
501 /**
502 * xe_guc_pc_get_rpa_freq - Get the RPa freq
503 * @pc: The GuC PC
504 *
505 * Returns: RPa freq.
506 */
xe_guc_pc_get_rpa_freq(struct xe_guc_pc * pc)507 u32 xe_guc_pc_get_rpa_freq(struct xe_guc_pc *pc)
508 {
509 pc_update_rp_values(pc);
510
511 return pc->rpa_freq;
512 }
513
514 /**
515 * xe_guc_pc_get_rpe_freq - Get the RPe freq
516 * @pc: The GuC PC
517 *
518 * Returns: RPe freq.
519 */
xe_guc_pc_get_rpe_freq(struct xe_guc_pc * pc)520 u32 xe_guc_pc_get_rpe_freq(struct xe_guc_pc *pc)
521 {
522 pc_update_rp_values(pc);
523
524 return pc->rpe_freq;
525 }
526
527 /**
528 * xe_guc_pc_get_rpn_freq - Get the RPn freq
529 * @pc: The GuC PC
530 *
531 * Returns: RPn freq.
532 */
xe_guc_pc_get_rpn_freq(struct xe_guc_pc * pc)533 u32 xe_guc_pc_get_rpn_freq(struct xe_guc_pc *pc)
534 {
535 return pc->rpn_freq;
536 }
537
538 /**
539 * xe_guc_pc_get_min_freq - Get the min operational frequency
540 * @pc: The GuC PC
541 * @freq: A pointer to a u32 where the freq value will be returned
542 *
543 * Returns: 0 on success,
544 * -EAGAIN if GuC PC not ready (likely in middle of a reset).
545 */
xe_guc_pc_get_min_freq(struct xe_guc_pc * pc,u32 * freq)546 int xe_guc_pc_get_min_freq(struct xe_guc_pc *pc, u32 *freq)
547 {
548 int ret;
549
550 xe_device_assert_mem_access(pc_to_xe(pc));
551
552 mutex_lock(&pc->freq_lock);
553 if (!pc->freq_ready) {
554 /* Might be in the middle of a gt reset */
555 ret = -EAGAIN;
556 goto out;
557 }
558
559 ret = pc_action_query_task_state(pc);
560 if (ret)
561 goto out;
562
563 *freq = pc_get_min_freq(pc);
564
565 out:
566 mutex_unlock(&pc->freq_lock);
567 return ret;
568 }
569
570 /**
571 * xe_guc_pc_set_min_freq - Set the minimal operational frequency
572 * @pc: The GuC PC
573 * @freq: The selected minimal frequency
574 *
575 * Returns: 0 on success,
576 * -EAGAIN if GuC PC not ready (likely in middle of a reset),
577 * -EINVAL if value out of bounds.
578 */
xe_guc_pc_set_min_freq(struct xe_guc_pc * pc,u32 freq)579 int xe_guc_pc_set_min_freq(struct xe_guc_pc *pc, u32 freq)
580 {
581 int ret;
582
583 mutex_lock(&pc->freq_lock);
584 if (!pc->freq_ready) {
585 /* Might be in the middle of a gt reset */
586 ret = -EAGAIN;
587 goto out;
588 }
589
590 ret = pc_set_min_freq(pc, freq);
591 if (ret)
592 goto out;
593
594 pc->user_requested_min = freq;
595
596 out:
597 mutex_unlock(&pc->freq_lock);
598 return ret;
599 }
600
601 /**
602 * xe_guc_pc_get_max_freq - Get Maximum operational frequency
603 * @pc: The GuC PC
604 * @freq: A pointer to a u32 where the freq value will be returned
605 *
606 * Returns: 0 on success,
607 * -EAGAIN if GuC PC not ready (likely in middle of a reset).
608 */
xe_guc_pc_get_max_freq(struct xe_guc_pc * pc,u32 * freq)609 int xe_guc_pc_get_max_freq(struct xe_guc_pc *pc, u32 *freq)
610 {
611 int ret;
612
613 mutex_lock(&pc->freq_lock);
614 if (!pc->freq_ready) {
615 /* Might be in the middle of a gt reset */
616 ret = -EAGAIN;
617 goto out;
618 }
619
620 ret = pc_action_query_task_state(pc);
621 if (ret)
622 goto out;
623
624 *freq = pc_get_max_freq(pc);
625
626 out:
627 mutex_unlock(&pc->freq_lock);
628 return ret;
629 }
630
631 /**
632 * xe_guc_pc_set_max_freq - Set the maximum operational frequency
633 * @pc: The GuC PC
634 * @freq: The selected maximum frequency value
635 *
636 * Returns: 0 on success,
637 * -EAGAIN if GuC PC not ready (likely in middle of a reset),
638 * -EINVAL if value out of bounds.
639 */
xe_guc_pc_set_max_freq(struct xe_guc_pc * pc,u32 freq)640 int xe_guc_pc_set_max_freq(struct xe_guc_pc *pc, u32 freq)
641 {
642 int ret;
643
644 mutex_lock(&pc->freq_lock);
645 if (!pc->freq_ready) {
646 /* Might be in the middle of a gt reset */
647 ret = -EAGAIN;
648 goto out;
649 }
650
651 ret = pc_set_max_freq(pc, freq);
652 if (ret)
653 goto out;
654
655 pc->user_requested_max = freq;
656
657 out:
658 mutex_unlock(&pc->freq_lock);
659 return ret;
660 }
661
662 /**
663 * xe_guc_pc_c_status - get the current GT C state
664 * @pc: XE_GuC_PC instance
665 */
xe_guc_pc_c_status(struct xe_guc_pc * pc)666 enum xe_gt_idle_state xe_guc_pc_c_status(struct xe_guc_pc *pc)
667 {
668 struct xe_gt *gt = pc_to_gt(pc);
669 u32 reg, gt_c_state;
670
671 if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 1270) {
672 reg = xe_mmio_read32(>->mmio, MTL_MIRROR_TARGET_WP1);
673 gt_c_state = REG_FIELD_GET(MTL_CC_MASK, reg);
674 } else {
675 reg = xe_mmio_read32(>->mmio, GT_CORE_STATUS);
676 gt_c_state = REG_FIELD_GET(RCN_MASK, reg);
677 }
678
679 switch (gt_c_state) {
680 case GT_C6:
681 return GT_IDLE_C6;
682 case GT_C0:
683 return GT_IDLE_C0;
684 default:
685 return GT_IDLE_UNKNOWN;
686 }
687 }
688
689 /**
690 * xe_guc_pc_rc6_residency - rc6 residency counter
691 * @pc: Xe_GuC_PC instance
692 */
xe_guc_pc_rc6_residency(struct xe_guc_pc * pc)693 u64 xe_guc_pc_rc6_residency(struct xe_guc_pc *pc)
694 {
695 struct xe_gt *gt = pc_to_gt(pc);
696 u32 reg;
697
698 reg = xe_mmio_read32(>->mmio, GT_GFX_RC6);
699
700 return reg;
701 }
702
703 /**
704 * xe_guc_pc_mc6_residency - mc6 residency counter
705 * @pc: Xe_GuC_PC instance
706 */
xe_guc_pc_mc6_residency(struct xe_guc_pc * pc)707 u64 xe_guc_pc_mc6_residency(struct xe_guc_pc *pc)
708 {
709 struct xe_gt *gt = pc_to_gt(pc);
710 u64 reg;
711
712 reg = xe_mmio_read32(>->mmio, MTL_MEDIA_MC6);
713
714 return reg;
715 }
716
mtl_init_fused_rp_values(struct xe_guc_pc * pc)717 static void mtl_init_fused_rp_values(struct xe_guc_pc *pc)
718 {
719 struct xe_gt *gt = pc_to_gt(pc);
720 u32 reg;
721
722 xe_device_assert_mem_access(pc_to_xe(pc));
723
724 if (xe_gt_is_media_type(gt))
725 reg = xe_mmio_read32(>->mmio, MTL_MEDIAP_STATE_CAP);
726 else
727 reg = xe_mmio_read32(>->mmio, MTL_RP_STATE_CAP);
728
729 pc->rp0_freq = decode_freq(REG_FIELD_GET(MTL_RP0_CAP_MASK, reg));
730
731 pc->rpn_freq = decode_freq(REG_FIELD_GET(MTL_RPN_CAP_MASK, reg));
732 }
733
tgl_init_fused_rp_values(struct xe_guc_pc * pc)734 static void tgl_init_fused_rp_values(struct xe_guc_pc *pc)
735 {
736 struct xe_gt *gt = pc_to_gt(pc);
737 struct xe_device *xe = gt_to_xe(gt);
738 u32 reg;
739
740 xe_device_assert_mem_access(pc_to_xe(pc));
741
742 if (xe->info.platform == XE_PVC)
743 reg = xe_mmio_read32(>->mmio, PVC_RP_STATE_CAP);
744 else
745 reg = xe_mmio_read32(>->mmio, RP_STATE_CAP);
746 pc->rp0_freq = REG_FIELD_GET(RP0_MASK, reg) * GT_FREQUENCY_MULTIPLIER;
747 pc->rpn_freq = REG_FIELD_GET(RPN_MASK, reg) * GT_FREQUENCY_MULTIPLIER;
748 }
749
pc_init_fused_rp_values(struct xe_guc_pc * pc)750 static void pc_init_fused_rp_values(struct xe_guc_pc *pc)
751 {
752 struct xe_gt *gt = pc_to_gt(pc);
753 struct xe_device *xe = gt_to_xe(gt);
754
755 if (GRAPHICS_VERx100(xe) >= 1270)
756 mtl_init_fused_rp_values(pc);
757 else
758 tgl_init_fused_rp_values(pc);
759 }
760
pc_max_freq_cap(struct xe_guc_pc * pc)761 static u32 pc_max_freq_cap(struct xe_guc_pc *pc)
762 {
763 struct xe_gt *gt = pc_to_gt(pc);
764
765 if (XE_WA(gt, 22019338487)) {
766 if (xe_gt_is_media_type(gt))
767 return min(LNL_MERT_FREQ_CAP, pc->rp0_freq);
768 else
769 return min(BMG_MERT_FREQ_CAP, pc->rp0_freq);
770 } else {
771 return pc->rp0_freq;
772 }
773 }
774
775 /**
776 * xe_guc_pc_raise_unslice - Initialize RPx values and request a higher GT
777 * frequency to allow faster GuC load times
778 * @pc: Xe_GuC_PC instance
779 */
xe_guc_pc_raise_unslice(struct xe_guc_pc * pc)780 void xe_guc_pc_raise_unslice(struct xe_guc_pc *pc)
781 {
782 struct xe_gt *gt = pc_to_gt(pc);
783
784 xe_force_wake_assert_held(gt_to_fw(gt), XE_FW_GT);
785 pc_set_cur_freq(pc, pc_max_freq_cap(pc));
786 }
787
788 /**
789 * xe_guc_pc_init_early - Initialize RPx values
790 * @pc: Xe_GuC_PC instance
791 */
xe_guc_pc_init_early(struct xe_guc_pc * pc)792 void xe_guc_pc_init_early(struct xe_guc_pc *pc)
793 {
794 struct xe_gt *gt = pc_to_gt(pc);
795
796 xe_force_wake_assert_held(gt_to_fw(gt), XE_FW_GT);
797 pc_init_fused_rp_values(pc);
798 }
799
pc_adjust_freq_bounds(struct xe_guc_pc * pc)800 static int pc_adjust_freq_bounds(struct xe_guc_pc *pc)
801 {
802 int ret;
803
804 lockdep_assert_held(&pc->freq_lock);
805
806 ret = pc_action_query_task_state(pc);
807 if (ret)
808 goto out;
809
810 /*
811 * GuC defaults to some RPmax that is not actually achievable without
812 * overclocking. Let's adjust it to the Hardware RP0, which is the
813 * regular maximum
814 */
815 if (pc_get_max_freq(pc) > pc->rp0_freq) {
816 ret = pc_set_max_freq(pc, pc->rp0_freq);
817 if (ret)
818 goto out;
819 }
820
821 /*
822 * Same thing happens for Server platforms where min is listed as
823 * RPMax
824 */
825 if (pc_get_min_freq(pc) > pc->rp0_freq)
826 ret = pc_set_min_freq(pc, pc->rp0_freq);
827
828 out:
829 return ret;
830 }
831
pc_adjust_requested_freq(struct xe_guc_pc * pc)832 static int pc_adjust_requested_freq(struct xe_guc_pc *pc)
833 {
834 int ret = 0;
835
836 lockdep_assert_held(&pc->freq_lock);
837
838 if (pc->user_requested_min != 0) {
839 ret = pc_set_min_freq(pc, pc->user_requested_min);
840 if (ret)
841 return ret;
842 }
843
844 if (pc->user_requested_max != 0) {
845 ret = pc_set_max_freq(pc, pc->user_requested_max);
846 if (ret)
847 return ret;
848 }
849
850 return ret;
851 }
852
pc_set_mert_freq_cap(struct xe_guc_pc * pc)853 static int pc_set_mert_freq_cap(struct xe_guc_pc *pc)
854 {
855 int ret = 0;
856
857 if (XE_WA(pc_to_gt(pc), 22019338487)) {
858 /*
859 * Get updated min/max and stash them.
860 */
861 ret = xe_guc_pc_get_min_freq(pc, &pc->stashed_min_freq);
862 if (!ret)
863 ret = xe_guc_pc_get_max_freq(pc, &pc->stashed_max_freq);
864 if (ret)
865 return ret;
866
867 /*
868 * Ensure min and max are bound by MERT_FREQ_CAP until driver loads.
869 */
870 mutex_lock(&pc->freq_lock);
871 ret = pc_set_min_freq(pc, min(pc->rpe_freq, pc_max_freq_cap(pc)));
872 if (!ret)
873 ret = pc_set_max_freq(pc, min(pc->rp0_freq, pc_max_freq_cap(pc)));
874 mutex_unlock(&pc->freq_lock);
875 }
876
877 return ret;
878 }
879
880 /**
881 * xe_guc_pc_restore_stashed_freq - Set min/max back to stashed values
882 * @pc: The GuC PC
883 *
884 * Returns: 0 on success,
885 * error code on failure
886 */
xe_guc_pc_restore_stashed_freq(struct xe_guc_pc * pc)887 int xe_guc_pc_restore_stashed_freq(struct xe_guc_pc *pc)
888 {
889 int ret = 0;
890
891 if (IS_SRIOV_VF(pc_to_xe(pc)) || pc_to_xe(pc)->info.skip_guc_pc)
892 return 0;
893
894 mutex_lock(&pc->freq_lock);
895 ret = pc_set_max_freq(pc, pc->stashed_max_freq);
896 if (!ret)
897 ret = pc_set_min_freq(pc, pc->stashed_min_freq);
898 mutex_unlock(&pc->freq_lock);
899
900 return ret;
901 }
902
903 /**
904 * xe_guc_pc_gucrc_disable - Disable GuC RC
905 * @pc: Xe_GuC_PC instance
906 *
907 * Disables GuC RC by taking control of RC6 back from GuC.
908 *
909 * Return: 0 on success, negative error code on error.
910 */
xe_guc_pc_gucrc_disable(struct xe_guc_pc * pc)911 int xe_guc_pc_gucrc_disable(struct xe_guc_pc *pc)
912 {
913 struct xe_device *xe = pc_to_xe(pc);
914 struct xe_gt *gt = pc_to_gt(pc);
915 unsigned int fw_ref;
916 int ret = 0;
917
918 if (xe->info.skip_guc_pc)
919 return 0;
920
921 ret = pc_action_setup_gucrc(pc, GUCRC_HOST_CONTROL);
922 if (ret)
923 return ret;
924
925 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
926 if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL)) {
927 xe_force_wake_put(gt_to_fw(gt), fw_ref);
928 return -ETIMEDOUT;
929 }
930
931 xe_gt_idle_disable_c6(gt);
932
933 xe_force_wake_put(gt_to_fw(gt), fw_ref);
934
935 return 0;
936 }
937
938 /**
939 * xe_guc_pc_override_gucrc_mode - override GUCRC mode
940 * @pc: Xe_GuC_PC instance
941 * @mode: new value of the mode.
942 *
943 * Return: 0 on success, negative error code on error
944 */
xe_guc_pc_override_gucrc_mode(struct xe_guc_pc * pc,enum slpc_gucrc_mode mode)945 int xe_guc_pc_override_gucrc_mode(struct xe_guc_pc *pc, enum slpc_gucrc_mode mode)
946 {
947 int ret;
948
949 xe_pm_runtime_get(pc_to_xe(pc));
950 ret = pc_action_set_param(pc, SLPC_PARAM_PWRGATE_RC_MODE, mode);
951 xe_pm_runtime_put(pc_to_xe(pc));
952
953 return ret;
954 }
955
956 /**
957 * xe_guc_pc_unset_gucrc_mode - unset GUCRC mode override
958 * @pc: Xe_GuC_PC instance
959 *
960 * Return: 0 on success, negative error code on error
961 */
xe_guc_pc_unset_gucrc_mode(struct xe_guc_pc * pc)962 int xe_guc_pc_unset_gucrc_mode(struct xe_guc_pc *pc)
963 {
964 int ret;
965
966 xe_pm_runtime_get(pc_to_xe(pc));
967 ret = pc_action_unset_param(pc, SLPC_PARAM_PWRGATE_RC_MODE);
968 xe_pm_runtime_put(pc_to_xe(pc));
969
970 return ret;
971 }
972
pc_init_pcode_freq(struct xe_guc_pc * pc)973 static void pc_init_pcode_freq(struct xe_guc_pc *pc)
974 {
975 u32 min = DIV_ROUND_CLOSEST(pc->rpn_freq, GT_FREQUENCY_MULTIPLIER);
976 u32 max = DIV_ROUND_CLOSEST(pc->rp0_freq, GT_FREQUENCY_MULTIPLIER);
977
978 XE_WARN_ON(xe_pcode_init_min_freq_table(gt_to_tile(pc_to_gt(pc)), min, max));
979 }
980
pc_init_freqs(struct xe_guc_pc * pc)981 static int pc_init_freqs(struct xe_guc_pc *pc)
982 {
983 int ret;
984
985 mutex_lock(&pc->freq_lock);
986
987 ret = pc_adjust_freq_bounds(pc);
988 if (ret)
989 goto out;
990
991 ret = pc_adjust_requested_freq(pc);
992 if (ret)
993 goto out;
994
995 pc_update_rp_values(pc);
996
997 pc_init_pcode_freq(pc);
998
999 /*
1000 * The frequencies are really ready for use only after the user
1001 * requested ones got restored.
1002 */
1003 pc->freq_ready = true;
1004
1005 out:
1006 mutex_unlock(&pc->freq_lock);
1007 return ret;
1008 }
1009
1010 /**
1011 * xe_guc_pc_start - Start GuC's Power Conservation component
1012 * @pc: Xe_GuC_PC instance
1013 */
xe_guc_pc_start(struct xe_guc_pc * pc)1014 int xe_guc_pc_start(struct xe_guc_pc *pc)
1015 {
1016 struct xe_device *xe = pc_to_xe(pc);
1017 struct xe_gt *gt = pc_to_gt(pc);
1018 u32 size = PAGE_ALIGN(sizeof(struct slpc_shared_data));
1019 unsigned int fw_ref;
1020 ktime_t earlier;
1021 int ret;
1022
1023 xe_gt_assert(gt, xe_device_uc_enabled(xe));
1024
1025 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
1026 if (!xe_force_wake_ref_has_domain(fw_ref, XE_FW_GT)) {
1027 xe_force_wake_put(gt_to_fw(gt), fw_ref);
1028 return -ETIMEDOUT;
1029 }
1030
1031 if (xe->info.skip_guc_pc) {
1032 if (xe->info.platform != XE_PVC)
1033 xe_gt_idle_enable_c6(gt);
1034
1035 /* Request max possible since dynamic freq mgmt is not enabled */
1036 pc_set_cur_freq(pc, UINT_MAX);
1037
1038 ret = 0;
1039 goto out;
1040 }
1041
1042 memset(pc->bo->vmap.vaddr, 0, size);
1043 slpc_shared_data_write(pc, header.size, size);
1044
1045 earlier = ktime_get();
1046 ret = pc_action_reset(pc);
1047 if (ret)
1048 goto out;
1049
1050 if (wait_for_pc_state(pc, SLPC_GLOBAL_STATE_RUNNING,
1051 SLPC_RESET_TIMEOUT_MS)) {
1052 xe_gt_warn(gt, "GuC PC start taking longer than normal [freq = %dMHz (req = %dMHz), perf_limit_reasons = 0x%08X]\n",
1053 xe_guc_pc_get_act_freq(pc), get_cur_freq(gt),
1054 xe_gt_throttle_get_limit_reasons(gt));
1055
1056 if (wait_for_pc_state(pc, SLPC_GLOBAL_STATE_RUNNING,
1057 SLPC_RESET_EXTENDED_TIMEOUT_MS)) {
1058 xe_gt_err(gt, "GuC PC Start failed: Dynamic GT frequency control and GT sleep states are now disabled.\n");
1059 goto out;
1060 }
1061
1062 xe_gt_warn(gt, "GuC PC excessive start time: %lldms",
1063 ktime_ms_delta(ktime_get(), earlier));
1064 }
1065
1066 ret = pc_init_freqs(pc);
1067 if (ret)
1068 goto out;
1069
1070 ret = pc_set_mert_freq_cap(pc);
1071 if (ret)
1072 goto out;
1073
1074 if (xe->info.platform == XE_PVC) {
1075 xe_guc_pc_gucrc_disable(pc);
1076 ret = 0;
1077 goto out;
1078 }
1079
1080 ret = pc_action_setup_gucrc(pc, GUCRC_FIRMWARE_CONTROL);
1081
1082 out:
1083 xe_force_wake_put(gt_to_fw(gt), fw_ref);
1084 return ret;
1085 }
1086
1087 /**
1088 * xe_guc_pc_stop - Stop GuC's Power Conservation component
1089 * @pc: Xe_GuC_PC instance
1090 */
xe_guc_pc_stop(struct xe_guc_pc * pc)1091 int xe_guc_pc_stop(struct xe_guc_pc *pc)
1092 {
1093 struct xe_device *xe = pc_to_xe(pc);
1094
1095 if (xe->info.skip_guc_pc) {
1096 xe_gt_idle_disable_c6(pc_to_gt(pc));
1097 return 0;
1098 }
1099
1100 mutex_lock(&pc->freq_lock);
1101 pc->freq_ready = false;
1102 mutex_unlock(&pc->freq_lock);
1103
1104 return 0;
1105 }
1106
1107 /**
1108 * xe_guc_pc_fini_hw - Finalize GuC's Power Conservation component
1109 * @arg: opaque pointer that should point to Xe_GuC_PC instance
1110 */
xe_guc_pc_fini_hw(void * arg)1111 static void xe_guc_pc_fini_hw(void *arg)
1112 {
1113 struct xe_guc_pc *pc = arg;
1114 struct xe_device *xe = pc_to_xe(pc);
1115 unsigned int fw_ref;
1116
1117 if (xe_device_wedged(xe))
1118 return;
1119
1120 fw_ref = xe_force_wake_get(gt_to_fw(pc_to_gt(pc)), XE_FORCEWAKE_ALL);
1121 xe_guc_pc_gucrc_disable(pc);
1122 XE_WARN_ON(xe_guc_pc_stop(pc));
1123
1124 /* Bind requested freq to mert_freq_cap before unload */
1125 pc_set_cur_freq(pc, min(pc_max_freq_cap(pc), pc->rpe_freq));
1126
1127 xe_force_wake_put(gt_to_fw(pc_to_gt(pc)), fw_ref);
1128 }
1129
1130 /**
1131 * xe_guc_pc_init - Initialize GuC's Power Conservation component
1132 * @pc: Xe_GuC_PC instance
1133 */
xe_guc_pc_init(struct xe_guc_pc * pc)1134 int xe_guc_pc_init(struct xe_guc_pc *pc)
1135 {
1136 struct xe_gt *gt = pc_to_gt(pc);
1137 struct xe_tile *tile = gt_to_tile(gt);
1138 struct xe_device *xe = gt_to_xe(gt);
1139 struct xe_bo *bo;
1140 u32 size = PAGE_ALIGN(sizeof(struct slpc_shared_data));
1141 int err;
1142
1143 if (xe->info.skip_guc_pc)
1144 return 0;
1145
1146 err = drmm_mutex_init(&xe->drm, &pc->freq_lock);
1147 if (err)
1148 return err;
1149
1150 bo = xe_managed_bo_create_pin_map(xe, tile, size,
1151 XE_BO_FLAG_VRAM_IF_DGFX(tile) |
1152 XE_BO_FLAG_GGTT |
1153 XE_BO_FLAG_GGTT_INVALIDATE);
1154 if (IS_ERR(bo))
1155 return PTR_ERR(bo);
1156
1157 pc->bo = bo;
1158
1159 return devm_add_action_or_reset(xe->drm.dev, xe_guc_pc_fini_hw, pc);
1160 }
1161