1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2022 Intel Corporation
4 */
5
6 #include "xe_guc_pc.h"
7
8 #include <linux/delay.h>
9
10 #include <drm/drm_managed.h>
11 #include <generated/xe_wa_oob.h>
12
13 #include "abi/guc_actions_slpc_abi.h"
14 #include "regs/xe_gt_regs.h"
15 #include "regs/xe_regs.h"
16 #include "xe_bo.h"
17 #include "xe_device.h"
18 #include "xe_force_wake.h"
19 #include "xe_gt.h"
20 #include "xe_gt_idle.h"
21 #include "xe_gt_printk.h"
22 #include "xe_gt_types.h"
23 #include "xe_guc.h"
24 #include "xe_guc_ct.h"
25 #include "xe_map.h"
26 #include "xe_mmio.h"
27 #include "xe_pcode.h"
28 #include "xe_pm.h"
29 #include "xe_sriov.h"
30 #include "xe_wa.h"
31
32 #define MCHBAR_MIRROR_BASE_SNB 0x140000
33
34 #define RP_STATE_CAP XE_REG(MCHBAR_MIRROR_BASE_SNB + 0x5998)
35 #define RP0_MASK REG_GENMASK(7, 0)
36 #define RP1_MASK REG_GENMASK(15, 8)
37 #define RPN_MASK REG_GENMASK(23, 16)
38
39 #define FREQ_INFO_REC XE_REG(MCHBAR_MIRROR_BASE_SNB + 0x5ef0)
40 #define RPE_MASK REG_GENMASK(15, 8)
41
42 #define GT_PERF_STATUS XE_REG(0x1381b4)
43 #define CAGF_MASK REG_GENMASK(19, 11)
44
45 #define GT_FREQUENCY_MULTIPLIER 50
46 #define GT_FREQUENCY_SCALER 3
47
48 #define LNL_MERT_FREQ_CAP 800
49 #define BMG_MERT_FREQ_CAP 2133
50
51 /**
52 * DOC: GuC Power Conservation (PC)
53 *
54 * GuC Power Conservation (PC) supports multiple features for the most
55 * efficient and performing use of the GT when GuC submission is enabled,
56 * including frequency management, Render-C states management, and various
57 * algorithms for power balancing.
58 *
59 * Single Loop Power Conservation (SLPC) is the name given to the suite of
60 * connected power conservation features in the GuC firmware. The firmware
61 * exposes a programming interface to the host for the control of SLPC.
62 *
63 * Frequency management:
64 * =====================
65 *
66 * Xe driver enables SLPC with all of its defaults features and frequency
67 * selection, which varies per platform.
68 *
69 * Render-C States:
70 * ================
71 *
72 * Render-C states is also a GuC PC feature that is now enabled in Xe for
73 * all platforms.
74 *
75 */
76
pc_to_guc(struct xe_guc_pc * pc)77 static struct xe_guc *pc_to_guc(struct xe_guc_pc *pc)
78 {
79 return container_of(pc, struct xe_guc, pc);
80 }
81
pc_to_ct(struct xe_guc_pc * pc)82 static struct xe_guc_ct *pc_to_ct(struct xe_guc_pc *pc)
83 {
84 return &pc_to_guc(pc)->ct;
85 }
86
pc_to_gt(struct xe_guc_pc * pc)87 static struct xe_gt *pc_to_gt(struct xe_guc_pc *pc)
88 {
89 return guc_to_gt(pc_to_guc(pc));
90 }
91
pc_to_xe(struct xe_guc_pc * pc)92 static struct xe_device *pc_to_xe(struct xe_guc_pc *pc)
93 {
94 return guc_to_xe(pc_to_guc(pc));
95 }
96
pc_to_maps(struct xe_guc_pc * pc)97 static struct iosys_map *pc_to_maps(struct xe_guc_pc *pc)
98 {
99 return &pc->bo->vmap;
100 }
101
102 #define slpc_shared_data_read(pc_, field_) \
103 xe_map_rd_field(pc_to_xe(pc_), pc_to_maps(pc_), 0, \
104 struct slpc_shared_data, field_)
105
106 #define slpc_shared_data_write(pc_, field_, val_) \
107 xe_map_wr_field(pc_to_xe(pc_), pc_to_maps(pc_), 0, \
108 struct slpc_shared_data, field_, val_)
109
110 #define SLPC_EVENT(id, count) \
111 (FIELD_PREP(HOST2GUC_PC_SLPC_REQUEST_MSG_1_EVENT_ID, id) | \
112 FIELD_PREP(HOST2GUC_PC_SLPC_REQUEST_MSG_1_EVENT_ARGC, count))
113
wait_for_pc_state(struct xe_guc_pc * pc,enum slpc_global_state state)114 static int wait_for_pc_state(struct xe_guc_pc *pc,
115 enum slpc_global_state state)
116 {
117 int timeout_us = 5000; /* rought 5ms, but no need for precision */
118 int slept, wait = 10;
119
120 xe_device_assert_mem_access(pc_to_xe(pc));
121
122 for (slept = 0; slept < timeout_us;) {
123 if (slpc_shared_data_read(pc, header.global_state) == state)
124 return 0;
125
126 usleep_range(wait, wait << 1);
127 slept += wait;
128 wait <<= 1;
129 if (slept + wait > timeout_us)
130 wait = timeout_us - slept;
131 }
132
133 return -ETIMEDOUT;
134 }
135
pc_action_reset(struct xe_guc_pc * pc)136 static int pc_action_reset(struct xe_guc_pc *pc)
137 {
138 struct xe_guc_ct *ct = pc_to_ct(pc);
139 u32 action[] = {
140 GUC_ACTION_HOST2GUC_PC_SLPC_REQUEST,
141 SLPC_EVENT(SLPC_EVENT_RESET, 2),
142 xe_bo_ggtt_addr(pc->bo),
143 0,
144 };
145 int ret;
146
147 ret = xe_guc_ct_send(ct, action, ARRAY_SIZE(action), 0, 0);
148 if (ret)
149 xe_gt_err(pc_to_gt(pc), "GuC PC reset failed: %pe\n",
150 ERR_PTR(ret));
151
152 return ret;
153 }
154
pc_action_query_task_state(struct xe_guc_pc * pc)155 static int pc_action_query_task_state(struct xe_guc_pc *pc)
156 {
157 struct xe_guc_ct *ct = pc_to_ct(pc);
158 u32 action[] = {
159 GUC_ACTION_HOST2GUC_PC_SLPC_REQUEST,
160 SLPC_EVENT(SLPC_EVENT_QUERY_TASK_STATE, 2),
161 xe_bo_ggtt_addr(pc->bo),
162 0,
163 };
164 int ret;
165
166 if (wait_for_pc_state(pc, SLPC_GLOBAL_STATE_RUNNING))
167 return -EAGAIN;
168
169 /* Blocking here to ensure the results are ready before reading them */
170 ret = xe_guc_ct_send_block(ct, action, ARRAY_SIZE(action));
171 if (ret)
172 xe_gt_err(pc_to_gt(pc), "GuC PC query task state failed: %pe\n",
173 ERR_PTR(ret));
174
175 return ret;
176 }
177
pc_action_set_param(struct xe_guc_pc * pc,u8 id,u32 value)178 static int pc_action_set_param(struct xe_guc_pc *pc, u8 id, u32 value)
179 {
180 struct xe_guc_ct *ct = pc_to_ct(pc);
181 u32 action[] = {
182 GUC_ACTION_HOST2GUC_PC_SLPC_REQUEST,
183 SLPC_EVENT(SLPC_EVENT_PARAMETER_SET, 2),
184 id,
185 value,
186 };
187 int ret;
188
189 if (wait_for_pc_state(pc, SLPC_GLOBAL_STATE_RUNNING))
190 return -EAGAIN;
191
192 ret = xe_guc_ct_send(ct, action, ARRAY_SIZE(action), 0, 0);
193 if (ret)
194 xe_gt_err(pc_to_gt(pc), "GuC PC set param[%u]=%u failed: %pe\n",
195 id, value, ERR_PTR(ret));
196
197 return ret;
198 }
199
pc_action_unset_param(struct xe_guc_pc * pc,u8 id)200 static int pc_action_unset_param(struct xe_guc_pc *pc, u8 id)
201 {
202 u32 action[] = {
203 GUC_ACTION_HOST2GUC_PC_SLPC_REQUEST,
204 SLPC_EVENT(SLPC_EVENT_PARAMETER_UNSET, 1),
205 id,
206 };
207 struct xe_guc_ct *ct = &pc_to_guc(pc)->ct;
208 int ret;
209
210 if (wait_for_pc_state(pc, SLPC_GLOBAL_STATE_RUNNING))
211 return -EAGAIN;
212
213 ret = xe_guc_ct_send(ct, action, ARRAY_SIZE(action), 0, 0);
214 if (ret)
215 xe_gt_err(pc_to_gt(pc), "GuC PC unset param failed: %pe",
216 ERR_PTR(ret));
217
218 return ret;
219 }
220
pc_action_setup_gucrc(struct xe_guc_pc * pc,u32 mode)221 static int pc_action_setup_gucrc(struct xe_guc_pc *pc, u32 mode)
222 {
223 struct xe_guc_ct *ct = pc_to_ct(pc);
224 u32 action[] = {
225 GUC_ACTION_HOST2GUC_SETUP_PC_GUCRC,
226 mode,
227 };
228 int ret;
229
230 ret = xe_guc_ct_send(ct, action, ARRAY_SIZE(action), 0, 0);
231 if (ret)
232 xe_gt_err(pc_to_gt(pc), "GuC RC enable mode=%u failed: %pe\n",
233 mode, ERR_PTR(ret));
234 return ret;
235 }
236
decode_freq(u32 raw)237 static u32 decode_freq(u32 raw)
238 {
239 return DIV_ROUND_CLOSEST(raw * GT_FREQUENCY_MULTIPLIER,
240 GT_FREQUENCY_SCALER);
241 }
242
encode_freq(u32 freq)243 static u32 encode_freq(u32 freq)
244 {
245 return DIV_ROUND_CLOSEST(freq * GT_FREQUENCY_SCALER,
246 GT_FREQUENCY_MULTIPLIER);
247 }
248
pc_get_min_freq(struct xe_guc_pc * pc)249 static u32 pc_get_min_freq(struct xe_guc_pc *pc)
250 {
251 u32 freq;
252
253 freq = FIELD_GET(SLPC_MIN_UNSLICE_FREQ_MASK,
254 slpc_shared_data_read(pc, task_state_data.freq));
255
256 return decode_freq(freq);
257 }
258
pc_set_manual_rp_ctrl(struct xe_guc_pc * pc,bool enable)259 static void pc_set_manual_rp_ctrl(struct xe_guc_pc *pc, bool enable)
260 {
261 struct xe_gt *gt = pc_to_gt(pc);
262 u32 state = enable ? RPSWCTL_ENABLE : RPSWCTL_DISABLE;
263
264 /* Allow/Disallow punit to process software freq requests */
265 xe_mmio_write32(>->mmio, RP_CONTROL, state);
266 }
267
pc_set_cur_freq(struct xe_guc_pc * pc,u32 freq)268 static void pc_set_cur_freq(struct xe_guc_pc *pc, u32 freq)
269 {
270 struct xe_gt *gt = pc_to_gt(pc);
271 u32 rpnswreq;
272
273 pc_set_manual_rp_ctrl(pc, true);
274
275 /* Req freq is in units of 16.66 Mhz */
276 rpnswreq = REG_FIELD_PREP(REQ_RATIO_MASK, encode_freq(freq));
277 xe_mmio_write32(>->mmio, RPNSWREQ, rpnswreq);
278
279 /* Sleep for a small time to allow pcode to respond */
280 usleep_range(100, 300);
281
282 pc_set_manual_rp_ctrl(pc, false);
283 }
284
pc_set_min_freq(struct xe_guc_pc * pc,u32 freq)285 static int pc_set_min_freq(struct xe_guc_pc *pc, u32 freq)
286 {
287 /*
288 * Let's only check for the rpn-rp0 range. If max < min,
289 * min becomes a fixed request.
290 */
291 if (freq < pc->rpn_freq || freq > pc->rp0_freq)
292 return -EINVAL;
293
294 /*
295 * GuC policy is to elevate minimum frequency to the efficient levels
296 * Our goal is to have the admin choices respected.
297 */
298 pc_action_set_param(pc, SLPC_PARAM_IGNORE_EFFICIENT_FREQUENCY,
299 freq < pc->rpe_freq);
300
301 return pc_action_set_param(pc,
302 SLPC_PARAM_GLOBAL_MIN_GT_UNSLICE_FREQ_MHZ,
303 freq);
304 }
305
pc_get_max_freq(struct xe_guc_pc * pc)306 static int pc_get_max_freq(struct xe_guc_pc *pc)
307 {
308 u32 freq;
309
310 freq = FIELD_GET(SLPC_MAX_UNSLICE_FREQ_MASK,
311 slpc_shared_data_read(pc, task_state_data.freq));
312
313 return decode_freq(freq);
314 }
315
pc_set_max_freq(struct xe_guc_pc * pc,u32 freq)316 static int pc_set_max_freq(struct xe_guc_pc *pc, u32 freq)
317 {
318 /*
319 * Let's only check for the rpn-rp0 range. If max < min,
320 * min becomes a fixed request.
321 * Also, overclocking is not supported.
322 */
323 if (freq < pc->rpn_freq || freq > pc->rp0_freq)
324 return -EINVAL;
325
326 return pc_action_set_param(pc,
327 SLPC_PARAM_GLOBAL_MAX_GT_UNSLICE_FREQ_MHZ,
328 freq);
329 }
330
mtl_update_rpe_value(struct xe_guc_pc * pc)331 static void mtl_update_rpe_value(struct xe_guc_pc *pc)
332 {
333 struct xe_gt *gt = pc_to_gt(pc);
334 u32 reg;
335
336 if (xe_gt_is_media_type(gt))
337 reg = xe_mmio_read32(>->mmio, MTL_MPE_FREQUENCY);
338 else
339 reg = xe_mmio_read32(>->mmio, MTL_GT_RPE_FREQUENCY);
340
341 pc->rpe_freq = decode_freq(REG_FIELD_GET(MTL_RPE_MASK, reg));
342 }
343
tgl_update_rpe_value(struct xe_guc_pc * pc)344 static void tgl_update_rpe_value(struct xe_guc_pc *pc)
345 {
346 struct xe_gt *gt = pc_to_gt(pc);
347 struct xe_device *xe = gt_to_xe(gt);
348 u32 reg;
349
350 /*
351 * For PVC we still need to use fused RP1 as the approximation for RPe
352 * For other platforms than PVC we get the resolved RPe directly from
353 * PCODE at a different register
354 */
355 if (xe->info.platform == XE_PVC)
356 reg = xe_mmio_read32(>->mmio, PVC_RP_STATE_CAP);
357 else
358 reg = xe_mmio_read32(>->mmio, FREQ_INFO_REC);
359
360 pc->rpe_freq = REG_FIELD_GET(RPE_MASK, reg) * GT_FREQUENCY_MULTIPLIER;
361 }
362
pc_update_rp_values(struct xe_guc_pc * pc)363 static void pc_update_rp_values(struct xe_guc_pc *pc)
364 {
365 struct xe_gt *gt = pc_to_gt(pc);
366 struct xe_device *xe = gt_to_xe(gt);
367
368 if (GRAPHICS_VERx100(xe) >= 1270)
369 mtl_update_rpe_value(pc);
370 else
371 tgl_update_rpe_value(pc);
372
373 /*
374 * RPe is decided at runtime by PCODE. In the rare case where that's
375 * smaller than the fused min, we will trust the PCODE and use that
376 * as our minimum one.
377 */
378 pc->rpn_freq = min(pc->rpn_freq, pc->rpe_freq);
379 }
380
381 /**
382 * xe_guc_pc_get_act_freq - Get Actual running frequency
383 * @pc: The GuC PC
384 *
385 * Returns: The Actual running frequency. Which might be 0 if GT is in Render-C sleep state (RC6).
386 */
xe_guc_pc_get_act_freq(struct xe_guc_pc * pc)387 u32 xe_guc_pc_get_act_freq(struct xe_guc_pc *pc)
388 {
389 struct xe_gt *gt = pc_to_gt(pc);
390 struct xe_device *xe = gt_to_xe(gt);
391 u32 freq;
392
393 /* When in RC6, actual frequency reported will be 0. */
394 if (GRAPHICS_VERx100(xe) >= 1270) {
395 freq = xe_mmio_read32(>->mmio, MTL_MIRROR_TARGET_WP1);
396 freq = REG_FIELD_GET(MTL_CAGF_MASK, freq);
397 } else {
398 freq = xe_mmio_read32(>->mmio, GT_PERF_STATUS);
399 freq = REG_FIELD_GET(CAGF_MASK, freq);
400 }
401
402 freq = decode_freq(freq);
403
404 return freq;
405 }
406
407 /**
408 * xe_guc_pc_get_cur_freq - Get Current requested frequency
409 * @pc: The GuC PC
410 * @freq: A pointer to a u32 where the freq value will be returned
411 *
412 * Returns: 0 on success,
413 * -EAGAIN if GuC PC not ready (likely in middle of a reset).
414 */
xe_guc_pc_get_cur_freq(struct xe_guc_pc * pc,u32 * freq)415 int xe_guc_pc_get_cur_freq(struct xe_guc_pc *pc, u32 *freq)
416 {
417 struct xe_gt *gt = pc_to_gt(pc);
418 unsigned int fw_ref;
419
420 /*
421 * GuC SLPC plays with cur freq request when GuCRC is enabled
422 * Block RC6 for a more reliable read.
423 */
424 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
425 if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL)) {
426 xe_force_wake_put(gt_to_fw(gt), fw_ref);
427 return -ETIMEDOUT;
428 }
429
430 *freq = xe_mmio_read32(>->mmio, RPNSWREQ);
431
432 *freq = REG_FIELD_GET(REQ_RATIO_MASK, *freq);
433 *freq = decode_freq(*freq);
434
435 xe_force_wake_put(gt_to_fw(gt), fw_ref);
436 return 0;
437 }
438
439 /**
440 * xe_guc_pc_get_rp0_freq - Get the RP0 freq
441 * @pc: The GuC PC
442 *
443 * Returns: RP0 freq.
444 */
xe_guc_pc_get_rp0_freq(struct xe_guc_pc * pc)445 u32 xe_guc_pc_get_rp0_freq(struct xe_guc_pc *pc)
446 {
447 return pc->rp0_freq;
448 }
449
450 /**
451 * xe_guc_pc_get_rpe_freq - Get the RPe freq
452 * @pc: The GuC PC
453 *
454 * Returns: RPe freq.
455 */
xe_guc_pc_get_rpe_freq(struct xe_guc_pc * pc)456 u32 xe_guc_pc_get_rpe_freq(struct xe_guc_pc *pc)
457 {
458 pc_update_rp_values(pc);
459
460 return pc->rpe_freq;
461 }
462
463 /**
464 * xe_guc_pc_get_rpn_freq - Get the RPn freq
465 * @pc: The GuC PC
466 *
467 * Returns: RPn freq.
468 */
xe_guc_pc_get_rpn_freq(struct xe_guc_pc * pc)469 u32 xe_guc_pc_get_rpn_freq(struct xe_guc_pc *pc)
470 {
471 return pc->rpn_freq;
472 }
473
474 /**
475 * xe_guc_pc_get_min_freq - Get the min operational frequency
476 * @pc: The GuC PC
477 * @freq: A pointer to a u32 where the freq value will be returned
478 *
479 * Returns: 0 on success,
480 * -EAGAIN if GuC PC not ready (likely in middle of a reset).
481 */
xe_guc_pc_get_min_freq(struct xe_guc_pc * pc,u32 * freq)482 int xe_guc_pc_get_min_freq(struct xe_guc_pc *pc, u32 *freq)
483 {
484 struct xe_gt *gt = pc_to_gt(pc);
485 unsigned int fw_ref;
486 int ret;
487
488 mutex_lock(&pc->freq_lock);
489 if (!pc->freq_ready) {
490 /* Might be in the middle of a gt reset */
491 ret = -EAGAIN;
492 goto out;
493 }
494
495 /*
496 * GuC SLPC plays with min freq request when GuCRC is enabled
497 * Block RC6 for a more reliable read.
498 */
499 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
500 if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL)) {
501 ret = -ETIMEDOUT;
502 goto fw;
503 }
504
505 ret = pc_action_query_task_state(pc);
506 if (ret)
507 goto fw;
508
509 *freq = pc_get_min_freq(pc);
510
511 fw:
512 xe_force_wake_put(gt_to_fw(gt), fw_ref);
513 out:
514 mutex_unlock(&pc->freq_lock);
515 return ret;
516 }
517
518 /**
519 * xe_guc_pc_set_min_freq - Set the minimal operational frequency
520 * @pc: The GuC PC
521 * @freq: The selected minimal frequency
522 *
523 * Returns: 0 on success,
524 * -EAGAIN if GuC PC not ready (likely in middle of a reset),
525 * -EINVAL if value out of bounds.
526 */
xe_guc_pc_set_min_freq(struct xe_guc_pc * pc,u32 freq)527 int xe_guc_pc_set_min_freq(struct xe_guc_pc *pc, u32 freq)
528 {
529 int ret;
530
531 mutex_lock(&pc->freq_lock);
532 if (!pc->freq_ready) {
533 /* Might be in the middle of a gt reset */
534 ret = -EAGAIN;
535 goto out;
536 }
537
538 ret = pc_set_min_freq(pc, freq);
539 if (ret)
540 goto out;
541
542 pc->user_requested_min = freq;
543
544 out:
545 mutex_unlock(&pc->freq_lock);
546 return ret;
547 }
548
549 /**
550 * xe_guc_pc_get_max_freq - Get Maximum operational frequency
551 * @pc: The GuC PC
552 * @freq: A pointer to a u32 where the freq value will be returned
553 *
554 * Returns: 0 on success,
555 * -EAGAIN if GuC PC not ready (likely in middle of a reset).
556 */
xe_guc_pc_get_max_freq(struct xe_guc_pc * pc,u32 * freq)557 int xe_guc_pc_get_max_freq(struct xe_guc_pc *pc, u32 *freq)
558 {
559 int ret;
560
561 mutex_lock(&pc->freq_lock);
562 if (!pc->freq_ready) {
563 /* Might be in the middle of a gt reset */
564 ret = -EAGAIN;
565 goto out;
566 }
567
568 ret = pc_action_query_task_state(pc);
569 if (ret)
570 goto out;
571
572 *freq = pc_get_max_freq(pc);
573
574 out:
575 mutex_unlock(&pc->freq_lock);
576 return ret;
577 }
578
579 /**
580 * xe_guc_pc_set_max_freq - Set the maximum operational frequency
581 * @pc: The GuC PC
582 * @freq: The selected maximum frequency value
583 *
584 * Returns: 0 on success,
585 * -EAGAIN if GuC PC not ready (likely in middle of a reset),
586 * -EINVAL if value out of bounds.
587 */
xe_guc_pc_set_max_freq(struct xe_guc_pc * pc,u32 freq)588 int xe_guc_pc_set_max_freq(struct xe_guc_pc *pc, u32 freq)
589 {
590 int ret;
591
592 mutex_lock(&pc->freq_lock);
593 if (!pc->freq_ready) {
594 /* Might be in the middle of a gt reset */
595 ret = -EAGAIN;
596 goto out;
597 }
598
599 ret = pc_set_max_freq(pc, freq);
600 if (ret)
601 goto out;
602
603 pc->user_requested_max = freq;
604
605 out:
606 mutex_unlock(&pc->freq_lock);
607 return ret;
608 }
609
610 /**
611 * xe_guc_pc_c_status - get the current GT C state
612 * @pc: XE_GuC_PC instance
613 */
xe_guc_pc_c_status(struct xe_guc_pc * pc)614 enum xe_gt_idle_state xe_guc_pc_c_status(struct xe_guc_pc *pc)
615 {
616 struct xe_gt *gt = pc_to_gt(pc);
617 u32 reg, gt_c_state;
618
619 if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 1270) {
620 reg = xe_mmio_read32(>->mmio, MTL_MIRROR_TARGET_WP1);
621 gt_c_state = REG_FIELD_GET(MTL_CC_MASK, reg);
622 } else {
623 reg = xe_mmio_read32(>->mmio, GT_CORE_STATUS);
624 gt_c_state = REG_FIELD_GET(RCN_MASK, reg);
625 }
626
627 switch (gt_c_state) {
628 case GT_C6:
629 return GT_IDLE_C6;
630 case GT_C0:
631 return GT_IDLE_C0;
632 default:
633 return GT_IDLE_UNKNOWN;
634 }
635 }
636
637 /**
638 * xe_guc_pc_rc6_residency - rc6 residency counter
639 * @pc: Xe_GuC_PC instance
640 */
xe_guc_pc_rc6_residency(struct xe_guc_pc * pc)641 u64 xe_guc_pc_rc6_residency(struct xe_guc_pc *pc)
642 {
643 struct xe_gt *gt = pc_to_gt(pc);
644 u32 reg;
645
646 reg = xe_mmio_read32(>->mmio, GT_GFX_RC6);
647
648 return reg;
649 }
650
651 /**
652 * xe_guc_pc_mc6_residency - mc6 residency counter
653 * @pc: Xe_GuC_PC instance
654 */
xe_guc_pc_mc6_residency(struct xe_guc_pc * pc)655 u64 xe_guc_pc_mc6_residency(struct xe_guc_pc *pc)
656 {
657 struct xe_gt *gt = pc_to_gt(pc);
658 u64 reg;
659
660 reg = xe_mmio_read32(>->mmio, MTL_MEDIA_MC6);
661
662 return reg;
663 }
664
mtl_init_fused_rp_values(struct xe_guc_pc * pc)665 static void mtl_init_fused_rp_values(struct xe_guc_pc *pc)
666 {
667 struct xe_gt *gt = pc_to_gt(pc);
668 u32 reg;
669
670 xe_device_assert_mem_access(pc_to_xe(pc));
671
672 if (xe_gt_is_media_type(gt))
673 reg = xe_mmio_read32(>->mmio, MTL_MEDIAP_STATE_CAP);
674 else
675 reg = xe_mmio_read32(>->mmio, MTL_RP_STATE_CAP);
676
677 pc->rp0_freq = decode_freq(REG_FIELD_GET(MTL_RP0_CAP_MASK, reg));
678
679 pc->rpn_freq = decode_freq(REG_FIELD_GET(MTL_RPN_CAP_MASK, reg));
680 }
681
tgl_init_fused_rp_values(struct xe_guc_pc * pc)682 static void tgl_init_fused_rp_values(struct xe_guc_pc *pc)
683 {
684 struct xe_gt *gt = pc_to_gt(pc);
685 struct xe_device *xe = gt_to_xe(gt);
686 u32 reg;
687
688 xe_device_assert_mem_access(pc_to_xe(pc));
689
690 if (xe->info.platform == XE_PVC)
691 reg = xe_mmio_read32(>->mmio, PVC_RP_STATE_CAP);
692 else
693 reg = xe_mmio_read32(>->mmio, RP_STATE_CAP);
694 pc->rp0_freq = REG_FIELD_GET(RP0_MASK, reg) * GT_FREQUENCY_MULTIPLIER;
695 pc->rpn_freq = REG_FIELD_GET(RPN_MASK, reg) * GT_FREQUENCY_MULTIPLIER;
696 }
697
pc_init_fused_rp_values(struct xe_guc_pc * pc)698 static void pc_init_fused_rp_values(struct xe_guc_pc *pc)
699 {
700 struct xe_gt *gt = pc_to_gt(pc);
701 struct xe_device *xe = gt_to_xe(gt);
702
703 if (GRAPHICS_VERx100(xe) >= 1270)
704 mtl_init_fused_rp_values(pc);
705 else
706 tgl_init_fused_rp_values(pc);
707 }
708
pc_max_freq_cap(struct xe_guc_pc * pc)709 static u32 pc_max_freq_cap(struct xe_guc_pc *pc)
710 {
711 struct xe_gt *gt = pc_to_gt(pc);
712
713 if (XE_WA(gt, 22019338487)) {
714 if (xe_gt_is_media_type(gt))
715 return min(LNL_MERT_FREQ_CAP, pc->rp0_freq);
716 else
717 return min(BMG_MERT_FREQ_CAP, pc->rp0_freq);
718 } else {
719 return pc->rp0_freq;
720 }
721 }
722
723 /**
724 * xe_guc_pc_raise_unslice - Initialize RPx values and request a higher GT
725 * frequency to allow faster GuC load times
726 * @pc: Xe_GuC_PC instance
727 */
xe_guc_pc_raise_unslice(struct xe_guc_pc * pc)728 void xe_guc_pc_raise_unslice(struct xe_guc_pc *pc)
729 {
730 struct xe_gt *gt = pc_to_gt(pc);
731
732 xe_force_wake_assert_held(gt_to_fw(gt), XE_FW_GT);
733 pc_set_cur_freq(pc, pc_max_freq_cap(pc));
734 }
735
736 /**
737 * xe_guc_pc_init_early - Initialize RPx values
738 * @pc: Xe_GuC_PC instance
739 */
xe_guc_pc_init_early(struct xe_guc_pc * pc)740 void xe_guc_pc_init_early(struct xe_guc_pc *pc)
741 {
742 struct xe_gt *gt = pc_to_gt(pc);
743
744 xe_force_wake_assert_held(gt_to_fw(gt), XE_FW_GT);
745 pc_init_fused_rp_values(pc);
746 }
747
pc_adjust_freq_bounds(struct xe_guc_pc * pc)748 static int pc_adjust_freq_bounds(struct xe_guc_pc *pc)
749 {
750 int ret;
751
752 lockdep_assert_held(&pc->freq_lock);
753
754 ret = pc_action_query_task_state(pc);
755 if (ret)
756 goto out;
757
758 /*
759 * GuC defaults to some RPmax that is not actually achievable without
760 * overclocking. Let's adjust it to the Hardware RP0, which is the
761 * regular maximum
762 */
763 if (pc_get_max_freq(pc) > pc->rp0_freq) {
764 ret = pc_set_max_freq(pc, pc->rp0_freq);
765 if (ret)
766 goto out;
767 }
768
769 /*
770 * Same thing happens for Server platforms where min is listed as
771 * RPMax
772 */
773 if (pc_get_min_freq(pc) > pc->rp0_freq)
774 ret = pc_set_min_freq(pc, pc->rp0_freq);
775
776 out:
777 return ret;
778 }
779
pc_adjust_requested_freq(struct xe_guc_pc * pc)780 static int pc_adjust_requested_freq(struct xe_guc_pc *pc)
781 {
782 int ret = 0;
783
784 lockdep_assert_held(&pc->freq_lock);
785
786 if (pc->user_requested_min != 0) {
787 ret = pc_set_min_freq(pc, pc->user_requested_min);
788 if (ret)
789 return ret;
790 }
791
792 if (pc->user_requested_max != 0) {
793 ret = pc_set_max_freq(pc, pc->user_requested_max);
794 if (ret)
795 return ret;
796 }
797
798 return ret;
799 }
800
pc_set_mert_freq_cap(struct xe_guc_pc * pc)801 static int pc_set_mert_freq_cap(struct xe_guc_pc *pc)
802 {
803 int ret = 0;
804
805 if (XE_WA(pc_to_gt(pc), 22019338487)) {
806 /*
807 * Get updated min/max and stash them.
808 */
809 ret = xe_guc_pc_get_min_freq(pc, &pc->stashed_min_freq);
810 if (!ret)
811 ret = xe_guc_pc_get_max_freq(pc, &pc->stashed_max_freq);
812 if (ret)
813 return ret;
814
815 /*
816 * Ensure min and max are bound by MERT_FREQ_CAP until driver loads.
817 */
818 mutex_lock(&pc->freq_lock);
819 ret = pc_set_min_freq(pc, min(pc->rpe_freq, pc_max_freq_cap(pc)));
820 if (!ret)
821 ret = pc_set_max_freq(pc, min(pc->rp0_freq, pc_max_freq_cap(pc)));
822 mutex_unlock(&pc->freq_lock);
823 }
824
825 return ret;
826 }
827
828 /**
829 * xe_guc_pc_restore_stashed_freq - Set min/max back to stashed values
830 * @pc: The GuC PC
831 *
832 * Returns: 0 on success,
833 * error code on failure
834 */
xe_guc_pc_restore_stashed_freq(struct xe_guc_pc * pc)835 int xe_guc_pc_restore_stashed_freq(struct xe_guc_pc *pc)
836 {
837 int ret = 0;
838
839 if (IS_SRIOV_VF(pc_to_xe(pc)) || pc_to_xe(pc)->info.skip_guc_pc)
840 return 0;
841
842 mutex_lock(&pc->freq_lock);
843 ret = pc_set_max_freq(pc, pc->stashed_max_freq);
844 if (!ret)
845 ret = pc_set_min_freq(pc, pc->stashed_min_freq);
846 mutex_unlock(&pc->freq_lock);
847
848 return ret;
849 }
850
851 /**
852 * xe_guc_pc_gucrc_disable - Disable GuC RC
853 * @pc: Xe_GuC_PC instance
854 *
855 * Disables GuC RC by taking control of RC6 back from GuC.
856 *
857 * Return: 0 on success, negative error code on error.
858 */
xe_guc_pc_gucrc_disable(struct xe_guc_pc * pc)859 int xe_guc_pc_gucrc_disable(struct xe_guc_pc *pc)
860 {
861 struct xe_device *xe = pc_to_xe(pc);
862 struct xe_gt *gt = pc_to_gt(pc);
863 unsigned int fw_ref;
864 int ret = 0;
865
866 if (xe->info.skip_guc_pc)
867 return 0;
868
869 ret = pc_action_setup_gucrc(pc, GUCRC_HOST_CONTROL);
870 if (ret)
871 return ret;
872
873 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
874 if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL)) {
875 xe_force_wake_put(gt_to_fw(gt), fw_ref);
876 return -ETIMEDOUT;
877 }
878
879 xe_gt_idle_disable_c6(gt);
880
881 xe_force_wake_put(gt_to_fw(gt), fw_ref);
882
883 return 0;
884 }
885
886 /**
887 * xe_guc_pc_override_gucrc_mode - override GUCRC mode
888 * @pc: Xe_GuC_PC instance
889 * @mode: new value of the mode.
890 *
891 * Return: 0 on success, negative error code on error
892 */
xe_guc_pc_override_gucrc_mode(struct xe_guc_pc * pc,enum slpc_gucrc_mode mode)893 int xe_guc_pc_override_gucrc_mode(struct xe_guc_pc *pc, enum slpc_gucrc_mode mode)
894 {
895 int ret;
896
897 xe_pm_runtime_get(pc_to_xe(pc));
898 ret = pc_action_set_param(pc, SLPC_PARAM_PWRGATE_RC_MODE, mode);
899 xe_pm_runtime_put(pc_to_xe(pc));
900
901 return ret;
902 }
903
904 /**
905 * xe_guc_pc_unset_gucrc_mode - unset GUCRC mode override
906 * @pc: Xe_GuC_PC instance
907 *
908 * Return: 0 on success, negative error code on error
909 */
xe_guc_pc_unset_gucrc_mode(struct xe_guc_pc * pc)910 int xe_guc_pc_unset_gucrc_mode(struct xe_guc_pc *pc)
911 {
912 int ret;
913
914 xe_pm_runtime_get(pc_to_xe(pc));
915 ret = pc_action_unset_param(pc, SLPC_PARAM_PWRGATE_RC_MODE);
916 xe_pm_runtime_put(pc_to_xe(pc));
917
918 return ret;
919 }
920
pc_init_pcode_freq(struct xe_guc_pc * pc)921 static void pc_init_pcode_freq(struct xe_guc_pc *pc)
922 {
923 u32 min = DIV_ROUND_CLOSEST(pc->rpn_freq, GT_FREQUENCY_MULTIPLIER);
924 u32 max = DIV_ROUND_CLOSEST(pc->rp0_freq, GT_FREQUENCY_MULTIPLIER);
925
926 XE_WARN_ON(xe_pcode_init_min_freq_table(gt_to_tile(pc_to_gt(pc)), min, max));
927 }
928
pc_init_freqs(struct xe_guc_pc * pc)929 static int pc_init_freqs(struct xe_guc_pc *pc)
930 {
931 int ret;
932
933 mutex_lock(&pc->freq_lock);
934
935 ret = pc_adjust_freq_bounds(pc);
936 if (ret)
937 goto out;
938
939 ret = pc_adjust_requested_freq(pc);
940 if (ret)
941 goto out;
942
943 pc_update_rp_values(pc);
944
945 pc_init_pcode_freq(pc);
946
947 /*
948 * The frequencies are really ready for use only after the user
949 * requested ones got restored.
950 */
951 pc->freq_ready = true;
952
953 out:
954 mutex_unlock(&pc->freq_lock);
955 return ret;
956 }
957
958 /**
959 * xe_guc_pc_start - Start GuC's Power Conservation component
960 * @pc: Xe_GuC_PC instance
961 */
xe_guc_pc_start(struct xe_guc_pc * pc)962 int xe_guc_pc_start(struct xe_guc_pc *pc)
963 {
964 struct xe_device *xe = pc_to_xe(pc);
965 struct xe_gt *gt = pc_to_gt(pc);
966 u32 size = PAGE_ALIGN(sizeof(struct slpc_shared_data));
967 unsigned int fw_ref;
968 int ret;
969
970 xe_gt_assert(gt, xe_device_uc_enabled(xe));
971
972 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
973 if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL)) {
974 xe_force_wake_put(gt_to_fw(gt), fw_ref);
975 return -ETIMEDOUT;
976 }
977
978 if (xe->info.skip_guc_pc) {
979 if (xe->info.platform != XE_PVC)
980 xe_gt_idle_enable_c6(gt);
981
982 /* Request max possible since dynamic freq mgmt is not enabled */
983 pc_set_cur_freq(pc, UINT_MAX);
984
985 ret = 0;
986 goto out;
987 }
988
989 memset(pc->bo->vmap.vaddr, 0, size);
990 slpc_shared_data_write(pc, header.size, size);
991
992 ret = pc_action_reset(pc);
993 if (ret)
994 goto out;
995
996 if (wait_for_pc_state(pc, SLPC_GLOBAL_STATE_RUNNING)) {
997 xe_gt_err(gt, "GuC PC Start failed\n");
998 ret = -EIO;
999 goto out;
1000 }
1001
1002 ret = pc_init_freqs(pc);
1003 if (ret)
1004 goto out;
1005
1006 ret = pc_set_mert_freq_cap(pc);
1007 if (ret)
1008 goto out;
1009
1010 if (xe->info.platform == XE_PVC) {
1011 xe_guc_pc_gucrc_disable(pc);
1012 ret = 0;
1013 goto out;
1014 }
1015
1016 ret = pc_action_setup_gucrc(pc, GUCRC_FIRMWARE_CONTROL);
1017
1018 out:
1019 xe_force_wake_put(gt_to_fw(gt), fw_ref);
1020 return ret;
1021 }
1022
1023 /**
1024 * xe_guc_pc_stop - Stop GuC's Power Conservation component
1025 * @pc: Xe_GuC_PC instance
1026 */
xe_guc_pc_stop(struct xe_guc_pc * pc)1027 int xe_guc_pc_stop(struct xe_guc_pc *pc)
1028 {
1029 struct xe_device *xe = pc_to_xe(pc);
1030
1031 if (xe->info.skip_guc_pc) {
1032 xe_gt_idle_disable_c6(pc_to_gt(pc));
1033 return 0;
1034 }
1035
1036 mutex_lock(&pc->freq_lock);
1037 pc->freq_ready = false;
1038 mutex_unlock(&pc->freq_lock);
1039
1040 return 0;
1041 }
1042
1043 /**
1044 * xe_guc_pc_fini_hw - Finalize GuC's Power Conservation component
1045 * @arg: opaque pointer that should point to Xe_GuC_PC instance
1046 */
xe_guc_pc_fini_hw(void * arg)1047 static void xe_guc_pc_fini_hw(void *arg)
1048 {
1049 struct xe_guc_pc *pc = arg;
1050 struct xe_device *xe = pc_to_xe(pc);
1051 unsigned int fw_ref;
1052
1053 if (xe_device_wedged(xe))
1054 return;
1055
1056 fw_ref = xe_force_wake_get(gt_to_fw(pc_to_gt(pc)), XE_FORCEWAKE_ALL);
1057 xe_guc_pc_gucrc_disable(pc);
1058 XE_WARN_ON(xe_guc_pc_stop(pc));
1059
1060 /* Bind requested freq to mert_freq_cap before unload */
1061 pc_set_cur_freq(pc, min(pc_max_freq_cap(pc), pc->rpe_freq));
1062
1063 xe_force_wake_put(gt_to_fw(pc_to_gt(pc)), fw_ref);
1064 }
1065
1066 /**
1067 * xe_guc_pc_init - Initialize GuC's Power Conservation component
1068 * @pc: Xe_GuC_PC instance
1069 */
xe_guc_pc_init(struct xe_guc_pc * pc)1070 int xe_guc_pc_init(struct xe_guc_pc *pc)
1071 {
1072 struct xe_gt *gt = pc_to_gt(pc);
1073 struct xe_tile *tile = gt_to_tile(gt);
1074 struct xe_device *xe = gt_to_xe(gt);
1075 struct xe_bo *bo;
1076 u32 size = PAGE_ALIGN(sizeof(struct slpc_shared_data));
1077 int err;
1078
1079 if (xe->info.skip_guc_pc)
1080 return 0;
1081
1082 err = drmm_mutex_init(&xe->drm, &pc->freq_lock);
1083 if (err)
1084 return err;
1085
1086 bo = xe_managed_bo_create_pin_map(xe, tile, size,
1087 XE_BO_FLAG_VRAM_IF_DGFX(tile) |
1088 XE_BO_FLAG_GGTT |
1089 XE_BO_FLAG_GGTT_INVALIDATE);
1090 if (IS_ERR(bo))
1091 return PTR_ERR(bo);
1092
1093 pc->bo = bo;
1094
1095 return devm_add_action_or_reset(xe->drm.dev, xe_guc_pc_fini_hw, pc);
1096 }
1097