1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2022 Intel Corporation
4 */
5
6 #include "xe_gt.h"
7
8 #include <linux/minmax.h>
9
10 #include <drm/drm_managed.h>
11 #include <uapi/drm/xe_drm.h>
12
13 #include <generated/xe_wa_oob.h>
14
15 #include "instructions/xe_gfxpipe_commands.h"
16 #include "instructions/xe_mi_commands.h"
17 #include "regs/xe_gt_regs.h"
18 #include "xe_assert.h"
19 #include "xe_bb.h"
20 #include "xe_bo.h"
21 #include "xe_device.h"
22 #include "xe_exec_queue.h"
23 #include "xe_execlist.h"
24 #include "xe_force_wake.h"
25 #include "xe_ggtt.h"
26 #include "xe_gsc.h"
27 #include "xe_gt_ccs_mode.h"
28 #include "xe_gt_clock.h"
29 #include "xe_gt_freq.h"
30 #include "xe_gt_idle.h"
31 #include "xe_gt_mcr.h"
32 #include "xe_gt_pagefault.h"
33 #include "xe_gt_printk.h"
34 #include "xe_gt_sriov_pf.h"
35 #include "xe_gt_sysfs.h"
36 #include "xe_gt_tlb_invalidation.h"
37 #include "xe_gt_topology.h"
38 #include "xe_guc_exec_queue_types.h"
39 #include "xe_guc_pc.h"
40 #include "xe_hw_fence.h"
41 #include "xe_hw_engine_class_sysfs.h"
42 #include "xe_irq.h"
43 #include "xe_lmtt.h"
44 #include "xe_lrc.h"
45 #include "xe_map.h"
46 #include "xe_migrate.h"
47 #include "xe_mmio.h"
48 #include "xe_pat.h"
49 #include "xe_pm.h"
50 #include "xe_mocs.h"
51 #include "xe_reg_sr.h"
52 #include "xe_ring_ops.h"
53 #include "xe_sa.h"
54 #include "xe_sched_job.h"
55 #include "xe_sriov.h"
56 #include "xe_tuning.h"
57 #include "xe_uc.h"
58 #include "xe_uc_fw.h"
59 #include "xe_vm.h"
60 #include "xe_wa.h"
61 #include "xe_wopcm.h"
62
gt_fini(struct drm_device * drm,void * arg)63 static void gt_fini(struct drm_device *drm, void *arg)
64 {
65 struct xe_gt *gt = arg;
66
67 destroy_workqueue(gt->ordered_wq);
68 }
69
xe_gt_alloc(struct xe_tile * tile)70 struct xe_gt *xe_gt_alloc(struct xe_tile *tile)
71 {
72 struct xe_gt *gt;
73 int err;
74
75 gt = drmm_kzalloc(&tile_to_xe(tile)->drm, sizeof(*gt), GFP_KERNEL);
76 if (!gt)
77 return ERR_PTR(-ENOMEM);
78
79 gt->tile = tile;
80 gt->ordered_wq = alloc_ordered_workqueue("gt-ordered-wq",
81 WQ_MEM_RECLAIM);
82
83 err = drmm_add_action_or_reset(>_to_xe(gt)->drm, gt_fini, gt);
84 if (err)
85 return ERR_PTR(err);
86
87 return gt;
88 }
89
xe_gt_sanitize(struct xe_gt * gt)90 void xe_gt_sanitize(struct xe_gt *gt)
91 {
92 /*
93 * FIXME: if xe_uc_sanitize is called here, on TGL driver will not
94 * reload
95 */
96 gt->uc.guc.submission_state.enabled = false;
97 }
98
xe_gt_enable_host_l2_vram(struct xe_gt * gt)99 static void xe_gt_enable_host_l2_vram(struct xe_gt *gt)
100 {
101 unsigned int fw_ref;
102 u32 reg;
103
104 if (!XE_WA(gt, 16023588340))
105 return;
106
107 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
108 if (!fw_ref)
109 return;
110
111 if (!xe_gt_is_media_type(gt)) {
112 reg = xe_gt_mcr_unicast_read_any(gt, XE2_GAMREQSTRM_CTRL);
113 reg |= CG_DIS_CNTLBUS;
114 xe_gt_mcr_multicast_write(gt, XE2_GAMREQSTRM_CTRL, reg);
115 }
116
117 xe_gt_mcr_multicast_write(gt, XEHPC_L3CLOS_MASK(3), 0x3);
118 xe_force_wake_put(gt_to_fw(gt), fw_ref);
119 }
120
xe_gt_disable_host_l2_vram(struct xe_gt * gt)121 static void xe_gt_disable_host_l2_vram(struct xe_gt *gt)
122 {
123 unsigned int fw_ref;
124 u32 reg;
125
126 if (!XE_WA(gt, 16023588340))
127 return;
128
129 if (xe_gt_is_media_type(gt))
130 return;
131
132 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
133 if (!fw_ref)
134 return;
135
136 reg = xe_gt_mcr_unicast_read_any(gt, XE2_GAMREQSTRM_CTRL);
137 reg &= ~CG_DIS_CNTLBUS;
138 xe_gt_mcr_multicast_write(gt, XE2_GAMREQSTRM_CTRL, reg);
139
140 xe_force_wake_put(gt_to_fw(gt), fw_ref);
141 }
142
143 /**
144 * xe_gt_remove() - Clean up the GT structures before driver removal
145 * @gt: the GT object
146 *
147 * This function should only act on objects/structures that must be cleaned
148 * before the driver removal callback is complete and therefore can't be
149 * deferred to a drmm action.
150 */
xe_gt_remove(struct xe_gt * gt)151 void xe_gt_remove(struct xe_gt *gt)
152 {
153 int i;
154
155 xe_uc_remove(>->uc);
156
157 for (i = 0; i < XE_ENGINE_CLASS_MAX; ++i)
158 xe_hw_fence_irq_finish(>->fence_irq[i]);
159
160 xe_gt_disable_host_l2_vram(gt);
161 }
162
163 static void gt_reset_worker(struct work_struct *w);
164
emit_nop_job(struct xe_gt * gt,struct xe_exec_queue * q)165 static int emit_nop_job(struct xe_gt *gt, struct xe_exec_queue *q)
166 {
167 struct xe_sched_job *job;
168 struct xe_bb *bb;
169 struct dma_fence *fence;
170 long timeout;
171
172 bb = xe_bb_new(gt, 4, false);
173 if (IS_ERR(bb))
174 return PTR_ERR(bb);
175
176 job = xe_bb_create_job(q, bb);
177 if (IS_ERR(job)) {
178 xe_bb_free(bb, NULL);
179 return PTR_ERR(job);
180 }
181
182 xe_sched_job_arm(job);
183 fence = dma_fence_get(&job->drm.s_fence->finished);
184 xe_sched_job_push(job);
185
186 timeout = dma_fence_wait_timeout(fence, false, HZ);
187 dma_fence_put(fence);
188 xe_bb_free(bb, NULL);
189 if (timeout < 0)
190 return timeout;
191 else if (!timeout)
192 return -ETIME;
193
194 return 0;
195 }
196
197 /*
198 * Convert back from encoded value to type-safe, only to be used when reg.mcr
199 * is true
200 */
to_xe_reg_mcr(const struct xe_reg reg)201 static struct xe_reg_mcr to_xe_reg_mcr(const struct xe_reg reg)
202 {
203 return (const struct xe_reg_mcr){.__reg.raw = reg.raw };
204 }
205
emit_wa_job(struct xe_gt * gt,struct xe_exec_queue * q)206 static int emit_wa_job(struct xe_gt *gt, struct xe_exec_queue *q)
207 {
208 struct xe_reg_sr *sr = &q->hwe->reg_lrc;
209 struct xe_reg_sr_entry *entry;
210 unsigned long idx;
211 struct xe_sched_job *job;
212 struct xe_bb *bb;
213 struct dma_fence *fence;
214 long timeout;
215 int count = 0;
216
217 if (q->hwe->class == XE_ENGINE_CLASS_RENDER)
218 /* Big enough to emit all of the context's 3DSTATE */
219 bb = xe_bb_new(gt, xe_gt_lrc_size(gt, q->hwe->class), false);
220 else
221 /* Just pick a large BB size */
222 bb = xe_bb_new(gt, SZ_4K, false);
223
224 if (IS_ERR(bb))
225 return PTR_ERR(bb);
226
227 xa_for_each(&sr->xa, idx, entry)
228 ++count;
229
230 if (count) {
231 xe_gt_dbg(gt, "LRC WA %s save-restore batch\n", sr->name);
232
233 bb->cs[bb->len++] = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(count);
234
235 xa_for_each(&sr->xa, idx, entry) {
236 struct xe_reg reg = entry->reg;
237 struct xe_reg_mcr reg_mcr = to_xe_reg_mcr(reg);
238 u32 val;
239
240 /*
241 * Skip reading the register if it's not really needed
242 */
243 if (reg.masked)
244 val = entry->clr_bits << 16;
245 else if (entry->clr_bits + 1)
246 val = (reg.mcr ?
247 xe_gt_mcr_unicast_read_any(gt, reg_mcr) :
248 xe_mmio_read32(>->mmio, reg)) & (~entry->clr_bits);
249 else
250 val = 0;
251
252 val |= entry->set_bits;
253
254 bb->cs[bb->len++] = reg.addr;
255 bb->cs[bb->len++] = val;
256 xe_gt_dbg(gt, "REG[0x%x] = 0x%08x", reg.addr, val);
257 }
258 }
259
260 xe_lrc_emit_hwe_state_instructions(q, bb);
261
262 job = xe_bb_create_job(q, bb);
263 if (IS_ERR(job)) {
264 xe_bb_free(bb, NULL);
265 return PTR_ERR(job);
266 }
267
268 xe_sched_job_arm(job);
269 fence = dma_fence_get(&job->drm.s_fence->finished);
270 xe_sched_job_push(job);
271
272 timeout = dma_fence_wait_timeout(fence, false, HZ);
273 dma_fence_put(fence);
274 xe_bb_free(bb, NULL);
275 if (timeout < 0)
276 return timeout;
277 else if (!timeout)
278 return -ETIME;
279
280 return 0;
281 }
282
xe_gt_record_default_lrcs(struct xe_gt * gt)283 int xe_gt_record_default_lrcs(struct xe_gt *gt)
284 {
285 struct xe_device *xe = gt_to_xe(gt);
286 struct xe_hw_engine *hwe;
287 enum xe_hw_engine_id id;
288 int err = 0;
289
290 for_each_hw_engine(hwe, gt, id) {
291 struct xe_exec_queue *q, *nop_q;
292 void *default_lrc;
293
294 if (gt->default_lrc[hwe->class])
295 continue;
296
297 xe_reg_sr_init(&hwe->reg_lrc, hwe->name, xe);
298 xe_wa_process_lrc(hwe);
299 xe_hw_engine_setup_default_lrc_state(hwe);
300 xe_tuning_process_lrc(hwe);
301
302 default_lrc = drmm_kzalloc(&xe->drm,
303 xe_gt_lrc_size(gt, hwe->class),
304 GFP_KERNEL);
305 if (!default_lrc)
306 return -ENOMEM;
307
308 q = xe_exec_queue_create(xe, NULL, BIT(hwe->logical_instance), 1,
309 hwe, EXEC_QUEUE_FLAG_KERNEL, 0);
310 if (IS_ERR(q)) {
311 err = PTR_ERR(q);
312 xe_gt_err(gt, "hwe %s: xe_exec_queue_create failed (%pe)\n",
313 hwe->name, q);
314 return err;
315 }
316
317 /* Prime golden LRC with known good state */
318 err = emit_wa_job(gt, q);
319 if (err) {
320 xe_gt_err(gt, "hwe %s: emit_wa_job failed (%pe) guc_id=%u\n",
321 hwe->name, ERR_PTR(err), q->guc->id);
322 goto put_exec_queue;
323 }
324
325 nop_q = xe_exec_queue_create(xe, NULL, BIT(hwe->logical_instance),
326 1, hwe, EXEC_QUEUE_FLAG_KERNEL, 0);
327 if (IS_ERR(nop_q)) {
328 err = PTR_ERR(nop_q);
329 xe_gt_err(gt, "hwe %s: nop xe_exec_queue_create failed (%pe)\n",
330 hwe->name, nop_q);
331 goto put_exec_queue;
332 }
333
334 /* Switch to different LRC */
335 err = emit_nop_job(gt, nop_q);
336 if (err) {
337 xe_gt_err(gt, "hwe %s: nop emit_nop_job failed (%pe) guc_id=%u\n",
338 hwe->name, ERR_PTR(err), nop_q->guc->id);
339 goto put_nop_q;
340 }
341
342 /* Reload golden LRC to record the effect of any indirect W/A */
343 err = emit_nop_job(gt, q);
344 if (err) {
345 xe_gt_err(gt, "hwe %s: emit_nop_job failed (%pe) guc_id=%u\n",
346 hwe->name, ERR_PTR(err), q->guc->id);
347 goto put_nop_q;
348 }
349
350 xe_map_memcpy_from(xe, default_lrc,
351 &q->lrc[0]->bo->vmap,
352 xe_lrc_pphwsp_offset(q->lrc[0]),
353 xe_gt_lrc_size(gt, hwe->class));
354
355 gt->default_lrc[hwe->class] = default_lrc;
356 put_nop_q:
357 xe_exec_queue_put(nop_q);
358 put_exec_queue:
359 xe_exec_queue_put(q);
360 if (err)
361 break;
362 }
363
364 return err;
365 }
366
xe_gt_init_early(struct xe_gt * gt)367 int xe_gt_init_early(struct xe_gt *gt)
368 {
369 int err;
370
371 if (IS_SRIOV_PF(gt_to_xe(gt))) {
372 err = xe_gt_sriov_pf_init_early(gt);
373 if (err)
374 return err;
375 }
376
377 xe_reg_sr_init(>->reg_sr, "GT", gt_to_xe(gt));
378
379 err = xe_wa_init(gt);
380 if (err)
381 return err;
382
383 xe_wa_process_oob(gt);
384
385 xe_force_wake_init_gt(gt, gt_to_fw(gt));
386 spin_lock_init(>->global_invl_lock);
387
388 err = xe_gt_tlb_invalidation_init_early(gt);
389 if (err)
390 return err;
391
392 return 0;
393 }
394
dump_pat_on_error(struct xe_gt * gt)395 static void dump_pat_on_error(struct xe_gt *gt)
396 {
397 struct drm_printer p;
398 char prefix[32];
399
400 snprintf(prefix, sizeof(prefix), "[GT%u Error]", gt->info.id);
401 p = drm_dbg_printer(>_to_xe(gt)->drm, DRM_UT_DRIVER, prefix);
402
403 xe_pat_dump(gt, &p);
404 }
405
gt_fw_domain_init(struct xe_gt * gt)406 static int gt_fw_domain_init(struct xe_gt *gt)
407 {
408 unsigned int fw_ref;
409 int err, i;
410
411 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
412 if (!fw_ref) {
413 err = -ETIMEDOUT;
414 goto err_hw_fence_irq;
415 }
416
417 if (!xe_gt_is_media_type(gt)) {
418 err = xe_ggtt_init(gt_to_tile(gt)->mem.ggtt);
419 if (err)
420 goto err_force_wake;
421 if (IS_SRIOV_PF(gt_to_xe(gt)))
422 xe_lmtt_init(>_to_tile(gt)->sriov.pf.lmtt);
423 }
424
425 /* Enable per hw engine IRQs */
426 xe_irq_enable_hwe(gt);
427
428 /* Rerun MCR init as we now have hw engine list */
429 xe_gt_mcr_init(gt);
430
431 err = xe_hw_engines_init_early(gt);
432 if (err)
433 goto err_force_wake;
434
435 err = xe_hw_engine_class_sysfs_init(gt);
436 if (err)
437 goto err_force_wake;
438
439 /* Initialize CCS mode sysfs after early initialization of HW engines */
440 err = xe_gt_ccs_mode_sysfs_init(gt);
441 if (err)
442 goto err_force_wake;
443
444 /*
445 * Stash hardware-reported version. Since this register does not exist
446 * on pre-MTL platforms, reading it there will (correctly) return 0.
447 */
448 gt->info.gmdid = xe_mmio_read32(>->mmio, GMD_ID);
449
450 xe_force_wake_put(gt_to_fw(gt), fw_ref);
451 return 0;
452
453 err_force_wake:
454 dump_pat_on_error(gt);
455 xe_force_wake_put(gt_to_fw(gt), fw_ref);
456 err_hw_fence_irq:
457 for (i = 0; i < XE_ENGINE_CLASS_MAX; ++i)
458 xe_hw_fence_irq_finish(>->fence_irq[i]);
459
460 return err;
461 }
462
all_fw_domain_init(struct xe_gt * gt)463 static int all_fw_domain_init(struct xe_gt *gt)
464 {
465 unsigned int fw_ref;
466 int err, i;
467
468 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
469 if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL)) {
470 err = -ETIMEDOUT;
471 goto err_force_wake;
472 }
473
474 xe_gt_mcr_set_implicit_defaults(gt);
475 xe_wa_process_gt(gt);
476 xe_tuning_process_gt(gt);
477 xe_reg_sr_apply_mmio(>->reg_sr, gt);
478
479 err = xe_gt_clock_init(gt);
480 if (err)
481 goto err_force_wake;
482
483 xe_mocs_init(gt);
484 err = xe_execlist_init(gt);
485 if (err)
486 goto err_force_wake;
487
488 err = xe_hw_engines_init(gt);
489 if (err)
490 goto err_force_wake;
491
492 err = xe_uc_init_post_hwconfig(>->uc);
493 if (err)
494 goto err_force_wake;
495
496 if (!xe_gt_is_media_type(gt)) {
497 /*
498 * USM has its only SA pool to non-block behind user operations
499 */
500 if (gt_to_xe(gt)->info.has_usm) {
501 struct xe_device *xe = gt_to_xe(gt);
502
503 gt->usm.bb_pool = xe_sa_bo_manager_init(gt_to_tile(gt),
504 IS_DGFX(xe) ? SZ_1M : SZ_512K, 16);
505 if (IS_ERR(gt->usm.bb_pool)) {
506 err = PTR_ERR(gt->usm.bb_pool);
507 goto err_force_wake;
508 }
509 }
510 }
511
512 if (!xe_gt_is_media_type(gt)) {
513 struct xe_tile *tile = gt_to_tile(gt);
514
515 tile->migrate = xe_migrate_init(tile);
516 if (IS_ERR(tile->migrate)) {
517 err = PTR_ERR(tile->migrate);
518 goto err_force_wake;
519 }
520 }
521
522 err = xe_uc_init_hw(>->uc);
523 if (err)
524 goto err_force_wake;
525
526 /* Configure default CCS mode of 1 engine with all resources */
527 if (xe_gt_ccs_mode_enabled(gt)) {
528 gt->ccs_mode = 1;
529 xe_gt_apply_ccs_mode(gt);
530 }
531
532 if (IS_SRIOV_PF(gt_to_xe(gt)) && !xe_gt_is_media_type(gt))
533 xe_lmtt_init_hw(>_to_tile(gt)->sriov.pf.lmtt);
534
535 if (IS_SRIOV_PF(gt_to_xe(gt))) {
536 xe_gt_sriov_pf_init(gt);
537 xe_gt_sriov_pf_init_hw(gt);
538 }
539
540 xe_force_wake_put(gt_to_fw(gt), fw_ref);
541
542 return 0;
543
544 err_force_wake:
545 xe_force_wake_put(gt_to_fw(gt), fw_ref);
546 for (i = 0; i < XE_ENGINE_CLASS_MAX; ++i)
547 xe_hw_fence_irq_finish(>->fence_irq[i]);
548
549 return err;
550 }
551
552 /*
553 * Initialize enough GT to be able to load GuC in order to obtain hwconfig and
554 * enable CTB communication.
555 */
xe_gt_init_hwconfig(struct xe_gt * gt)556 int xe_gt_init_hwconfig(struct xe_gt *gt)
557 {
558 unsigned int fw_ref;
559 int err;
560
561 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
562 if (!fw_ref)
563 return -ETIMEDOUT;
564
565 xe_gt_mcr_init_early(gt);
566 xe_pat_init(gt);
567
568 err = xe_uc_init(>->uc);
569 if (err)
570 goto out_fw;
571
572 err = xe_uc_init_hwconfig(>->uc);
573 if (err)
574 goto out_fw;
575
576 xe_gt_topology_init(gt);
577 xe_gt_mcr_init(gt);
578 xe_gt_enable_host_l2_vram(gt);
579
580 out_fw:
581 xe_force_wake_put(gt_to_fw(gt), fw_ref);
582 return err;
583 }
584
xe_gt_init(struct xe_gt * gt)585 int xe_gt_init(struct xe_gt *gt)
586 {
587 int err;
588 int i;
589
590 INIT_WORK(>->reset.worker, gt_reset_worker);
591
592 for (i = 0; i < XE_ENGINE_CLASS_MAX; ++i) {
593 gt->ring_ops[i] = xe_ring_ops_get(gt, i);
594 xe_hw_fence_irq_init(>->fence_irq[i]);
595 }
596
597 err = xe_gt_pagefault_init(gt);
598 if (err)
599 return err;
600
601 xe_mocs_init_early(gt);
602
603 err = xe_gt_sysfs_init(gt);
604 if (err)
605 return err;
606
607 err = gt_fw_domain_init(gt);
608 if (err)
609 return err;
610
611 err = xe_gt_idle_init(>->gtidle);
612 if (err)
613 return err;
614
615 err = xe_gt_freq_init(gt);
616 if (err)
617 return err;
618
619 xe_force_wake_init_engines(gt, gt_to_fw(gt));
620
621 err = all_fw_domain_init(gt);
622 if (err)
623 return err;
624
625 xe_gt_record_user_engines(gt);
626
627 return 0;
628 }
629
630 /**
631 * xe_gt_mmio_init() - Initialize GT's MMIO access
632 * @gt: the GT object
633 *
634 * Initialize GT's MMIO accessor, which will be used to access registers inside
635 * this GT.
636 */
xe_gt_mmio_init(struct xe_gt * gt)637 void xe_gt_mmio_init(struct xe_gt *gt)
638 {
639 struct xe_tile *tile = gt_to_tile(gt);
640
641 gt->mmio.regs = tile->mmio.regs;
642 gt->mmio.regs_size = tile->mmio.regs_size;
643 gt->mmio.tile = tile;
644
645 if (gt->info.type == XE_GT_TYPE_MEDIA) {
646 gt->mmio.adj_offset = MEDIA_GT_GSI_OFFSET;
647 gt->mmio.adj_limit = MEDIA_GT_GSI_LENGTH;
648 }
649
650 if (IS_SRIOV_VF(gt_to_xe(gt)))
651 gt->mmio.sriov_vf_gt = gt;
652 }
653
xe_gt_record_user_engines(struct xe_gt * gt)654 void xe_gt_record_user_engines(struct xe_gt *gt)
655 {
656 struct xe_hw_engine *hwe;
657 enum xe_hw_engine_id id;
658
659 gt->user_engines.mask = 0;
660 memset(gt->user_engines.instances_per_class, 0,
661 sizeof(gt->user_engines.instances_per_class));
662
663 for_each_hw_engine(hwe, gt, id) {
664 if (xe_hw_engine_is_reserved(hwe))
665 continue;
666
667 gt->user_engines.mask |= BIT_ULL(id);
668 gt->user_engines.instances_per_class[hwe->class]++;
669 }
670
671 xe_gt_assert(gt, (gt->user_engines.mask | gt->info.engine_mask)
672 == gt->info.engine_mask);
673 }
674
do_gt_reset(struct xe_gt * gt)675 static int do_gt_reset(struct xe_gt *gt)
676 {
677 int err;
678
679 xe_gsc_wa_14015076503(gt, true);
680
681 xe_mmio_write32(>->mmio, GDRST, GRDOM_FULL);
682 err = xe_mmio_wait32(>->mmio, GDRST, GRDOM_FULL, 0, 5000, NULL, false);
683 if (err)
684 xe_gt_err(gt, "failed to clear GRDOM_FULL (%pe)\n",
685 ERR_PTR(err));
686
687 xe_gsc_wa_14015076503(gt, false);
688
689 return err;
690 }
691
vf_gt_restart(struct xe_gt * gt)692 static int vf_gt_restart(struct xe_gt *gt)
693 {
694 int err;
695
696 err = xe_uc_sanitize_reset(>->uc);
697 if (err)
698 return err;
699
700 err = xe_uc_init_hw(>->uc);
701 if (err)
702 return err;
703
704 err = xe_uc_start(>->uc);
705 if (err)
706 return err;
707
708 return 0;
709 }
710
do_gt_restart(struct xe_gt * gt)711 static int do_gt_restart(struct xe_gt *gt)
712 {
713 struct xe_hw_engine *hwe;
714 enum xe_hw_engine_id id;
715 int err;
716
717 if (IS_SRIOV_VF(gt_to_xe(gt)))
718 return vf_gt_restart(gt);
719
720 xe_pat_init(gt);
721
722 xe_gt_enable_host_l2_vram(gt);
723
724 xe_gt_mcr_set_implicit_defaults(gt);
725 xe_reg_sr_apply_mmio(>->reg_sr, gt);
726
727 err = xe_wopcm_init(>->uc.wopcm);
728 if (err)
729 return err;
730
731 for_each_hw_engine(hwe, gt, id)
732 xe_hw_engine_enable_ring(hwe);
733
734 err = xe_uc_sanitize_reset(>->uc);
735 if (err)
736 return err;
737
738 err = xe_uc_init_hw(>->uc);
739 if (err)
740 return err;
741
742 if (IS_SRIOV_PF(gt_to_xe(gt)) && !xe_gt_is_media_type(gt))
743 xe_lmtt_init_hw(>_to_tile(gt)->sriov.pf.lmtt);
744
745 if (IS_SRIOV_PF(gt_to_xe(gt)))
746 xe_gt_sriov_pf_init_hw(gt);
747
748 xe_mocs_init(gt);
749 err = xe_uc_start(>->uc);
750 if (err)
751 return err;
752
753 for_each_hw_engine(hwe, gt, id)
754 xe_reg_sr_apply_mmio(&hwe->reg_sr, gt);
755
756 /* Get CCS mode in sync between sw/hw */
757 xe_gt_apply_ccs_mode(gt);
758
759 /* Restore GT freq to expected values */
760 xe_gt_sanitize_freq(gt);
761
762 if (IS_SRIOV_PF(gt_to_xe(gt)))
763 xe_gt_sriov_pf_restart(gt);
764
765 return 0;
766 }
767
gt_reset(struct xe_gt * gt)768 static int gt_reset(struct xe_gt *gt)
769 {
770 unsigned int fw_ref;
771 int err;
772
773 if (xe_device_wedged(gt_to_xe(gt)))
774 return -ECANCELED;
775
776 /* We only support GT resets with GuC submission */
777 if (!xe_device_uc_enabled(gt_to_xe(gt)))
778 return -ENODEV;
779
780 xe_gt_info(gt, "reset started\n");
781
782 xe_pm_runtime_get(gt_to_xe(gt));
783
784 if (xe_fault_inject_gt_reset()) {
785 err = -ECANCELED;
786 goto err_fail;
787 }
788
789 xe_gt_sanitize(gt);
790
791 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
792 if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL)) {
793 err = -ETIMEDOUT;
794 goto err_out;
795 }
796
797 xe_uc_gucrc_disable(>->uc);
798 xe_uc_stop_prepare(>->uc);
799 xe_gt_pagefault_reset(gt);
800
801 xe_uc_stop(>->uc);
802
803 xe_gt_tlb_invalidation_reset(gt);
804
805 err = do_gt_reset(gt);
806 if (err)
807 goto err_out;
808
809 err = do_gt_restart(gt);
810 if (err)
811 goto err_out;
812
813 xe_force_wake_put(gt_to_fw(gt), fw_ref);
814 xe_pm_runtime_put(gt_to_xe(gt));
815
816 xe_gt_info(gt, "reset done\n");
817
818 return 0;
819
820 err_out:
821 xe_force_wake_put(gt_to_fw(gt), fw_ref);
822 XE_WARN_ON(xe_uc_start(>->uc));
823 err_fail:
824 xe_gt_err(gt, "reset failed (%pe)\n", ERR_PTR(err));
825
826 xe_device_declare_wedged(gt_to_xe(gt));
827 xe_pm_runtime_put(gt_to_xe(gt));
828
829 return err;
830 }
831
gt_reset_worker(struct work_struct * w)832 static void gt_reset_worker(struct work_struct *w)
833 {
834 struct xe_gt *gt = container_of(w, typeof(*gt), reset.worker);
835
836 gt_reset(gt);
837 }
838
xe_gt_reset_async(struct xe_gt * gt)839 void xe_gt_reset_async(struct xe_gt *gt)
840 {
841 xe_gt_info(gt, "trying reset from %ps\n", __builtin_return_address(0));
842
843 /* Don't do a reset while one is already in flight */
844 if (!xe_fault_inject_gt_reset() && xe_uc_reset_prepare(>->uc))
845 return;
846
847 xe_gt_info(gt, "reset queued\n");
848 queue_work(gt->ordered_wq, >->reset.worker);
849 }
850
xe_gt_suspend_prepare(struct xe_gt * gt)851 void xe_gt_suspend_prepare(struct xe_gt *gt)
852 {
853 unsigned int fw_ref;
854
855 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
856
857 xe_uc_stop_prepare(>->uc);
858
859 xe_force_wake_put(gt_to_fw(gt), fw_ref);
860 }
861
xe_gt_suspend(struct xe_gt * gt)862 int xe_gt_suspend(struct xe_gt *gt)
863 {
864 unsigned int fw_ref;
865 int err;
866
867 xe_gt_dbg(gt, "suspending\n");
868 xe_gt_sanitize(gt);
869
870 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
871 if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL))
872 goto err_msg;
873
874 err = xe_uc_suspend(>->uc);
875 if (err)
876 goto err_force_wake;
877
878 xe_gt_idle_disable_pg(gt);
879
880 xe_gt_disable_host_l2_vram(gt);
881
882 xe_force_wake_put(gt_to_fw(gt), fw_ref);
883 xe_gt_dbg(gt, "suspended\n");
884
885 return 0;
886
887 err_msg:
888 err = -ETIMEDOUT;
889 err_force_wake:
890 xe_force_wake_put(gt_to_fw(gt), fw_ref);
891 xe_gt_err(gt, "suspend failed (%pe)\n", ERR_PTR(err));
892
893 return err;
894 }
895
xe_gt_shutdown(struct xe_gt * gt)896 void xe_gt_shutdown(struct xe_gt *gt)
897 {
898 unsigned int fw_ref;
899
900 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
901 do_gt_reset(gt);
902 xe_force_wake_put(gt_to_fw(gt), fw_ref);
903 }
904
905 /**
906 * xe_gt_sanitize_freq() - Restore saved frequencies if necessary.
907 * @gt: the GT object
908 *
909 * Called after driver init/GSC load completes to restore GT frequencies if we
910 * limited them for any WAs.
911 */
xe_gt_sanitize_freq(struct xe_gt * gt)912 int xe_gt_sanitize_freq(struct xe_gt *gt)
913 {
914 int ret = 0;
915
916 if ((!xe_uc_fw_is_available(>->uc.gsc.fw) ||
917 xe_uc_fw_is_loaded(>->uc.gsc.fw) ||
918 xe_uc_fw_is_in_error_state(>->uc.gsc.fw)) &&
919 XE_WA(gt, 22019338487))
920 ret = xe_guc_pc_restore_stashed_freq(>->uc.guc.pc);
921
922 return ret;
923 }
924
xe_gt_resume(struct xe_gt * gt)925 int xe_gt_resume(struct xe_gt *gt)
926 {
927 unsigned int fw_ref;
928 int err;
929
930 xe_gt_dbg(gt, "resuming\n");
931 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
932 if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL))
933 goto err_msg;
934
935 err = do_gt_restart(gt);
936 if (err)
937 goto err_force_wake;
938
939 xe_gt_idle_enable_pg(gt);
940
941 xe_force_wake_put(gt_to_fw(gt), fw_ref);
942 xe_gt_dbg(gt, "resumed\n");
943
944 return 0;
945
946 err_msg:
947 err = -ETIMEDOUT;
948 err_force_wake:
949 xe_force_wake_put(gt_to_fw(gt), fw_ref);
950 xe_gt_err(gt, "resume failed (%pe)\n", ERR_PTR(err));
951
952 return err;
953 }
954
xe_gt_hw_engine(struct xe_gt * gt,enum xe_engine_class class,u16 instance,bool logical)955 struct xe_hw_engine *xe_gt_hw_engine(struct xe_gt *gt,
956 enum xe_engine_class class,
957 u16 instance, bool logical)
958 {
959 struct xe_hw_engine *hwe;
960 enum xe_hw_engine_id id;
961
962 for_each_hw_engine(hwe, gt, id)
963 if (hwe->class == class &&
964 ((!logical && hwe->instance == instance) ||
965 (logical && hwe->logical_instance == instance)))
966 return hwe;
967
968 return NULL;
969 }
970
xe_gt_any_hw_engine_by_reset_domain(struct xe_gt * gt,enum xe_engine_class class)971 struct xe_hw_engine *xe_gt_any_hw_engine_by_reset_domain(struct xe_gt *gt,
972 enum xe_engine_class class)
973 {
974 struct xe_hw_engine *hwe;
975 enum xe_hw_engine_id id;
976
977 for_each_hw_engine(hwe, gt, id) {
978 switch (class) {
979 case XE_ENGINE_CLASS_RENDER:
980 case XE_ENGINE_CLASS_COMPUTE:
981 if (hwe->class == XE_ENGINE_CLASS_RENDER ||
982 hwe->class == XE_ENGINE_CLASS_COMPUTE)
983 return hwe;
984 break;
985 default:
986 if (hwe->class == class)
987 return hwe;
988 }
989 }
990
991 return NULL;
992 }
993
xe_gt_any_hw_engine(struct xe_gt * gt)994 struct xe_hw_engine *xe_gt_any_hw_engine(struct xe_gt *gt)
995 {
996 struct xe_hw_engine *hwe;
997 enum xe_hw_engine_id id;
998
999 for_each_hw_engine(hwe, gt, id)
1000 return hwe;
1001
1002 return NULL;
1003 }
1004
1005 /**
1006 * xe_gt_declare_wedged() - Declare GT wedged
1007 * @gt: the GT object
1008 *
1009 * Wedge the GT which stops all submission, saves desired debug state, and
1010 * cleans up anything which could timeout.
1011 */
xe_gt_declare_wedged(struct xe_gt * gt)1012 void xe_gt_declare_wedged(struct xe_gt *gt)
1013 {
1014 xe_gt_assert(gt, gt_to_xe(gt)->wedged.mode);
1015
1016 xe_uc_declare_wedged(>->uc);
1017 xe_gt_tlb_invalidation_reset(gt);
1018 }
1019