1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include <linux/minmax.h> 7 8 #include <drm/drm_managed.h> 9 10 #include "xe_bb.h" 11 #include "xe_bo.h" 12 #include "xe_device.h" 13 #include "xe_engine.h" 14 #include "xe_execlist.h" 15 #include "xe_force_wake.h" 16 #include "xe_ggtt.h" 17 #include "xe_gt.h" 18 #include "xe_gt_clock.h" 19 #include "xe_gt_mcr.h" 20 #include "xe_gt_pagefault.h" 21 #include "xe_gt_sysfs.h" 22 #include "xe_gt_topology.h" 23 #include "xe_hw_fence.h" 24 #include "xe_irq.h" 25 #include "xe_lrc.h" 26 #include "xe_map.h" 27 #include "xe_migrate.h" 28 #include "xe_mmio.h" 29 #include "xe_mocs.h" 30 #include "xe_reg_sr.h" 31 #include "xe_ring_ops.h" 32 #include "xe_sa.h" 33 #include "xe_sched_job.h" 34 #include "xe_ttm_gtt_mgr.h" 35 #include "xe_ttm_vram_mgr.h" 36 #include "xe_tuning.h" 37 #include "xe_uc.h" 38 #include "xe_vm.h" 39 #include "xe_wa.h" 40 #include "xe_wopcm.h" 41 42 #include "gt/intel_gt_regs.h" 43 44 struct xe_gt *xe_find_full_gt(struct xe_gt *gt) 45 { 46 struct xe_gt *search; 47 u8 id; 48 49 XE_BUG_ON(!xe_gt_is_media_type(gt)); 50 51 for_each_gt(search, gt_to_xe(gt), id) { 52 if (search->info.vram_id == gt->info.vram_id) 53 return search; 54 } 55 56 XE_BUG_ON("NOT POSSIBLE"); 57 return NULL; 58 } 59 60 int xe_gt_alloc(struct xe_device *xe, struct xe_gt *gt) 61 { 62 struct drm_device *drm = &xe->drm; 63 64 XE_BUG_ON(gt->info.type == XE_GT_TYPE_UNINITIALIZED); 65 66 if (!xe_gt_is_media_type(gt)) { 67 gt->mem.ggtt = drmm_kzalloc(drm, sizeof(*gt->mem.ggtt), 68 GFP_KERNEL); 69 if (!gt->mem.ggtt) 70 return -ENOMEM; 71 72 gt->mem.vram_mgr = drmm_kzalloc(drm, sizeof(*gt->mem.vram_mgr), 73 GFP_KERNEL); 74 if (!gt->mem.vram_mgr) 75 return -ENOMEM; 76 77 gt->mem.gtt_mgr = drmm_kzalloc(drm, sizeof(*gt->mem.gtt_mgr), 78 GFP_KERNEL); 79 if (!gt->mem.gtt_mgr) 80 return -ENOMEM; 81 } else { 82 struct xe_gt *full_gt = xe_find_full_gt(gt); 83 84 gt->mem.ggtt = full_gt->mem.ggtt; 85 gt->mem.vram_mgr = full_gt->mem.vram_mgr; 86 gt->mem.gtt_mgr = full_gt->mem.gtt_mgr; 87 } 88 89 gt->ordered_wq = alloc_ordered_workqueue("gt-ordered-wq", 0); 90 91 return 0; 92 } 93 94 /* FIXME: These should be in a common file */ 95 #define CHV_PPAT_SNOOP REG_BIT(6) 96 #define GEN8_PPAT_AGE(x) ((x)<<4) 97 #define GEN8_PPAT_LLCeLLC (3<<2) 98 #define GEN8_PPAT_LLCELLC (2<<2) 99 #define GEN8_PPAT_LLC (1<<2) 100 #define GEN8_PPAT_WB (3<<0) 101 #define GEN8_PPAT_WT (2<<0) 102 #define GEN8_PPAT_WC (1<<0) 103 #define GEN8_PPAT_UC (0<<0) 104 #define GEN8_PPAT_ELLC_OVERRIDE (0<<2) 105 #define GEN8_PPAT(i, x) ((u64)(x) << ((i) * 8)) 106 #define GEN12_PPAT_CLOS(x) ((x)<<2) 107 108 static void tgl_setup_private_ppat(struct xe_gt *gt) 109 { 110 /* TGL doesn't support LLC or AGE settings */ 111 xe_mmio_write32(gt, GEN12_PAT_INDEX(0).reg, GEN8_PPAT_WB); 112 xe_mmio_write32(gt, GEN12_PAT_INDEX(1).reg, GEN8_PPAT_WC); 113 xe_mmio_write32(gt, GEN12_PAT_INDEX(2).reg, GEN8_PPAT_WT); 114 xe_mmio_write32(gt, GEN12_PAT_INDEX(3).reg, GEN8_PPAT_UC); 115 xe_mmio_write32(gt, GEN12_PAT_INDEX(4).reg, GEN8_PPAT_WB); 116 xe_mmio_write32(gt, GEN12_PAT_INDEX(5).reg, GEN8_PPAT_WB); 117 xe_mmio_write32(gt, GEN12_PAT_INDEX(6).reg, GEN8_PPAT_WB); 118 xe_mmio_write32(gt, GEN12_PAT_INDEX(7).reg, GEN8_PPAT_WB); 119 } 120 121 static void pvc_setup_private_ppat(struct xe_gt *gt) 122 { 123 xe_mmio_write32(gt, GEN12_PAT_INDEX(0).reg, GEN8_PPAT_UC); 124 xe_mmio_write32(gt, GEN12_PAT_INDEX(1).reg, GEN8_PPAT_WC); 125 xe_mmio_write32(gt, GEN12_PAT_INDEX(2).reg, GEN8_PPAT_WT); 126 xe_mmio_write32(gt, GEN12_PAT_INDEX(3).reg, GEN8_PPAT_WB); 127 xe_mmio_write32(gt, GEN12_PAT_INDEX(4).reg, 128 GEN12_PPAT_CLOS(1) | GEN8_PPAT_WT); 129 xe_mmio_write32(gt, GEN12_PAT_INDEX(5).reg, 130 GEN12_PPAT_CLOS(1) | GEN8_PPAT_WB); 131 xe_mmio_write32(gt, GEN12_PAT_INDEX(6).reg, 132 GEN12_PPAT_CLOS(2) | GEN8_PPAT_WT); 133 xe_mmio_write32(gt, GEN12_PAT_INDEX(7).reg, 134 GEN12_PPAT_CLOS(2) | GEN8_PPAT_WB); 135 } 136 137 #define MTL_PPAT_L4_CACHE_POLICY_MASK REG_GENMASK(3, 2) 138 #define MTL_PAT_INDEX_COH_MODE_MASK REG_GENMASK(1, 0) 139 #define MTL_PPAT_3_UC REG_FIELD_PREP(MTL_PPAT_L4_CACHE_POLICY_MASK, 3) 140 #define MTL_PPAT_1_WT REG_FIELD_PREP(MTL_PPAT_L4_CACHE_POLICY_MASK, 1) 141 #define MTL_PPAT_0_WB REG_FIELD_PREP(MTL_PPAT_L4_CACHE_POLICY_MASK, 0) 142 #define MTL_3_COH_2W REG_FIELD_PREP(MTL_PAT_INDEX_COH_MODE_MASK, 3) 143 #define MTL_2_COH_1W REG_FIELD_PREP(MTL_PAT_INDEX_COH_MODE_MASK, 2) 144 #define MTL_0_COH_NON REG_FIELD_PREP(MTL_PAT_INDEX_COH_MODE_MASK, 0) 145 146 static void mtl_setup_private_ppat(struct xe_gt *gt) 147 { 148 xe_mmio_write32(gt, GEN12_PAT_INDEX(0).reg, MTL_PPAT_0_WB); 149 xe_mmio_write32(gt, GEN12_PAT_INDEX(1).reg, 150 MTL_PPAT_1_WT | MTL_2_COH_1W); 151 xe_mmio_write32(gt, GEN12_PAT_INDEX(2).reg, 152 MTL_PPAT_3_UC | MTL_2_COH_1W); 153 xe_mmio_write32(gt, GEN12_PAT_INDEX(3).reg, 154 MTL_PPAT_0_WB | MTL_2_COH_1W); 155 xe_mmio_write32(gt, GEN12_PAT_INDEX(4).reg, 156 MTL_PPAT_0_WB | MTL_3_COH_2W); 157 } 158 159 static void setup_private_ppat(struct xe_gt *gt) 160 { 161 struct xe_device *xe = gt_to_xe(gt); 162 163 if (xe->info.platform == XE_METEORLAKE) 164 mtl_setup_private_ppat(gt); 165 else if (xe->info.platform == XE_PVC) 166 pvc_setup_private_ppat(gt); 167 else 168 tgl_setup_private_ppat(gt); 169 } 170 171 static int gt_ttm_mgr_init(struct xe_gt *gt) 172 { 173 struct xe_device *xe = gt_to_xe(gt); 174 int err; 175 struct sysinfo si; 176 u64 gtt_size; 177 178 si_meminfo(&si); 179 gtt_size = (u64)si.totalram * si.mem_unit * 3/4; 180 181 if (gt->mem.vram.size) { 182 err = xe_ttm_vram_mgr_init(gt, gt->mem.vram_mgr); 183 if (err) 184 return err; 185 gtt_size = min(max((XE_DEFAULT_GTT_SIZE_MB << 20), 186 gt->mem.vram.size), 187 gtt_size); 188 xe->info.mem_region_mask |= BIT(gt->info.vram_id) << 1; 189 } 190 191 err = xe_ttm_gtt_mgr_init(gt, gt->mem.gtt_mgr, gtt_size); 192 if (err) 193 return err; 194 195 return 0; 196 } 197 198 static void gt_fini(struct drm_device *drm, void *arg) 199 { 200 struct xe_gt *gt = arg; 201 int i; 202 203 destroy_workqueue(gt->ordered_wq); 204 205 for (i = 0; i < XE_ENGINE_CLASS_MAX; ++i) 206 xe_hw_fence_irq_finish(>->fence_irq[i]); 207 } 208 209 static void gt_reset_worker(struct work_struct *w); 210 211 int emit_nop_job(struct xe_gt *gt, struct xe_engine *e) 212 { 213 struct xe_sched_job *job; 214 struct xe_bb *bb; 215 struct dma_fence *fence; 216 u64 batch_ofs; 217 long timeout; 218 219 bb = xe_bb_new(gt, 4, false); 220 if (IS_ERR(bb)) 221 return PTR_ERR(bb); 222 223 batch_ofs = xe_bo_ggtt_addr(gt->kernel_bb_pool.bo); 224 job = xe_bb_create_wa_job(e, bb, batch_ofs); 225 if (IS_ERR(job)) { 226 xe_bb_free(bb, NULL); 227 return PTR_ERR(bb); 228 } 229 230 xe_sched_job_arm(job); 231 fence = dma_fence_get(&job->drm.s_fence->finished); 232 xe_sched_job_push(job); 233 234 timeout = dma_fence_wait_timeout(fence, false, HZ); 235 dma_fence_put(fence); 236 xe_bb_free(bb, NULL); 237 if (timeout < 0) 238 return timeout; 239 else if (!timeout) 240 return -ETIME; 241 242 return 0; 243 } 244 245 int emit_wa_job(struct xe_gt *gt, struct xe_engine *e) 246 { 247 struct xe_reg_sr *sr = &e->hwe->reg_lrc; 248 struct xe_reg_sr_entry *entry; 249 unsigned long reg; 250 struct xe_sched_job *job; 251 struct xe_bb *bb; 252 struct dma_fence *fence; 253 u64 batch_ofs; 254 long timeout; 255 int count = 0; 256 257 bb = xe_bb_new(gt, SZ_4K, false); /* Just pick a large BB size */ 258 if (IS_ERR(bb)) 259 return PTR_ERR(bb); 260 261 xa_for_each(&sr->xa, reg, entry) 262 ++count; 263 264 if (count) { 265 bb->cs[bb->len++] = MI_LOAD_REGISTER_IMM(count); 266 xa_for_each(&sr->xa, reg, entry) { 267 bb->cs[bb->len++] = reg; 268 bb->cs[bb->len++] = entry->set_bits; 269 } 270 } 271 bb->cs[bb->len++] = MI_NOOP; 272 bb->cs[bb->len++] = MI_BATCH_BUFFER_END; 273 274 batch_ofs = xe_bo_ggtt_addr(gt->kernel_bb_pool.bo); 275 job = xe_bb_create_wa_job(e, bb, batch_ofs); 276 if (IS_ERR(job)) { 277 xe_bb_free(bb, NULL); 278 return PTR_ERR(bb); 279 } 280 281 xe_sched_job_arm(job); 282 fence = dma_fence_get(&job->drm.s_fence->finished); 283 xe_sched_job_push(job); 284 285 timeout = dma_fence_wait_timeout(fence, false, HZ); 286 dma_fence_put(fence); 287 xe_bb_free(bb, NULL); 288 if (timeout < 0) 289 return timeout; 290 else if (!timeout) 291 return -ETIME; 292 293 return 0; 294 } 295 296 int xe_gt_record_default_lrcs(struct xe_gt *gt) 297 { 298 struct xe_device *xe = gt_to_xe(gt); 299 struct xe_hw_engine *hwe; 300 enum xe_hw_engine_id id; 301 int err = 0; 302 303 for_each_hw_engine(hwe, gt, id) { 304 struct xe_engine *e, *nop_e; 305 struct xe_vm *vm; 306 void *default_lrc; 307 308 if (gt->default_lrc[hwe->class]) 309 continue; 310 311 xe_reg_sr_init(&hwe->reg_lrc, "LRC", xe); 312 xe_wa_process_lrc(hwe); 313 314 default_lrc = drmm_kzalloc(&xe->drm, 315 xe_lrc_size(xe, hwe->class), 316 GFP_KERNEL); 317 if (!default_lrc) 318 return -ENOMEM; 319 320 vm = xe_migrate_get_vm(gt->migrate); 321 e = xe_engine_create(xe, vm, BIT(hwe->logical_instance), 1, 322 hwe, ENGINE_FLAG_WA); 323 if (IS_ERR(e)) { 324 err = PTR_ERR(e); 325 goto put_vm; 326 } 327 328 /* Prime golden LRC with known good state */ 329 err = emit_wa_job(gt, e); 330 if (err) 331 goto put_engine; 332 333 nop_e = xe_engine_create(xe, vm, BIT(hwe->logical_instance), 334 1, hwe, ENGINE_FLAG_WA); 335 if (IS_ERR(nop_e)) { 336 err = PTR_ERR(nop_e); 337 goto put_engine; 338 } 339 340 /* Switch to different LRC */ 341 err = emit_nop_job(gt, nop_e); 342 if (err) 343 goto put_nop_e; 344 345 /* Reload golden LRC to record the effect of any indirect W/A */ 346 err = emit_nop_job(gt, e); 347 if (err) 348 goto put_nop_e; 349 350 xe_map_memcpy_from(xe, default_lrc, 351 &e->lrc[0].bo->vmap, 352 xe_lrc_pphwsp_offset(&e->lrc[0]), 353 xe_lrc_size(xe, hwe->class)); 354 355 gt->default_lrc[hwe->class] = default_lrc; 356 put_nop_e: 357 xe_engine_put(nop_e); 358 put_engine: 359 xe_engine_put(e); 360 put_vm: 361 xe_vm_put(vm); 362 if (err) 363 break; 364 } 365 366 return err; 367 } 368 369 int xe_gt_init_early(struct xe_gt *gt) 370 { 371 int err; 372 373 xe_force_wake_init_gt(gt, gt_to_fw(gt)); 374 375 err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); 376 if (err) 377 return err; 378 379 xe_gt_topology_init(gt); 380 xe_gt_mcr_init(gt); 381 382 err = xe_force_wake_put(gt_to_fw(gt), XE_FW_GT); 383 if (err) 384 return err; 385 386 xe_reg_sr_init(>->reg_sr, "GT", gt_to_xe(gt)); 387 xe_wa_process_gt(gt); 388 xe_tuning_process_gt(gt); 389 390 return 0; 391 } 392 393 /** 394 * xe_gt_init_noalloc - Init GT up to the point where allocations can happen. 395 * @gt: The GT to initialize. 396 * 397 * This function prepares the GT to allow memory allocations to VRAM, but is not 398 * allowed to allocate memory itself. This state is useful for display readout, 399 * because the inherited display framebuffer will otherwise be overwritten as it 400 * is usually put at the start of VRAM. 401 * 402 * Returns: 0 on success, negative error code on error. 403 */ 404 int xe_gt_init_noalloc(struct xe_gt *gt) 405 { 406 int err, err2; 407 408 if (xe_gt_is_media_type(gt)) 409 return 0; 410 411 xe_device_mem_access_get(gt_to_xe(gt)); 412 err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); 413 if (err) 414 goto err; 415 416 err = gt_ttm_mgr_init(gt); 417 if (err) 418 goto err_force_wake; 419 420 err = xe_ggtt_init_noalloc(gt, gt->mem.ggtt); 421 422 err_force_wake: 423 err2 = xe_force_wake_put(gt_to_fw(gt), XE_FW_GT); 424 XE_WARN_ON(err2); 425 xe_device_mem_access_put(gt_to_xe(gt)); 426 err: 427 return err; 428 } 429 430 static int gt_fw_domain_init(struct xe_gt *gt) 431 { 432 int err, i; 433 434 xe_device_mem_access_get(gt_to_xe(gt)); 435 err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); 436 if (err) 437 goto err_hw_fence_irq; 438 439 if (!xe_gt_is_media_type(gt)) { 440 err = xe_ggtt_init(gt, gt->mem.ggtt); 441 if (err) 442 goto err_force_wake; 443 } 444 445 /* Allow driver to load if uC init fails (likely missing firmware) */ 446 err = xe_uc_init(>->uc); 447 XE_WARN_ON(err); 448 449 err = xe_uc_init_hwconfig(>->uc); 450 if (err) 451 goto err_force_wake; 452 453 /* Enables per hw engine IRQs */ 454 xe_gt_irq_postinstall(gt); 455 456 /* Rerun MCR init as we now have hw engine list */ 457 xe_gt_mcr_init(gt); 458 459 err = xe_hw_engines_init_early(gt); 460 if (err) 461 goto err_force_wake; 462 463 err = xe_force_wake_put(gt_to_fw(gt), XE_FW_GT); 464 XE_WARN_ON(err); 465 xe_device_mem_access_put(gt_to_xe(gt)); 466 467 return 0; 468 469 err_force_wake: 470 xe_force_wake_put(gt_to_fw(gt), XE_FW_GT); 471 err_hw_fence_irq: 472 for (i = 0; i < XE_ENGINE_CLASS_MAX; ++i) 473 xe_hw_fence_irq_finish(>->fence_irq[i]); 474 xe_device_mem_access_put(gt_to_xe(gt)); 475 476 return err; 477 } 478 479 static int all_fw_domain_init(struct xe_gt *gt) 480 { 481 int err, i; 482 483 xe_device_mem_access_get(gt_to_xe(gt)); 484 err = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL); 485 if (err) 486 goto err_hw_fence_irq; 487 488 setup_private_ppat(gt); 489 490 xe_reg_sr_apply_mmio(>->reg_sr, gt); 491 492 err = xe_gt_clock_init(gt); 493 if (err) 494 goto err_force_wake; 495 496 xe_mocs_init(gt); 497 err = xe_execlist_init(gt); 498 if (err) 499 goto err_force_wake; 500 501 err = xe_hw_engines_init(gt); 502 if (err) 503 goto err_force_wake; 504 505 err = xe_uc_init_post_hwconfig(>->uc); 506 if (err) 507 goto err_force_wake; 508 509 /* 510 * FIXME: This should be ok as SA should only be used by gt->migrate and 511 * vm->gt->migrate and both should be pointing to a non-media GT. But to 512 * realy safe, convert gt->kernel_bb_pool to a pointer and point a media 513 * GT to the kernel_bb_pool on a real tile. 514 */ 515 if (!xe_gt_is_media_type(gt)) { 516 err = xe_sa_bo_manager_init(gt, >->kernel_bb_pool, SZ_1M, 16); 517 if (err) 518 goto err_force_wake; 519 520 /* 521 * USM has its only SA pool to non-block behind user operations 522 */ 523 if (gt_to_xe(gt)->info.supports_usm) { 524 err = xe_sa_bo_manager_init(gt, >->usm.bb_pool, 525 SZ_1M, 16); 526 if (err) 527 goto err_force_wake; 528 } 529 } 530 531 if (!xe_gt_is_media_type(gt)) { 532 gt->migrate = xe_migrate_init(gt); 533 if (IS_ERR(gt->migrate)) 534 goto err_force_wake; 535 } else { 536 gt->migrate = xe_find_full_gt(gt)->migrate; 537 } 538 539 err = xe_uc_init_hw(>->uc); 540 if (err) 541 goto err_force_wake; 542 543 err = xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL); 544 XE_WARN_ON(err); 545 xe_device_mem_access_put(gt_to_xe(gt)); 546 547 return 0; 548 549 err_force_wake: 550 xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL); 551 err_hw_fence_irq: 552 for (i = 0; i < XE_ENGINE_CLASS_MAX; ++i) 553 xe_hw_fence_irq_finish(>->fence_irq[i]); 554 xe_device_mem_access_put(gt_to_xe(gt)); 555 556 return err; 557 } 558 559 int xe_gt_init(struct xe_gt *gt) 560 { 561 int err; 562 int i; 563 564 INIT_WORK(>->reset.worker, gt_reset_worker); 565 566 for (i = 0; i < XE_ENGINE_CLASS_MAX; ++i) { 567 gt->ring_ops[i] = xe_ring_ops_get(gt, i); 568 xe_hw_fence_irq_init(>->fence_irq[i]); 569 } 570 571 err = xe_gt_pagefault_init(gt); 572 if (err) 573 return err; 574 575 xe_gt_sysfs_init(gt); 576 577 err = gt_fw_domain_init(gt); 578 if (err) 579 return err; 580 581 xe_force_wake_init_engines(gt, gt_to_fw(gt)); 582 583 err = all_fw_domain_init(gt); 584 if (err) 585 return err; 586 587 xe_force_wake_prune(gt, gt_to_fw(gt)); 588 589 err = drmm_add_action_or_reset(>_to_xe(gt)->drm, gt_fini, gt); 590 if (err) 591 return err; 592 593 return 0; 594 } 595 596 int do_gt_reset(struct xe_gt *gt) 597 { 598 struct xe_device *xe = gt_to_xe(gt); 599 int err; 600 601 xe_mmio_write32(gt, GEN6_GDRST.reg, GEN11_GRDOM_FULL); 602 err = xe_mmio_wait32(gt, GEN6_GDRST.reg, 0, GEN11_GRDOM_FULL, 5); 603 if (err) 604 drm_err(&xe->drm, 605 "GT reset failed to clear GEN11_GRDOM_FULL\n"); 606 607 return err; 608 } 609 610 static int do_gt_restart(struct xe_gt *gt) 611 { 612 struct xe_hw_engine *hwe; 613 enum xe_hw_engine_id id; 614 int err; 615 616 setup_private_ppat(gt); 617 618 xe_reg_sr_apply_mmio(>->reg_sr, gt); 619 620 err = xe_wopcm_init(>->uc.wopcm); 621 if (err) 622 return err; 623 624 for_each_hw_engine(hwe, gt, id) 625 xe_hw_engine_enable_ring(hwe); 626 627 err = xe_uc_init_hw(>->uc); 628 if (err) 629 return err; 630 631 xe_mocs_init(gt); 632 err = xe_uc_start(>->uc); 633 if (err) 634 return err; 635 636 for_each_hw_engine(hwe, gt, id) { 637 xe_reg_sr_apply_mmio(&hwe->reg_sr, gt); 638 xe_reg_sr_apply_whitelist(&hwe->reg_whitelist, 639 hwe->mmio_base, gt); 640 } 641 642 return 0; 643 } 644 645 static int gt_reset(struct xe_gt *gt) 646 { 647 struct xe_device *xe = gt_to_xe(gt); 648 int err; 649 650 /* We only support GT resets with GuC submission */ 651 if (!xe_device_guc_submission_enabled(gt_to_xe(gt))) 652 return -ENODEV; 653 654 drm_info(&xe->drm, "GT reset started\n"); 655 656 xe_device_mem_access_get(gt_to_xe(gt)); 657 err = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL); 658 if (err) 659 goto err_msg; 660 661 xe_uc_stop_prepare(>->uc); 662 xe_gt_pagefault_reset(gt); 663 664 err = xe_uc_stop(>->uc); 665 if (err) 666 goto err_out; 667 668 err = do_gt_reset(gt); 669 if (err) 670 goto err_out; 671 672 err = do_gt_restart(gt); 673 if (err) 674 goto err_out; 675 676 xe_device_mem_access_put(gt_to_xe(gt)); 677 err = xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL); 678 XE_WARN_ON(err); 679 680 drm_info(&xe->drm, "GT reset done\n"); 681 682 return 0; 683 684 err_out: 685 XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL)); 686 err_msg: 687 XE_WARN_ON(xe_uc_start(>->uc)); 688 xe_device_mem_access_put(gt_to_xe(gt)); 689 drm_err(&xe->drm, "GT reset failed, err=%d\n", err); 690 691 return err; 692 } 693 694 static void gt_reset_worker(struct work_struct *w) 695 { 696 struct xe_gt *gt = container_of(w, typeof(*gt), reset.worker); 697 698 gt_reset(gt); 699 } 700 701 void xe_gt_reset_async(struct xe_gt *gt) 702 { 703 struct xe_device *xe = gt_to_xe(gt); 704 705 drm_info(&xe->drm, "Try GT reset\n"); 706 707 /* Don't do a reset while one is already in flight */ 708 if (xe_uc_reset_prepare(>->uc)) 709 return; 710 711 drm_info(&xe->drm, "Doing GT reset\n"); 712 queue_work(gt->ordered_wq, >->reset.worker); 713 } 714 715 void xe_gt_suspend_prepare(struct xe_gt *gt) 716 { 717 xe_device_mem_access_get(gt_to_xe(gt)); 718 XE_WARN_ON(xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL)); 719 720 xe_uc_stop_prepare(>->uc); 721 722 xe_device_mem_access_put(gt_to_xe(gt)); 723 XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL)); 724 } 725 726 int xe_gt_suspend(struct xe_gt *gt) 727 { 728 struct xe_device *xe = gt_to_xe(gt); 729 int err; 730 731 /* For now suspend/resume is only allowed with GuC */ 732 if (!xe_device_guc_submission_enabled(gt_to_xe(gt))) 733 return -ENODEV; 734 735 xe_device_mem_access_get(gt_to_xe(gt)); 736 err = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL); 737 if (err) 738 goto err_msg; 739 740 err = xe_uc_suspend(>->uc); 741 if (err) 742 goto err_force_wake; 743 744 xe_device_mem_access_put(gt_to_xe(gt)); 745 XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL)); 746 drm_info(&xe->drm, "GT suspended\n"); 747 748 return 0; 749 750 err_force_wake: 751 XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL)); 752 err_msg: 753 xe_device_mem_access_put(gt_to_xe(gt)); 754 drm_err(&xe->drm, "GT suspend failed: %d\n", err); 755 756 return err; 757 } 758 759 int xe_gt_resume(struct xe_gt *gt) 760 { 761 struct xe_device *xe = gt_to_xe(gt); 762 int err; 763 764 xe_device_mem_access_get(gt_to_xe(gt)); 765 err = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL); 766 if (err) 767 goto err_msg; 768 769 err = do_gt_restart(gt); 770 if (err) 771 goto err_force_wake; 772 773 xe_device_mem_access_put(gt_to_xe(gt)); 774 XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL)); 775 drm_info(&xe->drm, "GT resumed\n"); 776 777 return 0; 778 779 err_force_wake: 780 XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL)); 781 err_msg: 782 xe_device_mem_access_put(gt_to_xe(gt)); 783 drm_err(&xe->drm, "GT resume failed: %d\n", err); 784 785 return err; 786 } 787 788 void xe_gt_migrate_wait(struct xe_gt *gt) 789 { 790 xe_migrate_wait(gt->migrate); 791 } 792 793 struct xe_hw_engine *xe_gt_hw_engine(struct xe_gt *gt, 794 enum xe_engine_class class, 795 u16 instance, bool logical) 796 { 797 struct xe_hw_engine *hwe; 798 enum xe_hw_engine_id id; 799 800 for_each_hw_engine(hwe, gt, id) 801 if (hwe->class == class && 802 ((!logical && hwe->instance == instance) || 803 (logical && hwe->logical_instance == instance))) 804 return hwe; 805 806 return NULL; 807 } 808 809 struct xe_hw_engine *xe_gt_any_hw_engine_by_reset_domain(struct xe_gt *gt, 810 enum xe_engine_class class) 811 { 812 struct xe_hw_engine *hwe; 813 enum xe_hw_engine_id id; 814 815 for_each_hw_engine(hwe, gt, id) { 816 switch (class) { 817 case XE_ENGINE_CLASS_RENDER: 818 case XE_ENGINE_CLASS_COMPUTE: 819 if (hwe->class == XE_ENGINE_CLASS_RENDER || 820 hwe->class == XE_ENGINE_CLASS_COMPUTE) 821 return hwe; 822 break; 823 default: 824 if (hwe->class == class) 825 return hwe; 826 } 827 } 828 829 return NULL; 830 } 831