1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include "xe_guc.h" 7 8 #include <drm/drm_managed.h> 9 10 #include <generated/xe_wa_oob.h> 11 12 #include "abi/guc_actions_abi.h" 13 #include "abi/guc_errors_abi.h" 14 #include "regs/xe_gt_regs.h" 15 #include "regs/xe_gtt_defs.h" 16 #include "regs/xe_guc_regs.h" 17 #include "xe_bo.h" 18 #include "xe_device.h" 19 #include "xe_force_wake.h" 20 #include "xe_gt.h" 21 #include "xe_gt_printk.h" 22 #include "xe_gt_sriov_vf.h" 23 #include "xe_gt_throttle.h" 24 #include "xe_guc_ads.h" 25 #include "xe_guc_ct.h" 26 #include "xe_guc_db_mgr.h" 27 #include "xe_guc_hwconfig.h" 28 #include "xe_guc_log.h" 29 #include "xe_guc_pc.h" 30 #include "xe_guc_relay.h" 31 #include "xe_guc_submit.h" 32 #include "xe_memirq.h" 33 #include "xe_mmio.h" 34 #include "xe_platform_types.h" 35 #include "xe_sriov.h" 36 #include "xe_uc.h" 37 #include "xe_uc_fw.h" 38 #include "xe_wa.h" 39 #include "xe_wopcm.h" 40 41 static u32 guc_bo_ggtt_addr(struct xe_guc *guc, 42 struct xe_bo *bo) 43 { 44 struct xe_device *xe = guc_to_xe(guc); 45 u32 addr = xe_bo_ggtt_addr(bo); 46 47 /* GuC addresses above GUC_GGTT_TOP don't map through the GTT */ 48 xe_assert(xe, addr >= xe_wopcm_size(guc_to_xe(guc))); 49 xe_assert(xe, addr < GUC_GGTT_TOP); 50 xe_assert(xe, bo->size <= GUC_GGTT_TOP - addr); 51 52 return addr; 53 } 54 55 static u32 guc_ctl_debug_flags(struct xe_guc *guc) 56 { 57 u32 level = xe_guc_log_get_level(&guc->log); 58 u32 flags = 0; 59 60 if (!GUC_LOG_LEVEL_IS_VERBOSE(level)) 61 flags |= GUC_LOG_DISABLED; 62 else 63 flags |= GUC_LOG_LEVEL_TO_VERBOSITY(level) << 64 GUC_LOG_VERBOSITY_SHIFT; 65 66 return flags; 67 } 68 69 static u32 guc_ctl_feature_flags(struct xe_guc *guc) 70 { 71 u32 flags = 0; 72 73 if (!guc_to_xe(guc)->info.skip_guc_pc) 74 flags |= GUC_CTL_ENABLE_SLPC; 75 76 return flags; 77 } 78 79 static u32 guc_ctl_log_params_flags(struct xe_guc *guc) 80 { 81 u32 offset = guc_bo_ggtt_addr(guc, guc->log.bo) >> PAGE_SHIFT; 82 u32 flags; 83 84 #if (((CRASH_BUFFER_SIZE) % SZ_1M) == 0) 85 #define LOG_UNIT SZ_1M 86 #define LOG_FLAG GUC_LOG_LOG_ALLOC_UNITS 87 #else 88 #define LOG_UNIT SZ_4K 89 #define LOG_FLAG 0 90 #endif 91 92 #if (((CAPTURE_BUFFER_SIZE) % SZ_1M) == 0) 93 #define CAPTURE_UNIT SZ_1M 94 #define CAPTURE_FLAG GUC_LOG_CAPTURE_ALLOC_UNITS 95 #else 96 #define CAPTURE_UNIT SZ_4K 97 #define CAPTURE_FLAG 0 98 #endif 99 100 BUILD_BUG_ON(!CRASH_BUFFER_SIZE); 101 BUILD_BUG_ON(!IS_ALIGNED(CRASH_BUFFER_SIZE, LOG_UNIT)); 102 BUILD_BUG_ON(!DEBUG_BUFFER_SIZE); 103 BUILD_BUG_ON(!IS_ALIGNED(DEBUG_BUFFER_SIZE, LOG_UNIT)); 104 BUILD_BUG_ON(!CAPTURE_BUFFER_SIZE); 105 BUILD_BUG_ON(!IS_ALIGNED(CAPTURE_BUFFER_SIZE, CAPTURE_UNIT)); 106 107 BUILD_BUG_ON((CRASH_BUFFER_SIZE / LOG_UNIT - 1) > 108 (GUC_LOG_CRASH_MASK >> GUC_LOG_CRASH_SHIFT)); 109 BUILD_BUG_ON((DEBUG_BUFFER_SIZE / LOG_UNIT - 1) > 110 (GUC_LOG_DEBUG_MASK >> GUC_LOG_DEBUG_SHIFT)); 111 BUILD_BUG_ON((CAPTURE_BUFFER_SIZE / CAPTURE_UNIT - 1) > 112 (GUC_LOG_CAPTURE_MASK >> GUC_LOG_CAPTURE_SHIFT)); 113 114 flags = GUC_LOG_VALID | 115 GUC_LOG_NOTIFY_ON_HALF_FULL | 116 CAPTURE_FLAG | 117 LOG_FLAG | 118 ((CRASH_BUFFER_SIZE / LOG_UNIT - 1) << GUC_LOG_CRASH_SHIFT) | 119 ((DEBUG_BUFFER_SIZE / LOG_UNIT - 1) << GUC_LOG_DEBUG_SHIFT) | 120 ((CAPTURE_BUFFER_SIZE / CAPTURE_UNIT - 1) << 121 GUC_LOG_CAPTURE_SHIFT) | 122 (offset << GUC_LOG_BUF_ADDR_SHIFT); 123 124 #undef LOG_UNIT 125 #undef LOG_FLAG 126 #undef CAPTURE_UNIT 127 #undef CAPTURE_FLAG 128 129 return flags; 130 } 131 132 static u32 guc_ctl_ads_flags(struct xe_guc *guc) 133 { 134 u32 ads = guc_bo_ggtt_addr(guc, guc->ads.bo) >> PAGE_SHIFT; 135 u32 flags = ads << GUC_ADS_ADDR_SHIFT; 136 137 return flags; 138 } 139 140 static u32 guc_ctl_wa_flags(struct xe_guc *guc) 141 { 142 struct xe_device *xe = guc_to_xe(guc); 143 struct xe_gt *gt = guc_to_gt(guc); 144 u32 flags = 0; 145 146 if (XE_WA(gt, 22012773006)) 147 flags |= GUC_WA_POLLCS; 148 149 if (XE_WA(gt, 14014475959)) 150 flags |= GUC_WA_HOLD_CCS_SWITCHOUT; 151 152 if (XE_WA(gt, 22011391025)) 153 flags |= GUC_WA_DUAL_QUEUE; 154 155 /* 156 * Wa_22011802037: FIXME - there's more to be done than simply setting 157 * this flag: make sure each CS is stopped when preparing for GT reset 158 * and wait for pending MI_FW. 159 */ 160 if (GRAPHICS_VERx100(xe) < 1270) 161 flags |= GUC_WA_PRE_PARSER; 162 163 if (XE_WA(gt, 22012727170) || XE_WA(gt, 22012727685)) 164 flags |= GUC_WA_CONTEXT_ISOLATION; 165 166 if (XE_WA(gt, 18020744125) && 167 !xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_RENDER)) 168 flags |= GUC_WA_RCS_REGS_IN_CCS_REGS_LIST; 169 170 if (XE_WA(gt, 1509372804)) 171 flags |= GUC_WA_RENDER_RST_RC6_EXIT; 172 173 if (XE_WA(gt, 14018913170)) 174 flags |= GUC_WA_ENABLE_TSC_CHECK_ON_RC6; 175 176 return flags; 177 } 178 179 static u32 guc_ctl_devid(struct xe_guc *guc) 180 { 181 struct xe_device *xe = guc_to_xe(guc); 182 183 return (((u32)xe->info.devid) << 16) | xe->info.revid; 184 } 185 186 static void guc_print_params(struct xe_guc *guc) 187 { 188 struct xe_gt *gt = guc_to_gt(guc); 189 u32 *params = guc->params; 190 int i; 191 192 BUILD_BUG_ON(sizeof(guc->params) != GUC_CTL_MAX_DWORDS * sizeof(u32)); 193 BUILD_BUG_ON(GUC_CTL_MAX_DWORDS + 2 != SOFT_SCRATCH_COUNT); 194 195 for (i = 0; i < GUC_CTL_MAX_DWORDS; i++) 196 xe_gt_dbg(gt, "GuC param[%2d] = 0x%08x\n", i, params[i]); 197 } 198 199 static void guc_init_params(struct xe_guc *guc) 200 { 201 u32 *params = guc->params; 202 203 params[GUC_CTL_LOG_PARAMS] = guc_ctl_log_params_flags(guc); 204 params[GUC_CTL_FEATURE] = 0; 205 params[GUC_CTL_DEBUG] = guc_ctl_debug_flags(guc); 206 params[GUC_CTL_ADS] = guc_ctl_ads_flags(guc); 207 params[GUC_CTL_WA] = 0; 208 params[GUC_CTL_DEVID] = guc_ctl_devid(guc); 209 210 guc_print_params(guc); 211 } 212 213 static void guc_init_params_post_hwconfig(struct xe_guc *guc) 214 { 215 u32 *params = guc->params; 216 217 params[GUC_CTL_LOG_PARAMS] = guc_ctl_log_params_flags(guc); 218 params[GUC_CTL_FEATURE] = guc_ctl_feature_flags(guc); 219 params[GUC_CTL_DEBUG] = guc_ctl_debug_flags(guc); 220 params[GUC_CTL_ADS] = guc_ctl_ads_flags(guc); 221 params[GUC_CTL_WA] = guc_ctl_wa_flags(guc); 222 params[GUC_CTL_DEVID] = guc_ctl_devid(guc); 223 224 guc_print_params(guc); 225 } 226 227 /* 228 * Initialize the GuC parameter block before starting the firmware 229 * transfer. These parameters are read by the firmware on startup 230 * and cannot be changed thereafter. 231 */ 232 static void guc_write_params(struct xe_guc *guc) 233 { 234 struct xe_gt *gt = guc_to_gt(guc); 235 int i; 236 237 xe_force_wake_assert_held(gt_to_fw(gt), XE_FW_GT); 238 239 xe_mmio_write32(gt, SOFT_SCRATCH(0), 0); 240 241 for (i = 0; i < GUC_CTL_MAX_DWORDS; i++) 242 xe_mmio_write32(gt, SOFT_SCRATCH(1 + i), guc->params[i]); 243 } 244 245 static void guc_fini_hw(void *arg) 246 { 247 struct xe_guc *guc = arg; 248 struct xe_gt *gt = guc_to_gt(guc); 249 250 xe_gt_WARN_ON(gt, xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL)); 251 xe_uc_fini_hw(&guc_to_gt(guc)->uc); 252 xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL); 253 } 254 255 /** 256 * xe_guc_comm_init_early - early initialization of GuC communication 257 * @guc: the &xe_guc to initialize 258 * 259 * Must be called prior to first MMIO communication with GuC firmware. 260 */ 261 void xe_guc_comm_init_early(struct xe_guc *guc) 262 { 263 struct xe_gt *gt = guc_to_gt(guc); 264 265 if (xe_gt_is_media_type(gt)) 266 guc->notify_reg = MED_GUC_HOST_INTERRUPT; 267 else 268 guc->notify_reg = GUC_HOST_INTERRUPT; 269 } 270 271 static int xe_guc_realloc_post_hwconfig(struct xe_guc *guc) 272 { 273 struct xe_tile *tile = gt_to_tile(guc_to_gt(guc)); 274 struct xe_device *xe = guc_to_xe(guc); 275 int ret; 276 277 if (!IS_DGFX(guc_to_xe(guc))) 278 return 0; 279 280 ret = xe_managed_bo_reinit_in_vram(xe, tile, &guc->fw.bo); 281 if (ret) 282 return ret; 283 284 ret = xe_managed_bo_reinit_in_vram(xe, tile, &guc->log.bo); 285 if (ret) 286 return ret; 287 288 ret = xe_managed_bo_reinit_in_vram(xe, tile, &guc->ads.bo); 289 if (ret) 290 return ret; 291 292 ret = xe_managed_bo_reinit_in_vram(xe, tile, &guc->ct.bo); 293 if (ret) 294 return ret; 295 296 return 0; 297 } 298 299 static int vf_guc_init(struct xe_guc *guc) 300 { 301 int err; 302 303 xe_guc_comm_init_early(guc); 304 305 err = xe_guc_ct_init(&guc->ct); 306 if (err) 307 return err; 308 309 err = xe_guc_relay_init(&guc->relay); 310 if (err) 311 return err; 312 313 return 0; 314 } 315 316 int xe_guc_init(struct xe_guc *guc) 317 { 318 struct xe_device *xe = guc_to_xe(guc); 319 struct xe_gt *gt = guc_to_gt(guc); 320 int ret; 321 322 guc->fw.type = XE_UC_FW_TYPE_GUC; 323 ret = xe_uc_fw_init(&guc->fw); 324 if (ret) 325 goto out; 326 327 if (!xe_uc_fw_is_enabled(&guc->fw)) 328 return 0; 329 330 if (IS_SRIOV_VF(xe)) { 331 ret = vf_guc_init(guc); 332 if (ret) 333 goto out; 334 return 0; 335 } 336 337 ret = xe_guc_log_init(&guc->log); 338 if (ret) 339 goto out; 340 341 ret = xe_guc_ads_init(&guc->ads); 342 if (ret) 343 goto out; 344 345 ret = xe_guc_ct_init(&guc->ct); 346 if (ret) 347 goto out; 348 349 ret = xe_guc_relay_init(&guc->relay); 350 if (ret) 351 goto out; 352 353 xe_uc_fw_change_status(&guc->fw, XE_UC_FIRMWARE_LOADABLE); 354 355 ret = devm_add_action_or_reset(xe->drm.dev, guc_fini_hw, guc); 356 if (ret) 357 goto out; 358 359 guc_init_params(guc); 360 361 xe_guc_comm_init_early(guc); 362 363 return 0; 364 365 out: 366 xe_gt_err(gt, "GuC init failed with %pe\n", ERR_PTR(ret)); 367 return ret; 368 } 369 370 static int vf_guc_init_post_hwconfig(struct xe_guc *guc) 371 { 372 int err; 373 374 err = xe_guc_submit_init(guc, xe_gt_sriov_vf_guc_ids(guc_to_gt(guc))); 375 if (err) 376 return err; 377 378 /* XXX xe_guc_db_mgr_init not needed for now */ 379 380 return 0; 381 } 382 383 /** 384 * xe_guc_init_post_hwconfig - initialize GuC post hwconfig load 385 * @guc: The GuC object 386 * 387 * Return: 0 on success, negative error code on error. 388 */ 389 int xe_guc_init_post_hwconfig(struct xe_guc *guc) 390 { 391 int ret; 392 393 if (IS_SRIOV_VF(guc_to_xe(guc))) 394 return vf_guc_init_post_hwconfig(guc); 395 396 ret = xe_guc_realloc_post_hwconfig(guc); 397 if (ret) 398 return ret; 399 400 guc_init_params_post_hwconfig(guc); 401 402 ret = xe_guc_submit_init(guc, ~0); 403 if (ret) 404 return ret; 405 406 ret = xe_guc_db_mgr_init(&guc->dbm, ~0); 407 if (ret) 408 return ret; 409 410 ret = xe_guc_pc_init(&guc->pc); 411 if (ret) 412 return ret; 413 414 return xe_guc_ads_init_post_hwconfig(&guc->ads); 415 } 416 417 int xe_guc_post_load_init(struct xe_guc *guc) 418 { 419 xe_guc_ads_populate_post_load(&guc->ads); 420 guc->submission_state.enabled = true; 421 422 return 0; 423 } 424 425 int xe_guc_reset(struct xe_guc *guc) 426 { 427 struct xe_gt *gt = guc_to_gt(guc); 428 u32 guc_status, gdrst; 429 int ret; 430 431 xe_force_wake_assert_held(gt_to_fw(gt), XE_FW_GT); 432 433 if (IS_SRIOV_VF(gt_to_xe(gt))) 434 return xe_gt_sriov_vf_bootstrap(gt); 435 436 xe_mmio_write32(gt, GDRST, GRDOM_GUC); 437 438 ret = xe_mmio_wait32(gt, GDRST, GRDOM_GUC, 0, 5000, &gdrst, false); 439 if (ret) { 440 xe_gt_err(gt, "GuC reset timed out, GDRST=%#x\n", gdrst); 441 goto err_out; 442 } 443 444 guc_status = xe_mmio_read32(gt, GUC_STATUS); 445 if (!(guc_status & GS_MIA_IN_RESET)) { 446 xe_gt_err(gt, "GuC status: %#x, MIA core expected to be in reset\n", 447 guc_status); 448 ret = -EIO; 449 goto err_out; 450 } 451 452 return 0; 453 454 err_out: 455 456 return ret; 457 } 458 459 static void guc_prepare_xfer(struct xe_guc *guc) 460 { 461 struct xe_gt *gt = guc_to_gt(guc); 462 struct xe_device *xe = guc_to_xe(guc); 463 u32 shim_flags = GUC_ENABLE_READ_CACHE_LOGIC | 464 GUC_ENABLE_READ_CACHE_FOR_SRAM_DATA | 465 GUC_ENABLE_READ_CACHE_FOR_WOPCM_DATA | 466 GUC_ENABLE_MIA_CLOCK_GATING; 467 468 if (GRAPHICS_VERx100(xe) < 1250) 469 shim_flags |= GUC_DISABLE_SRAM_INIT_TO_ZEROES | 470 GUC_ENABLE_MIA_CACHING; 471 472 if (GRAPHICS_VER(xe) >= 20 || xe->info.platform == XE_PVC) 473 shim_flags |= REG_FIELD_PREP(GUC_MOCS_INDEX_MASK, gt->mocs.uc_index); 474 475 /* Must program this register before loading the ucode with DMA */ 476 xe_mmio_write32(gt, GUC_SHIM_CONTROL, shim_flags); 477 478 xe_mmio_write32(gt, GT_PM_CONFIG, GT_DOORBELL_ENABLE); 479 480 /* Make sure GuC receives ARAT interrupts */ 481 xe_mmio_rmw32(gt, PMINTRMSK, ARAT_EXPIRED_INTRMSK, 0); 482 } 483 484 /* 485 * Supporting MMIO & in memory RSA 486 */ 487 static int guc_xfer_rsa(struct xe_guc *guc) 488 { 489 struct xe_gt *gt = guc_to_gt(guc); 490 u32 rsa[UOS_RSA_SCRATCH_COUNT]; 491 size_t copied; 492 int i; 493 494 if (guc->fw.rsa_size > 256) { 495 u32 rsa_ggtt_addr = xe_bo_ggtt_addr(guc->fw.bo) + 496 xe_uc_fw_rsa_offset(&guc->fw); 497 xe_mmio_write32(gt, UOS_RSA_SCRATCH(0), rsa_ggtt_addr); 498 return 0; 499 } 500 501 copied = xe_uc_fw_copy_rsa(&guc->fw, rsa, sizeof(rsa)); 502 if (copied < sizeof(rsa)) 503 return -ENOMEM; 504 505 for (i = 0; i < UOS_RSA_SCRATCH_COUNT; i++) 506 xe_mmio_write32(gt, UOS_RSA_SCRATCH(i), rsa[i]); 507 508 return 0; 509 } 510 511 /* 512 * Check a previously read GuC status register (GUC_STATUS) looking for 513 * known terminal states (either completion or failure) of either the 514 * microkernel status field or the boot ROM status field. Returns +1 for 515 * successful completion, -1 for failure and 0 for any intermediate state. 516 */ 517 static int guc_load_done(u32 status) 518 { 519 u32 uk_val = REG_FIELD_GET(GS_UKERNEL_MASK, status); 520 u32 br_val = REG_FIELD_GET(GS_BOOTROM_MASK, status); 521 522 switch (uk_val) { 523 case XE_GUC_LOAD_STATUS_READY: 524 return 1; 525 526 case XE_GUC_LOAD_STATUS_ERROR_DEVID_BUILD_MISMATCH: 527 case XE_GUC_LOAD_STATUS_GUC_PREPROD_BUILD_MISMATCH: 528 case XE_GUC_LOAD_STATUS_ERROR_DEVID_INVALID_GUCTYPE: 529 case XE_GUC_LOAD_STATUS_HWCONFIG_ERROR: 530 case XE_GUC_LOAD_STATUS_DPC_ERROR: 531 case XE_GUC_LOAD_STATUS_EXCEPTION: 532 case XE_GUC_LOAD_STATUS_INIT_DATA_INVALID: 533 case XE_GUC_LOAD_STATUS_MPU_DATA_INVALID: 534 case XE_GUC_LOAD_STATUS_INIT_MMIO_SAVE_RESTORE_INVALID: 535 return -1; 536 } 537 538 switch (br_val) { 539 case XE_BOOTROM_STATUS_NO_KEY_FOUND: 540 case XE_BOOTROM_STATUS_RSA_FAILED: 541 case XE_BOOTROM_STATUS_PAVPC_FAILED: 542 case XE_BOOTROM_STATUS_WOPCM_FAILED: 543 case XE_BOOTROM_STATUS_LOADLOC_FAILED: 544 case XE_BOOTROM_STATUS_JUMP_FAILED: 545 case XE_BOOTROM_STATUS_RC6CTXCONFIG_FAILED: 546 case XE_BOOTROM_STATUS_MPUMAP_INCORRECT: 547 case XE_BOOTROM_STATUS_EXCEPTION: 548 case XE_BOOTROM_STATUS_PROD_KEY_CHECK_FAILURE: 549 return -1; 550 } 551 552 return 0; 553 } 554 555 static s32 guc_pc_get_cur_freq(struct xe_guc_pc *guc_pc) 556 { 557 u32 freq; 558 int ret = xe_guc_pc_get_cur_freq(guc_pc, &freq); 559 560 return ret ? ret : freq; 561 } 562 563 /* 564 * Wait for the GuC to start up. 565 * 566 * Measurements indicate this should take no more than 20ms (assuming the GT 567 * clock is at maximum frequency). However, thermal throttling and other issues 568 * can prevent the clock hitting max and thus making the load take significantly 569 * longer. Allow up to 200ms as a safety margin for real world worst case situations. 570 * 571 * However, bugs anywhere from KMD to GuC to PCODE to fan failure in a CI farm can 572 * lead to even longer times. E.g. if the GT is clamped to minimum frequency then 573 * the load times can be in the seconds range. So the timeout is increased for debug 574 * builds to ensure that problems can be correctly analysed. For release builds, the 575 * timeout is kept short so that users don't wait forever to find out that there is a 576 * problem. In either case, if the load took longer than is reasonable even with some 577 * 'sensible' throttling, then flag a warning because something is not right. 578 * 579 * Note that there is a limit on how long an individual usleep_range() can wait for, 580 * hence longer waits require wrapping a shorter wait in a loop. 581 * 582 * Note that the only reason an end user should hit the shorter timeout is in case of 583 * extreme thermal throttling. And a system that is that hot during boot is probably 584 * dead anyway! 585 */ 586 #if defined(CONFIG_DRM_XE_DEBUG) 587 #define GUC_LOAD_RETRY_LIMIT 20 588 #else 589 #define GUC_LOAD_RETRY_LIMIT 3 590 #endif 591 #define GUC_LOAD_TIME_WARN_MS 200 592 593 static void guc_wait_ucode(struct xe_guc *guc) 594 { 595 struct xe_gt *gt = guc_to_gt(guc); 596 struct xe_guc_pc *guc_pc = >->uc.guc.pc; 597 ktime_t before, after, delta; 598 int load_done; 599 u32 status = 0; 600 int count = 0; 601 u64 delta_ms; 602 u32 before_freq; 603 604 before_freq = xe_guc_pc_get_act_freq(guc_pc); 605 before = ktime_get(); 606 /* 607 * Note, can't use any kind of timing information from the call to xe_mmio_wait. 608 * It could return a thousand intermediate stages at random times. Instead, must 609 * manually track the total time taken and locally implement the timeout. 610 */ 611 do { 612 u32 last_status = status & (GS_UKERNEL_MASK | GS_BOOTROM_MASK); 613 int ret; 614 615 /* 616 * Wait for any change (intermediate or terminal) in the status register. 617 * Note, the return value is a don't care. The only failure code is timeout 618 * but the timeouts need to be accumulated over all the intermediate partial 619 * timeouts rather than allowing a huge timeout each time. So basically, need 620 * to treat a timeout no different to a value change. 621 */ 622 ret = xe_mmio_wait32_not(gt, GUC_STATUS, GS_UKERNEL_MASK | GS_BOOTROM_MASK, 623 last_status, 1000 * 1000, &status, false); 624 if (ret < 0) 625 count++; 626 after = ktime_get(); 627 delta = ktime_sub(after, before); 628 delta_ms = ktime_to_ms(delta); 629 630 load_done = guc_load_done(status); 631 if (load_done != 0) 632 break; 633 634 if (delta_ms >= (GUC_LOAD_RETRY_LIMIT * 1000)) 635 break; 636 637 xe_gt_dbg(gt, "load still in progress, timeouts = %d, freq = %dMHz (req %dMHz), status = 0x%08X [0x%02X/%02X]\n", 638 count, xe_guc_pc_get_act_freq(guc_pc), 639 guc_pc_get_cur_freq(guc_pc), status, 640 REG_FIELD_GET(GS_BOOTROM_MASK, status), 641 REG_FIELD_GET(GS_UKERNEL_MASK, status)); 642 } while (1); 643 644 if (load_done != 1) { 645 u32 ukernel = REG_FIELD_GET(GS_UKERNEL_MASK, status); 646 u32 bootrom = REG_FIELD_GET(GS_BOOTROM_MASK, status); 647 648 xe_gt_err(gt, "load failed: status = 0x%08X, time = %lldms, freq = %dMHz (req %dMHz), done = %d\n", 649 status, delta_ms, xe_guc_pc_get_act_freq(guc_pc), 650 guc_pc_get_cur_freq(guc_pc), load_done); 651 xe_gt_err(gt, "load failed: status: Reset = %d, BootROM = 0x%02X, UKernel = 0x%02X, MIA = 0x%02X, Auth = 0x%02X\n", 652 REG_FIELD_GET(GS_MIA_IN_RESET, status), 653 bootrom, ukernel, 654 REG_FIELD_GET(GS_MIA_MASK, status), 655 REG_FIELD_GET(GS_AUTH_STATUS_MASK, status)); 656 657 switch (bootrom) { 658 case XE_BOOTROM_STATUS_NO_KEY_FOUND: 659 xe_gt_err(gt, "invalid key requested, header = 0x%08X\n", 660 xe_mmio_read32(gt, GUC_HEADER_INFO)); 661 break; 662 663 case XE_BOOTROM_STATUS_RSA_FAILED: 664 xe_gt_err(gt, "firmware signature verification failed\n"); 665 break; 666 667 case XE_BOOTROM_STATUS_PROD_KEY_CHECK_FAILURE: 668 xe_gt_err(gt, "firmware production part check failure\n"); 669 break; 670 } 671 672 switch (ukernel) { 673 case XE_GUC_LOAD_STATUS_EXCEPTION: 674 xe_gt_err(gt, "firmware exception. EIP: %#x\n", 675 xe_mmio_read32(gt, SOFT_SCRATCH(13))); 676 break; 677 678 case XE_GUC_LOAD_STATUS_INIT_MMIO_SAVE_RESTORE_INVALID: 679 xe_gt_err(gt, "illegal register in save/restore workaround list\n"); 680 break; 681 682 case XE_GUC_LOAD_STATUS_HWCONFIG_START: 683 xe_gt_err(gt, "still extracting hwconfig table.\n"); 684 break; 685 } 686 687 xe_device_declare_wedged(gt_to_xe(gt)); 688 } else if (delta_ms > GUC_LOAD_TIME_WARN_MS) { 689 xe_gt_warn(gt, "excessive init time: %lldms! [status = 0x%08X, timeouts = %d]\n", 690 delta_ms, status, count); 691 xe_gt_warn(gt, "excessive init time: [freq = %dMHz (req = %dMHz), before = %dMHz, perf_limit_reasons = 0x%08X]\n", 692 xe_guc_pc_get_act_freq(guc_pc), guc_pc_get_cur_freq(guc_pc), 693 before_freq, xe_gt_throttle_get_limit_reasons(gt)); 694 } else { 695 xe_gt_dbg(gt, "init took %lldms, freq = %dMHz (req = %dMHz), before = %dMHz, status = 0x%08X, timeouts = %d\n", 696 delta_ms, xe_guc_pc_get_act_freq(guc_pc), guc_pc_get_cur_freq(guc_pc), 697 before_freq, status, count); 698 } 699 } 700 701 static int __xe_guc_upload(struct xe_guc *guc) 702 { 703 int ret; 704 705 /* Raise GT freq to speed up HuC/GuC load */ 706 xe_guc_pc_raise_unslice(&guc->pc); 707 708 guc_write_params(guc); 709 guc_prepare_xfer(guc); 710 711 /* 712 * Note that GuC needs the CSS header plus uKernel code to be copied 713 * by the DMA engine in one operation, whereas the RSA signature is 714 * loaded separately, either by copying it to the UOS_RSA_SCRATCH 715 * register (if key size <= 256) or through a ggtt-pinned vma (if key 716 * size > 256). The RSA size and therefore the way we provide it to the 717 * HW is fixed for each platform and hard-coded in the bootrom. 718 */ 719 ret = guc_xfer_rsa(guc); 720 if (ret) 721 goto out; 722 /* 723 * Current uCode expects the code to be loaded at 8k; locations below 724 * this are used for the stack. 725 */ 726 ret = xe_uc_fw_upload(&guc->fw, 0x2000, UOS_MOVE); 727 if (ret) 728 goto out; 729 730 /* Wait for authentication */ 731 guc_wait_ucode(guc); 732 733 xe_uc_fw_change_status(&guc->fw, XE_UC_FIRMWARE_RUNNING); 734 return 0; 735 736 out: 737 xe_uc_fw_change_status(&guc->fw, XE_UC_FIRMWARE_LOAD_FAIL); 738 return 0 /* FIXME: ret, don't want to stop load currently */; 739 } 740 741 static int vf_guc_min_load_for_hwconfig(struct xe_guc *guc) 742 { 743 struct xe_gt *gt = guc_to_gt(guc); 744 int ret; 745 746 ret = xe_gt_sriov_vf_bootstrap(gt); 747 if (ret) 748 return ret; 749 750 ret = xe_gt_sriov_vf_query_config(gt); 751 if (ret) 752 return ret; 753 754 ret = xe_guc_hwconfig_init(guc); 755 if (ret) 756 return ret; 757 758 ret = xe_guc_enable_communication(guc); 759 if (ret) 760 return ret; 761 762 ret = xe_gt_sriov_vf_connect(gt); 763 if (ret) 764 return ret; 765 766 ret = xe_gt_sriov_vf_query_runtime(gt); 767 if (ret) 768 return ret; 769 770 return 0; 771 } 772 773 /** 774 * xe_guc_min_load_for_hwconfig - load minimal GuC and read hwconfig table 775 * @guc: The GuC object 776 * 777 * This function uploads a minimal GuC that does not support submissions but 778 * in a state where the hwconfig table can be read. Next, it reads and parses 779 * the hwconfig table so it can be used for subsequent steps in the driver load. 780 * Lastly, it enables CT communication (XXX: this is needed for PFs/VFs only). 781 * 782 * Return: 0 on success, negative error code on error. 783 */ 784 int xe_guc_min_load_for_hwconfig(struct xe_guc *guc) 785 { 786 int ret; 787 788 if (IS_SRIOV_VF(guc_to_xe(guc))) 789 return vf_guc_min_load_for_hwconfig(guc); 790 791 xe_guc_ads_populate_minimal(&guc->ads); 792 793 xe_guc_pc_init_early(&guc->pc); 794 795 ret = __xe_guc_upload(guc); 796 if (ret) 797 return ret; 798 799 ret = xe_guc_hwconfig_init(guc); 800 if (ret) 801 return ret; 802 803 ret = xe_guc_enable_communication(guc); 804 if (ret) 805 return ret; 806 807 return 0; 808 } 809 810 int xe_guc_upload(struct xe_guc *guc) 811 { 812 xe_guc_ads_populate(&guc->ads); 813 814 return __xe_guc_upload(guc); 815 } 816 817 static void guc_handle_mmio_msg(struct xe_guc *guc) 818 { 819 struct xe_gt *gt = guc_to_gt(guc); 820 u32 msg; 821 822 if (IS_SRIOV_VF(guc_to_xe(guc))) 823 return; 824 825 xe_force_wake_assert_held(gt_to_fw(gt), XE_FW_GT); 826 827 msg = xe_mmio_read32(gt, SOFT_SCRATCH(15)); 828 msg &= XE_GUC_RECV_MSG_EXCEPTION | 829 XE_GUC_RECV_MSG_CRASH_DUMP_POSTED; 830 xe_mmio_write32(gt, SOFT_SCRATCH(15), 0); 831 832 if (msg & XE_GUC_RECV_MSG_CRASH_DUMP_POSTED) 833 xe_gt_err(gt, "Received early GuC crash dump notification!\n"); 834 835 if (msg & XE_GUC_RECV_MSG_EXCEPTION) 836 xe_gt_err(gt, "Received early GuC exception notification!\n"); 837 } 838 839 static void guc_enable_irq(struct xe_guc *guc) 840 { 841 struct xe_gt *gt = guc_to_gt(guc); 842 u32 events = xe_gt_is_media_type(gt) ? 843 REG_FIELD_PREP(ENGINE0_MASK, GUC_INTR_GUC2HOST) : 844 REG_FIELD_PREP(ENGINE1_MASK, GUC_INTR_GUC2HOST); 845 846 /* Primary GuC and media GuC share a single enable bit */ 847 xe_mmio_write32(gt, GUC_SG_INTR_ENABLE, 848 REG_FIELD_PREP(ENGINE1_MASK, GUC_INTR_GUC2HOST)); 849 850 /* 851 * There are separate mask bits for primary and media GuCs, so use 852 * a RMW operation to avoid clobbering the other GuC's setting. 853 */ 854 xe_mmio_rmw32(gt, GUC_SG_INTR_MASK, events, 0); 855 } 856 857 int xe_guc_enable_communication(struct xe_guc *guc) 858 { 859 struct xe_device *xe = guc_to_xe(guc); 860 int err; 861 862 if (IS_SRIOV_VF(xe) && xe_device_has_memirq(xe)) { 863 struct xe_gt *gt = guc_to_gt(guc); 864 struct xe_tile *tile = gt_to_tile(gt); 865 866 err = xe_memirq_init_guc(&tile->sriov.vf.memirq, guc); 867 if (err) 868 return err; 869 } else { 870 guc_enable_irq(guc); 871 } 872 873 err = xe_guc_ct_enable(&guc->ct); 874 if (err) 875 return err; 876 877 guc_handle_mmio_msg(guc); 878 879 return 0; 880 } 881 882 int xe_guc_suspend(struct xe_guc *guc) 883 { 884 struct xe_gt *gt = guc_to_gt(guc); 885 u32 action[] = { 886 XE_GUC_ACTION_CLIENT_SOFT_RESET, 887 }; 888 int ret; 889 890 ret = xe_guc_mmio_send(guc, action, ARRAY_SIZE(action)); 891 if (ret) { 892 xe_gt_err(gt, "GuC suspend failed: %pe\n", ERR_PTR(ret)); 893 return ret; 894 } 895 896 xe_guc_sanitize(guc); 897 return 0; 898 } 899 900 void xe_guc_notify(struct xe_guc *guc) 901 { 902 struct xe_gt *gt = guc_to_gt(guc); 903 const u32 default_notify_data = 0; 904 905 /* 906 * Both GUC_HOST_INTERRUPT and MED_GUC_HOST_INTERRUPT can pass 907 * additional payload data to the GuC but this capability is not 908 * used by the firmware yet. Use default value in the meantime. 909 */ 910 xe_mmio_write32(gt, guc->notify_reg, default_notify_data); 911 } 912 913 int xe_guc_auth_huc(struct xe_guc *guc, u32 rsa_addr) 914 { 915 u32 action[] = { 916 XE_GUC_ACTION_AUTHENTICATE_HUC, 917 rsa_addr 918 }; 919 920 return xe_guc_ct_send_block(&guc->ct, action, ARRAY_SIZE(action)); 921 } 922 923 int xe_guc_mmio_send_recv(struct xe_guc *guc, const u32 *request, 924 u32 len, u32 *response_buf) 925 { 926 struct xe_device *xe = guc_to_xe(guc); 927 struct xe_gt *gt = guc_to_gt(guc); 928 u32 header, reply; 929 struct xe_reg reply_reg = xe_gt_is_media_type(gt) ? 930 MED_VF_SW_FLAG(0) : VF_SW_FLAG(0); 931 const u32 LAST_INDEX = VF_SW_FLAG_COUNT - 1; 932 int ret; 933 int i; 934 935 BUILD_BUG_ON(VF_SW_FLAG_COUNT != MED_VF_SW_FLAG_COUNT); 936 937 xe_assert(xe, !xe_guc_ct_enabled(&guc->ct)); 938 xe_assert(xe, len); 939 xe_assert(xe, len <= VF_SW_FLAG_COUNT); 940 xe_assert(xe, len <= MED_VF_SW_FLAG_COUNT); 941 xe_assert(xe, FIELD_GET(GUC_HXG_MSG_0_ORIGIN, request[0]) == 942 GUC_HXG_ORIGIN_HOST); 943 xe_assert(xe, FIELD_GET(GUC_HXG_MSG_0_TYPE, request[0]) == 944 GUC_HXG_TYPE_REQUEST); 945 946 retry: 947 /* Not in critical data-path, just do if else for GT type */ 948 if (xe_gt_is_media_type(gt)) { 949 for (i = 0; i < len; ++i) 950 xe_mmio_write32(gt, MED_VF_SW_FLAG(i), 951 request[i]); 952 xe_mmio_read32(gt, MED_VF_SW_FLAG(LAST_INDEX)); 953 } else { 954 for (i = 0; i < len; ++i) 955 xe_mmio_write32(gt, VF_SW_FLAG(i), 956 request[i]); 957 xe_mmio_read32(gt, VF_SW_FLAG(LAST_INDEX)); 958 } 959 960 xe_guc_notify(guc); 961 962 ret = xe_mmio_wait32(gt, reply_reg, GUC_HXG_MSG_0_ORIGIN, 963 FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_GUC), 964 50000, &reply, false); 965 if (ret) { 966 timeout: 967 xe_gt_err(gt, "GuC mmio request %#x: no reply %#x\n", 968 request[0], reply); 969 return ret; 970 } 971 972 header = xe_mmio_read32(gt, reply_reg); 973 if (FIELD_GET(GUC_HXG_MSG_0_TYPE, header) == 974 GUC_HXG_TYPE_NO_RESPONSE_BUSY) { 975 /* 976 * Once we got a BUSY reply we must wait again for the final 977 * response but this time we can't use ORIGIN mask anymore. 978 * To spot a right change in the reply, we take advantage that 979 * response SUCCESS and FAILURE differ only by the single bit 980 * and all other bits are set and can be used as a new mask. 981 */ 982 u32 resp_bits = GUC_HXG_TYPE_RESPONSE_SUCCESS & GUC_HXG_TYPE_RESPONSE_FAILURE; 983 u32 resp_mask = FIELD_PREP(GUC_HXG_MSG_0_TYPE, resp_bits); 984 985 BUILD_BUG_ON(FIELD_MAX(GUC_HXG_MSG_0_TYPE) != GUC_HXG_TYPE_RESPONSE_SUCCESS); 986 BUILD_BUG_ON((GUC_HXG_TYPE_RESPONSE_SUCCESS ^ GUC_HXG_TYPE_RESPONSE_FAILURE) != 1); 987 988 ret = xe_mmio_wait32(gt, reply_reg, resp_mask, resp_mask, 989 1000000, &header, false); 990 991 if (unlikely(FIELD_GET(GUC_HXG_MSG_0_ORIGIN, header) != 992 GUC_HXG_ORIGIN_GUC)) 993 goto proto; 994 if (unlikely(ret)) { 995 if (FIELD_GET(GUC_HXG_MSG_0_TYPE, header) != 996 GUC_HXG_TYPE_NO_RESPONSE_BUSY) 997 goto proto; 998 goto timeout; 999 } 1000 } 1001 1002 if (FIELD_GET(GUC_HXG_MSG_0_TYPE, header) == 1003 GUC_HXG_TYPE_NO_RESPONSE_RETRY) { 1004 u32 reason = FIELD_GET(GUC_HXG_RETRY_MSG_0_REASON, header); 1005 1006 xe_gt_dbg(gt, "GuC mmio request %#x: retrying, reason %#x\n", 1007 request[0], reason); 1008 goto retry; 1009 } 1010 1011 if (FIELD_GET(GUC_HXG_MSG_0_TYPE, header) == 1012 GUC_HXG_TYPE_RESPONSE_FAILURE) { 1013 u32 hint = FIELD_GET(GUC_HXG_FAILURE_MSG_0_HINT, header); 1014 u32 error = FIELD_GET(GUC_HXG_FAILURE_MSG_0_ERROR, header); 1015 1016 xe_gt_err(gt, "GuC mmio request %#x: failure %#x hint %#x\n", 1017 request[0], error, hint); 1018 return -ENXIO; 1019 } 1020 1021 if (FIELD_GET(GUC_HXG_MSG_0_TYPE, header) != 1022 GUC_HXG_TYPE_RESPONSE_SUCCESS) { 1023 proto: 1024 xe_gt_err(gt, "GuC mmio request %#x: unexpected reply %#x\n", 1025 request[0], header); 1026 return -EPROTO; 1027 } 1028 1029 /* Just copy entire possible message response */ 1030 if (response_buf) { 1031 response_buf[0] = header; 1032 1033 for (i = 1; i < VF_SW_FLAG_COUNT; i++) { 1034 reply_reg.addr += sizeof(u32); 1035 response_buf[i] = xe_mmio_read32(gt, reply_reg); 1036 } 1037 } 1038 1039 /* Use data from the GuC response as our return value */ 1040 return FIELD_GET(GUC_HXG_RESPONSE_MSG_0_DATA0, header); 1041 } 1042 1043 int xe_guc_mmio_send(struct xe_guc *guc, const u32 *request, u32 len) 1044 { 1045 return xe_guc_mmio_send_recv(guc, request, len, NULL); 1046 } 1047 1048 static int guc_self_cfg(struct xe_guc *guc, u16 key, u16 len, u64 val) 1049 { 1050 struct xe_device *xe = guc_to_xe(guc); 1051 u32 request[HOST2GUC_SELF_CFG_REQUEST_MSG_LEN] = { 1052 FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) | 1053 FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) | 1054 FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, 1055 GUC_ACTION_HOST2GUC_SELF_CFG), 1056 FIELD_PREP(HOST2GUC_SELF_CFG_REQUEST_MSG_1_KLV_KEY, key) | 1057 FIELD_PREP(HOST2GUC_SELF_CFG_REQUEST_MSG_1_KLV_LEN, len), 1058 FIELD_PREP(HOST2GUC_SELF_CFG_REQUEST_MSG_2_VALUE32, 1059 lower_32_bits(val)), 1060 FIELD_PREP(HOST2GUC_SELF_CFG_REQUEST_MSG_3_VALUE64, 1061 upper_32_bits(val)), 1062 }; 1063 int ret; 1064 1065 xe_assert(xe, len <= 2); 1066 xe_assert(xe, len != 1 || !upper_32_bits(val)); 1067 1068 /* Self config must go over MMIO */ 1069 ret = xe_guc_mmio_send(guc, request, ARRAY_SIZE(request)); 1070 1071 if (unlikely(ret < 0)) 1072 return ret; 1073 if (unlikely(ret > 1)) 1074 return -EPROTO; 1075 if (unlikely(!ret)) 1076 return -ENOKEY; 1077 1078 return 0; 1079 } 1080 1081 int xe_guc_self_cfg32(struct xe_guc *guc, u16 key, u32 val) 1082 { 1083 return guc_self_cfg(guc, key, 1, val); 1084 } 1085 1086 int xe_guc_self_cfg64(struct xe_guc *guc, u16 key, u64 val) 1087 { 1088 return guc_self_cfg(guc, key, 2, val); 1089 } 1090 1091 void xe_guc_irq_handler(struct xe_guc *guc, const u16 iir) 1092 { 1093 if (iir & GUC_INTR_GUC2HOST) 1094 xe_guc_ct_irq_handler(&guc->ct); 1095 } 1096 1097 void xe_guc_sanitize(struct xe_guc *guc) 1098 { 1099 xe_uc_fw_sanitize(&guc->fw); 1100 xe_guc_ct_disable(&guc->ct); 1101 guc->submission_state.enabled = false; 1102 } 1103 1104 int xe_guc_reset_prepare(struct xe_guc *guc) 1105 { 1106 return xe_guc_submit_reset_prepare(guc); 1107 } 1108 1109 void xe_guc_reset_wait(struct xe_guc *guc) 1110 { 1111 xe_guc_submit_reset_wait(guc); 1112 } 1113 1114 void xe_guc_stop_prepare(struct xe_guc *guc) 1115 { 1116 if (!IS_SRIOV_VF(guc_to_xe(guc))) { 1117 int err; 1118 1119 err = xe_guc_pc_stop(&guc->pc); 1120 xe_gt_WARN(guc_to_gt(guc), err, "Failed to stop GuC PC: %pe\n", 1121 ERR_PTR(err)); 1122 } 1123 } 1124 1125 void xe_guc_stop(struct xe_guc *guc) 1126 { 1127 xe_guc_ct_stop(&guc->ct); 1128 1129 xe_guc_submit_stop(guc); 1130 } 1131 1132 int xe_guc_start(struct xe_guc *guc) 1133 { 1134 if (!IS_SRIOV_VF(guc_to_xe(guc))) { 1135 int err; 1136 1137 err = xe_guc_pc_start(&guc->pc); 1138 xe_gt_WARN(guc_to_gt(guc), err, "Failed to start GuC PC: %pe\n", 1139 ERR_PTR(err)); 1140 } 1141 1142 return xe_guc_submit_start(guc); 1143 } 1144 1145 void xe_guc_print_info(struct xe_guc *guc, struct drm_printer *p) 1146 { 1147 struct xe_gt *gt = guc_to_gt(guc); 1148 u32 status; 1149 int err; 1150 int i; 1151 1152 xe_uc_fw_print(&guc->fw, p); 1153 1154 err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); 1155 if (err) 1156 return; 1157 1158 status = xe_mmio_read32(gt, GUC_STATUS); 1159 1160 drm_printf(p, "\nGuC status 0x%08x:\n", status); 1161 drm_printf(p, "\tBootrom status = 0x%x\n", 1162 REG_FIELD_GET(GS_BOOTROM_MASK, status)); 1163 drm_printf(p, "\tuKernel status = 0x%x\n", 1164 REG_FIELD_GET(GS_UKERNEL_MASK, status)); 1165 drm_printf(p, "\tMIA Core status = 0x%x\n", 1166 REG_FIELD_GET(GS_MIA_MASK, status)); 1167 drm_printf(p, "\tLog level = %d\n", 1168 xe_guc_log_get_level(&guc->log)); 1169 1170 drm_puts(p, "\nScratch registers:\n"); 1171 for (i = 0; i < SOFT_SCRATCH_COUNT; i++) { 1172 drm_printf(p, "\t%2d: \t0x%x\n", 1173 i, xe_mmio_read32(gt, SOFT_SCRATCH(i))); 1174 } 1175 1176 xe_force_wake_put(gt_to_fw(gt), XE_FW_GT); 1177 1178 xe_guc_ct_print(&guc->ct, p, false); 1179 xe_guc_submit_print(guc, p); 1180 } 1181 1182 /** 1183 * xe_guc_declare_wedged() - Declare GuC wedged 1184 * @guc: the GuC object 1185 * 1186 * Wedge the GuC which stops all submission, saves desired debug state, and 1187 * cleans up anything which could timeout. 1188 */ 1189 void xe_guc_declare_wedged(struct xe_guc *guc) 1190 { 1191 xe_gt_assert(guc_to_gt(guc), guc_to_xe(guc)->wedged.mode); 1192 1193 xe_guc_reset_prepare(guc); 1194 xe_guc_ct_stop(&guc->ct); 1195 xe_guc_submit_wedge(guc); 1196 } 1197