1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include "xe_guc_ads.h" 7 8 #include <linux/error-injection.h> 9 10 #include <drm/drm_managed.h> 11 12 #include <generated/xe_wa_oob.h> 13 14 #include "abi/guc_actions_abi.h" 15 #include "regs/xe_engine_regs.h" 16 #include "regs/xe_gt_regs.h" 17 #include "regs/xe_guc_regs.h" 18 #include "xe_bo.h" 19 #include "xe_configfs.h" 20 #include "xe_gt.h" 21 #include "xe_gt_ccs_mode.h" 22 #include "xe_gt_mcr.h" 23 #include "xe_gt_printk.h" 24 #include "xe_guc.h" 25 #include "xe_guc_buf.h" 26 #include "xe_guc_capture.h" 27 #include "xe_guc_ct.h" 28 #include "xe_hw_engine.h" 29 #include "xe_lrc.h" 30 #include "xe_map.h" 31 #include "xe_mmio.h" 32 #include "xe_wa.h" 33 34 /* Slack of a few additional entries per engine */ 35 #define ADS_REGSET_EXTRA_MAX 8 36 37 static struct xe_guc * 38 ads_to_guc(struct xe_guc_ads *ads) 39 { 40 return container_of(ads, struct xe_guc, ads); 41 } 42 43 static struct xe_gt * 44 ads_to_gt(struct xe_guc_ads *ads) 45 { 46 return container_of(ads, struct xe_gt, uc.guc.ads); 47 } 48 49 static struct xe_device * 50 ads_to_xe(struct xe_guc_ads *ads) 51 { 52 return gt_to_xe(ads_to_gt(ads)); 53 } 54 55 static struct iosys_map * 56 ads_to_map(struct xe_guc_ads *ads) 57 { 58 return &ads->bo->vmap; 59 } 60 61 /* UM Queue parameters: */ 62 #define GUC_UM_QUEUE_SIZE (SZ_64K) 63 #define GUC_PAGE_RES_TIMEOUT_US (-1) 64 65 /* 66 * The Additional Data Struct (ADS) has pointers for different buffers used by 67 * the GuC. One gem object (ads->bo) contains the ADS struct itself (guc_ads) 68 * and most of the extra buffers linked via the ADS struct's entries. The UM 69 * fault queues (PAGE_FAULT, PAGE_FAULT_RESPONSE, ACCESS_COUNTER rings) are 70 * kept in a separate BO (ads->um_queue_bo) so that the full memset of ads->bo 71 * performed on every GT reset does not discard fault descriptors already 72 * written into the rings by the GPU. 73 * 74 * Layout of the ADS blob (ads->bo): 75 * 76 * +---------------------------------------+ <== base 77 * | guc_ads | 78 * +---------------------------------------+ 79 * | guc_policies | 80 * +---------------------------------------+ 81 * | guc_gt_system_info | 82 * +---------------------------------------+ 83 * | guc_engine_usage | 84 * +---------------------------------------+ 85 * | guc_um_init_params | 86 * +---------------------------------------+ <== static 87 * | guc_mmio_reg[countA] (engine 0.0) | 88 * | guc_mmio_reg[countB] (engine 0.1) | 89 * | guc_mmio_reg[countC] (engine 1.0) | 90 * | ... | 91 * +---------------------------------------+ <== dynamic 92 * | padding | 93 * +---------------------------------------+ <== 4K aligned 94 * | golden contexts | 95 * +---------------------------------------+ 96 * | padding | 97 * +---------------------------------------+ <== 4K aligned 98 * | w/a KLVs | 99 * +---------------------------------------+ 100 * | padding | 101 * +---------------------------------------+ <== 4K aligned 102 * | capture lists | 103 * +---------------------------------------+ 104 * | padding | 105 * +---------------------------------------+ <== 4K aligned 106 * | private data | 107 * +---------------------------------------+ 108 * | padding | 109 * +---------------------------------------+ <== 4K aligned 110 */ 111 struct __guc_ads_blob { 112 struct guc_ads ads; 113 struct guc_policies policies; 114 struct guc_gt_system_info system_info; 115 struct guc_engine_usage engine_usage; 116 struct guc_um_init_params um_init_params; 117 /* From here on, location is dynamic! Refer to above diagram. */ 118 struct guc_mmio_reg regset[]; 119 } __packed; 120 121 #define ads_blob_read(ads_, field_) \ 122 xe_map_rd_field(ads_to_xe(ads_), ads_to_map(ads_), 0, \ 123 struct __guc_ads_blob, field_) 124 125 #define ads_blob_write(ads_, field_, val_) \ 126 xe_map_wr_field(ads_to_xe(ads_), ads_to_map(ads_), 0, \ 127 struct __guc_ads_blob, field_, val_) 128 129 #define info_map_write(xe_, map_, field_, val_) \ 130 xe_map_wr_field(xe_, map_, 0, struct guc_gt_system_info, field_, val_) 131 132 #define info_map_read(xe_, map_, field_) \ 133 xe_map_rd_field(xe_, map_, 0, struct guc_gt_system_info, field_) 134 135 static size_t guc_ads_regset_size(struct xe_guc_ads *ads) 136 { 137 struct xe_device *xe = ads_to_xe(ads); 138 139 xe_assert(xe, ads->regset_size); 140 141 return ads->regset_size; 142 } 143 144 static size_t guc_ads_golden_lrc_size(struct xe_guc_ads *ads) 145 { 146 return PAGE_ALIGN(ads->golden_lrc_size); 147 } 148 149 static u32 guc_ads_waklv_size(struct xe_guc_ads *ads) 150 { 151 return PAGE_ALIGN(ads->ads_waklv_size); 152 } 153 154 static size_t guc_ads_capture_size(struct xe_guc_ads *ads) 155 { 156 return PAGE_ALIGN(ads->capture_size); 157 } 158 159 static size_t guc_ads_private_data_size(struct xe_guc_ads *ads) 160 { 161 return PAGE_ALIGN(ads_to_guc(ads)->fw.private_data_size); 162 } 163 164 static size_t guc_ads_regset_offset(struct xe_guc_ads *ads) 165 { 166 return offsetof(struct __guc_ads_blob, regset); 167 } 168 169 static size_t guc_ads_golden_lrc_offset(struct xe_guc_ads *ads) 170 { 171 size_t offset; 172 173 offset = guc_ads_regset_offset(ads) + 174 guc_ads_regset_size(ads); 175 176 return PAGE_ALIGN(offset); 177 } 178 179 static size_t guc_ads_waklv_offset(struct xe_guc_ads *ads) 180 { 181 u32 offset; 182 183 offset = guc_ads_golden_lrc_offset(ads) + 184 guc_ads_golden_lrc_size(ads); 185 186 return PAGE_ALIGN(offset); 187 } 188 189 static size_t guc_ads_capture_offset(struct xe_guc_ads *ads) 190 { 191 size_t offset; 192 193 offset = guc_ads_waklv_offset(ads) + 194 guc_ads_waklv_size(ads); 195 196 return PAGE_ALIGN(offset); 197 } 198 199 static size_t guc_ads_private_data_offset(struct xe_guc_ads *ads) 200 { 201 size_t offset; 202 203 offset = guc_ads_capture_offset(ads) + 204 guc_ads_capture_size(ads); 205 206 return PAGE_ALIGN(offset); 207 } 208 209 static size_t guc_ads_size(struct xe_guc_ads *ads) 210 { 211 return guc_ads_private_data_offset(ads) + 212 guc_ads_private_data_size(ads); 213 } 214 215 static size_t calculate_regset_size(struct xe_gt *gt) 216 { 217 struct xe_reg_sr_entry *sr_entry; 218 unsigned long sr_idx; 219 struct xe_hw_engine *hwe; 220 enum xe_hw_engine_id id; 221 unsigned int count = 0; 222 223 for_each_hw_engine(hwe, gt, id) 224 xa_for_each(&hwe->reg_sr.xa, sr_idx, sr_entry) 225 count++; 226 227 count += ADS_REGSET_EXTRA_MAX * XE_NUM_HW_ENGINES; 228 229 if (XE_GT_WA(gt, 1607983814)) 230 count += LNCFCMOCS_REG_COUNT; 231 232 return count * sizeof(struct guc_mmio_reg); 233 } 234 235 static inline enum xe_engine_class guc_class_to_engine_class(u16 guc_class) 236 { 237 switch (guc_class) { 238 case GUC_RENDER_CLASS: 239 return XE_ENGINE_CLASS_RENDER; 240 case GUC_VIDEO_CLASS: 241 return XE_ENGINE_CLASS_VIDEO_DECODE; 242 case GUC_VIDEOENHANCE_CLASS: 243 return XE_ENGINE_CLASS_VIDEO_ENHANCE; 244 case GUC_BLITTER_CLASS: 245 case GUC_PAGING_CLASS: 246 return XE_ENGINE_CLASS_COPY; 247 case GUC_COMPUTE_CLASS: 248 return XE_ENGINE_CLASS_COMPUTE; 249 case GUC_GSC_OTHER_CLASS: 250 return XE_ENGINE_CLASS_OTHER; 251 default: 252 XE_WARN_ON(guc_class); 253 return -1; 254 } 255 } 256 257 static u32 engine_enable_mask(struct xe_gt *gt, u16 guc_class) 258 { 259 struct xe_hw_engine *hwe; 260 enum xe_hw_engine_id id; 261 u32 mask = 0; 262 263 for_each_hw_engine(hwe, gt, id) 264 if (xe_hwe_to_guc_class(hwe) == guc_class) 265 mask |= BIT(hwe->instance); 266 267 /* We expect at most one paging engine per GuC instance, for now */ 268 if (guc_class == GUC_PAGING_CLASS) 269 xe_gt_assert(gt, !mask || is_power_of_2(mask)); 270 271 return mask; 272 } 273 274 static size_t calculate_golden_lrc_size(struct xe_guc_ads *ads) 275 { 276 struct xe_gt *gt = ads_to_gt(ads); 277 size_t total_size = 0, alloc_size, real_size; 278 u16 guc_class; 279 280 for (guc_class = 0; guc_class <= GUC_LAST_ENGINE_CLASS; ++guc_class) { 281 enum xe_engine_class class = 282 guc_class_to_engine_class(guc_class); 283 284 if (!engine_enable_mask(gt, guc_class)) 285 continue; 286 287 real_size = xe_gt_lrc_size(gt, class); 288 alloc_size = PAGE_ALIGN(real_size); 289 total_size += alloc_size; 290 } 291 292 return total_size; 293 } 294 295 static void guc_waklv_enable(struct xe_guc_ads *ads, 296 u32 data[], u32 data_len_dw, 297 u32 *offset, u32 *remain, 298 enum xe_guc_klv_ids klv_id) 299 { 300 size_t size = sizeof(u32) * (1 + data_len_dw); 301 302 if (*remain < size) { 303 drm_warn(&ads_to_xe(ads)->drm, 304 "w/a klv buffer too small to add klv id 0x%04X\n", klv_id); 305 return; 306 } 307 308 /* 16:16 key/length */ 309 xe_map_wr(ads_to_xe(ads), ads_to_map(ads), *offset, u32, 310 FIELD_PREP(GUC_KLV_0_KEY, klv_id) | FIELD_PREP(GUC_KLV_0_LEN, data_len_dw)); 311 /* data_len_dw dwords of data */ 312 xe_map_memcpy_to(ads_to_xe(ads), ads_to_map(ads), 313 *offset + sizeof(u32), data, data_len_dw * sizeof(u32)); 314 315 *offset += size; 316 *remain -= size; 317 } 318 319 static void guc_waklv_init(struct xe_guc_ads *ads) 320 { 321 struct xe_gt *gt = ads_to_gt(ads); 322 u64 addr_ggtt; 323 u32 offset, remain, size; 324 325 offset = guc_ads_waklv_offset(ads); 326 remain = guc_ads_waklv_size(ads); 327 328 if (XE_GT_WA(gt, 16021333562)) 329 guc_waklv_enable(ads, NULL, 0, &offset, &remain, 330 GUC_WORKAROUND_KLV_BLOCK_INTERRUPTS_WHEN_MGSR_BLOCKED); 331 if (XE_GT_WA(gt, 18024947630)) 332 guc_waklv_enable(ads, NULL, 0, &offset, &remain, 333 GUC_WORKAROUND_KLV_ID_GAM_PFQ_SHADOW_TAIL_POLLING); 334 if (XE_GT_WA(gt, 16022287689)) 335 guc_waklv_enable(ads, NULL, 0, &offset, &remain, 336 GUC_WORKAROUND_KLV_ID_DISABLE_MTP_DURING_ASYNC_COMPUTE); 337 338 if (XE_GT_WA(gt, 14022866841)) 339 guc_waklv_enable(ads, NULL, 0, &offset, &remain, 340 GUC_WA_KLV_WAKE_POWER_DOMAINS_FOR_OUTBOUND_MMIO); 341 342 /* 343 * On RC6 exit, GuC will write register 0xB04 with the default value provided. As of now, 344 * the default value for this register is determined to be 0xC40. This could change in the 345 * future, so GuC depends on KMD to send it the correct value. 346 */ 347 if (XE_GT_WA(gt, 13011645652)) { 348 u32 data = 0xC40; 349 350 guc_waklv_enable(ads, &data, 1, &offset, &remain, 351 GUC_WA_KLV_NP_RD_WRITE_TO_CLEAR_RCSM_AT_CGP_LATE_RESTORE); 352 } 353 354 if (XE_GT_WA(gt, 14022293748) || XE_GT_WA(gt, 22019794406)) 355 guc_waklv_enable(ads, NULL, 0, &offset, &remain, 356 GUC_WORKAROUND_KLV_ID_BACK_TO_BACK_RCS_ENGINE_RESET); 357 358 if (GUC_FIRMWARE_VER_AT_LEAST(>->uc.guc, 70, 44) && XE_GT_WA(gt, 16026508708)) 359 guc_waklv_enable(ads, NULL, 0, &offset, &remain, 360 GUC_WA_KLV_RESET_BB_STACK_PTR_ON_VF_SWITCH); 361 if (GUC_FIRMWARE_VER_AT_LEAST(>->uc.guc, 70, 47) && XE_GT_WA(gt, 16026007364)) { 362 u32 data[] = { 363 0x0, 364 0xF, 365 }; 366 guc_waklv_enable(ads, data, ARRAY_SIZE(data), &offset, &remain, 367 GUC_WA_KLV_RESTORE_UNSAVED_MEDIA_CONTROL_REG); 368 } 369 370 if (XE_GT_WA(gt, 14020001231)) 371 guc_waklv_enable(ads, NULL, 0, &offset, &remain, 372 GUC_WORKAROUND_KLV_DISABLE_PSMI_INTERRUPTS_AT_C6_ENTRY_RESTORE_AT_EXIT); 373 if (XE_GT_WA(gt, 14025515070) && GUC_FIRMWARE_VER_AT_LEAST(>->uc.guc, 70, 53)) 374 guc_waklv_enable(ads, NULL, 0, &offset, &remain, 375 GUC_WA_KLV_CLR_CS_INDIRECT_RING_STATE_IF_IDLE_AT_CTX_REG); 376 377 if (XE_GT_WA(gt, 22022079272) && GUC_FIRMWARE_VER_AT_LEAST(>->uc.guc, 70, 62)) 378 guc_waklv_enable(ads, NULL, 0, &offset, &remain, GUC_WA_KLV_REMAP_RANGED_TLB_INV); 379 380 /* The GuC does not enable the sem_tok_64 feature on NVL-S */ 381 if (XE_GT_WA(gt, 16029897822) && gt_to_xe(gt)->info.platform != XE_NOVALAKE_S && 382 GUC_FIRMWARE_VER_AT_LEAST(>->uc.guc, 70, 69)) 383 guc_waklv_enable(ads, NULL, 0, &offset, &remain, 384 GUC_WA_KLV_IGNORE_MMIO_READ_SEM_TOKEN_64); 385 386 /* GuC only applies this WA for scheduled MultiQ contexts; no KMD-side gating needed. */ 387 if (XE_GT_WA(gt, 16030224309) && 388 GUC_FIRMWARE_VER_AT_LEAST(>->uc.guc, 70, 73)) 389 guc_waklv_enable(ads, NULL, 0, &offset, &remain, 390 GUC_WA_KLV_IGNORE_MULTIQ_CTX_COMPLETE_WITH_PREEMPT_IN_CSB); 391 392 /* 393 * On GuC firmware 70.66 and above, use the Feature KLV (shared with the 394 * WA KLV buffer); older firmware uses GUC_CTL_DISABLE_MULTI_QUEUE in 395 * the init params instead. 396 */ 397 if (!xe_configfs_get_enable_multi_queue(to_pci_dev(gt_to_xe(gt)->drm.dev)) && 398 GUC_FIRMWARE_VER_AT_LEAST(>->uc.guc, 70, 66)) { 399 u32 data = 1; 400 401 guc_waklv_enable(ads, &data, 1, &offset, &remain, 402 GUC_FEATURE_KLV_DISABLE_MULTI_QUEUE); 403 } 404 405 size = guc_ads_waklv_size(ads) - remain; 406 if (!size) 407 return; 408 409 offset = guc_ads_waklv_offset(ads); 410 addr_ggtt = xe_bo_ggtt_addr(ads->bo) + offset; 411 412 ads_blob_write(ads, ads.wa_klv_addr_lo, lower_32_bits(addr_ggtt)); 413 ads_blob_write(ads, ads.wa_klv_addr_hi, upper_32_bits(addr_ggtt)); 414 ads_blob_write(ads, ads.wa_klv_size, size); 415 } 416 417 static int calculate_waklv_size(struct xe_guc_ads *ads) 418 { 419 /* 420 * A single page is both the minimum size possible and 421 * is sufficiently large enough for all current platforms. 422 */ 423 return SZ_4K; 424 } 425 426 #define MAX_GOLDEN_LRC_SIZE (SZ_4K * 64) 427 428 int xe_guc_ads_init(struct xe_guc_ads *ads) 429 { 430 struct xe_device *xe = ads_to_xe(ads); 431 struct xe_gt *gt = ads_to_gt(ads); 432 struct xe_tile *tile = gt_to_tile(gt); 433 struct xe_bo *bo; 434 435 ads->golden_lrc_size = calculate_golden_lrc_size(ads); 436 ads->capture_size = xe_guc_capture_ads_input_worst_size(ads_to_guc(ads)); 437 ads->regset_size = calculate_regset_size(gt); 438 ads->ads_waklv_size = calculate_waklv_size(ads); 439 440 bo = xe_managed_bo_create_pin_map(xe, tile, guc_ads_size(ads) + MAX_GOLDEN_LRC_SIZE, 441 XE_BO_FLAG_SYSTEM | 442 XE_BO_FLAG_GGTT | 443 XE_BO_FLAG_GGTT_INVALIDATE | 444 XE_BO_FLAG_PINNED_NORESTORE | 445 xe_guc_bo_wa_flags(gt)); 446 if (IS_ERR(bo)) 447 return PTR_ERR(bo); 448 449 ads->bo = bo; 450 451 if (xe->info.has_usm) { 452 /* 453 * Allocate a separate BO for the HW fault ring (UM queues). 454 * 455 * Round the size up to the next power of two so that on iGPU 456 * (system memory, no IOMMU) the TTM pool issues a single 457 * alloc_pages(order=N) call, maximising the chance of getting 458 * a physically contiguous block. GuC requires contiguous DPA. 459 */ 460 size_t um_size = IS_DGFX(xe) ? 461 GUC_UM_QUEUE_SIZE * GUC_UM_HW_QUEUE_MAX : 462 roundup_pow_of_two(GUC_UM_QUEUE_SIZE * 463 GUC_UM_HW_QUEUE_MAX); 464 465 u32 um_flags = XE_BO_FLAG_VRAM_IF_DGFX(tile) | 466 XE_BO_FLAG_GGTT | 467 XE_BO_FLAG_GGTT_INVALIDATE | 468 XE_BO_FLAG_PINNED_NORESTORE | 469 XE_BO_FLAG_NEEDS_UC; 470 471 bo = xe_managed_bo_create_pin_map(xe, tile, um_size, um_flags); 472 if (IS_ERR(bo)) 473 return PTR_ERR(bo); 474 475 /* 476 * On pre-Xe3p platforms, GAM (not GuC) accesses the UM queue 477 * ring via base_dpa, which must be a contiguous DMA address 478 * range. Verify that the allocated pages are contiguous in 479 * DMA address space. 480 */ 481 if (!xe_bo_is_vram(bo) && 482 !xe_guc_using_main_gamctrl_queues(ads_to_guc(ads)) && 483 unlikely(!xe_bo_sg_is_contiguous(bo, 484 GUC_UM_QUEUE_SIZE * 485 GUC_UM_HW_QUEUE_MAX))) { 486 drm_err(&xe->drm, 487 "UM fault queue memory is not contiguous in DMA address space; GAM requires contiguous DPA\n"); 488 return -ENOMEM; 489 } 490 491 ads->um_queue_bo = bo; 492 } 493 494 return 0; 495 } 496 ALLOW_ERROR_INJECTION(xe_guc_ads_init, ERRNO); /* See xe_pci_probe() */ 497 498 /** 499 * xe_guc_ads_init_post_hwconfig - initialize ADS post hwconfig load 500 * @ads: Additional data structures object 501 * 502 * Recalculate golden_lrc_size, capture_size and regset_size as the number 503 * hardware engines may have changed after the hwconfig was loaded. Also verify 504 * the new sizes fit in the already allocated ADS buffer object. 505 * 506 * Return: 0 on success, negative error code on error. 507 */ 508 int xe_guc_ads_init_post_hwconfig(struct xe_guc_ads *ads) 509 { 510 struct xe_gt *gt = ads_to_gt(ads); 511 u32 prev_regset_size = ads->regset_size; 512 513 xe_gt_assert(gt, ads->bo); 514 515 ads->golden_lrc_size = calculate_golden_lrc_size(ads); 516 /* Calculate Capture size with worst size */ 517 ads->capture_size = xe_guc_capture_ads_input_worst_size(ads_to_guc(ads)); 518 ads->regset_size = calculate_regset_size(gt); 519 520 xe_gt_assert(gt, ads->golden_lrc_size + 521 (ads->regset_size - prev_regset_size) <= 522 MAX_GOLDEN_LRC_SIZE); 523 524 return 0; 525 } 526 527 static void guc_policies_init(struct xe_guc_ads *ads) 528 { 529 struct xe_device *xe = ads_to_xe(ads); 530 u32 global_flags = 0; 531 532 ads_blob_write(ads, policies.dpc_promote_time, 533 GLOBAL_POLICY_DEFAULT_DPC_PROMOTE_TIME_US); 534 ads_blob_write(ads, policies.max_num_work_items, 535 GLOBAL_POLICY_MAX_NUM_WI); 536 537 if (xe->wedged.mode == XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET) 538 global_flags |= GLOBAL_POLICY_DISABLE_ENGINE_RESET; 539 540 ads_blob_write(ads, policies.global_flags, global_flags); 541 ads_blob_write(ads, policies.is_valid, 1); 542 } 543 544 static void fill_engine_enable_masks(struct xe_gt *gt, 545 struct iosys_map *info_map) 546 { 547 struct xe_device *xe = gt_to_xe(gt); 548 u16 guc_class; 549 550 for (guc_class = 0; guc_class <= GUC_LAST_ENGINE_CLASS; ++guc_class) 551 info_map_write(xe, info_map, engine_enabled_masks[guc_class], 552 engine_enable_mask(gt, guc_class)); 553 } 554 555 u16 xe_hwe_to_guc_class(struct xe_hw_engine *hwe) 556 { 557 if (xe_guc_has_paging_engine(&hwe->gt->uc.guc) && 558 xe_gt_is_usm_hwe(hwe->gt, hwe)) 559 return GUC_PAGING_CLASS; 560 561 switch (hwe->class) { 562 case XE_ENGINE_CLASS_RENDER: 563 return GUC_RENDER_CLASS; 564 case XE_ENGINE_CLASS_VIDEO_DECODE: 565 return GUC_VIDEO_CLASS; 566 case XE_ENGINE_CLASS_VIDEO_ENHANCE: 567 return GUC_VIDEOENHANCE_CLASS; 568 case XE_ENGINE_CLASS_COPY: 569 return GUC_BLITTER_CLASS; 570 case XE_ENGINE_CLASS_COMPUTE: 571 return GUC_COMPUTE_CLASS; 572 case XE_ENGINE_CLASS_OTHER: 573 return GUC_GSC_OTHER_CLASS; 574 default: 575 XE_WARN_ON(hwe->class); 576 return -1; 577 } 578 } 579 580 /* 581 * Write the offsets corresponding to the golden LRCs. The actual data is 582 * populated later by guc_golden_lrc_populate() 583 */ 584 static void guc_golden_lrc_init(struct xe_guc_ads *ads) 585 { 586 struct xe_device *xe = ads_to_xe(ads); 587 struct xe_gt *gt = ads_to_gt(ads); 588 struct iosys_map info_map = IOSYS_MAP_INIT_OFFSET(ads_to_map(ads), 589 offsetof(struct __guc_ads_blob, system_info)); 590 size_t alloc_size, real_size; 591 u32 addr_ggtt, offset; 592 u16 guc_class; 593 594 offset = guc_ads_golden_lrc_offset(ads); 595 addr_ggtt = xe_bo_ggtt_addr(ads->bo) + offset; 596 597 for (guc_class = 0; guc_class <= GUC_LAST_ENGINE_CLASS; ++guc_class) { 598 enum xe_engine_class class = 599 guc_class_to_engine_class(guc_class); 600 601 if (!info_map_read(xe, &info_map, 602 engine_enabled_masks[guc_class])) 603 continue; 604 605 real_size = xe_gt_lrc_size(gt, class); 606 alloc_size = PAGE_ALIGN(real_size); 607 608 /* 609 * This interface is slightly confusing. We need to pass the 610 * base address of the full golden context and the size of just 611 * the engine state, which is the section of the context image 612 * that starts after the execlists LRC registers. This is 613 * required to allow the GuC to restore just the engine state 614 * when a watchdog reset occurs. 615 */ 616 ads_blob_write(ads, ads.eng_state_size[guc_class], 617 xe_lrc_engine_state_size(gt, class)); 618 ads_blob_write(ads, ads.golden_context_lrca[guc_class], 619 addr_ggtt); 620 621 addr_ggtt += alloc_size; 622 } 623 } 624 625 static void guc_mapping_table_init_invalid(struct xe_gt *gt, 626 struct iosys_map *info_map) 627 { 628 struct xe_device *xe = gt_to_xe(gt); 629 unsigned int i, j; 630 631 /* Table must be set to invalid values for entries not used */ 632 for (i = 0; i < GUC_MAX_ENGINE_CLASSES; ++i) 633 for (j = 0; j < GUC_MAX_INSTANCES_PER_CLASS; ++j) 634 info_map_write(xe, info_map, mapping_table[i][j], 635 GUC_MAX_INSTANCES_PER_CLASS); 636 } 637 638 static void guc_mapping_table_init(struct xe_gt *gt, 639 struct iosys_map *info_map) 640 { 641 struct xe_device *xe = gt_to_xe(gt); 642 struct xe_hw_engine *hwe; 643 enum xe_hw_engine_id id; 644 645 guc_mapping_table_init_invalid(gt, info_map); 646 647 for_each_hw_engine(hwe, gt, id) { 648 u16 guc_logical_instance; 649 u8 guc_class; 650 651 guc_class = xe_hwe_to_guc_class(hwe); 652 guc_logical_instance = xe_hwe_guc_logical_instance(hwe); 653 654 info_map_write(xe, info_map, 655 mapping_table[guc_class][guc_logical_instance], 656 hwe->instance); 657 } 658 } 659 660 static u32 guc_get_capture_engine_mask(struct xe_gt *gt, struct iosys_map *info_map, 661 enum guc_capture_list_class_type capture_class) 662 { 663 struct xe_device *xe = gt_to_xe(gt); 664 u32 mask; 665 666 switch (capture_class) { 667 case GUC_CAPTURE_LIST_CLASS_RENDER_COMPUTE: 668 mask = info_map_read(xe, info_map, engine_enabled_masks[GUC_RENDER_CLASS]); 669 mask |= info_map_read(xe, info_map, engine_enabled_masks[GUC_COMPUTE_CLASS]); 670 break; 671 case GUC_CAPTURE_LIST_CLASS_VIDEO: 672 mask = info_map_read(xe, info_map, engine_enabled_masks[GUC_VIDEO_CLASS]); 673 break; 674 case GUC_CAPTURE_LIST_CLASS_VIDEOENHANCE: 675 mask = info_map_read(xe, info_map, engine_enabled_masks[GUC_VIDEOENHANCE_CLASS]); 676 break; 677 case GUC_CAPTURE_LIST_CLASS_BLITTER: 678 mask = info_map_read(xe, info_map, engine_enabled_masks[GUC_BLITTER_CLASS]); 679 break; 680 case GUC_CAPTURE_LIST_CLASS_GSC_OTHER: 681 mask = info_map_read(xe, info_map, engine_enabled_masks[GUC_GSC_OTHER_CLASS]); 682 break; 683 case GUC_CAPTURE_LIST_CLASS_PAGING: 684 mask = info_map_read(xe, info_map, engine_enabled_masks[GUC_PAGING_CLASS]); 685 break; 686 default: 687 mask = 0; 688 } 689 690 return mask; 691 } 692 693 static inline bool get_capture_list(struct xe_guc_ads *ads, struct xe_guc *guc, struct xe_gt *gt, 694 int owner, int type, int class, u32 *total_size, size_t *size, 695 void **pptr) 696 { 697 *size = 0; 698 699 if (!xe_guc_capture_getlistsize(guc, owner, type, class, size)) { 700 if (*total_size + *size > ads->capture_size) 701 xe_gt_dbg(gt, "Capture size overflow :%zu vs %d\n", 702 *total_size + *size, ads->capture_size); 703 else if (!xe_guc_capture_getlist(guc, owner, type, class, pptr)) 704 return false; 705 } 706 707 return true; 708 } 709 710 static int guc_capture_prep_lists(struct xe_guc_ads *ads) 711 { 712 struct xe_guc *guc = ads_to_guc(ads); 713 struct xe_gt *gt = ads_to_gt(ads); 714 u32 ads_ggtt, capture_offset, null_ggtt, total_size = 0; 715 struct iosys_map info_map; 716 size_t size = 0; 717 void *ptr; 718 int i, j; 719 720 /* 721 * GuC Capture's steered reg-list needs to be allocated and initialized 722 * after the GuC-hwconfig is available which guaranteed from here. 723 */ 724 xe_guc_capture_steered_list_init(ads_to_guc(ads)); 725 726 capture_offset = guc_ads_capture_offset(ads); 727 ads_ggtt = xe_bo_ggtt_addr(ads->bo); 728 info_map = IOSYS_MAP_INIT_OFFSET(ads_to_map(ads), 729 offsetof(struct __guc_ads_blob, system_info)); 730 731 /* first, set aside the first page for a capture_list with zero descriptors */ 732 total_size = PAGE_SIZE; 733 if (!xe_guc_capture_getnullheader(guc, &ptr, &size)) 734 xe_map_memcpy_to(ads_to_xe(ads), ads_to_map(ads), capture_offset, ptr, size); 735 736 null_ggtt = ads_ggtt + capture_offset; 737 capture_offset += PAGE_SIZE; 738 739 /* 740 * Populate capture list : at this point adps is already allocated and 741 * mapped to worst case size 742 */ 743 for (i = 0; i < GUC_CAPTURE_LIST_INDEX_MAX; i++) { 744 bool write_empty_list; 745 746 for (j = 0; j < GUC_CAPTURE_LIST_CLASS_MAX; j++) { 747 u32 engine_mask = guc_get_capture_engine_mask(gt, &info_map, j); 748 /* null list if we dont have said engine or list */ 749 if (!engine_mask) { 750 ads_blob_write(ads, ads.capture_class[i][j], null_ggtt); 751 ads_blob_write(ads, ads.capture_instance[i][j], null_ggtt); 752 continue; 753 } 754 755 /* engine exists: start with engine-class registers */ 756 write_empty_list = get_capture_list(ads, guc, gt, i, 757 GUC_STATE_CAPTURE_TYPE_ENGINE_CLASS, 758 j, &total_size, &size, &ptr); 759 if (!write_empty_list) { 760 ads_blob_write(ads, ads.capture_class[i][j], 761 ads_ggtt + capture_offset); 762 xe_map_memcpy_to(ads_to_xe(ads), ads_to_map(ads), capture_offset, 763 ptr, size); 764 total_size += size; 765 capture_offset += size; 766 } else { 767 ads_blob_write(ads, ads.capture_class[i][j], null_ggtt); 768 } 769 770 /* engine exists: next, engine-instance registers */ 771 write_empty_list = get_capture_list(ads, guc, gt, i, 772 GUC_STATE_CAPTURE_TYPE_ENGINE_INSTANCE, 773 j, &total_size, &size, &ptr); 774 if (!write_empty_list) { 775 ads_blob_write(ads, ads.capture_instance[i][j], 776 ads_ggtt + capture_offset); 777 xe_map_memcpy_to(ads_to_xe(ads), ads_to_map(ads), capture_offset, 778 ptr, size); 779 total_size += size; 780 capture_offset += size; 781 } else { 782 ads_blob_write(ads, ads.capture_instance[i][j], null_ggtt); 783 } 784 } 785 786 /* global registers is last in our PF/VF loops */ 787 write_empty_list = get_capture_list(ads, guc, gt, i, 788 GUC_STATE_CAPTURE_TYPE_GLOBAL, 789 0, &total_size, &size, &ptr); 790 if (!write_empty_list) { 791 ads_blob_write(ads, ads.capture_global[i], ads_ggtt + capture_offset); 792 xe_map_memcpy_to(ads_to_xe(ads), ads_to_map(ads), capture_offset, ptr, 793 size); 794 total_size += size; 795 capture_offset += size; 796 } else { 797 ads_blob_write(ads, ads.capture_global[i], null_ggtt); 798 } 799 } 800 801 if (ads->capture_size != PAGE_ALIGN(total_size)) 802 xe_gt_dbg(gt, "Updated ADS capture size %d (was %d)\n", 803 PAGE_ALIGN(total_size), ads->capture_size); 804 return PAGE_ALIGN(total_size); 805 } 806 807 static void guc_mmio_regset_write_one(struct xe_guc_ads *ads, 808 struct iosys_map *regset_map, 809 struct xe_reg reg, 810 unsigned int n_entry) 811 { 812 struct guc_mmio_reg entry = { 813 .offset = reg.addr, 814 .flags = reg.masked ? GUC_REGSET_MASKED : 0, 815 }; 816 817 if (reg.mcr) { 818 struct xe_reg_mcr mcr_reg = XE_REG_MCR(reg.addr); 819 u8 group, instance; 820 821 bool steer = xe_gt_mcr_get_nonterminated_steering(ads_to_gt(ads), mcr_reg, 822 &group, &instance); 823 824 if (steer) { 825 entry.flags |= FIELD_PREP(GUC_REGSET_STEERING_GROUP, group); 826 entry.flags |= FIELD_PREP(GUC_REGSET_STEERING_INSTANCE, instance); 827 entry.flags |= GUC_REGSET_STEERING_NEEDED; 828 } 829 } 830 831 xe_map_memcpy_to(ads_to_xe(ads), regset_map, n_entry * sizeof(entry), 832 &entry, sizeof(entry)); 833 } 834 835 static unsigned int guc_mmio_regset_write(struct xe_guc_ads *ads, 836 struct iosys_map *regset_map, 837 struct xe_hw_engine *hwe) 838 { 839 struct xe_hw_engine *hwe_rcs_reset_domain = 840 xe_gt_any_hw_engine_by_reset_domain(hwe->gt, XE_ENGINE_CLASS_RENDER); 841 struct xe_reg_sr_entry *entry; 842 unsigned long idx; 843 unsigned int count = 0; 844 const struct { 845 struct xe_reg reg; 846 bool skip; 847 } *e, extra_regs[] = { 848 { .reg = RING_HWS_PGA(hwe->mmio_base), }, 849 { .reg = RING_IMR(hwe->mmio_base), }, 850 { .reg = CCS_MODE, 851 .skip = hwe != hwe_rcs_reset_domain || !xe_gt_ccs_mode_enabled(hwe->gt) }, 852 }; 853 u32 i; 854 855 BUILD_BUG_ON(ARRAY_SIZE(extra_regs) > ADS_REGSET_EXTRA_MAX); 856 857 xa_for_each(&hwe->reg_sr.xa, idx, entry) 858 guc_mmio_regset_write_one(ads, regset_map, entry->reg, count++); 859 860 for (e = extra_regs; e < extra_regs + ARRAY_SIZE(extra_regs); e++) { 861 if (e->skip) 862 continue; 863 864 guc_mmio_regset_write_one(ads, regset_map, e->reg, count++); 865 } 866 867 if (XE_GT_WA(hwe->gt, 1607983814) && hwe->class == XE_ENGINE_CLASS_RENDER) { 868 for (i = 0; i < LNCFCMOCS_REG_COUNT; i++) { 869 guc_mmio_regset_write_one(ads, regset_map, 870 XELP_LNCFCMOCS(i), count++); 871 } 872 } 873 874 if (XE_GT_WA(hwe->gt, 16023105232) || XE_GT_WA(hwe->gt, 14025941587)) 875 guc_mmio_regset_write_one(ads, regset_map, 876 RING_IDLEDLY(hwe->mmio_base), 877 count++); 878 879 return count; 880 } 881 882 static void guc_mmio_reg_state_init(struct xe_guc_ads *ads) 883 { 884 size_t regset_offset = guc_ads_regset_offset(ads); 885 struct xe_gt *gt = ads_to_gt(ads); 886 struct xe_hw_engine *hwe; 887 enum xe_hw_engine_id id; 888 u32 addr = xe_bo_ggtt_addr(ads->bo) + regset_offset; 889 struct iosys_map regset_map = IOSYS_MAP_INIT_OFFSET(ads_to_map(ads), 890 regset_offset); 891 unsigned int regset_used = 0; 892 893 for_each_hw_engine(hwe, gt, id) { 894 unsigned int count; 895 u8 gc; 896 897 /* 898 * 1. Write all MMIO entries for this exec queue to the table. No 899 * need to worry about fused-off engines and when there are 900 * entries in the regset: the reg_state_list has been zero'ed 901 * by xe_guc_ads_populate() 902 */ 903 count = guc_mmio_regset_write(ads, ®set_map, hwe); 904 if (!count) 905 continue; 906 907 /* 908 * 2. Record in the header (ads.reg_state_list) the address 909 * location and number of entries 910 */ 911 gc = xe_hwe_to_guc_class(hwe); 912 ads_blob_write(ads, ads.reg_state_list[gc][hwe->instance].address, addr); 913 ads_blob_write(ads, ads.reg_state_list[gc][hwe->instance].count, count); 914 915 addr += count * sizeof(struct guc_mmio_reg); 916 iosys_map_incr(®set_map, count * sizeof(struct guc_mmio_reg)); 917 918 regset_used += count * sizeof(struct guc_mmio_reg); 919 } 920 921 xe_gt_assert(gt, regset_used <= ads->regset_size); 922 } 923 924 static void guc_um_init_params(struct xe_guc_ads *ads) 925 { 926 struct xe_bo *um_bo = ads->um_queue_bo; 927 struct xe_guc *guc = ads_to_guc(ads); 928 struct xe_device *xe = ads_to_xe(ads); 929 u64 base_dpa; 930 u32 base_ggtt; 931 bool with_dpa; 932 int i; 933 934 with_dpa = !xe_guc_using_main_gamctrl_queues(guc); 935 936 if (um_bo) { 937 /* All USM platforms: UM queues in dedicated um_queue_bo */ 938 base_ggtt = xe_bo_ggtt_addr(um_bo); 939 base_dpa = xe_bo_main_addr(um_bo, PAGE_SIZE); 940 } else { 941 /* Platform does not support USM: no UM queues, nothing to do */ 942 return; 943 } 944 945 for (i = 0; i < GUC_UM_HW_QUEUE_MAX; ++i) { 946 /* 947 * Some platforms support USM but not access counters. 948 * Skip ACCESS_COUNTER queue initialization for such 949 * platforms, leaving queue_params[2] zero-initialized 950 * to signal unavailability to the GuC. 951 */ 952 if (i == GUC_UM_HW_QUEUE_ACCESS_COUNTER && 953 !xe->info.has_access_counter) 954 continue; 955 956 ads_blob_write(ads, um_init_params.queue_params[i].base_dpa, 957 with_dpa ? (base_dpa + (i * GUC_UM_QUEUE_SIZE)) : 0); 958 ads_blob_write(ads, um_init_params.queue_params[i].base_ggtt_address, 959 base_ggtt + (i * GUC_UM_QUEUE_SIZE)); 960 ads_blob_write(ads, um_init_params.queue_params[i].size_in_bytes, 961 GUC_UM_QUEUE_SIZE); 962 } 963 964 ads_blob_write(ads, um_init_params.page_response_timeout_in_us, 965 GUC_PAGE_RES_TIMEOUT_US); 966 } 967 968 static void guc_doorbell_init(struct xe_guc_ads *ads) 969 { 970 struct xe_device *xe = ads_to_xe(ads); 971 struct xe_gt *gt = ads_to_gt(ads); 972 973 if (GRAPHICS_VER(xe) >= 12 && !IS_DGFX(xe)) { 974 u32 distdbreg = 975 xe_mmio_read32(>->mmio, DIST_DBS_POPULATED); 976 977 ads_blob_write(ads, 978 system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_DOORBELL_COUNT_PER_SQIDI], 979 REG_FIELD_GET(DOORBELLS_PER_SQIDI_MASK, distdbreg) + 1); 980 } 981 } 982 983 /** 984 * xe_guc_ads_populate_minimal - populate minimal ADS 985 * @ads: Additional data structures object 986 * 987 * This function populates a minimal ADS that does not support submissions but 988 * enough so the GuC can load and the hwconfig table can be read. 989 */ 990 void xe_guc_ads_populate_minimal(struct xe_guc_ads *ads) 991 { 992 struct xe_gt *gt = ads_to_gt(ads); 993 struct iosys_map info_map = IOSYS_MAP_INIT_OFFSET(ads_to_map(ads), 994 offsetof(struct __guc_ads_blob, system_info)); 995 u32 base = xe_bo_ggtt_addr(ads->bo); 996 997 xe_gt_assert(gt, ads->bo); 998 999 xe_map_memset(ads_to_xe(ads), ads_to_map(ads), 0, 0, xe_bo_size(ads->bo)); 1000 guc_policies_init(ads); 1001 guc_golden_lrc_init(ads); 1002 guc_mapping_table_init_invalid(gt, &info_map); 1003 guc_doorbell_init(ads); 1004 1005 ads_blob_write(ads, ads.scheduler_policies, base + 1006 offsetof(struct __guc_ads_blob, policies)); 1007 ads_blob_write(ads, ads.gt_system_info, base + 1008 offsetof(struct __guc_ads_blob, system_info)); 1009 ads_blob_write(ads, ads.private_data, base + 1010 guc_ads_private_data_offset(ads)); 1011 } 1012 1013 void xe_guc_ads_populate(struct xe_guc_ads *ads) 1014 { 1015 struct xe_device *xe = ads_to_xe(ads); 1016 struct xe_gt *gt = ads_to_gt(ads); 1017 struct iosys_map info_map = IOSYS_MAP_INIT_OFFSET(ads_to_map(ads), 1018 offsetof(struct __guc_ads_blob, system_info)); 1019 u32 base = xe_bo_ggtt_addr(ads->bo); 1020 1021 xe_gt_assert(gt, ads->bo); 1022 1023 xe_map_memset(ads_to_xe(ads), ads_to_map(ads), 0, 0, xe_bo_size(ads->bo)); 1024 guc_policies_init(ads); 1025 fill_engine_enable_masks(gt, &info_map); 1026 guc_mmio_reg_state_init(ads); 1027 guc_golden_lrc_init(ads); 1028 guc_mapping_table_init(gt, &info_map); 1029 guc_capture_prep_lists(ads); 1030 guc_doorbell_init(ads); 1031 guc_waklv_init(ads); 1032 1033 if (xe->info.has_usm) { 1034 guc_um_init_params(ads); 1035 ads_blob_write(ads, ads.um_init_data, base + 1036 offsetof(struct __guc_ads_blob, um_init_params)); 1037 } 1038 1039 ads_blob_write(ads, ads.scheduler_policies, base + 1040 offsetof(struct __guc_ads_blob, policies)); 1041 ads_blob_write(ads, ads.gt_system_info, base + 1042 offsetof(struct __guc_ads_blob, system_info)); 1043 ads_blob_write(ads, ads.private_data, base + 1044 guc_ads_private_data_offset(ads)); 1045 } 1046 1047 /* 1048 * After the golden LRC's are recorded for each engine class by the first 1049 * submission, copy them to the ADS, as initialized earlier by 1050 * guc_golden_lrc_init(). 1051 */ 1052 static void guc_golden_lrc_populate(struct xe_guc_ads *ads) 1053 { 1054 struct xe_device *xe = ads_to_xe(ads); 1055 struct xe_gt *gt = ads_to_gt(ads); 1056 struct iosys_map info_map = IOSYS_MAP_INIT_OFFSET(ads_to_map(ads), 1057 offsetof(struct __guc_ads_blob, system_info)); 1058 size_t total_size = 0, alloc_size, real_size; 1059 u32 offset; 1060 u16 guc_class; 1061 1062 offset = guc_ads_golden_lrc_offset(ads); 1063 1064 for (guc_class = 0; guc_class <= GUC_LAST_ENGINE_CLASS; ++guc_class) { 1065 enum xe_engine_class class = 1066 guc_class_to_engine_class(guc_class); 1067 1068 if (!info_map_read(xe, &info_map, 1069 engine_enabled_masks[guc_class])) 1070 continue; 1071 1072 xe_gt_assert(gt, gt->default_lrc[class]); 1073 1074 real_size = xe_gt_lrc_size(gt, class); 1075 alloc_size = PAGE_ALIGN(real_size); 1076 total_size += alloc_size; 1077 1078 xe_map_memcpy_to(xe, ads_to_map(ads), offset, 1079 gt->default_lrc[class], real_size); 1080 1081 offset += alloc_size; 1082 } 1083 1084 xe_gt_assert(gt, total_size == ads->golden_lrc_size); 1085 } 1086 1087 void xe_guc_ads_populate_post_load(struct xe_guc_ads *ads) 1088 { 1089 guc_golden_lrc_populate(ads); 1090 } 1091 1092 static int guc_ads_action_update_policies(struct xe_guc_ads *ads, u32 policy_offset) 1093 { 1094 struct xe_guc_ct *ct = &ads_to_guc(ads)->ct; 1095 u32 action[] = { 1096 XE_GUC_ACTION_GLOBAL_SCHED_POLICY_CHANGE, 1097 policy_offset 1098 }; 1099 1100 return xe_guc_ct_send(ct, action, ARRAY_SIZE(action), 0, 0); 1101 } 1102 1103 /** 1104 * xe_guc_ads_scheduler_policy_toggle_reset - Toggle reset policy 1105 * @ads: Additional data structures object 1106 * @enable_engine_reset: true to enable engine resets, false otherwise 1107 * 1108 * This function update the GuC's engine reset policy. 1109 * 1110 * Return: 0 on success, and negative error code otherwise. 1111 */ 1112 int xe_guc_ads_scheduler_policy_toggle_reset(struct xe_guc_ads *ads, 1113 bool enable_engine_reset) 1114 { 1115 struct guc_policies *policies; 1116 struct xe_guc *guc = ads_to_guc(ads); 1117 CLASS(xe_guc_buf, buf)(&guc->buf, sizeof(*policies)); 1118 1119 if (!xe_guc_buf_is_valid(buf)) 1120 return -ENOBUFS; 1121 1122 policies = xe_guc_buf_cpu_ptr(buf); 1123 memset(policies, 0, sizeof(*policies)); 1124 1125 policies->dpc_promote_time = ads_blob_read(ads, policies.dpc_promote_time); 1126 policies->max_num_work_items = ads_blob_read(ads, policies.max_num_work_items); 1127 policies->is_valid = 1; 1128 1129 if (enable_engine_reset) 1130 policies->global_flags &= ~GLOBAL_POLICY_DISABLE_ENGINE_RESET; 1131 else 1132 policies->global_flags |= GLOBAL_POLICY_DISABLE_ENGINE_RESET; 1133 1134 return guc_ads_action_update_policies(ads, xe_guc_buf_flush(buf)); 1135 } 1136