1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include "xe_guc_ads.h" 7 8 #include <linux/fault-inject.h> 9 10 #include <drm/drm_managed.h> 11 12 #include <generated/xe_wa_oob.h> 13 14 #include "abi/guc_actions_abi.h" 15 #include "regs/xe_engine_regs.h" 16 #include "regs/xe_gt_regs.h" 17 #include "regs/xe_guc_regs.h" 18 #include "xe_bo.h" 19 #include "xe_gt.h" 20 #include "xe_gt_ccs_mode.h" 21 #include "xe_gt_printk.h" 22 #include "xe_guc.h" 23 #include "xe_guc_buf.h" 24 #include "xe_guc_capture.h" 25 #include "xe_guc_ct.h" 26 #include "xe_hw_engine.h" 27 #include "xe_lrc.h" 28 #include "xe_map.h" 29 #include "xe_mmio.h" 30 #include "xe_platform_types.h" 31 #include "xe_uc_fw.h" 32 #include "xe_wa.h" 33 #include "xe_gt_mcr.h" 34 35 /* Slack of a few additional entries per engine */ 36 #define ADS_REGSET_EXTRA_MAX 8 37 38 static struct xe_guc * 39 ads_to_guc(struct xe_guc_ads *ads) 40 { 41 return container_of(ads, struct xe_guc, ads); 42 } 43 44 static struct xe_gt * 45 ads_to_gt(struct xe_guc_ads *ads) 46 { 47 return container_of(ads, struct xe_gt, uc.guc.ads); 48 } 49 50 static struct xe_device * 51 ads_to_xe(struct xe_guc_ads *ads) 52 { 53 return gt_to_xe(ads_to_gt(ads)); 54 } 55 56 static struct iosys_map * 57 ads_to_map(struct xe_guc_ads *ads) 58 { 59 return &ads->bo->vmap; 60 } 61 62 /* UM Queue parameters: */ 63 #define GUC_UM_QUEUE_SIZE (SZ_64K) 64 #define GUC_PAGE_RES_TIMEOUT_US (-1) 65 66 /* 67 * The Additional Data Struct (ADS) has pointers for different buffers used by 68 * the GuC. One single gem object contains the ADS struct itself (guc_ads) and 69 * all the extra buffers indirectly linked via the ADS struct's entries. 70 * 71 * Layout of the ADS blob allocated for the GuC: 72 * 73 * +---------------------------------------+ <== base 74 * | guc_ads | 75 * +---------------------------------------+ 76 * | guc_policies | 77 * +---------------------------------------+ 78 * | guc_gt_system_info | 79 * +---------------------------------------+ 80 * | guc_engine_usage | 81 * +---------------------------------------+ 82 * | guc_um_init_params | 83 * +---------------------------------------+ <== static 84 * | guc_mmio_reg[countA] (engine 0.0) | 85 * | guc_mmio_reg[countB] (engine 0.1) | 86 * | guc_mmio_reg[countC] (engine 1.0) | 87 * | ... | 88 * +---------------------------------------+ <== dynamic 89 * | padding | 90 * +---------------------------------------+ <== 4K aligned 91 * | golden contexts | 92 * +---------------------------------------+ 93 * | padding | 94 * +---------------------------------------+ <== 4K aligned 95 * | w/a KLVs | 96 * +---------------------------------------+ 97 * | padding | 98 * +---------------------------------------+ <== 4K aligned 99 * | capture lists | 100 * +---------------------------------------+ 101 * | padding | 102 * +---------------------------------------+ <== 4K aligned 103 * | UM queues | 104 * +---------------------------------------+ 105 * | padding | 106 * +---------------------------------------+ <== 4K aligned 107 * | private data | 108 * +---------------------------------------+ 109 * | padding | 110 * +---------------------------------------+ <== 4K aligned 111 */ 112 struct __guc_ads_blob { 113 struct guc_ads ads; 114 struct guc_policies policies; 115 struct guc_gt_system_info system_info; 116 struct guc_engine_usage engine_usage; 117 struct guc_um_init_params um_init_params; 118 /* From here on, location is dynamic! Refer to above diagram. */ 119 struct guc_mmio_reg regset[]; 120 } __packed; 121 122 #define ads_blob_read(ads_, field_) \ 123 xe_map_rd_field(ads_to_xe(ads_), ads_to_map(ads_), 0, \ 124 struct __guc_ads_blob, field_) 125 126 #define ads_blob_write(ads_, field_, val_) \ 127 xe_map_wr_field(ads_to_xe(ads_), ads_to_map(ads_), 0, \ 128 struct __guc_ads_blob, field_, val_) 129 130 #define info_map_write(xe_, map_, field_, val_) \ 131 xe_map_wr_field(xe_, map_, 0, struct guc_gt_system_info, field_, val_) 132 133 #define info_map_read(xe_, map_, field_) \ 134 xe_map_rd_field(xe_, map_, 0, struct guc_gt_system_info, field_) 135 136 static size_t guc_ads_regset_size(struct xe_guc_ads *ads) 137 { 138 struct xe_device *xe = ads_to_xe(ads); 139 140 xe_assert(xe, ads->regset_size); 141 142 return ads->regset_size; 143 } 144 145 static size_t guc_ads_golden_lrc_size(struct xe_guc_ads *ads) 146 { 147 return PAGE_ALIGN(ads->golden_lrc_size); 148 } 149 150 static u32 guc_ads_waklv_size(struct xe_guc_ads *ads) 151 { 152 return PAGE_ALIGN(ads->ads_waklv_size); 153 } 154 155 static size_t guc_ads_capture_size(struct xe_guc_ads *ads) 156 { 157 return PAGE_ALIGN(ads->capture_size); 158 } 159 160 static size_t guc_ads_um_queues_size(struct xe_guc_ads *ads) 161 { 162 struct xe_device *xe = ads_to_xe(ads); 163 164 if (!xe->info.has_usm) 165 return 0; 166 167 return GUC_UM_QUEUE_SIZE * GUC_UM_HW_QUEUE_MAX; 168 } 169 170 static size_t guc_ads_private_data_size(struct xe_guc_ads *ads) 171 { 172 return PAGE_ALIGN(ads_to_guc(ads)->fw.private_data_size); 173 } 174 175 static size_t guc_ads_regset_offset(struct xe_guc_ads *ads) 176 { 177 return offsetof(struct __guc_ads_blob, regset); 178 } 179 180 static size_t guc_ads_golden_lrc_offset(struct xe_guc_ads *ads) 181 { 182 size_t offset; 183 184 offset = guc_ads_regset_offset(ads) + 185 guc_ads_regset_size(ads); 186 187 return PAGE_ALIGN(offset); 188 } 189 190 static size_t guc_ads_waklv_offset(struct xe_guc_ads *ads) 191 { 192 u32 offset; 193 194 offset = guc_ads_golden_lrc_offset(ads) + 195 guc_ads_golden_lrc_size(ads); 196 197 return PAGE_ALIGN(offset); 198 } 199 200 static size_t guc_ads_capture_offset(struct xe_guc_ads *ads) 201 { 202 size_t offset; 203 204 offset = guc_ads_waklv_offset(ads) + 205 guc_ads_waklv_size(ads); 206 207 return PAGE_ALIGN(offset); 208 } 209 210 static size_t guc_ads_um_queues_offset(struct xe_guc_ads *ads) 211 { 212 u32 offset; 213 214 offset = guc_ads_capture_offset(ads) + 215 guc_ads_capture_size(ads); 216 217 return PAGE_ALIGN(offset); 218 } 219 220 static size_t guc_ads_private_data_offset(struct xe_guc_ads *ads) 221 { 222 size_t offset; 223 224 offset = guc_ads_um_queues_offset(ads) + 225 guc_ads_um_queues_size(ads); 226 227 return PAGE_ALIGN(offset); 228 } 229 230 static size_t guc_ads_size(struct xe_guc_ads *ads) 231 { 232 return guc_ads_private_data_offset(ads) + 233 guc_ads_private_data_size(ads); 234 } 235 236 static size_t calculate_regset_size(struct xe_gt *gt) 237 { 238 struct xe_reg_sr_entry *sr_entry; 239 unsigned long sr_idx; 240 struct xe_hw_engine *hwe; 241 enum xe_hw_engine_id id; 242 unsigned int count = 0; 243 244 for_each_hw_engine(hwe, gt, id) 245 xa_for_each(&hwe->reg_sr.xa, sr_idx, sr_entry) 246 count++; 247 248 count += ADS_REGSET_EXTRA_MAX * XE_NUM_HW_ENGINES; 249 250 if (XE_WA(gt, 1607983814)) 251 count += LNCFCMOCS_REG_COUNT; 252 253 return count * sizeof(struct guc_mmio_reg); 254 } 255 256 static u32 engine_enable_mask(struct xe_gt *gt, enum xe_engine_class class) 257 { 258 struct xe_hw_engine *hwe; 259 enum xe_hw_engine_id id; 260 u32 mask = 0; 261 262 for_each_hw_engine(hwe, gt, id) 263 if (hwe->class == class) 264 mask |= BIT(hwe->instance); 265 266 return mask; 267 } 268 269 static size_t calculate_golden_lrc_size(struct xe_guc_ads *ads) 270 { 271 struct xe_gt *gt = ads_to_gt(ads); 272 size_t total_size = 0, alloc_size, real_size; 273 int class; 274 275 for (class = 0; class < XE_ENGINE_CLASS_MAX; ++class) { 276 if (!engine_enable_mask(gt, class)) 277 continue; 278 279 real_size = xe_gt_lrc_size(gt, class); 280 alloc_size = PAGE_ALIGN(real_size); 281 total_size += alloc_size; 282 } 283 284 return total_size; 285 } 286 287 static void guc_waklv_enable_one_word(struct xe_guc_ads *ads, 288 enum xe_guc_klv_ids klv_id, 289 u32 value, 290 u32 *offset, u32 *remain) 291 { 292 u32 size; 293 u32 klv_entry[] = { 294 /* 16:16 key/length */ 295 FIELD_PREP(GUC_KLV_0_KEY, klv_id) | 296 FIELD_PREP(GUC_KLV_0_LEN, 1), 297 value, 298 /* 1 dword data */ 299 }; 300 301 size = sizeof(klv_entry); 302 303 if (*remain < size) { 304 drm_warn(&ads_to_xe(ads)->drm, 305 "w/a klv buffer too small to add klv id %d\n", klv_id); 306 } else { 307 xe_map_memcpy_to(ads_to_xe(ads), ads_to_map(ads), *offset, 308 klv_entry, size); 309 *offset += size; 310 *remain -= size; 311 } 312 } 313 314 static void guc_waklv_enable_simple(struct xe_guc_ads *ads, 315 enum xe_guc_klv_ids klv_id, u32 *offset, u32 *remain) 316 { 317 u32 klv_entry[] = { 318 /* 16:16 key/length */ 319 FIELD_PREP(GUC_KLV_0_KEY, klv_id) | 320 FIELD_PREP(GUC_KLV_0_LEN, 0), 321 /* 0 dwords data */ 322 }; 323 u32 size; 324 325 size = sizeof(klv_entry); 326 327 if (xe_gt_WARN(ads_to_gt(ads), *remain < size, 328 "w/a klv buffer too small to add klv id %d\n", klv_id)) 329 return; 330 331 xe_map_memcpy_to(ads_to_xe(ads), ads_to_map(ads), *offset, 332 klv_entry, size); 333 *offset += size; 334 *remain -= size; 335 } 336 337 static void guc_waklv_init(struct xe_guc_ads *ads) 338 { 339 struct xe_gt *gt = ads_to_gt(ads); 340 u64 addr_ggtt; 341 u32 offset, remain, size; 342 343 offset = guc_ads_waklv_offset(ads); 344 remain = guc_ads_waklv_size(ads); 345 346 if (XE_WA(gt, 14019882105) || XE_WA(gt, 16021333562)) 347 guc_waklv_enable_simple(ads, 348 GUC_WORKAROUND_KLV_BLOCK_INTERRUPTS_WHEN_MGSR_BLOCKED, 349 &offset, &remain); 350 if (XE_WA(gt, 18024947630)) 351 guc_waklv_enable_simple(ads, 352 GUC_WORKAROUND_KLV_ID_GAM_PFQ_SHADOW_TAIL_POLLING, 353 &offset, &remain); 354 if (XE_WA(gt, 16022287689)) 355 guc_waklv_enable_simple(ads, 356 GUC_WORKAROUND_KLV_ID_DISABLE_MTP_DURING_ASYNC_COMPUTE, 357 &offset, &remain); 358 359 if (XE_WA(gt, 14022866841)) 360 guc_waklv_enable_simple(ads, 361 GUC_WA_KLV_WAKE_POWER_DOMAINS_FOR_OUTBOUND_MMIO, 362 &offset, &remain); 363 364 /* 365 * On RC6 exit, GuC will write register 0xB04 with the default value provided. As of now, 366 * the default value for this register is determined to be 0xC40. This could change in the 367 * future, so GuC depends on KMD to send it the correct value. 368 */ 369 if (XE_WA(gt, 13011645652)) 370 guc_waklv_enable_one_word(ads, 371 GUC_WA_KLV_NP_RD_WRITE_TO_CLEAR_RCSM_AT_CGP_LATE_RESTORE, 372 0xC40, 373 &offset, &remain); 374 375 if (XE_WA(gt, 14022293748) || XE_WA(gt, 22019794406)) 376 guc_waklv_enable_simple(ads, 377 GUC_WORKAROUND_KLV_ID_BACK_TO_BACK_RCS_ENGINE_RESET, 378 &offset, &remain); 379 380 if (GUC_FIRMWARE_VER(>->uc.guc) >= MAKE_GUC_VER(70, 44, 0) && XE_WA(gt, 16026508708)) 381 guc_waklv_enable_simple(ads, 382 GUC_WA_KLV_RESET_BB_STACK_PTR_ON_VF_SWITCH, 383 &offset, &remain); 384 385 size = guc_ads_waklv_size(ads) - remain; 386 if (!size) 387 return; 388 389 offset = guc_ads_waklv_offset(ads); 390 addr_ggtt = xe_bo_ggtt_addr(ads->bo) + offset; 391 392 ads_blob_write(ads, ads.wa_klv_addr_lo, lower_32_bits(addr_ggtt)); 393 ads_blob_write(ads, ads.wa_klv_addr_hi, upper_32_bits(addr_ggtt)); 394 ads_blob_write(ads, ads.wa_klv_size, size); 395 } 396 397 static int calculate_waklv_size(struct xe_guc_ads *ads) 398 { 399 /* 400 * A single page is both the minimum size possible and 401 * is sufficiently large enough for all current platforms. 402 */ 403 return SZ_4K; 404 } 405 406 #define MAX_GOLDEN_LRC_SIZE (SZ_4K * 64) 407 408 int xe_guc_ads_init(struct xe_guc_ads *ads) 409 { 410 struct xe_device *xe = ads_to_xe(ads); 411 struct xe_gt *gt = ads_to_gt(ads); 412 struct xe_tile *tile = gt_to_tile(gt); 413 struct xe_bo *bo; 414 415 ads->golden_lrc_size = calculate_golden_lrc_size(ads); 416 ads->capture_size = xe_guc_capture_ads_input_worst_size(ads_to_guc(ads)); 417 ads->regset_size = calculate_regset_size(gt); 418 ads->ads_waklv_size = calculate_waklv_size(ads); 419 420 bo = xe_managed_bo_create_pin_map(xe, tile, guc_ads_size(ads) + MAX_GOLDEN_LRC_SIZE, 421 XE_BO_FLAG_SYSTEM | 422 XE_BO_FLAG_GGTT | 423 XE_BO_FLAG_GGTT_INVALIDATE | 424 XE_BO_FLAG_PINNED_NORESTORE); 425 if (IS_ERR(bo)) 426 return PTR_ERR(bo); 427 428 ads->bo = bo; 429 430 return 0; 431 } 432 ALLOW_ERROR_INJECTION(xe_guc_ads_init, ERRNO); /* See xe_pci_probe() */ 433 434 /** 435 * xe_guc_ads_init_post_hwconfig - initialize ADS post hwconfig load 436 * @ads: Additional data structures object 437 * 438 * Recalculate golden_lrc_size, capture_size and regset_size as the number 439 * hardware engines may have changed after the hwconfig was loaded. Also verify 440 * the new sizes fit in the already allocated ADS buffer object. 441 * 442 * Return: 0 on success, negative error code on error. 443 */ 444 int xe_guc_ads_init_post_hwconfig(struct xe_guc_ads *ads) 445 { 446 struct xe_gt *gt = ads_to_gt(ads); 447 u32 prev_regset_size = ads->regset_size; 448 449 xe_gt_assert(gt, ads->bo); 450 451 ads->golden_lrc_size = calculate_golden_lrc_size(ads); 452 /* Calculate Capture size with worst size */ 453 ads->capture_size = xe_guc_capture_ads_input_worst_size(ads_to_guc(ads)); 454 ads->regset_size = calculate_regset_size(gt); 455 456 xe_gt_assert(gt, ads->golden_lrc_size + 457 (ads->regset_size - prev_regset_size) <= 458 MAX_GOLDEN_LRC_SIZE); 459 460 return 0; 461 } 462 463 static void guc_policies_init(struct xe_guc_ads *ads) 464 { 465 struct xe_device *xe = ads_to_xe(ads); 466 u32 global_flags = 0; 467 468 ads_blob_write(ads, policies.dpc_promote_time, 469 GLOBAL_POLICY_DEFAULT_DPC_PROMOTE_TIME_US); 470 ads_blob_write(ads, policies.max_num_work_items, 471 GLOBAL_POLICY_MAX_NUM_WI); 472 473 if (xe->wedged.mode == 2) 474 global_flags |= GLOBAL_POLICY_DISABLE_ENGINE_RESET; 475 476 ads_blob_write(ads, policies.global_flags, global_flags); 477 ads_blob_write(ads, policies.is_valid, 1); 478 } 479 480 static void fill_engine_enable_masks(struct xe_gt *gt, 481 struct iosys_map *info_map) 482 { 483 struct xe_device *xe = gt_to_xe(gt); 484 485 info_map_write(xe, info_map, engine_enabled_masks[GUC_RENDER_CLASS], 486 engine_enable_mask(gt, XE_ENGINE_CLASS_RENDER)); 487 info_map_write(xe, info_map, engine_enabled_masks[GUC_BLITTER_CLASS], 488 engine_enable_mask(gt, XE_ENGINE_CLASS_COPY)); 489 info_map_write(xe, info_map, engine_enabled_masks[GUC_VIDEO_CLASS], 490 engine_enable_mask(gt, XE_ENGINE_CLASS_VIDEO_DECODE)); 491 info_map_write(xe, info_map, 492 engine_enabled_masks[GUC_VIDEOENHANCE_CLASS], 493 engine_enable_mask(gt, XE_ENGINE_CLASS_VIDEO_ENHANCE)); 494 info_map_write(xe, info_map, engine_enabled_masks[GUC_COMPUTE_CLASS], 495 engine_enable_mask(gt, XE_ENGINE_CLASS_COMPUTE)); 496 info_map_write(xe, info_map, engine_enabled_masks[GUC_GSC_OTHER_CLASS], 497 engine_enable_mask(gt, XE_ENGINE_CLASS_OTHER)); 498 } 499 500 /* 501 * Write the offsets corresponding to the golden LRCs. The actual data is 502 * populated later by guc_golden_lrc_populate() 503 */ 504 static void guc_golden_lrc_init(struct xe_guc_ads *ads) 505 { 506 struct xe_device *xe = ads_to_xe(ads); 507 struct xe_gt *gt = ads_to_gt(ads); 508 struct iosys_map info_map = IOSYS_MAP_INIT_OFFSET(ads_to_map(ads), 509 offsetof(struct __guc_ads_blob, system_info)); 510 size_t alloc_size, real_size; 511 u32 addr_ggtt, offset; 512 int class; 513 514 offset = guc_ads_golden_lrc_offset(ads); 515 addr_ggtt = xe_bo_ggtt_addr(ads->bo) + offset; 516 517 for (class = 0; class < XE_ENGINE_CLASS_MAX; ++class) { 518 u8 guc_class; 519 520 guc_class = xe_engine_class_to_guc_class(class); 521 522 if (!info_map_read(xe, &info_map, 523 engine_enabled_masks[guc_class])) 524 continue; 525 526 real_size = xe_gt_lrc_size(gt, class); 527 alloc_size = PAGE_ALIGN(real_size); 528 529 /* 530 * This interface is slightly confusing. We need to pass the 531 * base address of the full golden context and the size of just 532 * the engine state, which is the section of the context image 533 * that starts after the execlists LRC registers. This is 534 * required to allow the GuC to restore just the engine state 535 * when a watchdog reset occurs. 536 * We calculate the engine state size by removing the size of 537 * what comes before it in the context image (which is identical 538 * on all engines). 539 */ 540 ads_blob_write(ads, ads.eng_state_size[guc_class], 541 real_size - xe_lrc_skip_size(xe)); 542 ads_blob_write(ads, ads.golden_context_lrca[guc_class], 543 addr_ggtt); 544 545 addr_ggtt += alloc_size; 546 } 547 } 548 549 static void guc_mapping_table_init_invalid(struct xe_gt *gt, 550 struct iosys_map *info_map) 551 { 552 struct xe_device *xe = gt_to_xe(gt); 553 unsigned int i, j; 554 555 /* Table must be set to invalid values for entries not used */ 556 for (i = 0; i < GUC_MAX_ENGINE_CLASSES; ++i) 557 for (j = 0; j < GUC_MAX_INSTANCES_PER_CLASS; ++j) 558 info_map_write(xe, info_map, mapping_table[i][j], 559 GUC_MAX_INSTANCES_PER_CLASS); 560 } 561 562 static void guc_mapping_table_init(struct xe_gt *gt, 563 struct iosys_map *info_map) 564 { 565 struct xe_device *xe = gt_to_xe(gt); 566 struct xe_hw_engine *hwe; 567 enum xe_hw_engine_id id; 568 569 guc_mapping_table_init_invalid(gt, info_map); 570 571 for_each_hw_engine(hwe, gt, id) { 572 u8 guc_class; 573 574 guc_class = xe_engine_class_to_guc_class(hwe->class); 575 info_map_write(xe, info_map, 576 mapping_table[guc_class][hwe->logical_instance], 577 hwe->instance); 578 } 579 } 580 581 static u32 guc_get_capture_engine_mask(struct xe_gt *gt, struct iosys_map *info_map, 582 enum guc_capture_list_class_type capture_class) 583 { 584 struct xe_device *xe = gt_to_xe(gt); 585 u32 mask; 586 587 switch (capture_class) { 588 case GUC_CAPTURE_LIST_CLASS_RENDER_COMPUTE: 589 mask = info_map_read(xe, info_map, engine_enabled_masks[GUC_RENDER_CLASS]); 590 mask |= info_map_read(xe, info_map, engine_enabled_masks[GUC_COMPUTE_CLASS]); 591 break; 592 case GUC_CAPTURE_LIST_CLASS_VIDEO: 593 mask = info_map_read(xe, info_map, engine_enabled_masks[GUC_VIDEO_CLASS]); 594 break; 595 case GUC_CAPTURE_LIST_CLASS_VIDEOENHANCE: 596 mask = info_map_read(xe, info_map, engine_enabled_masks[GUC_VIDEOENHANCE_CLASS]); 597 break; 598 case GUC_CAPTURE_LIST_CLASS_BLITTER: 599 mask = info_map_read(xe, info_map, engine_enabled_masks[GUC_BLITTER_CLASS]); 600 break; 601 case GUC_CAPTURE_LIST_CLASS_GSC_OTHER: 602 mask = info_map_read(xe, info_map, engine_enabled_masks[GUC_GSC_OTHER_CLASS]); 603 break; 604 default: 605 mask = 0; 606 } 607 608 return mask; 609 } 610 611 static inline bool get_capture_list(struct xe_guc_ads *ads, struct xe_guc *guc, struct xe_gt *gt, 612 int owner, int type, int class, u32 *total_size, size_t *size, 613 void **pptr) 614 { 615 *size = 0; 616 617 if (!xe_guc_capture_getlistsize(guc, owner, type, class, size)) { 618 if (*total_size + *size > ads->capture_size) 619 xe_gt_dbg(gt, "Capture size overflow :%zu vs %d\n", 620 *total_size + *size, ads->capture_size); 621 else if (!xe_guc_capture_getlist(guc, owner, type, class, pptr)) 622 return false; 623 } 624 625 return true; 626 } 627 628 static int guc_capture_prep_lists(struct xe_guc_ads *ads) 629 { 630 struct xe_guc *guc = ads_to_guc(ads); 631 struct xe_gt *gt = ads_to_gt(ads); 632 u32 ads_ggtt, capture_offset, null_ggtt, total_size = 0; 633 struct iosys_map info_map; 634 size_t size = 0; 635 void *ptr; 636 int i, j; 637 638 /* 639 * GuC Capture's steered reg-list needs to be allocated and initialized 640 * after the GuC-hwconfig is available which guaranteed from here. 641 */ 642 xe_guc_capture_steered_list_init(ads_to_guc(ads)); 643 644 capture_offset = guc_ads_capture_offset(ads); 645 ads_ggtt = xe_bo_ggtt_addr(ads->bo); 646 info_map = IOSYS_MAP_INIT_OFFSET(ads_to_map(ads), 647 offsetof(struct __guc_ads_blob, system_info)); 648 649 /* first, set aside the first page for a capture_list with zero descriptors */ 650 total_size = PAGE_SIZE; 651 if (!xe_guc_capture_getnullheader(guc, &ptr, &size)) 652 xe_map_memcpy_to(ads_to_xe(ads), ads_to_map(ads), capture_offset, ptr, size); 653 654 null_ggtt = ads_ggtt + capture_offset; 655 capture_offset += PAGE_SIZE; 656 657 /* 658 * Populate capture list : at this point adps is already allocated and 659 * mapped to worst case size 660 */ 661 for (i = 0; i < GUC_CAPTURE_LIST_INDEX_MAX; i++) { 662 bool write_empty_list; 663 664 for (j = 0; j < GUC_CAPTURE_LIST_CLASS_MAX; j++) { 665 u32 engine_mask = guc_get_capture_engine_mask(gt, &info_map, j); 666 /* null list if we dont have said engine or list */ 667 if (!engine_mask) { 668 ads_blob_write(ads, ads.capture_class[i][j], null_ggtt); 669 ads_blob_write(ads, ads.capture_instance[i][j], null_ggtt); 670 continue; 671 } 672 673 /* engine exists: start with engine-class registers */ 674 write_empty_list = get_capture_list(ads, guc, gt, i, 675 GUC_STATE_CAPTURE_TYPE_ENGINE_CLASS, 676 j, &total_size, &size, &ptr); 677 if (!write_empty_list) { 678 ads_blob_write(ads, ads.capture_class[i][j], 679 ads_ggtt + capture_offset); 680 xe_map_memcpy_to(ads_to_xe(ads), ads_to_map(ads), capture_offset, 681 ptr, size); 682 total_size += size; 683 capture_offset += size; 684 } else { 685 ads_blob_write(ads, ads.capture_class[i][j], null_ggtt); 686 } 687 688 /* engine exists: next, engine-instance registers */ 689 write_empty_list = get_capture_list(ads, guc, gt, i, 690 GUC_STATE_CAPTURE_TYPE_ENGINE_INSTANCE, 691 j, &total_size, &size, &ptr); 692 if (!write_empty_list) { 693 ads_blob_write(ads, ads.capture_instance[i][j], 694 ads_ggtt + capture_offset); 695 xe_map_memcpy_to(ads_to_xe(ads), ads_to_map(ads), capture_offset, 696 ptr, size); 697 total_size += size; 698 capture_offset += size; 699 } else { 700 ads_blob_write(ads, ads.capture_instance[i][j], null_ggtt); 701 } 702 } 703 704 /* global registers is last in our PF/VF loops */ 705 write_empty_list = get_capture_list(ads, guc, gt, i, 706 GUC_STATE_CAPTURE_TYPE_GLOBAL, 707 0, &total_size, &size, &ptr); 708 if (!write_empty_list) { 709 ads_blob_write(ads, ads.capture_global[i], ads_ggtt + capture_offset); 710 xe_map_memcpy_to(ads_to_xe(ads), ads_to_map(ads), capture_offset, ptr, 711 size); 712 total_size += size; 713 capture_offset += size; 714 } else { 715 ads_blob_write(ads, ads.capture_global[i], null_ggtt); 716 } 717 } 718 719 if (ads->capture_size != PAGE_ALIGN(total_size)) 720 xe_gt_dbg(gt, "Updated ADS capture size %d (was %d)\n", 721 PAGE_ALIGN(total_size), ads->capture_size); 722 return PAGE_ALIGN(total_size); 723 } 724 725 static void guc_mmio_regset_write_one(struct xe_guc_ads *ads, 726 struct iosys_map *regset_map, 727 struct xe_reg reg, 728 unsigned int n_entry) 729 { 730 struct guc_mmio_reg entry = { 731 .offset = reg.addr, 732 .flags = reg.masked ? GUC_REGSET_MASKED : 0, 733 }; 734 735 if (reg.mcr) { 736 struct xe_reg_mcr mcr_reg = XE_REG_MCR(reg.addr); 737 u8 group, instance; 738 739 bool steer = xe_gt_mcr_get_nonterminated_steering(ads_to_gt(ads), mcr_reg, 740 &group, &instance); 741 742 if (steer) { 743 entry.flags |= FIELD_PREP(GUC_REGSET_STEERING_GROUP, group); 744 entry.flags |= FIELD_PREP(GUC_REGSET_STEERING_INSTANCE, instance); 745 entry.flags |= GUC_REGSET_STEERING_NEEDED; 746 } 747 } 748 749 xe_map_memcpy_to(ads_to_xe(ads), regset_map, n_entry * sizeof(entry), 750 &entry, sizeof(entry)); 751 } 752 753 static unsigned int guc_mmio_regset_write(struct xe_guc_ads *ads, 754 struct iosys_map *regset_map, 755 struct xe_hw_engine *hwe) 756 { 757 struct xe_hw_engine *hwe_rcs_reset_domain = 758 xe_gt_any_hw_engine_by_reset_domain(hwe->gt, XE_ENGINE_CLASS_RENDER); 759 struct xe_reg_sr_entry *entry; 760 unsigned long idx; 761 unsigned int count = 0; 762 const struct { 763 struct xe_reg reg; 764 bool skip; 765 } *e, extra_regs[] = { 766 { .reg = RING_MODE(hwe->mmio_base), }, 767 { .reg = RING_HWS_PGA(hwe->mmio_base), }, 768 { .reg = RING_IMR(hwe->mmio_base), }, 769 { .reg = RCU_MODE, .skip = hwe != hwe_rcs_reset_domain }, 770 { .reg = CCS_MODE, 771 .skip = hwe != hwe_rcs_reset_domain || !xe_gt_ccs_mode_enabled(hwe->gt) }, 772 }; 773 u32 i; 774 775 BUILD_BUG_ON(ARRAY_SIZE(extra_regs) > ADS_REGSET_EXTRA_MAX); 776 777 xa_for_each(&hwe->reg_sr.xa, idx, entry) 778 guc_mmio_regset_write_one(ads, regset_map, entry->reg, count++); 779 780 for (e = extra_regs; e < extra_regs + ARRAY_SIZE(extra_regs); e++) { 781 if (e->skip) 782 continue; 783 784 guc_mmio_regset_write_one(ads, regset_map, e->reg, count++); 785 } 786 787 if (XE_WA(hwe->gt, 1607983814) && hwe->class == XE_ENGINE_CLASS_RENDER) { 788 for (i = 0; i < LNCFCMOCS_REG_COUNT; i++) { 789 guc_mmio_regset_write_one(ads, regset_map, 790 XELP_LNCFCMOCS(i), count++); 791 } 792 } 793 794 return count; 795 } 796 797 static void guc_mmio_reg_state_init(struct xe_guc_ads *ads) 798 { 799 size_t regset_offset = guc_ads_regset_offset(ads); 800 struct xe_gt *gt = ads_to_gt(ads); 801 struct xe_hw_engine *hwe; 802 enum xe_hw_engine_id id; 803 u32 addr = xe_bo_ggtt_addr(ads->bo) + regset_offset; 804 struct iosys_map regset_map = IOSYS_MAP_INIT_OFFSET(ads_to_map(ads), 805 regset_offset); 806 unsigned int regset_used = 0; 807 808 for_each_hw_engine(hwe, gt, id) { 809 unsigned int count; 810 u8 gc; 811 812 /* 813 * 1. Write all MMIO entries for this exec queue to the table. No 814 * need to worry about fused-off engines and when there are 815 * entries in the regset: the reg_state_list has been zero'ed 816 * by xe_guc_ads_populate() 817 */ 818 count = guc_mmio_regset_write(ads, ®set_map, hwe); 819 if (!count) 820 continue; 821 822 /* 823 * 2. Record in the header (ads.reg_state_list) the address 824 * location and number of entries 825 */ 826 gc = xe_engine_class_to_guc_class(hwe->class); 827 ads_blob_write(ads, ads.reg_state_list[gc][hwe->instance].address, addr); 828 ads_blob_write(ads, ads.reg_state_list[gc][hwe->instance].count, count); 829 830 addr += count * sizeof(struct guc_mmio_reg); 831 iosys_map_incr(®set_map, count * sizeof(struct guc_mmio_reg)); 832 833 regset_used += count * sizeof(struct guc_mmio_reg); 834 } 835 836 xe_gt_assert(gt, regset_used <= ads->regset_size); 837 } 838 839 static void guc_um_init_params(struct xe_guc_ads *ads) 840 { 841 u32 um_queue_offset = guc_ads_um_queues_offset(ads); 842 u64 base_dpa; 843 u32 base_ggtt; 844 int i; 845 846 base_ggtt = xe_bo_ggtt_addr(ads->bo) + um_queue_offset; 847 base_dpa = xe_bo_main_addr(ads->bo, PAGE_SIZE) + um_queue_offset; 848 849 for (i = 0; i < GUC_UM_HW_QUEUE_MAX; ++i) { 850 ads_blob_write(ads, um_init_params.queue_params[i].base_dpa, 851 base_dpa + (i * GUC_UM_QUEUE_SIZE)); 852 ads_blob_write(ads, um_init_params.queue_params[i].base_ggtt_address, 853 base_ggtt + (i * GUC_UM_QUEUE_SIZE)); 854 ads_blob_write(ads, um_init_params.queue_params[i].size_in_bytes, 855 GUC_UM_QUEUE_SIZE); 856 } 857 858 ads_blob_write(ads, um_init_params.page_response_timeout_in_us, 859 GUC_PAGE_RES_TIMEOUT_US); 860 } 861 862 static void guc_doorbell_init(struct xe_guc_ads *ads) 863 { 864 struct xe_device *xe = ads_to_xe(ads); 865 struct xe_gt *gt = ads_to_gt(ads); 866 867 if (GRAPHICS_VER(xe) >= 12 && !IS_DGFX(xe)) { 868 u32 distdbreg = 869 xe_mmio_read32(>->mmio, DIST_DBS_POPULATED); 870 871 ads_blob_write(ads, 872 system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_DOORBELL_COUNT_PER_SQIDI], 873 REG_FIELD_GET(DOORBELLS_PER_SQIDI_MASK, distdbreg) + 1); 874 } 875 } 876 877 /** 878 * xe_guc_ads_populate_minimal - populate minimal ADS 879 * @ads: Additional data structures object 880 * 881 * This function populates a minimal ADS that does not support submissions but 882 * enough so the GuC can load and the hwconfig table can be read. 883 */ 884 void xe_guc_ads_populate_minimal(struct xe_guc_ads *ads) 885 { 886 struct xe_gt *gt = ads_to_gt(ads); 887 struct iosys_map info_map = IOSYS_MAP_INIT_OFFSET(ads_to_map(ads), 888 offsetof(struct __guc_ads_blob, system_info)); 889 u32 base = xe_bo_ggtt_addr(ads->bo); 890 891 xe_gt_assert(gt, ads->bo); 892 893 xe_map_memset(ads_to_xe(ads), ads_to_map(ads), 0, 0, xe_bo_size(ads->bo)); 894 guc_policies_init(ads); 895 guc_golden_lrc_init(ads); 896 guc_mapping_table_init_invalid(gt, &info_map); 897 guc_doorbell_init(ads); 898 899 ads_blob_write(ads, ads.scheduler_policies, base + 900 offsetof(struct __guc_ads_blob, policies)); 901 ads_blob_write(ads, ads.gt_system_info, base + 902 offsetof(struct __guc_ads_blob, system_info)); 903 ads_blob_write(ads, ads.private_data, base + 904 guc_ads_private_data_offset(ads)); 905 } 906 907 void xe_guc_ads_populate(struct xe_guc_ads *ads) 908 { 909 struct xe_device *xe = ads_to_xe(ads); 910 struct xe_gt *gt = ads_to_gt(ads); 911 struct iosys_map info_map = IOSYS_MAP_INIT_OFFSET(ads_to_map(ads), 912 offsetof(struct __guc_ads_blob, system_info)); 913 u32 base = xe_bo_ggtt_addr(ads->bo); 914 915 xe_gt_assert(gt, ads->bo); 916 917 xe_map_memset(ads_to_xe(ads), ads_to_map(ads), 0, 0, xe_bo_size(ads->bo)); 918 guc_policies_init(ads); 919 fill_engine_enable_masks(gt, &info_map); 920 guc_mmio_reg_state_init(ads); 921 guc_golden_lrc_init(ads); 922 guc_mapping_table_init(gt, &info_map); 923 guc_capture_prep_lists(ads); 924 guc_doorbell_init(ads); 925 guc_waklv_init(ads); 926 927 if (xe->info.has_usm) { 928 guc_um_init_params(ads); 929 ads_blob_write(ads, ads.um_init_data, base + 930 offsetof(struct __guc_ads_blob, um_init_params)); 931 } 932 933 ads_blob_write(ads, ads.scheduler_policies, base + 934 offsetof(struct __guc_ads_blob, policies)); 935 ads_blob_write(ads, ads.gt_system_info, base + 936 offsetof(struct __guc_ads_blob, system_info)); 937 ads_blob_write(ads, ads.private_data, base + 938 guc_ads_private_data_offset(ads)); 939 } 940 941 /* 942 * After the golden LRC's are recorded for each engine class by the first 943 * submission, copy them to the ADS, as initialized earlier by 944 * guc_golden_lrc_init(). 945 */ 946 static void guc_golden_lrc_populate(struct xe_guc_ads *ads) 947 { 948 struct xe_device *xe = ads_to_xe(ads); 949 struct xe_gt *gt = ads_to_gt(ads); 950 struct iosys_map info_map = IOSYS_MAP_INIT_OFFSET(ads_to_map(ads), 951 offsetof(struct __guc_ads_blob, system_info)); 952 size_t total_size = 0, alloc_size, real_size; 953 u32 offset; 954 int class; 955 956 offset = guc_ads_golden_lrc_offset(ads); 957 958 for (class = 0; class < XE_ENGINE_CLASS_MAX; ++class) { 959 u8 guc_class; 960 961 guc_class = xe_engine_class_to_guc_class(class); 962 963 if (!info_map_read(xe, &info_map, 964 engine_enabled_masks[guc_class])) 965 continue; 966 967 xe_gt_assert(gt, gt->default_lrc[class]); 968 969 real_size = xe_gt_lrc_size(gt, class); 970 alloc_size = PAGE_ALIGN(real_size); 971 total_size += alloc_size; 972 973 xe_map_memcpy_to(xe, ads_to_map(ads), offset, 974 gt->default_lrc[class], real_size); 975 976 offset += alloc_size; 977 } 978 979 xe_gt_assert(gt, total_size == ads->golden_lrc_size); 980 } 981 982 void xe_guc_ads_populate_post_load(struct xe_guc_ads *ads) 983 { 984 guc_golden_lrc_populate(ads); 985 } 986 987 static int guc_ads_action_update_policies(struct xe_guc_ads *ads, u32 policy_offset) 988 { 989 struct xe_guc_ct *ct = &ads_to_guc(ads)->ct; 990 u32 action[] = { 991 XE_GUC_ACTION_GLOBAL_SCHED_POLICY_CHANGE, 992 policy_offset 993 }; 994 995 return xe_guc_ct_send(ct, action, ARRAY_SIZE(action), 0, 0); 996 } 997 998 /** 999 * xe_guc_ads_scheduler_policy_toggle_reset - Toggle reset policy 1000 * @ads: Additional data structures object 1001 * 1002 * This function update the GuC's engine reset policy based on wedged.mode. 1003 * 1004 * Return: 0 on success, and negative error code otherwise. 1005 */ 1006 int xe_guc_ads_scheduler_policy_toggle_reset(struct xe_guc_ads *ads) 1007 { 1008 struct guc_policies *policies; 1009 struct xe_guc *guc = ads_to_guc(ads); 1010 struct xe_device *xe = ads_to_xe(ads); 1011 CLASS(xe_guc_buf, buf)(&guc->buf, sizeof(*policies)); 1012 1013 if (!xe_guc_buf_is_valid(buf)) 1014 return -ENOBUFS; 1015 1016 policies = xe_guc_buf_cpu_ptr(buf); 1017 memset(policies, 0, sizeof(*policies)); 1018 1019 policies->dpc_promote_time = ads_blob_read(ads, policies.dpc_promote_time); 1020 policies->max_num_work_items = ads_blob_read(ads, policies.max_num_work_items); 1021 policies->is_valid = 1; 1022 if (xe->wedged.mode == 2) 1023 policies->global_flags |= GLOBAL_POLICY_DISABLE_ENGINE_RESET; 1024 else 1025 policies->global_flags &= ~GLOBAL_POLICY_DISABLE_ENGINE_RESET; 1026 1027 return guc_ads_action_update_policies(ads, xe_guc_buf_flush(buf)); 1028 } 1029