1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include "xe_guc_ads.h" 7 8 #include <linux/fault-inject.h> 9 10 #include <drm/drm_managed.h> 11 12 #include <generated/xe_wa_oob.h> 13 14 #include "abi/guc_actions_abi.h" 15 #include "regs/xe_engine_regs.h" 16 #include "regs/xe_gt_regs.h" 17 #include "regs/xe_guc_regs.h" 18 #include "xe_bo.h" 19 #include "xe_gt.h" 20 #include "xe_gt_ccs_mode.h" 21 #include "xe_gt_mcr.h" 22 #include "xe_gt_printk.h" 23 #include "xe_guc.h" 24 #include "xe_guc_buf.h" 25 #include "xe_guc_capture.h" 26 #include "xe_guc_ct.h" 27 #include "xe_hw_engine.h" 28 #include "xe_lrc.h" 29 #include "xe_map.h" 30 #include "xe_mmio.h" 31 #include "xe_wa.h" 32 33 /* Slack of a few additional entries per engine */ 34 #define ADS_REGSET_EXTRA_MAX 8 35 36 static struct xe_guc * 37 ads_to_guc(struct xe_guc_ads *ads) 38 { 39 return container_of(ads, struct xe_guc, ads); 40 } 41 42 static struct xe_gt * 43 ads_to_gt(struct xe_guc_ads *ads) 44 { 45 return container_of(ads, struct xe_gt, uc.guc.ads); 46 } 47 48 static struct xe_device * 49 ads_to_xe(struct xe_guc_ads *ads) 50 { 51 return gt_to_xe(ads_to_gt(ads)); 52 } 53 54 static struct iosys_map * 55 ads_to_map(struct xe_guc_ads *ads) 56 { 57 return &ads->bo->vmap; 58 } 59 60 /* UM Queue parameters: */ 61 #define GUC_UM_QUEUE_SIZE (SZ_64K) 62 #define GUC_PAGE_RES_TIMEOUT_US (-1) 63 64 /* 65 * The Additional Data Struct (ADS) has pointers for different buffers used by 66 * the GuC. One single gem object contains the ADS struct itself (guc_ads) and 67 * all the extra buffers indirectly linked via the ADS struct's entries. 68 * 69 * Layout of the ADS blob allocated for the GuC: 70 * 71 * +---------------------------------------+ <== base 72 * | guc_ads | 73 * +---------------------------------------+ 74 * | guc_policies | 75 * +---------------------------------------+ 76 * | guc_gt_system_info | 77 * +---------------------------------------+ 78 * | guc_engine_usage | 79 * +---------------------------------------+ 80 * | guc_um_init_params | 81 * +---------------------------------------+ <== static 82 * | guc_mmio_reg[countA] (engine 0.0) | 83 * | guc_mmio_reg[countB] (engine 0.1) | 84 * | guc_mmio_reg[countC] (engine 1.0) | 85 * | ... | 86 * +---------------------------------------+ <== dynamic 87 * | padding | 88 * +---------------------------------------+ <== 4K aligned 89 * | golden contexts | 90 * +---------------------------------------+ 91 * | padding | 92 * +---------------------------------------+ <== 4K aligned 93 * | w/a KLVs | 94 * +---------------------------------------+ 95 * | padding | 96 * +---------------------------------------+ <== 4K aligned 97 * | capture lists | 98 * +---------------------------------------+ 99 * | padding | 100 * +---------------------------------------+ <== 4K aligned 101 * | UM queues | 102 * +---------------------------------------+ 103 * | padding | 104 * +---------------------------------------+ <== 4K aligned 105 * | private data | 106 * +---------------------------------------+ 107 * | padding | 108 * +---------------------------------------+ <== 4K aligned 109 */ 110 struct __guc_ads_blob { 111 struct guc_ads ads; 112 struct guc_policies policies; 113 struct guc_gt_system_info system_info; 114 struct guc_engine_usage engine_usage; 115 struct guc_um_init_params um_init_params; 116 /* From here on, location is dynamic! Refer to above diagram. */ 117 struct guc_mmio_reg regset[]; 118 } __packed; 119 120 #define ads_blob_read(ads_, field_) \ 121 xe_map_rd_field(ads_to_xe(ads_), ads_to_map(ads_), 0, \ 122 struct __guc_ads_blob, field_) 123 124 #define ads_blob_write(ads_, field_, val_) \ 125 xe_map_wr_field(ads_to_xe(ads_), ads_to_map(ads_), 0, \ 126 struct __guc_ads_blob, field_, val_) 127 128 #define info_map_write(xe_, map_, field_, val_) \ 129 xe_map_wr_field(xe_, map_, 0, struct guc_gt_system_info, field_, val_) 130 131 #define info_map_read(xe_, map_, field_) \ 132 xe_map_rd_field(xe_, map_, 0, struct guc_gt_system_info, field_) 133 134 static size_t guc_ads_regset_size(struct xe_guc_ads *ads) 135 { 136 struct xe_device *xe = ads_to_xe(ads); 137 138 xe_assert(xe, ads->regset_size); 139 140 return ads->regset_size; 141 } 142 143 static size_t guc_ads_golden_lrc_size(struct xe_guc_ads *ads) 144 { 145 return PAGE_ALIGN(ads->golden_lrc_size); 146 } 147 148 static u32 guc_ads_waklv_size(struct xe_guc_ads *ads) 149 { 150 return PAGE_ALIGN(ads->ads_waklv_size); 151 } 152 153 static size_t guc_ads_capture_size(struct xe_guc_ads *ads) 154 { 155 return PAGE_ALIGN(ads->capture_size); 156 } 157 158 static size_t guc_ads_um_queues_size(struct xe_guc_ads *ads) 159 { 160 struct xe_device *xe = ads_to_xe(ads); 161 162 if (!xe->info.has_usm) 163 return 0; 164 165 return GUC_UM_QUEUE_SIZE * GUC_UM_HW_QUEUE_MAX; 166 } 167 168 static size_t guc_ads_private_data_size(struct xe_guc_ads *ads) 169 { 170 return PAGE_ALIGN(ads_to_guc(ads)->fw.private_data_size); 171 } 172 173 static size_t guc_ads_regset_offset(struct xe_guc_ads *ads) 174 { 175 return offsetof(struct __guc_ads_blob, regset); 176 } 177 178 static size_t guc_ads_golden_lrc_offset(struct xe_guc_ads *ads) 179 { 180 size_t offset; 181 182 offset = guc_ads_regset_offset(ads) + 183 guc_ads_regset_size(ads); 184 185 return PAGE_ALIGN(offset); 186 } 187 188 static size_t guc_ads_waklv_offset(struct xe_guc_ads *ads) 189 { 190 u32 offset; 191 192 offset = guc_ads_golden_lrc_offset(ads) + 193 guc_ads_golden_lrc_size(ads); 194 195 return PAGE_ALIGN(offset); 196 } 197 198 static size_t guc_ads_capture_offset(struct xe_guc_ads *ads) 199 { 200 size_t offset; 201 202 offset = guc_ads_waklv_offset(ads) + 203 guc_ads_waklv_size(ads); 204 205 return PAGE_ALIGN(offset); 206 } 207 208 static size_t guc_ads_um_queues_offset(struct xe_guc_ads *ads) 209 { 210 u32 offset; 211 212 offset = guc_ads_capture_offset(ads) + 213 guc_ads_capture_size(ads); 214 215 return PAGE_ALIGN(offset); 216 } 217 218 static size_t guc_ads_private_data_offset(struct xe_guc_ads *ads) 219 { 220 size_t offset; 221 222 offset = guc_ads_um_queues_offset(ads) + 223 guc_ads_um_queues_size(ads); 224 225 return PAGE_ALIGN(offset); 226 } 227 228 static size_t guc_ads_size(struct xe_guc_ads *ads) 229 { 230 return guc_ads_private_data_offset(ads) + 231 guc_ads_private_data_size(ads); 232 } 233 234 static size_t calculate_regset_size(struct xe_gt *gt) 235 { 236 struct xe_reg_sr_entry *sr_entry; 237 unsigned long sr_idx; 238 struct xe_hw_engine *hwe; 239 enum xe_hw_engine_id id; 240 unsigned int count = 0; 241 242 for_each_hw_engine(hwe, gt, id) 243 xa_for_each(&hwe->reg_sr.xa, sr_idx, sr_entry) 244 count++; 245 246 count += ADS_REGSET_EXTRA_MAX * XE_NUM_HW_ENGINES; 247 248 if (XE_GT_WA(gt, 1607983814)) 249 count += LNCFCMOCS_REG_COUNT; 250 251 return count * sizeof(struct guc_mmio_reg); 252 } 253 254 static u32 engine_enable_mask(struct xe_gt *gt, enum xe_engine_class class) 255 { 256 struct xe_hw_engine *hwe; 257 enum xe_hw_engine_id id; 258 u32 mask = 0; 259 260 for_each_hw_engine(hwe, gt, id) 261 if (hwe->class == class) 262 mask |= BIT(hwe->instance); 263 264 return mask; 265 } 266 267 static size_t calculate_golden_lrc_size(struct xe_guc_ads *ads) 268 { 269 struct xe_gt *gt = ads_to_gt(ads); 270 size_t total_size = 0, alloc_size, real_size; 271 int class; 272 273 for (class = 0; class < XE_ENGINE_CLASS_MAX; ++class) { 274 if (!engine_enable_mask(gt, class)) 275 continue; 276 277 real_size = xe_gt_lrc_size(gt, class); 278 alloc_size = PAGE_ALIGN(real_size); 279 total_size += alloc_size; 280 } 281 282 return total_size; 283 } 284 285 static void guc_waklv_enable(struct xe_guc_ads *ads, 286 u32 data[], u32 data_len_dw, 287 u32 *offset, u32 *remain, 288 enum xe_guc_klv_ids klv_id) 289 { 290 size_t size = sizeof(u32) * (1 + data_len_dw); 291 292 if (*remain < size) { 293 drm_warn(&ads_to_xe(ads)->drm, 294 "w/a klv buffer too small to add klv id 0x%04X\n", klv_id); 295 return; 296 } 297 298 /* 16:16 key/length */ 299 xe_map_wr(ads_to_xe(ads), ads_to_map(ads), *offset, u32, 300 FIELD_PREP(GUC_KLV_0_KEY, klv_id) | FIELD_PREP(GUC_KLV_0_LEN, data_len_dw)); 301 /* data_len_dw dwords of data */ 302 xe_map_memcpy_to(ads_to_xe(ads), ads_to_map(ads), 303 *offset + sizeof(u32), data, data_len_dw * sizeof(u32)); 304 305 *offset += size; 306 *remain -= size; 307 } 308 309 static void guc_waklv_init(struct xe_guc_ads *ads) 310 { 311 struct xe_gt *gt = ads_to_gt(ads); 312 u64 addr_ggtt; 313 u32 offset, remain, size; 314 315 offset = guc_ads_waklv_offset(ads); 316 remain = guc_ads_waklv_size(ads); 317 318 if (XE_GT_WA(gt, 16021333562)) 319 guc_waklv_enable(ads, NULL, 0, &offset, &remain, 320 GUC_WORKAROUND_KLV_BLOCK_INTERRUPTS_WHEN_MGSR_BLOCKED); 321 if (XE_GT_WA(gt, 18024947630)) 322 guc_waklv_enable(ads, NULL, 0, &offset, &remain, 323 GUC_WORKAROUND_KLV_ID_GAM_PFQ_SHADOW_TAIL_POLLING); 324 if (XE_GT_WA(gt, 16022287689)) 325 guc_waklv_enable(ads, NULL, 0, &offset, &remain, 326 GUC_WORKAROUND_KLV_ID_DISABLE_MTP_DURING_ASYNC_COMPUTE); 327 328 if (XE_GT_WA(gt, 14022866841)) 329 guc_waklv_enable(ads, NULL, 0, &offset, &remain, 330 GUC_WA_KLV_WAKE_POWER_DOMAINS_FOR_OUTBOUND_MMIO); 331 332 /* 333 * On RC6 exit, GuC will write register 0xB04 with the default value provided. As of now, 334 * the default value for this register is determined to be 0xC40. This could change in the 335 * future, so GuC depends on KMD to send it the correct value. 336 */ 337 if (XE_GT_WA(gt, 13011645652)) { 338 u32 data = 0xC40; 339 340 guc_waklv_enable(ads, &data, 1, &offset, &remain, 341 GUC_WA_KLV_NP_RD_WRITE_TO_CLEAR_RCSM_AT_CGP_LATE_RESTORE); 342 } 343 344 if (XE_GT_WA(gt, 14022293748) || XE_GT_WA(gt, 22019794406)) 345 guc_waklv_enable(ads, NULL, 0, &offset, &remain, 346 GUC_WORKAROUND_KLV_ID_BACK_TO_BACK_RCS_ENGINE_RESET); 347 348 if (GUC_FIRMWARE_VER_AT_LEAST(>->uc.guc, 70, 44) && XE_GT_WA(gt, 16026508708)) 349 guc_waklv_enable(ads, NULL, 0, &offset, &remain, 350 GUC_WA_KLV_RESET_BB_STACK_PTR_ON_VF_SWITCH); 351 if (GUC_FIRMWARE_VER_AT_LEAST(>->uc.guc, 70, 47) && XE_GT_WA(gt, 16026007364)) { 352 u32 data[] = { 353 0x0, 354 0xF, 355 }; 356 guc_waklv_enable(ads, data, ARRAY_SIZE(data), &offset, &remain, 357 GUC_WA_KLV_RESTORE_UNSAVED_MEDIA_CONTROL_REG); 358 } 359 360 if (XE_GT_WA(gt, 14020001231)) 361 guc_waklv_enable(ads, NULL, 0, &offset, &remain, 362 GUC_WORKAROUND_KLV_DISABLE_PSMI_INTERRUPTS_AT_C6_ENTRY_RESTORE_AT_EXIT); 363 if (XE_GT_WA(gt, 14025515070) && GUC_FIRMWARE_VER_AT_LEAST(>->uc.guc, 70, 53)) 364 guc_waklv_enable(ads, NULL, 0, &offset, &remain, 365 GUC_WA_KLV_CLR_CS_INDIRECT_RING_STATE_IF_IDLE_AT_CTX_REG); 366 367 size = guc_ads_waklv_size(ads) - remain; 368 if (!size) 369 return; 370 371 offset = guc_ads_waklv_offset(ads); 372 addr_ggtt = xe_bo_ggtt_addr(ads->bo) + offset; 373 374 ads_blob_write(ads, ads.wa_klv_addr_lo, lower_32_bits(addr_ggtt)); 375 ads_blob_write(ads, ads.wa_klv_addr_hi, upper_32_bits(addr_ggtt)); 376 ads_blob_write(ads, ads.wa_klv_size, size); 377 } 378 379 static int calculate_waklv_size(struct xe_guc_ads *ads) 380 { 381 /* 382 * A single page is both the minimum size possible and 383 * is sufficiently large enough for all current platforms. 384 */ 385 return SZ_4K; 386 } 387 388 #define MAX_GOLDEN_LRC_SIZE (SZ_4K * 64) 389 390 int xe_guc_ads_init(struct xe_guc_ads *ads) 391 { 392 struct xe_device *xe = ads_to_xe(ads); 393 struct xe_gt *gt = ads_to_gt(ads); 394 struct xe_tile *tile = gt_to_tile(gt); 395 struct xe_bo *bo; 396 397 ads->golden_lrc_size = calculate_golden_lrc_size(ads); 398 ads->capture_size = xe_guc_capture_ads_input_worst_size(ads_to_guc(ads)); 399 ads->regset_size = calculate_regset_size(gt); 400 ads->ads_waklv_size = calculate_waklv_size(ads); 401 402 bo = xe_managed_bo_create_pin_map(xe, tile, guc_ads_size(ads) + MAX_GOLDEN_LRC_SIZE, 403 XE_BO_FLAG_SYSTEM | 404 XE_BO_FLAG_GGTT | 405 XE_BO_FLAG_GGTT_INVALIDATE | 406 XE_BO_FLAG_PINNED_NORESTORE); 407 if (IS_ERR(bo)) 408 return PTR_ERR(bo); 409 410 ads->bo = bo; 411 412 return 0; 413 } 414 ALLOW_ERROR_INJECTION(xe_guc_ads_init, ERRNO); /* See xe_pci_probe() */ 415 416 /** 417 * xe_guc_ads_init_post_hwconfig - initialize ADS post hwconfig load 418 * @ads: Additional data structures object 419 * 420 * Recalculate golden_lrc_size, capture_size and regset_size as the number 421 * hardware engines may have changed after the hwconfig was loaded. Also verify 422 * the new sizes fit in the already allocated ADS buffer object. 423 * 424 * Return: 0 on success, negative error code on error. 425 */ 426 int xe_guc_ads_init_post_hwconfig(struct xe_guc_ads *ads) 427 { 428 struct xe_gt *gt = ads_to_gt(ads); 429 u32 prev_regset_size = ads->regset_size; 430 431 xe_gt_assert(gt, ads->bo); 432 433 ads->golden_lrc_size = calculate_golden_lrc_size(ads); 434 /* Calculate Capture size with worst size */ 435 ads->capture_size = xe_guc_capture_ads_input_worst_size(ads_to_guc(ads)); 436 ads->regset_size = calculate_regset_size(gt); 437 438 xe_gt_assert(gt, ads->golden_lrc_size + 439 (ads->regset_size - prev_regset_size) <= 440 MAX_GOLDEN_LRC_SIZE); 441 442 return 0; 443 } 444 445 static void guc_policies_init(struct xe_guc_ads *ads) 446 { 447 struct xe_device *xe = ads_to_xe(ads); 448 u32 global_flags = 0; 449 450 ads_blob_write(ads, policies.dpc_promote_time, 451 GLOBAL_POLICY_DEFAULT_DPC_PROMOTE_TIME_US); 452 ads_blob_write(ads, policies.max_num_work_items, 453 GLOBAL_POLICY_MAX_NUM_WI); 454 455 if (xe->wedged.mode == XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET) 456 global_flags |= GLOBAL_POLICY_DISABLE_ENGINE_RESET; 457 458 ads_blob_write(ads, policies.global_flags, global_flags); 459 ads_blob_write(ads, policies.is_valid, 1); 460 } 461 462 static void fill_engine_enable_masks(struct xe_gt *gt, 463 struct iosys_map *info_map) 464 { 465 struct xe_device *xe = gt_to_xe(gt); 466 467 info_map_write(xe, info_map, engine_enabled_masks[GUC_RENDER_CLASS], 468 engine_enable_mask(gt, XE_ENGINE_CLASS_RENDER)); 469 info_map_write(xe, info_map, engine_enabled_masks[GUC_BLITTER_CLASS], 470 engine_enable_mask(gt, XE_ENGINE_CLASS_COPY)); 471 info_map_write(xe, info_map, engine_enabled_masks[GUC_VIDEO_CLASS], 472 engine_enable_mask(gt, XE_ENGINE_CLASS_VIDEO_DECODE)); 473 info_map_write(xe, info_map, 474 engine_enabled_masks[GUC_VIDEOENHANCE_CLASS], 475 engine_enable_mask(gt, XE_ENGINE_CLASS_VIDEO_ENHANCE)); 476 info_map_write(xe, info_map, engine_enabled_masks[GUC_COMPUTE_CLASS], 477 engine_enable_mask(gt, XE_ENGINE_CLASS_COMPUTE)); 478 info_map_write(xe, info_map, engine_enabled_masks[GUC_GSC_OTHER_CLASS], 479 engine_enable_mask(gt, XE_ENGINE_CLASS_OTHER)); 480 } 481 482 /* 483 * Write the offsets corresponding to the golden LRCs. The actual data is 484 * populated later by guc_golden_lrc_populate() 485 */ 486 static void guc_golden_lrc_init(struct xe_guc_ads *ads) 487 { 488 struct xe_device *xe = ads_to_xe(ads); 489 struct xe_gt *gt = ads_to_gt(ads); 490 struct iosys_map info_map = IOSYS_MAP_INIT_OFFSET(ads_to_map(ads), 491 offsetof(struct __guc_ads_blob, system_info)); 492 size_t alloc_size, real_size; 493 u32 addr_ggtt, offset; 494 int class; 495 496 offset = guc_ads_golden_lrc_offset(ads); 497 addr_ggtt = xe_bo_ggtt_addr(ads->bo) + offset; 498 499 for (class = 0; class < XE_ENGINE_CLASS_MAX; ++class) { 500 u8 guc_class; 501 502 guc_class = xe_engine_class_to_guc_class(class); 503 504 if (!info_map_read(xe, &info_map, 505 engine_enabled_masks[guc_class])) 506 continue; 507 508 real_size = xe_gt_lrc_size(gt, class); 509 alloc_size = PAGE_ALIGN(real_size); 510 511 /* 512 * This interface is slightly confusing. We need to pass the 513 * base address of the full golden context and the size of just 514 * the engine state, which is the section of the context image 515 * that starts after the execlists LRC registers. This is 516 * required to allow the GuC to restore just the engine state 517 * when a watchdog reset occurs. 518 */ 519 ads_blob_write(ads, ads.eng_state_size[guc_class], 520 xe_lrc_engine_state_size(gt, class)); 521 ads_blob_write(ads, ads.golden_context_lrca[guc_class], 522 addr_ggtt); 523 524 addr_ggtt += alloc_size; 525 } 526 } 527 528 static void guc_mapping_table_init_invalid(struct xe_gt *gt, 529 struct iosys_map *info_map) 530 { 531 struct xe_device *xe = gt_to_xe(gt); 532 unsigned int i, j; 533 534 /* Table must be set to invalid values for entries not used */ 535 for (i = 0; i < GUC_MAX_ENGINE_CLASSES; ++i) 536 for (j = 0; j < GUC_MAX_INSTANCES_PER_CLASS; ++j) 537 info_map_write(xe, info_map, mapping_table[i][j], 538 GUC_MAX_INSTANCES_PER_CLASS); 539 } 540 541 static void guc_mapping_table_init(struct xe_gt *gt, 542 struct iosys_map *info_map) 543 { 544 struct xe_device *xe = gt_to_xe(gt); 545 struct xe_hw_engine *hwe; 546 enum xe_hw_engine_id id; 547 548 guc_mapping_table_init_invalid(gt, info_map); 549 550 for_each_hw_engine(hwe, gt, id) { 551 u8 guc_class; 552 553 guc_class = xe_engine_class_to_guc_class(hwe->class); 554 info_map_write(xe, info_map, 555 mapping_table[guc_class][hwe->logical_instance], 556 hwe->instance); 557 } 558 } 559 560 static u32 guc_get_capture_engine_mask(struct xe_gt *gt, struct iosys_map *info_map, 561 enum guc_capture_list_class_type capture_class) 562 { 563 struct xe_device *xe = gt_to_xe(gt); 564 u32 mask; 565 566 switch (capture_class) { 567 case GUC_CAPTURE_LIST_CLASS_RENDER_COMPUTE: 568 mask = info_map_read(xe, info_map, engine_enabled_masks[GUC_RENDER_CLASS]); 569 mask |= info_map_read(xe, info_map, engine_enabled_masks[GUC_COMPUTE_CLASS]); 570 break; 571 case GUC_CAPTURE_LIST_CLASS_VIDEO: 572 mask = info_map_read(xe, info_map, engine_enabled_masks[GUC_VIDEO_CLASS]); 573 break; 574 case GUC_CAPTURE_LIST_CLASS_VIDEOENHANCE: 575 mask = info_map_read(xe, info_map, engine_enabled_masks[GUC_VIDEOENHANCE_CLASS]); 576 break; 577 case GUC_CAPTURE_LIST_CLASS_BLITTER: 578 mask = info_map_read(xe, info_map, engine_enabled_masks[GUC_BLITTER_CLASS]); 579 break; 580 case GUC_CAPTURE_LIST_CLASS_GSC_OTHER: 581 mask = info_map_read(xe, info_map, engine_enabled_masks[GUC_GSC_OTHER_CLASS]); 582 break; 583 default: 584 mask = 0; 585 } 586 587 return mask; 588 } 589 590 static inline bool get_capture_list(struct xe_guc_ads *ads, struct xe_guc *guc, struct xe_gt *gt, 591 int owner, int type, int class, u32 *total_size, size_t *size, 592 void **pptr) 593 { 594 *size = 0; 595 596 if (!xe_guc_capture_getlistsize(guc, owner, type, class, size)) { 597 if (*total_size + *size > ads->capture_size) 598 xe_gt_dbg(gt, "Capture size overflow :%zu vs %d\n", 599 *total_size + *size, ads->capture_size); 600 else if (!xe_guc_capture_getlist(guc, owner, type, class, pptr)) 601 return false; 602 } 603 604 return true; 605 } 606 607 static int guc_capture_prep_lists(struct xe_guc_ads *ads) 608 { 609 struct xe_guc *guc = ads_to_guc(ads); 610 struct xe_gt *gt = ads_to_gt(ads); 611 u32 ads_ggtt, capture_offset, null_ggtt, total_size = 0; 612 struct iosys_map info_map; 613 size_t size = 0; 614 void *ptr; 615 int i, j; 616 617 /* 618 * GuC Capture's steered reg-list needs to be allocated and initialized 619 * after the GuC-hwconfig is available which guaranteed from here. 620 */ 621 xe_guc_capture_steered_list_init(ads_to_guc(ads)); 622 623 capture_offset = guc_ads_capture_offset(ads); 624 ads_ggtt = xe_bo_ggtt_addr(ads->bo); 625 info_map = IOSYS_MAP_INIT_OFFSET(ads_to_map(ads), 626 offsetof(struct __guc_ads_blob, system_info)); 627 628 /* first, set aside the first page for a capture_list with zero descriptors */ 629 total_size = PAGE_SIZE; 630 if (!xe_guc_capture_getnullheader(guc, &ptr, &size)) 631 xe_map_memcpy_to(ads_to_xe(ads), ads_to_map(ads), capture_offset, ptr, size); 632 633 null_ggtt = ads_ggtt + capture_offset; 634 capture_offset += PAGE_SIZE; 635 636 /* 637 * Populate capture list : at this point adps is already allocated and 638 * mapped to worst case size 639 */ 640 for (i = 0; i < GUC_CAPTURE_LIST_INDEX_MAX; i++) { 641 bool write_empty_list; 642 643 for (j = 0; j < GUC_CAPTURE_LIST_CLASS_MAX; j++) { 644 u32 engine_mask = guc_get_capture_engine_mask(gt, &info_map, j); 645 /* null list if we dont have said engine or list */ 646 if (!engine_mask) { 647 ads_blob_write(ads, ads.capture_class[i][j], null_ggtt); 648 ads_blob_write(ads, ads.capture_instance[i][j], null_ggtt); 649 continue; 650 } 651 652 /* engine exists: start with engine-class registers */ 653 write_empty_list = get_capture_list(ads, guc, gt, i, 654 GUC_STATE_CAPTURE_TYPE_ENGINE_CLASS, 655 j, &total_size, &size, &ptr); 656 if (!write_empty_list) { 657 ads_blob_write(ads, ads.capture_class[i][j], 658 ads_ggtt + capture_offset); 659 xe_map_memcpy_to(ads_to_xe(ads), ads_to_map(ads), capture_offset, 660 ptr, size); 661 total_size += size; 662 capture_offset += size; 663 } else { 664 ads_blob_write(ads, ads.capture_class[i][j], null_ggtt); 665 } 666 667 /* engine exists: next, engine-instance registers */ 668 write_empty_list = get_capture_list(ads, guc, gt, i, 669 GUC_STATE_CAPTURE_TYPE_ENGINE_INSTANCE, 670 j, &total_size, &size, &ptr); 671 if (!write_empty_list) { 672 ads_blob_write(ads, ads.capture_instance[i][j], 673 ads_ggtt + capture_offset); 674 xe_map_memcpy_to(ads_to_xe(ads), ads_to_map(ads), capture_offset, 675 ptr, size); 676 total_size += size; 677 capture_offset += size; 678 } else { 679 ads_blob_write(ads, ads.capture_instance[i][j], null_ggtt); 680 } 681 } 682 683 /* global registers is last in our PF/VF loops */ 684 write_empty_list = get_capture_list(ads, guc, gt, i, 685 GUC_STATE_CAPTURE_TYPE_GLOBAL, 686 0, &total_size, &size, &ptr); 687 if (!write_empty_list) { 688 ads_blob_write(ads, ads.capture_global[i], ads_ggtt + capture_offset); 689 xe_map_memcpy_to(ads_to_xe(ads), ads_to_map(ads), capture_offset, ptr, 690 size); 691 total_size += size; 692 capture_offset += size; 693 } else { 694 ads_blob_write(ads, ads.capture_global[i], null_ggtt); 695 } 696 } 697 698 if (ads->capture_size != PAGE_ALIGN(total_size)) 699 xe_gt_dbg(gt, "Updated ADS capture size %d (was %d)\n", 700 PAGE_ALIGN(total_size), ads->capture_size); 701 return PAGE_ALIGN(total_size); 702 } 703 704 static void guc_mmio_regset_write_one(struct xe_guc_ads *ads, 705 struct iosys_map *regset_map, 706 struct xe_reg reg, 707 unsigned int n_entry) 708 { 709 struct guc_mmio_reg entry = { 710 .offset = reg.addr, 711 .flags = reg.masked ? GUC_REGSET_MASKED : 0, 712 }; 713 714 if (reg.mcr) { 715 struct xe_reg_mcr mcr_reg = XE_REG_MCR(reg.addr); 716 u8 group, instance; 717 718 bool steer = xe_gt_mcr_get_nonterminated_steering(ads_to_gt(ads), mcr_reg, 719 &group, &instance); 720 721 if (steer) { 722 entry.flags |= FIELD_PREP(GUC_REGSET_STEERING_GROUP, group); 723 entry.flags |= FIELD_PREP(GUC_REGSET_STEERING_INSTANCE, instance); 724 entry.flags |= GUC_REGSET_STEERING_NEEDED; 725 } 726 } 727 728 xe_map_memcpy_to(ads_to_xe(ads), regset_map, n_entry * sizeof(entry), 729 &entry, sizeof(entry)); 730 } 731 732 static unsigned int guc_mmio_regset_write(struct xe_guc_ads *ads, 733 struct iosys_map *regset_map, 734 struct xe_hw_engine *hwe) 735 { 736 struct xe_hw_engine *hwe_rcs_reset_domain = 737 xe_gt_any_hw_engine_by_reset_domain(hwe->gt, XE_ENGINE_CLASS_RENDER); 738 struct xe_reg_sr_entry *entry; 739 unsigned long idx; 740 unsigned int count = 0; 741 const struct { 742 struct xe_reg reg; 743 bool skip; 744 } *e, extra_regs[] = { 745 { .reg = RING_HWS_PGA(hwe->mmio_base), }, 746 { .reg = RING_IMR(hwe->mmio_base), }, 747 { .reg = CCS_MODE, 748 .skip = hwe != hwe_rcs_reset_domain || !xe_gt_ccs_mode_enabled(hwe->gt) }, 749 }; 750 u32 i; 751 752 BUILD_BUG_ON(ARRAY_SIZE(extra_regs) > ADS_REGSET_EXTRA_MAX); 753 754 xa_for_each(&hwe->reg_sr.xa, idx, entry) 755 guc_mmio_regset_write_one(ads, regset_map, entry->reg, count++); 756 757 for (e = extra_regs; e < extra_regs + ARRAY_SIZE(extra_regs); e++) { 758 if (e->skip) 759 continue; 760 761 guc_mmio_regset_write_one(ads, regset_map, e->reg, count++); 762 } 763 764 if (XE_GT_WA(hwe->gt, 1607983814) && hwe->class == XE_ENGINE_CLASS_RENDER) { 765 for (i = 0; i < LNCFCMOCS_REG_COUNT; i++) { 766 guc_mmio_regset_write_one(ads, regset_map, 767 XELP_LNCFCMOCS(i), count++); 768 } 769 } 770 771 if (XE_GT_WA(hwe->gt, 16023105232)) 772 guc_mmio_regset_write_one(ads, regset_map, 773 RING_IDLEDLY(hwe->mmio_base), 774 count++); 775 776 return count; 777 } 778 779 static void guc_mmio_reg_state_init(struct xe_guc_ads *ads) 780 { 781 size_t regset_offset = guc_ads_regset_offset(ads); 782 struct xe_gt *gt = ads_to_gt(ads); 783 struct xe_hw_engine *hwe; 784 enum xe_hw_engine_id id; 785 u32 addr = xe_bo_ggtt_addr(ads->bo) + regset_offset; 786 struct iosys_map regset_map = IOSYS_MAP_INIT_OFFSET(ads_to_map(ads), 787 regset_offset); 788 unsigned int regset_used = 0; 789 790 for_each_hw_engine(hwe, gt, id) { 791 unsigned int count; 792 u8 gc; 793 794 /* 795 * 1. Write all MMIO entries for this exec queue to the table. No 796 * need to worry about fused-off engines and when there are 797 * entries in the regset: the reg_state_list has been zero'ed 798 * by xe_guc_ads_populate() 799 */ 800 count = guc_mmio_regset_write(ads, ®set_map, hwe); 801 if (!count) 802 continue; 803 804 /* 805 * 2. Record in the header (ads.reg_state_list) the address 806 * location and number of entries 807 */ 808 gc = xe_engine_class_to_guc_class(hwe->class); 809 ads_blob_write(ads, ads.reg_state_list[gc][hwe->instance].address, addr); 810 ads_blob_write(ads, ads.reg_state_list[gc][hwe->instance].count, count); 811 812 addr += count * sizeof(struct guc_mmio_reg); 813 iosys_map_incr(®set_map, count * sizeof(struct guc_mmio_reg)); 814 815 regset_used += count * sizeof(struct guc_mmio_reg); 816 } 817 818 xe_gt_assert(gt, regset_used <= ads->regset_size); 819 } 820 821 static void guc_um_init_params(struct xe_guc_ads *ads) 822 { 823 u32 um_queue_offset = guc_ads_um_queues_offset(ads); 824 struct xe_guc *guc = ads_to_guc(ads); 825 struct xe_device *xe = ads_to_xe(ads); 826 u64 base_dpa; 827 u32 base_ggtt; 828 bool with_dpa; 829 int i; 830 831 with_dpa = !xe_guc_using_main_gamctrl_queues(guc); 832 833 base_ggtt = xe_bo_ggtt_addr(ads->bo) + um_queue_offset; 834 base_dpa = xe_bo_main_addr(ads->bo, PAGE_SIZE) + um_queue_offset; 835 836 for (i = 0; i < GUC_UM_HW_QUEUE_MAX; ++i) { 837 /* 838 * Some platforms support USM but not access counters. 839 * Skip ACCESS_COUNTER queue initialization for such 840 * platforms, leaving queue_params[2] zero-initialized 841 * to signal unavailability to the GuC. 842 */ 843 if (i == GUC_UM_HW_QUEUE_ACCESS_COUNTER && 844 !xe->info.has_access_counter) 845 continue; 846 847 ads_blob_write(ads, um_init_params.queue_params[i].base_dpa, 848 with_dpa ? (base_dpa + (i * GUC_UM_QUEUE_SIZE)) : 0); 849 ads_blob_write(ads, um_init_params.queue_params[i].base_ggtt_address, 850 base_ggtt + (i * GUC_UM_QUEUE_SIZE)); 851 ads_blob_write(ads, um_init_params.queue_params[i].size_in_bytes, 852 GUC_UM_QUEUE_SIZE); 853 } 854 855 ads_blob_write(ads, um_init_params.page_response_timeout_in_us, 856 GUC_PAGE_RES_TIMEOUT_US); 857 } 858 859 static void guc_doorbell_init(struct xe_guc_ads *ads) 860 { 861 struct xe_device *xe = ads_to_xe(ads); 862 struct xe_gt *gt = ads_to_gt(ads); 863 864 if (GRAPHICS_VER(xe) >= 12 && !IS_DGFX(xe)) { 865 u32 distdbreg = 866 xe_mmio_read32(>->mmio, DIST_DBS_POPULATED); 867 868 ads_blob_write(ads, 869 system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_DOORBELL_COUNT_PER_SQIDI], 870 REG_FIELD_GET(DOORBELLS_PER_SQIDI_MASK, distdbreg) + 1); 871 } 872 } 873 874 /** 875 * xe_guc_ads_populate_minimal - populate minimal ADS 876 * @ads: Additional data structures object 877 * 878 * This function populates a minimal ADS that does not support submissions but 879 * enough so the GuC can load and the hwconfig table can be read. 880 */ 881 void xe_guc_ads_populate_minimal(struct xe_guc_ads *ads) 882 { 883 struct xe_gt *gt = ads_to_gt(ads); 884 struct iosys_map info_map = IOSYS_MAP_INIT_OFFSET(ads_to_map(ads), 885 offsetof(struct __guc_ads_blob, system_info)); 886 u32 base = xe_bo_ggtt_addr(ads->bo); 887 888 xe_gt_assert(gt, ads->bo); 889 890 xe_map_memset(ads_to_xe(ads), ads_to_map(ads), 0, 0, xe_bo_size(ads->bo)); 891 guc_policies_init(ads); 892 guc_golden_lrc_init(ads); 893 guc_mapping_table_init_invalid(gt, &info_map); 894 guc_doorbell_init(ads); 895 896 ads_blob_write(ads, ads.scheduler_policies, base + 897 offsetof(struct __guc_ads_blob, policies)); 898 ads_blob_write(ads, ads.gt_system_info, base + 899 offsetof(struct __guc_ads_blob, system_info)); 900 ads_blob_write(ads, ads.private_data, base + 901 guc_ads_private_data_offset(ads)); 902 } 903 904 void xe_guc_ads_populate(struct xe_guc_ads *ads) 905 { 906 struct xe_device *xe = ads_to_xe(ads); 907 struct xe_gt *gt = ads_to_gt(ads); 908 struct iosys_map info_map = IOSYS_MAP_INIT_OFFSET(ads_to_map(ads), 909 offsetof(struct __guc_ads_blob, system_info)); 910 u32 base = xe_bo_ggtt_addr(ads->bo); 911 912 xe_gt_assert(gt, ads->bo); 913 914 xe_map_memset(ads_to_xe(ads), ads_to_map(ads), 0, 0, xe_bo_size(ads->bo)); 915 guc_policies_init(ads); 916 fill_engine_enable_masks(gt, &info_map); 917 guc_mmio_reg_state_init(ads); 918 guc_golden_lrc_init(ads); 919 guc_mapping_table_init(gt, &info_map); 920 guc_capture_prep_lists(ads); 921 guc_doorbell_init(ads); 922 guc_waklv_init(ads); 923 924 if (xe->info.has_usm) { 925 guc_um_init_params(ads); 926 ads_blob_write(ads, ads.um_init_data, base + 927 offsetof(struct __guc_ads_blob, um_init_params)); 928 } 929 930 ads_blob_write(ads, ads.scheduler_policies, base + 931 offsetof(struct __guc_ads_blob, policies)); 932 ads_blob_write(ads, ads.gt_system_info, base + 933 offsetof(struct __guc_ads_blob, system_info)); 934 ads_blob_write(ads, ads.private_data, base + 935 guc_ads_private_data_offset(ads)); 936 } 937 938 /* 939 * After the golden LRC's are recorded for each engine class by the first 940 * submission, copy them to the ADS, as initialized earlier by 941 * guc_golden_lrc_init(). 942 */ 943 static void guc_golden_lrc_populate(struct xe_guc_ads *ads) 944 { 945 struct xe_device *xe = ads_to_xe(ads); 946 struct xe_gt *gt = ads_to_gt(ads); 947 struct iosys_map info_map = IOSYS_MAP_INIT_OFFSET(ads_to_map(ads), 948 offsetof(struct __guc_ads_blob, system_info)); 949 size_t total_size = 0, alloc_size, real_size; 950 u32 offset; 951 int class; 952 953 offset = guc_ads_golden_lrc_offset(ads); 954 955 for (class = 0; class < XE_ENGINE_CLASS_MAX; ++class) { 956 u8 guc_class; 957 958 guc_class = xe_engine_class_to_guc_class(class); 959 960 if (!info_map_read(xe, &info_map, 961 engine_enabled_masks[guc_class])) 962 continue; 963 964 xe_gt_assert(gt, gt->default_lrc[class]); 965 966 real_size = xe_gt_lrc_size(gt, class); 967 alloc_size = PAGE_ALIGN(real_size); 968 total_size += alloc_size; 969 970 xe_map_memcpy_to(xe, ads_to_map(ads), offset, 971 gt->default_lrc[class], real_size); 972 973 offset += alloc_size; 974 } 975 976 xe_gt_assert(gt, total_size == ads->golden_lrc_size); 977 } 978 979 void xe_guc_ads_populate_post_load(struct xe_guc_ads *ads) 980 { 981 guc_golden_lrc_populate(ads); 982 } 983 984 static int guc_ads_action_update_policies(struct xe_guc_ads *ads, u32 policy_offset) 985 { 986 struct xe_guc_ct *ct = &ads_to_guc(ads)->ct; 987 u32 action[] = { 988 XE_GUC_ACTION_GLOBAL_SCHED_POLICY_CHANGE, 989 policy_offset 990 }; 991 992 return xe_guc_ct_send(ct, action, ARRAY_SIZE(action), 0, 0); 993 } 994 995 /** 996 * xe_guc_ads_scheduler_policy_toggle_reset - Toggle reset policy 997 * @ads: Additional data structures object 998 * @enable_engine_reset: true to enable engine resets, false otherwise 999 * 1000 * This function update the GuC's engine reset policy. 1001 * 1002 * Return: 0 on success, and negative error code otherwise. 1003 */ 1004 int xe_guc_ads_scheduler_policy_toggle_reset(struct xe_guc_ads *ads, 1005 bool enable_engine_reset) 1006 { 1007 struct guc_policies *policies; 1008 struct xe_guc *guc = ads_to_guc(ads); 1009 CLASS(xe_guc_buf, buf)(&guc->buf, sizeof(*policies)); 1010 1011 if (!xe_guc_buf_is_valid(buf)) 1012 return -ENOBUFS; 1013 1014 policies = xe_guc_buf_cpu_ptr(buf); 1015 memset(policies, 0, sizeof(*policies)); 1016 1017 policies->dpc_promote_time = ads_blob_read(ads, policies.dpc_promote_time); 1018 policies->max_num_work_items = ads_blob_read(ads, policies.max_num_work_items); 1019 policies->is_valid = 1; 1020 1021 if (enable_engine_reset) 1022 policies->global_flags &= ~GLOBAL_POLICY_DISABLE_ENGINE_RESET; 1023 else 1024 policies->global_flags |= GLOBAL_POLICY_DISABLE_ENGINE_RESET; 1025 1026 return guc_ads_action_update_policies(ads, xe_guc_buf_flush(buf)); 1027 } 1028