1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2014-2019 Intel Corporation 4 */ 5 6 #include <linux/bsearch.h> 7 8 #include "gem/i915_gem_lmem.h" 9 #include "gt/intel_engine_regs.h" 10 #include "gt/intel_gt.h" 11 #include "gt/intel_gt_mcr.h" 12 #include "gt/intel_gt_regs.h" 13 #include "gt/intel_lrc.h" 14 #include "gt/shmem_utils.h" 15 #include "intel_guc_ads.h" 16 #include "intel_guc_capture.h" 17 #include "intel_guc_fwif.h" 18 #include "intel_guc_print.h" 19 #include "intel_uc.h" 20 #include "i915_drv.h" 21 22 /* 23 * The Additional Data Struct (ADS) has pointers for different buffers used by 24 * the GuC. One single gem object contains the ADS struct itself (guc_ads) and 25 * all the extra buffers indirectly linked via the ADS struct's entries. 26 * 27 * Layout of the ADS blob allocated for the GuC: 28 * 29 * +---------------------------------------+ <== base 30 * | guc_ads | 31 * +---------------------------------------+ 32 * | guc_policies | 33 * +---------------------------------------+ 34 * | guc_gt_system_info | 35 * +---------------------------------------+ 36 * | guc_engine_usage | 37 * +---------------------------------------+ <== static 38 * | guc_mmio_reg[countA] (engine 0.0) | 39 * | guc_mmio_reg[countB] (engine 0.1) | 40 * | guc_mmio_reg[countC] (engine 1.0) | 41 * | ... | 42 * +---------------------------------------+ <== dynamic 43 * | padding | 44 * +---------------------------------------+ <== 4K aligned 45 * | golden contexts | 46 * +---------------------------------------+ 47 * | padding | 48 * +---------------------------------------+ <== 4K aligned 49 * | capture lists | 50 * +---------------------------------------+ 51 * | padding | 52 * +---------------------------------------+ <== 4K aligned 53 * | private data | 54 * +---------------------------------------+ 55 * | padding | 56 * +---------------------------------------+ <== 4K aligned 57 */ 58 struct __guc_ads_blob { 59 struct guc_ads ads; 60 struct guc_policies policies; 61 struct guc_gt_system_info system_info; 62 struct guc_engine_usage engine_usage; 63 /* From here on, location is dynamic! Refer to above diagram. */ 64 struct guc_mmio_reg regset[]; 65 } __packed; 66 67 #define ads_blob_read(guc_, field_) \ 68 iosys_map_rd_field(&(guc_)->ads_map, 0, struct __guc_ads_blob, field_) 69 70 #define ads_blob_write(guc_, field_, val_) \ 71 iosys_map_wr_field(&(guc_)->ads_map, 0, struct __guc_ads_blob, \ 72 field_, val_) 73 74 #define info_map_write(map_, field_, val_) \ 75 iosys_map_wr_field(map_, 0, struct guc_gt_system_info, field_, val_) 76 77 #define info_map_read(map_, field_) \ 78 iosys_map_rd_field(map_, 0, struct guc_gt_system_info, field_) 79 80 static u32 guc_ads_regset_size(struct intel_guc *guc) 81 { 82 GEM_BUG_ON(!guc->ads_regset_size); 83 return guc->ads_regset_size; 84 } 85 86 static u32 guc_ads_golden_ctxt_size(struct intel_guc *guc) 87 { 88 return PAGE_ALIGN(guc->ads_golden_ctxt_size); 89 } 90 91 static u32 guc_ads_capture_size(struct intel_guc *guc) 92 { 93 return PAGE_ALIGN(guc->ads_capture_size); 94 } 95 96 static u32 guc_ads_private_data_size(struct intel_guc *guc) 97 { 98 return PAGE_ALIGN(guc->fw.private_data_size); 99 } 100 101 static u32 guc_ads_regset_offset(struct intel_guc *guc) 102 { 103 return offsetof(struct __guc_ads_blob, regset); 104 } 105 106 static u32 guc_ads_golden_ctxt_offset(struct intel_guc *guc) 107 { 108 u32 offset; 109 110 offset = guc_ads_regset_offset(guc) + 111 guc_ads_regset_size(guc); 112 113 return PAGE_ALIGN(offset); 114 } 115 116 static u32 guc_ads_capture_offset(struct intel_guc *guc) 117 { 118 u32 offset; 119 120 offset = guc_ads_golden_ctxt_offset(guc) + 121 guc_ads_golden_ctxt_size(guc); 122 123 return PAGE_ALIGN(offset); 124 } 125 126 static u32 guc_ads_private_data_offset(struct intel_guc *guc) 127 { 128 u32 offset; 129 130 offset = guc_ads_capture_offset(guc) + 131 guc_ads_capture_size(guc); 132 133 return PAGE_ALIGN(offset); 134 } 135 136 static u32 guc_ads_blob_size(struct intel_guc *guc) 137 { 138 return guc_ads_private_data_offset(guc) + 139 guc_ads_private_data_size(guc); 140 } 141 142 static void guc_policies_init(struct intel_guc *guc) 143 { 144 struct intel_gt *gt = guc_to_gt(guc); 145 struct drm_i915_private *i915 = gt->i915; 146 u32 global_flags = 0; 147 148 ads_blob_write(guc, policies.dpc_promote_time, 149 GLOBAL_POLICY_DEFAULT_DPC_PROMOTE_TIME_US); 150 ads_blob_write(guc, policies.max_num_work_items, 151 GLOBAL_POLICY_MAX_NUM_WI); 152 153 if (i915->params.reset < 2) 154 global_flags |= GLOBAL_POLICY_DISABLE_ENGINE_RESET; 155 156 ads_blob_write(guc, policies.global_flags, global_flags); 157 ads_blob_write(guc, policies.is_valid, 1); 158 } 159 160 void intel_guc_ads_print_policy_info(struct intel_guc *guc, 161 struct drm_printer *dp) 162 { 163 if (unlikely(iosys_map_is_null(&guc->ads_map))) 164 return; 165 166 drm_printf(dp, "Global scheduling policies:\n"); 167 drm_printf(dp, " DPC promote time = %u\n", 168 ads_blob_read(guc, policies.dpc_promote_time)); 169 drm_printf(dp, " Max num work items = %u\n", 170 ads_blob_read(guc, policies.max_num_work_items)); 171 drm_printf(dp, " Flags = %u\n", 172 ads_blob_read(guc, policies.global_flags)); 173 } 174 175 static int guc_action_policies_update(struct intel_guc *guc, u32 policy_offset) 176 { 177 u32 action[] = { 178 INTEL_GUC_ACTION_GLOBAL_SCHED_POLICY_CHANGE, 179 policy_offset 180 }; 181 182 return intel_guc_send_busy_loop(guc, action, ARRAY_SIZE(action), 0, true); 183 } 184 185 int intel_guc_global_policies_update(struct intel_guc *guc) 186 { 187 struct intel_gt *gt = guc_to_gt(guc); 188 u32 scheduler_policies; 189 intel_wakeref_t wakeref; 190 int ret; 191 192 if (iosys_map_is_null(&guc->ads_map)) 193 return -EOPNOTSUPP; 194 195 scheduler_policies = ads_blob_read(guc, ads.scheduler_policies); 196 GEM_BUG_ON(!scheduler_policies); 197 198 guc_policies_init(guc); 199 200 if (!intel_guc_is_ready(guc)) 201 return 0; 202 203 with_intel_runtime_pm(>->i915->runtime_pm, wakeref) 204 ret = guc_action_policies_update(guc, scheduler_policies); 205 206 return ret; 207 } 208 209 static void guc_mapping_table_init(struct intel_gt *gt, 210 struct iosys_map *info_map) 211 { 212 unsigned int i, j; 213 struct intel_engine_cs *engine; 214 enum intel_engine_id id; 215 216 /* Table must be set to invalid values for entries not used */ 217 for (i = 0; i < GUC_MAX_ENGINE_CLASSES; ++i) 218 for (j = 0; j < GUC_MAX_INSTANCES_PER_CLASS; ++j) 219 info_map_write(info_map, mapping_table[i][j], 220 GUC_MAX_INSTANCES_PER_CLASS); 221 222 for_each_engine(engine, gt, id) { 223 u8 guc_class = engine_class_to_guc_class(engine->class); 224 225 info_map_write(info_map, mapping_table[guc_class][ilog2(engine->logical_mask)], 226 engine->instance); 227 } 228 } 229 230 /* 231 * The save/restore register list must be pre-calculated to a temporary 232 * buffer before it can be copied inside the ADS. 233 */ 234 struct temp_regset { 235 /* 236 * ptr to the section of the storage for the engine currently being 237 * worked on 238 */ 239 struct guc_mmio_reg *registers; 240 /* ptr to the base of the allocated storage for all engines */ 241 struct guc_mmio_reg *storage; 242 u32 storage_used; 243 u32 storage_max; 244 }; 245 246 static int guc_mmio_reg_cmp(const void *a, const void *b) 247 { 248 const struct guc_mmio_reg *ra = a; 249 const struct guc_mmio_reg *rb = b; 250 251 return (int)ra->offset - (int)rb->offset; 252 } 253 254 static struct guc_mmio_reg * __must_check 255 __mmio_reg_add(struct temp_regset *regset, struct guc_mmio_reg *reg) 256 { 257 u32 pos = regset->storage_used; 258 struct guc_mmio_reg *slot; 259 260 if (pos >= regset->storage_max) { 261 size_t size = ALIGN((pos + 1) * sizeof(*slot), PAGE_SIZE); 262 struct guc_mmio_reg *r = krealloc(regset->storage, 263 size, GFP_KERNEL); 264 if (!r) { 265 WARN_ONCE(1, "Incomplete regset list: can't add register (%d)\n", 266 -ENOMEM); 267 return ERR_PTR(-ENOMEM); 268 } 269 270 regset->registers = r + (regset->registers - regset->storage); 271 regset->storage = r; 272 regset->storage_max = size / sizeof(*slot); 273 } 274 275 slot = ®set->storage[pos]; 276 regset->storage_used++; 277 *slot = *reg; 278 279 return slot; 280 } 281 282 static long __must_check guc_mmio_reg_add(struct intel_gt *gt, 283 struct temp_regset *regset, 284 u32 offset, u32 flags) 285 { 286 u32 count = regset->storage_used - (regset->registers - regset->storage); 287 struct guc_mmio_reg entry = { 288 .offset = offset, 289 .flags = flags, 290 }; 291 struct guc_mmio_reg *slot; 292 293 /* 294 * The mmio list is built using separate lists within the driver. 295 * It's possible that at some point we may attempt to add the same 296 * register more than once. Do not consider this an error; silently 297 * move on if the register is already in the list. 298 */ 299 if (bsearch(&entry, regset->registers, count, 300 sizeof(entry), guc_mmio_reg_cmp)) 301 return 0; 302 303 slot = __mmio_reg_add(regset, &entry); 304 if (IS_ERR(slot)) 305 return PTR_ERR(slot); 306 307 while (slot-- > regset->registers) { 308 GEM_BUG_ON(slot[0].offset == slot[1].offset); 309 if (slot[1].offset > slot[0].offset) 310 break; 311 312 swap(slot[1], slot[0]); 313 } 314 315 return 0; 316 } 317 318 #define GUC_MMIO_REG_ADD(gt, regset, reg, masked) \ 319 guc_mmio_reg_add(gt, \ 320 regset, \ 321 i915_mmio_reg_offset(reg), \ 322 (masked) ? GUC_REGSET_MASKED : 0) 323 324 #define GUC_REGSET_STEERING(group, instance) ( \ 325 FIELD_PREP(GUC_REGSET_STEERING_GROUP, (group)) | \ 326 FIELD_PREP(GUC_REGSET_STEERING_INSTANCE, (instance)) | \ 327 GUC_REGSET_NEEDS_STEERING \ 328 ) 329 330 static long __must_check guc_mcr_reg_add(struct intel_gt *gt, 331 struct temp_regset *regset, 332 i915_mcr_reg_t reg, u32 flags) 333 { 334 u8 group, inst; 335 336 /* 337 * The GuC doesn't have a default steering, so we need to explicitly 338 * steer all registers that need steering. However, we do not keep track 339 * of all the steering ranges, only of those that have a chance of using 340 * a non-default steering from the i915 pov. Instead of adding such 341 * tracking, it is easier to just program the default steering for all 342 * regs that don't need a non-default one. 343 */ 344 intel_gt_mcr_get_nonterminated_steering(gt, reg, &group, &inst); 345 flags |= GUC_REGSET_STEERING(group, inst); 346 347 return guc_mmio_reg_add(gt, regset, i915_mmio_reg_offset(reg), flags); 348 } 349 350 #define GUC_MCR_REG_ADD(gt, regset, reg, masked) \ 351 guc_mcr_reg_add(gt, \ 352 regset, \ 353 (reg), \ 354 (masked) ? GUC_REGSET_MASKED : 0) 355 356 static int guc_mmio_regset_init(struct temp_regset *regset, 357 struct intel_engine_cs *engine) 358 { 359 struct intel_gt *gt = engine->gt; 360 const u32 base = engine->mmio_base; 361 struct i915_wa_list *wal = &engine->wa_list; 362 struct i915_wa *wa; 363 unsigned int i; 364 int ret = 0; 365 366 /* 367 * Each engine's registers point to a new start relative to 368 * storage 369 */ 370 regset->registers = regset->storage + regset->storage_used; 371 372 ret |= GUC_MMIO_REG_ADD(gt, regset, RING_MODE_GEN7(base), true); 373 ret |= GUC_MMIO_REG_ADD(gt, regset, RING_HWS_PGA(base), false); 374 ret |= GUC_MMIO_REG_ADD(gt, regset, RING_IMR(base), false); 375 376 if ((engine->flags & I915_ENGINE_FIRST_RENDER_COMPUTE) && 377 CCS_MASK(engine->gt)) 378 ret |= GUC_MMIO_REG_ADD(gt, regset, GEN12_RCU_MODE, true); 379 380 /* 381 * some of the WA registers are MCR registers. As it is safe to 382 * use MCR form for non-MCR registers, for code simplicity, all 383 * WA registers are added with MCR form. 384 */ 385 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) 386 ret |= GUC_MCR_REG_ADD(gt, regset, wa->mcr_reg, wa->masked_reg); 387 388 /* Be extra paranoid and include all whitelist registers. */ 389 for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) 390 ret |= GUC_MMIO_REG_ADD(gt, regset, 391 RING_FORCE_TO_NONPRIV(base, i), 392 false); 393 394 /* add in local MOCS registers */ 395 for (i = 0; i < LNCFCMOCS_REG_COUNT; i++) 396 if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50)) 397 ret |= GUC_MCR_REG_ADD(gt, regset, XEHP_LNCFCMOCS(i), false); 398 else 399 ret |= GUC_MMIO_REG_ADD(gt, regset, GEN9_LNCFCMOCS(i), false); 400 401 if (GRAPHICS_VER(engine->i915) >= 12) { 402 ret |= GUC_MCR_REG_ADD(gt, regset, MCR_REG(i915_mmio_reg_offset(EU_PERF_CNTL0)), false); 403 ret |= GUC_MCR_REG_ADD(gt, regset, MCR_REG(i915_mmio_reg_offset(EU_PERF_CNTL1)), false); 404 ret |= GUC_MCR_REG_ADD(gt, regset, MCR_REG(i915_mmio_reg_offset(EU_PERF_CNTL2)), false); 405 ret |= GUC_MCR_REG_ADD(gt, regset, MCR_REG(i915_mmio_reg_offset(EU_PERF_CNTL3)), false); 406 ret |= GUC_MCR_REG_ADD(gt, regset, MCR_REG(i915_mmio_reg_offset(EU_PERF_CNTL4)), false); 407 ret |= GUC_MCR_REG_ADD(gt, regset, MCR_REG(i915_mmio_reg_offset(EU_PERF_CNTL5)), false); 408 ret |= GUC_MCR_REG_ADD(gt, regset, MCR_REG(i915_mmio_reg_offset(EU_PERF_CNTL6)), false); 409 } 410 411 return ret ? -1 : 0; 412 } 413 414 static long guc_mmio_reg_state_create(struct intel_guc *guc) 415 { 416 struct intel_gt *gt = guc_to_gt(guc); 417 struct intel_engine_cs *engine; 418 enum intel_engine_id id; 419 struct temp_regset temp_set = {}; 420 long total = 0; 421 long ret; 422 423 for_each_engine(engine, gt, id) { 424 u32 used = temp_set.storage_used; 425 426 ret = guc_mmio_regset_init(&temp_set, engine); 427 if (ret < 0) 428 goto fail_regset_init; 429 430 guc->ads_regset_count[id] = temp_set.storage_used - used; 431 total += guc->ads_regset_count[id]; 432 } 433 434 guc->ads_regset = temp_set.storage; 435 436 guc_dbg(guc, "Used %zu KB for temporary ADS regset\n", 437 (temp_set.storage_max * sizeof(struct guc_mmio_reg)) >> 10); 438 439 return total * sizeof(struct guc_mmio_reg); 440 441 fail_regset_init: 442 kfree(temp_set.storage); 443 return ret; 444 } 445 446 static void guc_mmio_reg_state_init(struct intel_guc *guc) 447 { 448 struct intel_gt *gt = guc_to_gt(guc); 449 struct intel_engine_cs *engine; 450 enum intel_engine_id id; 451 u32 addr_ggtt, offset; 452 453 offset = guc_ads_regset_offset(guc); 454 addr_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma) + offset; 455 456 iosys_map_memcpy_to(&guc->ads_map, offset, guc->ads_regset, 457 guc->ads_regset_size); 458 459 for_each_engine(engine, gt, id) { 460 u32 count = guc->ads_regset_count[id]; 461 u8 guc_class; 462 463 /* Class index is checked in class converter */ 464 GEM_BUG_ON(engine->instance >= GUC_MAX_INSTANCES_PER_CLASS); 465 466 guc_class = engine_class_to_guc_class(engine->class); 467 468 if (!count) { 469 ads_blob_write(guc, 470 ads.reg_state_list[guc_class][engine->instance].address, 471 0); 472 ads_blob_write(guc, 473 ads.reg_state_list[guc_class][engine->instance].count, 474 0); 475 continue; 476 } 477 478 ads_blob_write(guc, 479 ads.reg_state_list[guc_class][engine->instance].address, 480 addr_ggtt); 481 ads_blob_write(guc, 482 ads.reg_state_list[guc_class][engine->instance].count, 483 count); 484 485 addr_ggtt += count * sizeof(struct guc_mmio_reg); 486 } 487 } 488 489 static void fill_engine_enable_masks(struct intel_gt *gt, 490 struct iosys_map *info_map) 491 { 492 info_map_write(info_map, engine_enabled_masks[GUC_RENDER_CLASS], RCS_MASK(gt)); 493 info_map_write(info_map, engine_enabled_masks[GUC_COMPUTE_CLASS], CCS_MASK(gt)); 494 info_map_write(info_map, engine_enabled_masks[GUC_BLITTER_CLASS], BCS_MASK(gt)); 495 info_map_write(info_map, engine_enabled_masks[GUC_VIDEO_CLASS], VDBOX_MASK(gt)); 496 info_map_write(info_map, engine_enabled_masks[GUC_VIDEOENHANCE_CLASS], VEBOX_MASK(gt)); 497 498 /* The GSC engine is an instance (6) of OTHER_CLASS */ 499 if (gt->engine[GSC0]) 500 info_map_write(info_map, engine_enabled_masks[GUC_GSC_OTHER_CLASS], 501 BIT(gt->engine[GSC0]->instance)); 502 } 503 504 #define LR_HW_CONTEXT_SIZE (80 * sizeof(u32)) 505 #define XEHP_LR_HW_CONTEXT_SIZE (96 * sizeof(u32)) 506 #define LR_HW_CONTEXT_SZ(i915) (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 50) ? \ 507 XEHP_LR_HW_CONTEXT_SIZE : \ 508 LR_HW_CONTEXT_SIZE) 509 #define LRC_SKIP_SIZE(i915) (LRC_PPHWSP_SZ * PAGE_SIZE + LR_HW_CONTEXT_SZ(i915)) 510 static int guc_prep_golden_context(struct intel_guc *guc) 511 { 512 struct intel_gt *gt = guc_to_gt(guc); 513 u32 addr_ggtt, offset; 514 u32 total_size = 0, alloc_size, real_size; 515 u8 engine_class, guc_class; 516 struct guc_gt_system_info local_info; 517 struct iosys_map info_map; 518 519 /* 520 * Reserve the memory for the golden contexts and point GuC at it but 521 * leave it empty for now. The context data will be filled in later 522 * once there is something available to put there. 523 * 524 * Note that the HWSP and ring context are not included. 525 * 526 * Note also that the storage must be pinned in the GGTT, so that the 527 * address won't change after GuC has been told where to find it. The 528 * GuC will also validate that the LRC base + size fall within the 529 * allowed GGTT range. 530 */ 531 if (!iosys_map_is_null(&guc->ads_map)) { 532 offset = guc_ads_golden_ctxt_offset(guc); 533 addr_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma) + offset; 534 info_map = IOSYS_MAP_INIT_OFFSET(&guc->ads_map, 535 offsetof(struct __guc_ads_blob, system_info)); 536 } else { 537 memset(&local_info, 0, sizeof(local_info)); 538 iosys_map_set_vaddr(&info_map, &local_info); 539 fill_engine_enable_masks(gt, &info_map); 540 } 541 542 for (engine_class = 0; engine_class <= MAX_ENGINE_CLASS; ++engine_class) { 543 guc_class = engine_class_to_guc_class(engine_class); 544 545 if (!info_map_read(&info_map, engine_enabled_masks[guc_class])) 546 continue; 547 548 real_size = intel_engine_context_size(gt, engine_class); 549 alloc_size = PAGE_ALIGN(real_size); 550 total_size += alloc_size; 551 552 if (iosys_map_is_null(&guc->ads_map)) 553 continue; 554 555 /* 556 * This interface is slightly confusing. We need to pass the 557 * base address of the full golden context and the size of just 558 * the engine state, which is the section of the context image 559 * that starts after the execlists context. This is required to 560 * allow the GuC to restore just the engine state when a 561 * watchdog reset occurs. 562 * We calculate the engine state size by removing the size of 563 * what comes before it in the context image (which is identical 564 * on all engines). 565 */ 566 ads_blob_write(guc, ads.eng_state_size[guc_class], 567 real_size - LRC_SKIP_SIZE(gt->i915)); 568 ads_blob_write(guc, ads.golden_context_lrca[guc_class], 569 addr_ggtt); 570 571 addr_ggtt += alloc_size; 572 } 573 574 /* Make sure current size matches what we calculated previously */ 575 if (guc->ads_golden_ctxt_size) 576 GEM_BUG_ON(guc->ads_golden_ctxt_size != total_size); 577 578 return total_size; 579 } 580 581 static struct intel_engine_cs *find_engine_state(struct intel_gt *gt, u8 engine_class) 582 { 583 struct intel_engine_cs *engine; 584 enum intel_engine_id id; 585 586 for_each_engine(engine, gt, id) { 587 if (engine->class != engine_class) 588 continue; 589 590 if (!engine->default_state) 591 continue; 592 593 return engine; 594 } 595 596 return NULL; 597 } 598 599 static void guc_init_golden_context(struct intel_guc *guc) 600 { 601 struct intel_engine_cs *engine; 602 struct intel_gt *gt = guc_to_gt(guc); 603 unsigned long offset; 604 u32 addr_ggtt, total_size = 0, alloc_size, real_size; 605 u8 engine_class, guc_class; 606 607 if (!intel_uc_uses_guc_submission(>->uc)) 608 return; 609 610 GEM_BUG_ON(iosys_map_is_null(&guc->ads_map)); 611 612 /* 613 * Go back and fill in the golden context data now that it is 614 * available. 615 */ 616 offset = guc_ads_golden_ctxt_offset(guc); 617 addr_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma) + offset; 618 619 for (engine_class = 0; engine_class <= MAX_ENGINE_CLASS; ++engine_class) { 620 guc_class = engine_class_to_guc_class(engine_class); 621 if (!ads_blob_read(guc, system_info.engine_enabled_masks[guc_class])) 622 continue; 623 624 real_size = intel_engine_context_size(gt, engine_class); 625 alloc_size = PAGE_ALIGN(real_size); 626 total_size += alloc_size; 627 628 engine = find_engine_state(gt, engine_class); 629 if (!engine) { 630 guc_err(guc, "No engine state recorded for class %d!\n", 631 engine_class); 632 ads_blob_write(guc, ads.eng_state_size[guc_class], 0); 633 ads_blob_write(guc, ads.golden_context_lrca[guc_class], 0); 634 continue; 635 } 636 637 GEM_BUG_ON(ads_blob_read(guc, ads.eng_state_size[guc_class]) != 638 real_size - LRC_SKIP_SIZE(gt->i915)); 639 GEM_BUG_ON(ads_blob_read(guc, ads.golden_context_lrca[guc_class]) != addr_ggtt); 640 641 addr_ggtt += alloc_size; 642 643 shmem_read_to_iosys_map(engine->default_state, 0, &guc->ads_map, 644 offset, real_size); 645 offset += alloc_size; 646 } 647 648 GEM_BUG_ON(guc->ads_golden_ctxt_size != total_size); 649 } 650 651 static u32 guc_get_capture_engine_mask(struct iosys_map *info_map, u32 capture_class) 652 { 653 u32 mask; 654 655 switch (capture_class) { 656 case GUC_CAPTURE_LIST_CLASS_RENDER_COMPUTE: 657 mask = info_map_read(info_map, engine_enabled_masks[GUC_RENDER_CLASS]); 658 mask |= info_map_read(info_map, engine_enabled_masks[GUC_COMPUTE_CLASS]); 659 break; 660 661 case GUC_CAPTURE_LIST_CLASS_VIDEO: 662 mask = info_map_read(info_map, engine_enabled_masks[GUC_VIDEO_CLASS]); 663 break; 664 665 case GUC_CAPTURE_LIST_CLASS_VIDEOENHANCE: 666 mask = info_map_read(info_map, engine_enabled_masks[GUC_VIDEOENHANCE_CLASS]); 667 break; 668 669 case GUC_CAPTURE_LIST_CLASS_BLITTER: 670 mask = info_map_read(info_map, engine_enabled_masks[GUC_BLITTER_CLASS]); 671 break; 672 673 case GUC_CAPTURE_LIST_CLASS_GSC_OTHER: 674 mask = info_map_read(info_map, engine_enabled_masks[GUC_GSC_OTHER_CLASS]); 675 break; 676 677 default: 678 mask = 0; 679 } 680 681 return mask; 682 } 683 684 static int 685 guc_capture_prep_lists(struct intel_guc *guc) 686 { 687 struct intel_gt *gt = guc_to_gt(guc); 688 u32 ads_ggtt, capture_offset, null_ggtt, total_size = 0; 689 struct guc_gt_system_info local_info; 690 struct iosys_map info_map; 691 bool ads_is_mapped; 692 size_t size = 0; 693 void *ptr; 694 int i, j; 695 696 ads_is_mapped = !iosys_map_is_null(&guc->ads_map); 697 if (ads_is_mapped) { 698 capture_offset = guc_ads_capture_offset(guc); 699 ads_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma); 700 info_map = IOSYS_MAP_INIT_OFFSET(&guc->ads_map, 701 offsetof(struct __guc_ads_blob, system_info)); 702 } else { 703 memset(&local_info, 0, sizeof(local_info)); 704 iosys_map_set_vaddr(&info_map, &local_info); 705 fill_engine_enable_masks(gt, &info_map); 706 } 707 708 /* first, set aside the first page for a capture_list with zero descriptors */ 709 total_size = PAGE_SIZE; 710 if (ads_is_mapped) { 711 if (!intel_guc_capture_getnullheader(guc, &ptr, &size)) 712 iosys_map_memcpy_to(&guc->ads_map, capture_offset, ptr, size); 713 null_ggtt = ads_ggtt + capture_offset; 714 capture_offset += PAGE_SIZE; 715 } 716 717 for (i = 0; i < GUC_CAPTURE_LIST_INDEX_MAX; i++) { 718 for (j = 0; j < GUC_MAX_ENGINE_CLASSES; j++) { 719 u32 engine_mask = guc_get_capture_engine_mask(&info_map, j); 720 721 /* null list if we dont have said engine or list */ 722 if (!engine_mask) { 723 if (ads_is_mapped) { 724 ads_blob_write(guc, ads.capture_class[i][j], null_ggtt); 725 ads_blob_write(guc, ads.capture_instance[i][j], null_ggtt); 726 } 727 continue; 728 } 729 if (intel_guc_capture_getlistsize(guc, i, 730 GUC_CAPTURE_LIST_TYPE_ENGINE_CLASS, 731 j, &size)) { 732 if (ads_is_mapped) 733 ads_blob_write(guc, ads.capture_class[i][j], null_ggtt); 734 goto engine_instance_list; 735 } 736 total_size += size; 737 if (ads_is_mapped) { 738 if (total_size > guc->ads_capture_size || 739 intel_guc_capture_getlist(guc, i, 740 GUC_CAPTURE_LIST_TYPE_ENGINE_CLASS, 741 j, &ptr)) { 742 ads_blob_write(guc, ads.capture_class[i][j], null_ggtt); 743 continue; 744 } 745 ads_blob_write(guc, ads.capture_class[i][j], ads_ggtt + 746 capture_offset); 747 iosys_map_memcpy_to(&guc->ads_map, capture_offset, ptr, size); 748 capture_offset += size; 749 } 750 engine_instance_list: 751 if (intel_guc_capture_getlistsize(guc, i, 752 GUC_CAPTURE_LIST_TYPE_ENGINE_INSTANCE, 753 j, &size)) { 754 if (ads_is_mapped) 755 ads_blob_write(guc, ads.capture_instance[i][j], null_ggtt); 756 continue; 757 } 758 total_size += size; 759 if (ads_is_mapped) { 760 if (total_size > guc->ads_capture_size || 761 intel_guc_capture_getlist(guc, i, 762 GUC_CAPTURE_LIST_TYPE_ENGINE_INSTANCE, 763 j, &ptr)) { 764 ads_blob_write(guc, ads.capture_instance[i][j], null_ggtt); 765 continue; 766 } 767 ads_blob_write(guc, ads.capture_instance[i][j], ads_ggtt + 768 capture_offset); 769 iosys_map_memcpy_to(&guc->ads_map, capture_offset, ptr, size); 770 capture_offset += size; 771 } 772 } 773 if (intel_guc_capture_getlistsize(guc, i, GUC_CAPTURE_LIST_TYPE_GLOBAL, 0, &size)) { 774 if (ads_is_mapped) 775 ads_blob_write(guc, ads.capture_global[i], null_ggtt); 776 continue; 777 } 778 total_size += size; 779 if (ads_is_mapped) { 780 if (total_size > guc->ads_capture_size || 781 intel_guc_capture_getlist(guc, i, GUC_CAPTURE_LIST_TYPE_GLOBAL, 0, 782 &ptr)) { 783 ads_blob_write(guc, ads.capture_global[i], null_ggtt); 784 continue; 785 } 786 ads_blob_write(guc, ads.capture_global[i], ads_ggtt + capture_offset); 787 iosys_map_memcpy_to(&guc->ads_map, capture_offset, ptr, size); 788 capture_offset += size; 789 } 790 } 791 792 if (guc->ads_capture_size && guc->ads_capture_size != PAGE_ALIGN(total_size)) 793 guc_warn(guc, "ADS capture alloc size changed from %d to %d\n", 794 guc->ads_capture_size, PAGE_ALIGN(total_size)); 795 796 return PAGE_ALIGN(total_size); 797 } 798 799 static void __guc_ads_init(struct intel_guc *guc) 800 { 801 struct intel_gt *gt = guc_to_gt(guc); 802 struct drm_i915_private *i915 = gt->i915; 803 struct iosys_map info_map = IOSYS_MAP_INIT_OFFSET(&guc->ads_map, 804 offsetof(struct __guc_ads_blob, system_info)); 805 u32 base; 806 807 /* GuC scheduling policies */ 808 guc_policies_init(guc); 809 810 /* System info */ 811 fill_engine_enable_masks(gt, &info_map); 812 813 ads_blob_write(guc, system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_SLICE_ENABLED], 814 hweight8(gt->info.sseu.slice_mask)); 815 ads_blob_write(guc, system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_VDBOX_SFC_SUPPORT_MASK], 816 gt->info.vdbox_sfc_access); 817 818 if (GRAPHICS_VER(i915) >= 12 && !IS_DGFX(i915)) { 819 u32 distdbreg = intel_uncore_read(gt->uncore, 820 GEN12_DIST_DBS_POPULATED); 821 ads_blob_write(guc, 822 system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_DOORBELL_COUNT_PER_SQIDI], 823 ((distdbreg >> GEN12_DOORBELLS_PER_SQIDI_SHIFT) 824 & GEN12_DOORBELLS_PER_SQIDI) + 1); 825 } 826 827 /* Golden contexts for re-initialising after a watchdog reset */ 828 guc_prep_golden_context(guc); 829 830 guc_mapping_table_init(guc_to_gt(guc), &info_map); 831 832 base = intel_guc_ggtt_offset(guc, guc->ads_vma); 833 834 /* Lists for error capture debug */ 835 guc_capture_prep_lists(guc); 836 837 /* ADS */ 838 ads_blob_write(guc, ads.scheduler_policies, base + 839 offsetof(struct __guc_ads_blob, policies)); 840 ads_blob_write(guc, ads.gt_system_info, base + 841 offsetof(struct __guc_ads_blob, system_info)); 842 843 /* MMIO save/restore list */ 844 guc_mmio_reg_state_init(guc); 845 846 /* Private Data */ 847 ads_blob_write(guc, ads.private_data, base + 848 guc_ads_private_data_offset(guc)); 849 850 i915_gem_object_flush_map(guc->ads_vma->obj); 851 } 852 853 /** 854 * intel_guc_ads_create() - allocates and initializes GuC ADS. 855 * @guc: intel_guc struct 856 * 857 * GuC needs memory block (Additional Data Struct), where it will store 858 * some data. Allocate and initialize such memory block for GuC use. 859 */ 860 int intel_guc_ads_create(struct intel_guc *guc) 861 { 862 void *ads_blob; 863 u32 size; 864 int ret; 865 866 GEM_BUG_ON(guc->ads_vma); 867 868 /* 869 * Create reg state size dynamically on system memory to be copied to 870 * the final ads blob on gt init/reset 871 */ 872 ret = guc_mmio_reg_state_create(guc); 873 if (ret < 0) 874 return ret; 875 guc->ads_regset_size = ret; 876 877 /* Likewise the golden contexts: */ 878 ret = guc_prep_golden_context(guc); 879 if (ret < 0) 880 return ret; 881 guc->ads_golden_ctxt_size = ret; 882 883 /* Likewise the capture lists: */ 884 ret = guc_capture_prep_lists(guc); 885 if (ret < 0) 886 return ret; 887 guc->ads_capture_size = ret; 888 889 /* Now the total size can be determined: */ 890 size = guc_ads_blob_size(guc); 891 892 ret = intel_guc_allocate_and_map_vma(guc, size, &guc->ads_vma, 893 &ads_blob); 894 if (ret) 895 return ret; 896 897 if (i915_gem_object_is_lmem(guc->ads_vma->obj)) 898 iosys_map_set_vaddr_iomem(&guc->ads_map, (void __iomem *)ads_blob); 899 else 900 iosys_map_set_vaddr(&guc->ads_map, ads_blob); 901 902 __guc_ads_init(guc); 903 904 return 0; 905 } 906 907 void intel_guc_ads_init_late(struct intel_guc *guc) 908 { 909 /* 910 * The golden context setup requires the saved engine state from 911 * __engines_record_defaults(). However, that requires engines to be 912 * operational which means the ADS must already have been configured. 913 * Fortunately, the golden context state is not needed until a hang 914 * occurs, so it can be filled in during this late init phase. 915 */ 916 guc_init_golden_context(guc); 917 } 918 919 void intel_guc_ads_destroy(struct intel_guc *guc) 920 { 921 i915_vma_unpin_and_release(&guc->ads_vma, I915_VMA_RELEASE_MAP); 922 iosys_map_clear(&guc->ads_map); 923 kfree(guc->ads_regset); 924 } 925 926 static void guc_ads_private_data_reset(struct intel_guc *guc) 927 { 928 u32 size; 929 930 size = guc_ads_private_data_size(guc); 931 if (!size) 932 return; 933 934 iosys_map_memset(&guc->ads_map, guc_ads_private_data_offset(guc), 935 0, size); 936 } 937 938 /** 939 * intel_guc_ads_reset() - prepares GuC Additional Data Struct for reuse 940 * @guc: intel_guc struct 941 * 942 * GuC stores some data in ADS, which might be stale after a reset. 943 * Reinitialize whole ADS in case any part of it was corrupted during 944 * previous GuC run. 945 */ 946 void intel_guc_ads_reset(struct intel_guc *guc) 947 { 948 if (!guc->ads_vma) 949 return; 950 951 __guc_ads_init(guc); 952 953 guc_ads_private_data_reset(guc); 954 } 955 956 u32 intel_guc_engine_usage_offset(struct intel_guc *guc) 957 { 958 return intel_guc_ggtt_offset(guc, guc->ads_vma) + 959 offsetof(struct __guc_ads_blob, engine_usage); 960 } 961 962 struct iosys_map intel_guc_engine_usage_record_map(struct intel_engine_cs *engine) 963 { 964 struct intel_guc *guc = &engine->gt->uc.guc; 965 u8 guc_class = engine_class_to_guc_class(engine->class); 966 size_t offset = offsetof(struct __guc_ads_blob, 967 engine_usage.engines[guc_class][ilog2(engine->logical_mask)]); 968 969 return IOSYS_MAP_INIT_OFFSET(&guc->ads_map, offset); 970 } 971