1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2014-2019 Intel Corporation 4 */ 5 6 #include <linux/bsearch.h> 7 8 #include "gem/i915_gem_lmem.h" 9 #include "gt/intel_engine_regs.h" 10 #include "gt/intel_gt.h" 11 #include "gt/intel_gt_mcr.h" 12 #include "gt/intel_gt_regs.h" 13 #include "gt/intel_lrc.h" 14 #include "gt/shmem_utils.h" 15 #include "intel_guc_ads.h" 16 #include "intel_guc_capture.h" 17 #include "intel_guc_fwif.h" 18 #include "intel_guc_print.h" 19 #include "intel_uc.h" 20 #include "i915_drv.h" 21 22 /* 23 * The Additional Data Struct (ADS) has pointers for different buffers used by 24 * the GuC. One single gem object contains the ADS struct itself (guc_ads) and 25 * all the extra buffers indirectly linked via the ADS struct's entries. 26 * 27 * Layout of the ADS blob allocated for the GuC: 28 * 29 * +---------------------------------------+ <== base 30 * | guc_ads | 31 * +---------------------------------------+ 32 * | guc_policies | 33 * +---------------------------------------+ 34 * | guc_gt_system_info | 35 * +---------------------------------------+ 36 * | guc_engine_usage | 37 * +---------------------------------------+ <== static 38 * | guc_mmio_reg[countA] (engine 0.0) | 39 * | guc_mmio_reg[countB] (engine 0.1) | 40 * | guc_mmio_reg[countC] (engine 1.0) | 41 * | ... | 42 * +---------------------------------------+ <== dynamic 43 * | padding | 44 * +---------------------------------------+ <== 4K aligned 45 * | golden contexts | 46 * +---------------------------------------+ 47 * | padding | 48 * +---------------------------------------+ <== 4K aligned 49 * | w/a KLVs | 50 * +---------------------------------------+ 51 * | padding | 52 * +---------------------------------------+ <== 4K aligned 53 * | capture lists | 54 * +---------------------------------------+ 55 * | padding | 56 * +---------------------------------------+ <== 4K aligned 57 * | private data | 58 * +---------------------------------------+ 59 * | padding | 60 * +---------------------------------------+ <== 4K aligned 61 */ 62 struct __guc_ads_blob { 63 struct guc_ads ads; 64 struct guc_policies policies; 65 struct guc_gt_system_info system_info; 66 struct guc_engine_usage engine_usage; 67 /* From here on, location is dynamic! Refer to above diagram. */ 68 struct guc_mmio_reg regset[]; 69 } __packed; 70 71 #define ads_blob_read(guc_, field_) \ 72 iosys_map_rd_field(&(guc_)->ads_map, 0, struct __guc_ads_blob, field_) 73 74 #define ads_blob_write(guc_, field_, val_) \ 75 iosys_map_wr_field(&(guc_)->ads_map, 0, struct __guc_ads_blob, \ 76 field_, val_) 77 78 #define info_map_write(map_, field_, val_) \ 79 iosys_map_wr_field(map_, 0, struct guc_gt_system_info, field_, val_) 80 81 #define info_map_read(map_, field_) \ 82 iosys_map_rd_field(map_, 0, struct guc_gt_system_info, field_) 83 84 static u32 guc_ads_regset_size(struct intel_guc *guc) 85 { 86 GEM_BUG_ON(!guc->ads_regset_size); 87 return guc->ads_regset_size; 88 } 89 90 static u32 guc_ads_golden_ctxt_size(struct intel_guc *guc) 91 { 92 return PAGE_ALIGN(guc->ads_golden_ctxt_size); 93 } 94 95 static u32 guc_ads_waklv_size(struct intel_guc *guc) 96 { 97 return PAGE_ALIGN(guc->ads_waklv_size); 98 } 99 100 static u32 guc_ads_capture_size(struct intel_guc *guc) 101 { 102 return PAGE_ALIGN(guc->ads_capture_size); 103 } 104 105 static u32 guc_ads_private_data_size(struct intel_guc *guc) 106 { 107 return PAGE_ALIGN(guc->fw.private_data_size); 108 } 109 110 static u32 guc_ads_regset_offset(struct intel_guc *guc) 111 { 112 return offsetof(struct __guc_ads_blob, regset); 113 } 114 115 static u32 guc_ads_golden_ctxt_offset(struct intel_guc *guc) 116 { 117 u32 offset; 118 119 offset = guc_ads_regset_offset(guc) + 120 guc_ads_regset_size(guc); 121 122 return PAGE_ALIGN(offset); 123 } 124 125 static u32 guc_ads_waklv_offset(struct intel_guc *guc) 126 { 127 u32 offset; 128 129 offset = guc_ads_golden_ctxt_offset(guc) + 130 guc_ads_golden_ctxt_size(guc); 131 132 return PAGE_ALIGN(offset); 133 } 134 135 static u32 guc_ads_capture_offset(struct intel_guc *guc) 136 { 137 u32 offset; 138 139 offset = guc_ads_waklv_offset(guc) + 140 guc_ads_waklv_size(guc); 141 142 return PAGE_ALIGN(offset); 143 } 144 145 static u32 guc_ads_private_data_offset(struct intel_guc *guc) 146 { 147 u32 offset; 148 149 offset = guc_ads_capture_offset(guc) + 150 guc_ads_capture_size(guc); 151 152 return PAGE_ALIGN(offset); 153 } 154 155 static u32 guc_ads_blob_size(struct intel_guc *guc) 156 { 157 return guc_ads_private_data_offset(guc) + 158 guc_ads_private_data_size(guc); 159 } 160 161 static void guc_policies_init(struct intel_guc *guc) 162 { 163 struct intel_gt *gt = guc_to_gt(guc); 164 struct drm_i915_private *i915 = gt->i915; 165 u32 global_flags = 0; 166 167 ads_blob_write(guc, policies.dpc_promote_time, 168 GLOBAL_POLICY_DEFAULT_DPC_PROMOTE_TIME_US); 169 ads_blob_write(guc, policies.max_num_work_items, 170 GLOBAL_POLICY_MAX_NUM_WI); 171 172 if (i915->params.reset < 2) 173 global_flags |= GLOBAL_POLICY_DISABLE_ENGINE_RESET; 174 175 ads_blob_write(guc, policies.global_flags, global_flags); 176 ads_blob_write(guc, policies.is_valid, 1); 177 } 178 179 void intel_guc_ads_print_policy_info(struct intel_guc *guc, 180 struct drm_printer *dp) 181 { 182 if (unlikely(iosys_map_is_null(&guc->ads_map))) 183 return; 184 185 drm_printf(dp, "Global scheduling policies:\n"); 186 drm_printf(dp, " DPC promote time = %u\n", 187 ads_blob_read(guc, policies.dpc_promote_time)); 188 drm_printf(dp, " Max num work items = %u\n", 189 ads_blob_read(guc, policies.max_num_work_items)); 190 drm_printf(dp, " Flags = %u\n", 191 ads_blob_read(guc, policies.global_flags)); 192 } 193 194 static int guc_action_policies_update(struct intel_guc *guc, u32 policy_offset) 195 { 196 u32 action[] = { 197 INTEL_GUC_ACTION_GLOBAL_SCHED_POLICY_CHANGE, 198 policy_offset 199 }; 200 201 return intel_guc_send_busy_loop(guc, action, ARRAY_SIZE(action), 0, true); 202 } 203 204 int intel_guc_global_policies_update(struct intel_guc *guc) 205 { 206 struct intel_gt *gt = guc_to_gt(guc); 207 u32 scheduler_policies; 208 intel_wakeref_t wakeref; 209 int ret; 210 211 if (iosys_map_is_null(&guc->ads_map)) 212 return -EOPNOTSUPP; 213 214 scheduler_policies = ads_blob_read(guc, ads.scheduler_policies); 215 GEM_BUG_ON(!scheduler_policies); 216 217 guc_policies_init(guc); 218 219 if (!intel_guc_is_ready(guc)) 220 return 0; 221 222 with_intel_runtime_pm(>->i915->runtime_pm, wakeref) 223 ret = guc_action_policies_update(guc, scheduler_policies); 224 225 return ret; 226 } 227 228 static void guc_mapping_table_init(struct intel_gt *gt, 229 struct iosys_map *info_map) 230 { 231 unsigned int i, j; 232 struct intel_engine_cs *engine; 233 enum intel_engine_id id; 234 235 /* Table must be set to invalid values for entries not used */ 236 for (i = 0; i < GUC_MAX_ENGINE_CLASSES; ++i) 237 for (j = 0; j < GUC_MAX_INSTANCES_PER_CLASS; ++j) 238 info_map_write(info_map, mapping_table[i][j], 239 GUC_MAX_INSTANCES_PER_CLASS); 240 241 for_each_engine(engine, gt, id) { 242 u8 guc_class = engine_class_to_guc_class(engine->class); 243 244 info_map_write(info_map, mapping_table[guc_class][ilog2(engine->logical_mask)], 245 engine->instance); 246 } 247 } 248 249 /* 250 * The save/restore register list must be pre-calculated to a temporary 251 * buffer before it can be copied inside the ADS. 252 */ 253 struct temp_regset { 254 /* 255 * ptr to the section of the storage for the engine currently being 256 * worked on 257 */ 258 struct guc_mmio_reg *registers; 259 /* ptr to the base of the allocated storage for all engines */ 260 struct guc_mmio_reg *storage; 261 u32 storage_used; 262 u32 storage_max; 263 }; 264 265 static int guc_mmio_reg_cmp(const void *a, const void *b) 266 { 267 const struct guc_mmio_reg *ra = a; 268 const struct guc_mmio_reg *rb = b; 269 270 return (int)ra->offset - (int)rb->offset; 271 } 272 273 static struct guc_mmio_reg * __must_check 274 __mmio_reg_add(struct temp_regset *regset, struct guc_mmio_reg *reg) 275 { 276 u32 pos = regset->storage_used; 277 struct guc_mmio_reg *slot; 278 279 if (pos >= regset->storage_max) { 280 size_t size = ALIGN((pos + 1) * sizeof(*slot), PAGE_SIZE); 281 struct guc_mmio_reg *r = krealloc(regset->storage, 282 size, GFP_KERNEL); 283 if (!r) { 284 WARN_ONCE(1, "Incomplete regset list: can't add register (%d)\n", 285 -ENOMEM); 286 return ERR_PTR(-ENOMEM); 287 } 288 289 regset->registers = r + (regset->registers - regset->storage); 290 regset->storage = r; 291 regset->storage_max = size / sizeof(*slot); 292 } 293 294 slot = ®set->storage[pos]; 295 regset->storage_used++; 296 *slot = *reg; 297 298 return slot; 299 } 300 301 static long __must_check guc_mmio_reg_add(struct intel_gt *gt, 302 struct temp_regset *regset, 303 u32 offset, u32 flags) 304 { 305 u32 count = regset->storage_used - (regset->registers - regset->storage); 306 struct guc_mmio_reg entry = { 307 .offset = offset, 308 .flags = flags, 309 }; 310 struct guc_mmio_reg *slot; 311 312 /* 313 * The mmio list is built using separate lists within the driver. 314 * It's possible that at some point we may attempt to add the same 315 * register more than once. Do not consider this an error; silently 316 * move on if the register is already in the list. 317 */ 318 if (bsearch(&entry, regset->registers, count, 319 sizeof(entry), guc_mmio_reg_cmp)) 320 return 0; 321 322 slot = __mmio_reg_add(regset, &entry); 323 if (IS_ERR(slot)) 324 return PTR_ERR(slot); 325 326 while (slot-- > regset->registers) { 327 GEM_BUG_ON(slot[0].offset == slot[1].offset); 328 if (slot[1].offset > slot[0].offset) 329 break; 330 331 swap(slot[1], slot[0]); 332 } 333 334 return 0; 335 } 336 337 #define GUC_MMIO_REG_ADD(gt, regset, reg, masked) \ 338 guc_mmio_reg_add(gt, \ 339 regset, \ 340 i915_mmio_reg_offset(reg), \ 341 (masked) ? GUC_REGSET_MASKED : 0) 342 343 #define GUC_REGSET_STEERING(group, instance) ( \ 344 FIELD_PREP(GUC_REGSET_STEERING_GROUP, (group)) | \ 345 FIELD_PREP(GUC_REGSET_STEERING_INSTANCE, (instance)) | \ 346 GUC_REGSET_NEEDS_STEERING \ 347 ) 348 349 static long __must_check guc_mcr_reg_add(struct intel_gt *gt, 350 struct temp_regset *regset, 351 i915_mcr_reg_t reg, u32 flags) 352 { 353 u8 group, inst; 354 355 /* 356 * The GuC doesn't have a default steering, so we need to explicitly 357 * steer all registers that need steering. However, we do not keep track 358 * of all the steering ranges, only of those that have a chance of using 359 * a non-default steering from the i915 pov. Instead of adding such 360 * tracking, it is easier to just program the default steering for all 361 * regs that don't need a non-default one. 362 */ 363 intel_gt_mcr_get_nonterminated_steering(gt, reg, &group, &inst); 364 flags |= GUC_REGSET_STEERING(group, inst); 365 366 return guc_mmio_reg_add(gt, regset, i915_mmio_reg_offset(reg), flags); 367 } 368 369 #define GUC_MCR_REG_ADD(gt, regset, reg, masked) \ 370 guc_mcr_reg_add(gt, \ 371 regset, \ 372 (reg), \ 373 (masked) ? GUC_REGSET_MASKED : 0) 374 375 static int guc_mmio_regset_init(struct temp_regset *regset, 376 struct intel_engine_cs *engine) 377 { 378 struct intel_gt *gt = engine->gt; 379 const u32 base = engine->mmio_base; 380 struct i915_wa_list *wal = &engine->wa_list; 381 struct i915_wa *wa; 382 unsigned int i; 383 int ret = 0; 384 385 /* 386 * Each engine's registers point to a new start relative to 387 * storage 388 */ 389 regset->registers = regset->storage + regset->storage_used; 390 391 ret |= GUC_MMIO_REG_ADD(gt, regset, RING_MODE_GEN7(base), true); 392 ret |= GUC_MMIO_REG_ADD(gt, regset, RING_HWS_PGA(base), false); 393 ret |= GUC_MMIO_REG_ADD(gt, regset, RING_IMR(base), false); 394 395 if ((engine->flags & I915_ENGINE_FIRST_RENDER_COMPUTE) && 396 CCS_MASK(engine->gt)) 397 ret |= GUC_MMIO_REG_ADD(gt, regset, GEN12_RCU_MODE, true); 398 399 /* 400 * some of the WA registers are MCR registers. As it is safe to 401 * use MCR form for non-MCR registers, for code simplicity, all 402 * WA registers are added with MCR form. 403 */ 404 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) 405 ret |= GUC_MCR_REG_ADD(gt, regset, wa->mcr_reg, wa->masked_reg); 406 407 /* Be extra paranoid and include all whitelist registers. */ 408 for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) 409 ret |= GUC_MMIO_REG_ADD(gt, regset, 410 RING_FORCE_TO_NONPRIV(base, i), 411 false); 412 413 /* add in local MOCS registers */ 414 for (i = 0; i < LNCFCMOCS_REG_COUNT; i++) 415 if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 55)) 416 ret |= GUC_MCR_REG_ADD(gt, regset, XEHP_LNCFCMOCS(i), false); 417 else 418 ret |= GUC_MMIO_REG_ADD(gt, regset, GEN9_LNCFCMOCS(i), false); 419 420 if (GRAPHICS_VER(engine->i915) >= 12) { 421 ret |= GUC_MCR_REG_ADD(gt, regset, MCR_REG(i915_mmio_reg_offset(EU_PERF_CNTL0)), false); 422 ret |= GUC_MCR_REG_ADD(gt, regset, MCR_REG(i915_mmio_reg_offset(EU_PERF_CNTL1)), false); 423 ret |= GUC_MCR_REG_ADD(gt, regset, MCR_REG(i915_mmio_reg_offset(EU_PERF_CNTL2)), false); 424 ret |= GUC_MCR_REG_ADD(gt, regset, MCR_REG(i915_mmio_reg_offset(EU_PERF_CNTL3)), false); 425 ret |= GUC_MCR_REG_ADD(gt, regset, MCR_REG(i915_mmio_reg_offset(EU_PERF_CNTL4)), false); 426 ret |= GUC_MCR_REG_ADD(gt, regset, MCR_REG(i915_mmio_reg_offset(EU_PERF_CNTL5)), false); 427 ret |= GUC_MCR_REG_ADD(gt, regset, MCR_REG(i915_mmio_reg_offset(EU_PERF_CNTL6)), false); 428 } 429 430 return ret ? -1 : 0; 431 } 432 433 static long guc_mmio_reg_state_create(struct intel_guc *guc) 434 { 435 struct intel_gt *gt = guc_to_gt(guc); 436 struct intel_engine_cs *engine; 437 enum intel_engine_id id; 438 struct temp_regset temp_set = {}; 439 long total = 0; 440 long ret; 441 442 for_each_engine(engine, gt, id) { 443 u32 used = temp_set.storage_used; 444 445 ret = guc_mmio_regset_init(&temp_set, engine); 446 if (ret < 0) 447 goto fail_regset_init; 448 449 guc->ads_regset_count[id] = temp_set.storage_used - used; 450 total += guc->ads_regset_count[id]; 451 } 452 453 guc->ads_regset = temp_set.storage; 454 455 guc_dbg(guc, "Used %zu KB for temporary ADS regset\n", 456 (temp_set.storage_max * sizeof(struct guc_mmio_reg)) >> 10); 457 458 return total * sizeof(struct guc_mmio_reg); 459 460 fail_regset_init: 461 kfree(temp_set.storage); 462 return ret; 463 } 464 465 static void guc_mmio_reg_state_init(struct intel_guc *guc) 466 { 467 struct intel_gt *gt = guc_to_gt(guc); 468 struct intel_engine_cs *engine; 469 enum intel_engine_id id; 470 u32 addr_ggtt, offset; 471 472 offset = guc_ads_regset_offset(guc); 473 addr_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma) + offset; 474 475 iosys_map_memcpy_to(&guc->ads_map, offset, guc->ads_regset, 476 guc->ads_regset_size); 477 478 for_each_engine(engine, gt, id) { 479 u32 count = guc->ads_regset_count[id]; 480 u8 guc_class; 481 482 /* Class index is checked in class converter */ 483 GEM_BUG_ON(engine->instance >= GUC_MAX_INSTANCES_PER_CLASS); 484 485 guc_class = engine_class_to_guc_class(engine->class); 486 487 if (!count) { 488 ads_blob_write(guc, 489 ads.reg_state_list[guc_class][engine->instance].address, 490 0); 491 ads_blob_write(guc, 492 ads.reg_state_list[guc_class][engine->instance].count, 493 0); 494 continue; 495 } 496 497 ads_blob_write(guc, 498 ads.reg_state_list[guc_class][engine->instance].address, 499 addr_ggtt); 500 ads_blob_write(guc, 501 ads.reg_state_list[guc_class][engine->instance].count, 502 count); 503 504 addr_ggtt += count * sizeof(struct guc_mmio_reg); 505 } 506 } 507 508 static void fill_engine_enable_masks(struct intel_gt *gt, 509 struct iosys_map *info_map) 510 { 511 info_map_write(info_map, engine_enabled_masks[GUC_RENDER_CLASS], RCS_MASK(gt)); 512 info_map_write(info_map, engine_enabled_masks[GUC_COMPUTE_CLASS], CCS_MASK(gt)); 513 info_map_write(info_map, engine_enabled_masks[GUC_BLITTER_CLASS], BCS_MASK(gt)); 514 info_map_write(info_map, engine_enabled_masks[GUC_VIDEO_CLASS], VDBOX_MASK(gt)); 515 info_map_write(info_map, engine_enabled_masks[GUC_VIDEOENHANCE_CLASS], VEBOX_MASK(gt)); 516 517 /* The GSC engine is an instance (6) of OTHER_CLASS */ 518 if (gt->engine[GSC0]) 519 info_map_write(info_map, engine_enabled_masks[GUC_GSC_OTHER_CLASS], 520 BIT(gt->engine[GSC0]->instance)); 521 } 522 523 #define LR_HW_CONTEXT_SIZE (80 * sizeof(u32)) 524 #define XEHP_LR_HW_CONTEXT_SIZE (96 * sizeof(u32)) 525 #define LR_HW_CONTEXT_SZ(i915) (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 55) ? \ 526 XEHP_LR_HW_CONTEXT_SIZE : \ 527 LR_HW_CONTEXT_SIZE) 528 #define LRC_SKIP_SIZE(i915) (LRC_PPHWSP_SZ * PAGE_SIZE + LR_HW_CONTEXT_SZ(i915)) 529 static int guc_prep_golden_context(struct intel_guc *guc) 530 { 531 struct intel_gt *gt = guc_to_gt(guc); 532 u32 addr_ggtt, offset; 533 u32 total_size = 0, alloc_size, real_size; 534 u8 engine_class, guc_class; 535 struct guc_gt_system_info local_info; 536 struct iosys_map info_map; 537 538 /* 539 * Reserve the memory for the golden contexts and point GuC at it but 540 * leave it empty for now. The context data will be filled in later 541 * once there is something available to put there. 542 * 543 * Note that the HWSP and ring context are not included. 544 * 545 * Note also that the storage must be pinned in the GGTT, so that the 546 * address won't change after GuC has been told where to find it. The 547 * GuC will also validate that the LRC base + size fall within the 548 * allowed GGTT range. 549 */ 550 if (!iosys_map_is_null(&guc->ads_map)) { 551 offset = guc_ads_golden_ctxt_offset(guc); 552 addr_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma) + offset; 553 info_map = IOSYS_MAP_INIT_OFFSET(&guc->ads_map, 554 offsetof(struct __guc_ads_blob, system_info)); 555 } else { 556 memset(&local_info, 0, sizeof(local_info)); 557 iosys_map_set_vaddr(&info_map, &local_info); 558 fill_engine_enable_masks(gt, &info_map); 559 } 560 561 for (engine_class = 0; engine_class <= MAX_ENGINE_CLASS; ++engine_class) { 562 guc_class = engine_class_to_guc_class(engine_class); 563 564 if (!info_map_read(&info_map, engine_enabled_masks[guc_class])) 565 continue; 566 567 real_size = intel_engine_context_size(gt, engine_class); 568 alloc_size = PAGE_ALIGN(real_size); 569 total_size += alloc_size; 570 571 if (iosys_map_is_null(&guc->ads_map)) 572 continue; 573 574 /* 575 * This interface is slightly confusing. We need to pass the 576 * base address of the full golden context and the size of just 577 * the engine state, which is the section of the context image 578 * that starts after the execlists context. This is required to 579 * allow the GuC to restore just the engine state when a 580 * watchdog reset occurs. 581 * We calculate the engine state size by removing the size of 582 * what comes before it in the context image (which is identical 583 * on all engines). 584 */ 585 ads_blob_write(guc, ads.eng_state_size[guc_class], 586 real_size - LRC_SKIP_SIZE(gt->i915)); 587 ads_blob_write(guc, ads.golden_context_lrca[guc_class], 588 addr_ggtt); 589 590 addr_ggtt += alloc_size; 591 } 592 593 /* Make sure current size matches what we calculated previously */ 594 if (guc->ads_golden_ctxt_size) 595 GEM_BUG_ON(guc->ads_golden_ctxt_size != total_size); 596 597 return total_size; 598 } 599 600 static struct intel_engine_cs *find_engine_state(struct intel_gt *gt, u8 engine_class) 601 { 602 struct intel_engine_cs *engine; 603 enum intel_engine_id id; 604 605 for_each_engine(engine, gt, id) { 606 if (engine->class != engine_class) 607 continue; 608 609 if (!engine->default_state) 610 continue; 611 612 return engine; 613 } 614 615 return NULL; 616 } 617 618 static void guc_init_golden_context(struct intel_guc *guc) 619 { 620 struct intel_engine_cs *engine; 621 struct intel_gt *gt = guc_to_gt(guc); 622 unsigned long offset; 623 u32 addr_ggtt, total_size = 0, alloc_size, real_size; 624 u8 engine_class, guc_class; 625 626 if (!intel_uc_uses_guc_submission(>->uc)) 627 return; 628 629 GEM_BUG_ON(iosys_map_is_null(&guc->ads_map)); 630 631 /* 632 * Go back and fill in the golden context data now that it is 633 * available. 634 */ 635 offset = guc_ads_golden_ctxt_offset(guc); 636 addr_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma) + offset; 637 638 for (engine_class = 0; engine_class <= MAX_ENGINE_CLASS; ++engine_class) { 639 guc_class = engine_class_to_guc_class(engine_class); 640 if (!ads_blob_read(guc, system_info.engine_enabled_masks[guc_class])) 641 continue; 642 643 real_size = intel_engine_context_size(gt, engine_class); 644 alloc_size = PAGE_ALIGN(real_size); 645 total_size += alloc_size; 646 647 engine = find_engine_state(gt, engine_class); 648 if (!engine) { 649 guc_err(guc, "No engine state recorded for class %d!\n", 650 engine_class); 651 ads_blob_write(guc, ads.eng_state_size[guc_class], 0); 652 ads_blob_write(guc, ads.golden_context_lrca[guc_class], 0); 653 continue; 654 } 655 656 GEM_BUG_ON(ads_blob_read(guc, ads.eng_state_size[guc_class]) != 657 real_size - LRC_SKIP_SIZE(gt->i915)); 658 GEM_BUG_ON(ads_blob_read(guc, ads.golden_context_lrca[guc_class]) != addr_ggtt); 659 660 addr_ggtt += alloc_size; 661 662 shmem_read_to_iosys_map(engine->default_state, 0, &guc->ads_map, 663 offset, real_size); 664 offset += alloc_size; 665 } 666 667 GEM_BUG_ON(guc->ads_golden_ctxt_size != total_size); 668 } 669 670 static u32 guc_get_capture_engine_mask(struct iosys_map *info_map, u32 capture_class) 671 { 672 u32 mask; 673 674 switch (capture_class) { 675 case GUC_CAPTURE_LIST_CLASS_RENDER_COMPUTE: 676 mask = info_map_read(info_map, engine_enabled_masks[GUC_RENDER_CLASS]); 677 mask |= info_map_read(info_map, engine_enabled_masks[GUC_COMPUTE_CLASS]); 678 break; 679 680 case GUC_CAPTURE_LIST_CLASS_VIDEO: 681 mask = info_map_read(info_map, engine_enabled_masks[GUC_VIDEO_CLASS]); 682 break; 683 684 case GUC_CAPTURE_LIST_CLASS_VIDEOENHANCE: 685 mask = info_map_read(info_map, engine_enabled_masks[GUC_VIDEOENHANCE_CLASS]); 686 break; 687 688 case GUC_CAPTURE_LIST_CLASS_BLITTER: 689 mask = info_map_read(info_map, engine_enabled_masks[GUC_BLITTER_CLASS]); 690 break; 691 692 case GUC_CAPTURE_LIST_CLASS_GSC_OTHER: 693 mask = info_map_read(info_map, engine_enabled_masks[GUC_GSC_OTHER_CLASS]); 694 break; 695 696 default: 697 mask = 0; 698 } 699 700 return mask; 701 } 702 703 static int 704 guc_capture_prep_lists(struct intel_guc *guc) 705 { 706 struct intel_gt *gt = guc_to_gt(guc); 707 u32 ads_ggtt, capture_offset, null_ggtt, total_size = 0; 708 struct guc_gt_system_info local_info; 709 struct iosys_map info_map; 710 bool ads_is_mapped; 711 size_t size = 0; 712 void *ptr; 713 int i, j; 714 715 ads_is_mapped = !iosys_map_is_null(&guc->ads_map); 716 if (ads_is_mapped) { 717 capture_offset = guc_ads_capture_offset(guc); 718 ads_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma); 719 info_map = IOSYS_MAP_INIT_OFFSET(&guc->ads_map, 720 offsetof(struct __guc_ads_blob, system_info)); 721 } else { 722 memset(&local_info, 0, sizeof(local_info)); 723 iosys_map_set_vaddr(&info_map, &local_info); 724 fill_engine_enable_masks(gt, &info_map); 725 } 726 727 /* first, set aside the first page for a capture_list with zero descriptors */ 728 total_size = PAGE_SIZE; 729 if (ads_is_mapped) { 730 if (!intel_guc_capture_getnullheader(guc, &ptr, &size)) 731 iosys_map_memcpy_to(&guc->ads_map, capture_offset, ptr, size); 732 null_ggtt = ads_ggtt + capture_offset; 733 capture_offset += PAGE_SIZE; 734 } 735 736 for (i = 0; i < GUC_CAPTURE_LIST_INDEX_MAX; i++) { 737 for (j = 0; j < GUC_MAX_ENGINE_CLASSES; j++) { 738 u32 engine_mask = guc_get_capture_engine_mask(&info_map, j); 739 740 /* null list if we dont have said engine or list */ 741 if (!engine_mask) { 742 if (ads_is_mapped) { 743 ads_blob_write(guc, ads.capture_class[i][j], null_ggtt); 744 ads_blob_write(guc, ads.capture_instance[i][j], null_ggtt); 745 } 746 continue; 747 } 748 if (intel_guc_capture_getlistsize(guc, i, 749 GUC_CAPTURE_LIST_TYPE_ENGINE_CLASS, 750 j, &size)) { 751 if (ads_is_mapped) 752 ads_blob_write(guc, ads.capture_class[i][j], null_ggtt); 753 goto engine_instance_list; 754 } 755 total_size += size; 756 if (ads_is_mapped) { 757 if (total_size > guc->ads_capture_size || 758 intel_guc_capture_getlist(guc, i, 759 GUC_CAPTURE_LIST_TYPE_ENGINE_CLASS, 760 j, &ptr)) { 761 ads_blob_write(guc, ads.capture_class[i][j], null_ggtt); 762 continue; 763 } 764 ads_blob_write(guc, ads.capture_class[i][j], ads_ggtt + 765 capture_offset); 766 iosys_map_memcpy_to(&guc->ads_map, capture_offset, ptr, size); 767 capture_offset += size; 768 } 769 engine_instance_list: 770 if (intel_guc_capture_getlistsize(guc, i, 771 GUC_CAPTURE_LIST_TYPE_ENGINE_INSTANCE, 772 j, &size)) { 773 if (ads_is_mapped) 774 ads_blob_write(guc, ads.capture_instance[i][j], null_ggtt); 775 continue; 776 } 777 total_size += size; 778 if (ads_is_mapped) { 779 if (total_size > guc->ads_capture_size || 780 intel_guc_capture_getlist(guc, i, 781 GUC_CAPTURE_LIST_TYPE_ENGINE_INSTANCE, 782 j, &ptr)) { 783 ads_blob_write(guc, ads.capture_instance[i][j], null_ggtt); 784 continue; 785 } 786 ads_blob_write(guc, ads.capture_instance[i][j], ads_ggtt + 787 capture_offset); 788 iosys_map_memcpy_to(&guc->ads_map, capture_offset, ptr, size); 789 capture_offset += size; 790 } 791 } 792 if (intel_guc_capture_getlistsize(guc, i, GUC_CAPTURE_LIST_TYPE_GLOBAL, 0, &size)) { 793 if (ads_is_mapped) 794 ads_blob_write(guc, ads.capture_global[i], null_ggtt); 795 continue; 796 } 797 total_size += size; 798 if (ads_is_mapped) { 799 if (total_size > guc->ads_capture_size || 800 intel_guc_capture_getlist(guc, i, GUC_CAPTURE_LIST_TYPE_GLOBAL, 0, 801 &ptr)) { 802 ads_blob_write(guc, ads.capture_global[i], null_ggtt); 803 continue; 804 } 805 ads_blob_write(guc, ads.capture_global[i], ads_ggtt + capture_offset); 806 iosys_map_memcpy_to(&guc->ads_map, capture_offset, ptr, size); 807 capture_offset += size; 808 } 809 } 810 811 if (guc->ads_capture_size && guc->ads_capture_size != PAGE_ALIGN(total_size)) 812 guc_warn(guc, "ADS capture alloc size changed from %d to %d\n", 813 guc->ads_capture_size, PAGE_ALIGN(total_size)); 814 815 return PAGE_ALIGN(total_size); 816 } 817 818 static void guc_waklv_enable_simple(struct intel_guc *guc, u32 *offset, u32 *remain, u32 klv_id) 819 { 820 u32 size; 821 u32 klv_entry[] = { 822 /* 16:16 key/length */ 823 FIELD_PREP(GUC_KLV_0_KEY, klv_id) | 824 FIELD_PREP(GUC_KLV_0_LEN, 0), 825 /* 0 dwords data */ 826 }; 827 828 size = sizeof(klv_entry); 829 GEM_BUG_ON(*remain < size); 830 831 iosys_map_memcpy_to(&guc->ads_map, *offset, klv_entry, size); 832 *offset += size; 833 *remain -= size; 834 } 835 836 static void guc_waklv_init(struct intel_guc *guc) 837 { 838 struct intel_gt *gt = guc_to_gt(guc); 839 u32 offset, addr_ggtt, remain, size; 840 841 if (!intel_uc_uses_guc_submission(>->uc)) 842 return; 843 844 if (GUC_FIRMWARE_VER(guc) < MAKE_GUC_VER(70, 10, 0)) 845 return; 846 847 GEM_BUG_ON(iosys_map_is_null(&guc->ads_map)); 848 offset = guc_ads_waklv_offset(guc); 849 remain = guc_ads_waklv_size(guc); 850 851 /* Wa_14019159160 */ 852 if (IS_GFX_GT_IP_RANGE(gt, IP_VER(12, 70), IP_VER(12, 74))) { 853 guc_waklv_enable_simple(guc, &offset, &remain, 854 GUC_WORKAROUND_KLV_SERIALIZED_RA_MODE); 855 guc_waklv_enable_simple(guc, &offset, &remain, 856 GUC_WORKAROUND_KLV_AVOID_GFX_CLEAR_WHILE_ACTIVE); 857 } 858 859 /* Wa_16021333562 */ 860 if ((GUC_FIRMWARE_VER(guc) >= MAKE_GUC_VER(70, 21, 1)) && 861 (IS_GFX_GT_IP_RANGE(gt, IP_VER(12, 70), IP_VER(12, 74)) || 862 IS_MEDIA_GT_IP_RANGE(gt, IP_VER(13, 0), IP_VER(13, 0)) || 863 IS_DG2(gt->i915))) 864 guc_waklv_enable_simple(guc, &offset, &remain, 865 GUC_WORKAROUND_KLV_BLOCK_INTERRUPTS_WHEN_MGSR_BLOCKED); 866 867 size = guc_ads_waklv_size(guc) - remain; 868 if (!size) 869 return; 870 871 offset = guc_ads_waklv_offset(guc); 872 addr_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma) + offset; 873 874 ads_blob_write(guc, ads.wa_klv_addr_lo, addr_ggtt); 875 ads_blob_write(guc, ads.wa_klv_addr_hi, 0); 876 ads_blob_write(guc, ads.wa_klv_size, size); 877 } 878 879 static int guc_prep_waklv(struct intel_guc *guc) 880 { 881 /* Fudge something chunky for now: */ 882 return PAGE_SIZE; 883 } 884 885 static void __guc_ads_init(struct intel_guc *guc) 886 { 887 struct intel_gt *gt = guc_to_gt(guc); 888 struct drm_i915_private *i915 = gt->i915; 889 struct iosys_map info_map = IOSYS_MAP_INIT_OFFSET(&guc->ads_map, 890 offsetof(struct __guc_ads_blob, system_info)); 891 u32 base; 892 893 /* GuC scheduling policies */ 894 guc_policies_init(guc); 895 896 /* System info */ 897 fill_engine_enable_masks(gt, &info_map); 898 899 ads_blob_write(guc, system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_SLICE_ENABLED], 900 hweight8(gt->info.sseu.slice_mask)); 901 ads_blob_write(guc, system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_VDBOX_SFC_SUPPORT_MASK], 902 gt->info.vdbox_sfc_access); 903 904 if (GRAPHICS_VER(i915) >= 12 && !IS_DGFX(i915)) { 905 u32 distdbreg = intel_uncore_read(gt->uncore, 906 GEN12_DIST_DBS_POPULATED); 907 ads_blob_write(guc, 908 system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_DOORBELL_COUNT_PER_SQIDI], 909 ((distdbreg >> GEN12_DOORBELLS_PER_SQIDI_SHIFT) 910 & GEN12_DOORBELLS_PER_SQIDI) + 1); 911 } 912 913 /* Golden contexts for re-initialising after a watchdog reset */ 914 guc_prep_golden_context(guc); 915 916 guc_mapping_table_init(guc_to_gt(guc), &info_map); 917 918 base = intel_guc_ggtt_offset(guc, guc->ads_vma); 919 920 /* Lists for error capture debug */ 921 guc_capture_prep_lists(guc); 922 923 /* ADS */ 924 ads_blob_write(guc, ads.scheduler_policies, base + 925 offsetof(struct __guc_ads_blob, policies)); 926 ads_blob_write(guc, ads.gt_system_info, base + 927 offsetof(struct __guc_ads_blob, system_info)); 928 929 /* MMIO save/restore list */ 930 guc_mmio_reg_state_init(guc); 931 932 /* Workaround KLV list */ 933 guc_waklv_init(guc); 934 935 /* Private Data */ 936 ads_blob_write(guc, ads.private_data, base + 937 guc_ads_private_data_offset(guc)); 938 939 i915_gem_object_flush_map(guc->ads_vma->obj); 940 } 941 942 /** 943 * intel_guc_ads_create() - allocates and initializes GuC ADS. 944 * @guc: intel_guc struct 945 * 946 * GuC needs memory block (Additional Data Struct), where it will store 947 * some data. Allocate and initialize such memory block for GuC use. 948 */ 949 int intel_guc_ads_create(struct intel_guc *guc) 950 { 951 void *ads_blob; 952 u32 size; 953 int ret; 954 955 GEM_BUG_ON(guc->ads_vma); 956 957 /* 958 * Create reg state size dynamically on system memory to be copied to 959 * the final ads blob on gt init/reset 960 */ 961 ret = guc_mmio_reg_state_create(guc); 962 if (ret < 0) 963 return ret; 964 guc->ads_regset_size = ret; 965 966 /* Likewise the golden contexts: */ 967 ret = guc_prep_golden_context(guc); 968 if (ret < 0) 969 return ret; 970 guc->ads_golden_ctxt_size = ret; 971 972 /* Likewise the capture lists: */ 973 ret = guc_capture_prep_lists(guc); 974 if (ret < 0) 975 return ret; 976 guc->ads_capture_size = ret; 977 978 /* And don't forget the workaround KLVs: */ 979 ret = guc_prep_waklv(guc); 980 if (ret < 0) 981 return ret; 982 guc->ads_waklv_size = ret; 983 984 /* Now the total size can be determined: */ 985 size = guc_ads_blob_size(guc); 986 987 ret = intel_guc_allocate_and_map_vma(guc, size, &guc->ads_vma, 988 &ads_blob); 989 if (ret) 990 return ret; 991 992 if (i915_gem_object_is_lmem(guc->ads_vma->obj)) 993 iosys_map_set_vaddr_iomem(&guc->ads_map, (void __iomem *)ads_blob); 994 else 995 iosys_map_set_vaddr(&guc->ads_map, ads_blob); 996 997 __guc_ads_init(guc); 998 999 return 0; 1000 } 1001 1002 void intel_guc_ads_init_late(struct intel_guc *guc) 1003 { 1004 /* 1005 * The golden context setup requires the saved engine state from 1006 * __engines_record_defaults(). However, that requires engines to be 1007 * operational which means the ADS must already have been configured. 1008 * Fortunately, the golden context state is not needed until a hang 1009 * occurs, so it can be filled in during this late init phase. 1010 */ 1011 guc_init_golden_context(guc); 1012 } 1013 1014 void intel_guc_ads_destroy(struct intel_guc *guc) 1015 { 1016 i915_vma_unpin_and_release(&guc->ads_vma, I915_VMA_RELEASE_MAP); 1017 iosys_map_clear(&guc->ads_map); 1018 kfree(guc->ads_regset); 1019 } 1020 1021 static void guc_ads_private_data_reset(struct intel_guc *guc) 1022 { 1023 u32 size; 1024 1025 size = guc_ads_private_data_size(guc); 1026 if (!size) 1027 return; 1028 1029 iosys_map_memset(&guc->ads_map, guc_ads_private_data_offset(guc), 1030 0, size); 1031 } 1032 1033 /** 1034 * intel_guc_ads_reset() - prepares GuC Additional Data Struct for reuse 1035 * @guc: intel_guc struct 1036 * 1037 * GuC stores some data in ADS, which might be stale after a reset. 1038 * Reinitialize whole ADS in case any part of it was corrupted during 1039 * previous GuC run. 1040 */ 1041 void intel_guc_ads_reset(struct intel_guc *guc) 1042 { 1043 if (!guc->ads_vma) 1044 return; 1045 1046 __guc_ads_init(guc); 1047 1048 guc_ads_private_data_reset(guc); 1049 } 1050 1051 u32 intel_guc_engine_usage_offset(struct intel_guc *guc) 1052 { 1053 return intel_guc_ggtt_offset(guc, guc->ads_vma) + 1054 offsetof(struct __guc_ads_blob, engine_usage); 1055 } 1056 1057 struct iosys_map intel_guc_engine_usage_record_map(struct intel_engine_cs *engine) 1058 { 1059 struct intel_guc *guc = gt_to_guc(engine->gt); 1060 u8 guc_class = engine_class_to_guc_class(engine->class); 1061 size_t offset = offsetof(struct __guc_ads_blob, 1062 engine_usage.engines[guc_class][ilog2(engine->logical_mask)]); 1063 1064 return IOSYS_MAP_INIT_OFFSET(&guc->ads_map, offset); 1065 } 1066