1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include "xe_hw_engine.h" 7 8 #include <drm/drm_managed.h> 9 10 #include "regs/xe_engine_regs.h" 11 #include "regs/xe_gt_regs.h" 12 #include "xe_assert.h" 13 #include "xe_bo.h" 14 #include "xe_device.h" 15 #include "xe_execlist.h" 16 #include "xe_force_wake.h" 17 #include "xe_gt.h" 18 #include "xe_gt_ccs_mode.h" 19 #include "xe_gt_topology.h" 20 #include "xe_hw_fence.h" 21 #include "xe_irq.h" 22 #include "xe_lrc.h" 23 #include "xe_macros.h" 24 #include "xe_mmio.h" 25 #include "xe_reg_sr.h" 26 #include "xe_rtp.h" 27 #include "xe_sched_job.h" 28 #include "xe_tuning.h" 29 #include "xe_uc_fw.h" 30 #include "xe_wa.h" 31 32 #define MAX_MMIO_BASES 3 33 struct engine_info { 34 const char *name; 35 unsigned int class : 8; 36 unsigned int instance : 8; 37 enum xe_force_wake_domains domain; 38 u32 mmio_base; 39 }; 40 41 static const struct engine_info engine_infos[] = { 42 [XE_HW_ENGINE_RCS0] = { 43 .name = "rcs0", 44 .class = XE_ENGINE_CLASS_RENDER, 45 .instance = 0, 46 .domain = XE_FW_RENDER, 47 .mmio_base = RENDER_RING_BASE, 48 }, 49 [XE_HW_ENGINE_BCS0] = { 50 .name = "bcs0", 51 .class = XE_ENGINE_CLASS_COPY, 52 .instance = 0, 53 .domain = XE_FW_RENDER, 54 .mmio_base = BLT_RING_BASE, 55 }, 56 [XE_HW_ENGINE_BCS1] = { 57 .name = "bcs1", 58 .class = XE_ENGINE_CLASS_COPY, 59 .instance = 1, 60 .domain = XE_FW_RENDER, 61 .mmio_base = XEHPC_BCS1_RING_BASE, 62 }, 63 [XE_HW_ENGINE_BCS2] = { 64 .name = "bcs2", 65 .class = XE_ENGINE_CLASS_COPY, 66 .instance = 2, 67 .domain = XE_FW_RENDER, 68 .mmio_base = XEHPC_BCS2_RING_BASE, 69 }, 70 [XE_HW_ENGINE_BCS3] = { 71 .name = "bcs3", 72 .class = XE_ENGINE_CLASS_COPY, 73 .instance = 3, 74 .domain = XE_FW_RENDER, 75 .mmio_base = XEHPC_BCS3_RING_BASE, 76 }, 77 [XE_HW_ENGINE_BCS4] = { 78 .name = "bcs4", 79 .class = XE_ENGINE_CLASS_COPY, 80 .instance = 4, 81 .domain = XE_FW_RENDER, 82 .mmio_base = XEHPC_BCS4_RING_BASE, 83 }, 84 [XE_HW_ENGINE_BCS5] = { 85 .name = "bcs5", 86 .class = XE_ENGINE_CLASS_COPY, 87 .instance = 5, 88 .domain = XE_FW_RENDER, 89 .mmio_base = XEHPC_BCS5_RING_BASE, 90 }, 91 [XE_HW_ENGINE_BCS6] = { 92 .name = "bcs6", 93 .class = XE_ENGINE_CLASS_COPY, 94 .instance = 6, 95 .domain = XE_FW_RENDER, 96 .mmio_base = XEHPC_BCS6_RING_BASE, 97 }, 98 [XE_HW_ENGINE_BCS7] = { 99 .name = "bcs7", 100 .class = XE_ENGINE_CLASS_COPY, 101 .instance = 7, 102 .domain = XE_FW_RENDER, 103 .mmio_base = XEHPC_BCS7_RING_BASE, 104 }, 105 [XE_HW_ENGINE_BCS8] = { 106 .name = "bcs8", 107 .class = XE_ENGINE_CLASS_COPY, 108 .instance = 8, 109 .domain = XE_FW_RENDER, 110 .mmio_base = XEHPC_BCS8_RING_BASE, 111 }, 112 113 [XE_HW_ENGINE_VCS0] = { 114 .name = "vcs0", 115 .class = XE_ENGINE_CLASS_VIDEO_DECODE, 116 .instance = 0, 117 .domain = XE_FW_MEDIA_VDBOX0, 118 .mmio_base = BSD_RING_BASE, 119 }, 120 [XE_HW_ENGINE_VCS1] = { 121 .name = "vcs1", 122 .class = XE_ENGINE_CLASS_VIDEO_DECODE, 123 .instance = 1, 124 .domain = XE_FW_MEDIA_VDBOX1, 125 .mmio_base = BSD2_RING_BASE, 126 }, 127 [XE_HW_ENGINE_VCS2] = { 128 .name = "vcs2", 129 .class = XE_ENGINE_CLASS_VIDEO_DECODE, 130 .instance = 2, 131 .domain = XE_FW_MEDIA_VDBOX2, 132 .mmio_base = BSD3_RING_BASE, 133 }, 134 [XE_HW_ENGINE_VCS3] = { 135 .name = "vcs3", 136 .class = XE_ENGINE_CLASS_VIDEO_DECODE, 137 .instance = 3, 138 .domain = XE_FW_MEDIA_VDBOX3, 139 .mmio_base = BSD4_RING_BASE, 140 }, 141 [XE_HW_ENGINE_VCS4] = { 142 .name = "vcs4", 143 .class = XE_ENGINE_CLASS_VIDEO_DECODE, 144 .instance = 4, 145 .domain = XE_FW_MEDIA_VDBOX4, 146 .mmio_base = XEHP_BSD5_RING_BASE, 147 }, 148 [XE_HW_ENGINE_VCS5] = { 149 .name = "vcs5", 150 .class = XE_ENGINE_CLASS_VIDEO_DECODE, 151 .instance = 5, 152 .domain = XE_FW_MEDIA_VDBOX5, 153 .mmio_base = XEHP_BSD6_RING_BASE, 154 }, 155 [XE_HW_ENGINE_VCS6] = { 156 .name = "vcs6", 157 .class = XE_ENGINE_CLASS_VIDEO_DECODE, 158 .instance = 6, 159 .domain = XE_FW_MEDIA_VDBOX6, 160 .mmio_base = XEHP_BSD7_RING_BASE, 161 }, 162 [XE_HW_ENGINE_VCS7] = { 163 .name = "vcs7", 164 .class = XE_ENGINE_CLASS_VIDEO_DECODE, 165 .instance = 7, 166 .domain = XE_FW_MEDIA_VDBOX7, 167 .mmio_base = XEHP_BSD8_RING_BASE, 168 }, 169 [XE_HW_ENGINE_VECS0] = { 170 .name = "vecs0", 171 .class = XE_ENGINE_CLASS_VIDEO_ENHANCE, 172 .instance = 0, 173 .domain = XE_FW_MEDIA_VEBOX0, 174 .mmio_base = VEBOX_RING_BASE, 175 }, 176 [XE_HW_ENGINE_VECS1] = { 177 .name = "vecs1", 178 .class = XE_ENGINE_CLASS_VIDEO_ENHANCE, 179 .instance = 1, 180 .domain = XE_FW_MEDIA_VEBOX1, 181 .mmio_base = VEBOX2_RING_BASE, 182 }, 183 [XE_HW_ENGINE_VECS2] = { 184 .name = "vecs2", 185 .class = XE_ENGINE_CLASS_VIDEO_ENHANCE, 186 .instance = 2, 187 .domain = XE_FW_MEDIA_VEBOX2, 188 .mmio_base = XEHP_VEBOX3_RING_BASE, 189 }, 190 [XE_HW_ENGINE_VECS3] = { 191 .name = "vecs3", 192 .class = XE_ENGINE_CLASS_VIDEO_ENHANCE, 193 .instance = 3, 194 .domain = XE_FW_MEDIA_VEBOX3, 195 .mmio_base = XEHP_VEBOX4_RING_BASE, 196 }, 197 [XE_HW_ENGINE_CCS0] = { 198 .name = "ccs0", 199 .class = XE_ENGINE_CLASS_COMPUTE, 200 .instance = 0, 201 .domain = XE_FW_RENDER, 202 .mmio_base = COMPUTE0_RING_BASE, 203 }, 204 [XE_HW_ENGINE_CCS1] = { 205 .name = "ccs1", 206 .class = XE_ENGINE_CLASS_COMPUTE, 207 .instance = 1, 208 .domain = XE_FW_RENDER, 209 .mmio_base = COMPUTE1_RING_BASE, 210 }, 211 [XE_HW_ENGINE_CCS2] = { 212 .name = "ccs2", 213 .class = XE_ENGINE_CLASS_COMPUTE, 214 .instance = 2, 215 .domain = XE_FW_RENDER, 216 .mmio_base = COMPUTE2_RING_BASE, 217 }, 218 [XE_HW_ENGINE_CCS3] = { 219 .name = "ccs3", 220 .class = XE_ENGINE_CLASS_COMPUTE, 221 .instance = 3, 222 .domain = XE_FW_RENDER, 223 .mmio_base = COMPUTE3_RING_BASE, 224 }, 225 [XE_HW_ENGINE_GSCCS0] = { 226 .name = "gsccs0", 227 .class = XE_ENGINE_CLASS_OTHER, 228 .instance = OTHER_GSC_INSTANCE, 229 .domain = XE_FW_GSC, 230 .mmio_base = GSCCS_RING_BASE, 231 }, 232 }; 233 234 static void hw_engine_fini(struct drm_device *drm, void *arg) 235 { 236 struct xe_hw_engine *hwe = arg; 237 238 if (hwe->exl_port) 239 xe_execlist_port_destroy(hwe->exl_port); 240 xe_lrc_finish(&hwe->kernel_lrc); 241 242 hwe->gt = NULL; 243 } 244 245 static void hw_engine_mmio_write32(struct xe_hw_engine *hwe, struct xe_reg reg, 246 u32 val) 247 { 248 xe_gt_assert(hwe->gt, !(reg.addr & hwe->mmio_base)); 249 xe_force_wake_assert_held(gt_to_fw(hwe->gt), hwe->domain); 250 251 reg.addr += hwe->mmio_base; 252 253 xe_mmio_write32(hwe->gt, reg, val); 254 } 255 256 static u32 hw_engine_mmio_read32(struct xe_hw_engine *hwe, struct xe_reg reg) 257 { 258 xe_gt_assert(hwe->gt, !(reg.addr & hwe->mmio_base)); 259 xe_force_wake_assert_held(gt_to_fw(hwe->gt), hwe->domain); 260 261 reg.addr += hwe->mmio_base; 262 263 return xe_mmio_read32(hwe->gt, reg); 264 } 265 266 void xe_hw_engine_enable_ring(struct xe_hw_engine *hwe) 267 { 268 u32 ccs_mask = 269 xe_hw_engine_mask_per_class(hwe->gt, XE_ENGINE_CLASS_COMPUTE); 270 271 if (hwe->class == XE_ENGINE_CLASS_COMPUTE && ccs_mask) 272 xe_mmio_write32(hwe->gt, RCU_MODE, 273 _MASKED_BIT_ENABLE(RCU_MODE_CCS_ENABLE)); 274 275 hw_engine_mmio_write32(hwe, RING_HWSTAM(0), ~0x0); 276 hw_engine_mmio_write32(hwe, RING_HWS_PGA(0), 277 xe_bo_ggtt_addr(hwe->hwsp)); 278 hw_engine_mmio_write32(hwe, RING_MODE(0), 279 _MASKED_BIT_ENABLE(GFX_DISABLE_LEGACY_MODE)); 280 hw_engine_mmio_write32(hwe, RING_MI_MODE(0), 281 _MASKED_BIT_DISABLE(STOP_RING)); 282 hw_engine_mmio_read32(hwe, RING_MI_MODE(0)); 283 } 284 285 static bool xe_hw_engine_match_fixed_cslice_mode(const struct xe_gt *gt, 286 const struct xe_hw_engine *hwe) 287 { 288 return xe_gt_ccs_mode_enabled(gt) && 289 xe_rtp_match_first_render_or_compute(gt, hwe); 290 } 291 292 void 293 xe_hw_engine_setup_default_lrc_state(struct xe_hw_engine *hwe) 294 { 295 struct xe_gt *gt = hwe->gt; 296 const u8 mocs_write_idx = gt->mocs.uc_index; 297 const u8 mocs_read_idx = gt->mocs.uc_index; 298 u32 blit_cctl_val = REG_FIELD_PREP(BLIT_CCTL_DST_MOCS_MASK, mocs_write_idx) | 299 REG_FIELD_PREP(BLIT_CCTL_SRC_MOCS_MASK, mocs_read_idx); 300 struct xe_rtp_process_ctx ctx = XE_RTP_PROCESS_CTX_INITIALIZER(hwe); 301 const struct xe_rtp_entry_sr lrc_was[] = { 302 /* 303 * Some blitter commands do not have a field for MOCS, those 304 * commands will use MOCS index pointed by BLIT_CCTL. 305 * BLIT_CCTL registers are needed to be programmed to un-cached. 306 */ 307 { XE_RTP_NAME("BLIT_CCTL_default_MOCS"), 308 XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, XE_RTP_END_VERSION_UNDEFINED), 309 ENGINE_CLASS(COPY)), 310 XE_RTP_ACTIONS(FIELD_SET(BLIT_CCTL(0), 311 BLIT_CCTL_DST_MOCS_MASK | 312 BLIT_CCTL_SRC_MOCS_MASK, 313 blit_cctl_val, 314 XE_RTP_ACTION_FLAG(ENGINE_BASE))) 315 }, 316 /* Use Fixed slice CCS mode */ 317 { XE_RTP_NAME("RCU_MODE_FIXED_SLICE_CCS_MODE"), 318 XE_RTP_RULES(FUNC(xe_hw_engine_match_fixed_cslice_mode)), 319 XE_RTP_ACTIONS(FIELD_SET(RCU_MODE, RCU_MODE_FIXED_SLICE_CCS_MODE, 320 RCU_MODE_FIXED_SLICE_CCS_MODE)) 321 }, 322 {} 323 }; 324 325 xe_rtp_process_to_sr(&ctx, lrc_was, &hwe->reg_lrc); 326 } 327 328 static void 329 hw_engine_setup_default_state(struct xe_hw_engine *hwe) 330 { 331 struct xe_gt *gt = hwe->gt; 332 struct xe_device *xe = gt_to_xe(gt); 333 /* 334 * RING_CMD_CCTL specifies the default MOCS entry that will be 335 * used by the command streamer when executing commands that 336 * don't have a way to explicitly specify a MOCS setting. 337 * The default should usually reference whichever MOCS entry 338 * corresponds to uncached behavior, although use of a WB cached 339 * entry is recommended by the spec in certain circumstances on 340 * specific platforms. 341 * Bspec: 72161 342 */ 343 const u8 mocs_write_idx = gt->mocs.uc_index; 344 const u8 mocs_read_idx = hwe->class == XE_ENGINE_CLASS_COMPUTE && 345 (GRAPHICS_VER(xe) >= 20 || xe->info.platform == XE_PVC) ? 346 gt->mocs.wb_index : gt->mocs.uc_index; 347 u32 ring_cmd_cctl_val = REG_FIELD_PREP(CMD_CCTL_WRITE_OVERRIDE_MASK, mocs_write_idx) | 348 REG_FIELD_PREP(CMD_CCTL_READ_OVERRIDE_MASK, mocs_read_idx); 349 struct xe_rtp_process_ctx ctx = XE_RTP_PROCESS_CTX_INITIALIZER(hwe); 350 const struct xe_rtp_entry_sr engine_entries[] = { 351 { XE_RTP_NAME("RING_CMD_CCTL_default_MOCS"), 352 XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, XE_RTP_END_VERSION_UNDEFINED)), 353 XE_RTP_ACTIONS(FIELD_SET(RING_CMD_CCTL(0), 354 CMD_CCTL_WRITE_OVERRIDE_MASK | 355 CMD_CCTL_READ_OVERRIDE_MASK, 356 ring_cmd_cctl_val, 357 XE_RTP_ACTION_FLAG(ENGINE_BASE))) 358 }, 359 /* 360 * To allow the GSC engine to go idle on MTL we need to enable 361 * idle messaging and set the hysteresis value (we use 0xA=5us 362 * as recommended in spec). On platforms after MTL this is 363 * enabled by default. 364 */ 365 { XE_RTP_NAME("MTL GSCCS IDLE MSG enable"), 366 XE_RTP_RULES(MEDIA_VERSION(1300), ENGINE_CLASS(OTHER)), 367 XE_RTP_ACTIONS(CLR(RING_PSMI_CTL(0), 368 IDLE_MSG_DISABLE, 369 XE_RTP_ACTION_FLAG(ENGINE_BASE)), 370 FIELD_SET(RING_PWRCTX_MAXCNT(0), 371 IDLE_WAIT_TIME, 372 0xA, 373 XE_RTP_ACTION_FLAG(ENGINE_BASE))) 374 }, 375 {} 376 }; 377 378 xe_rtp_process_to_sr(&ctx, engine_entries, &hwe->reg_sr); 379 } 380 381 static void hw_engine_init_early(struct xe_gt *gt, struct xe_hw_engine *hwe, 382 enum xe_hw_engine_id id) 383 { 384 const struct engine_info *info; 385 386 if (WARN_ON(id >= ARRAY_SIZE(engine_infos) || !engine_infos[id].name)) 387 return; 388 389 if (!(gt->info.engine_mask & BIT(id))) 390 return; 391 392 info = &engine_infos[id]; 393 394 xe_gt_assert(gt, !hwe->gt); 395 396 hwe->gt = gt; 397 hwe->class = info->class; 398 hwe->instance = info->instance; 399 hwe->mmio_base = info->mmio_base; 400 hwe->domain = info->domain; 401 hwe->name = info->name; 402 hwe->fence_irq = >->fence_irq[info->class]; 403 hwe->engine_id = id; 404 405 hwe->eclass = >->eclass[hwe->class]; 406 if (!hwe->eclass->sched_props.job_timeout_ms) { 407 hwe->eclass->sched_props.job_timeout_ms = 5 * 1000; 408 hwe->eclass->sched_props.job_timeout_min = XE_HW_ENGINE_JOB_TIMEOUT_MIN; 409 hwe->eclass->sched_props.job_timeout_max = XE_HW_ENGINE_JOB_TIMEOUT_MAX; 410 hwe->eclass->sched_props.timeslice_us = 1 * 1000; 411 hwe->eclass->sched_props.timeslice_min = XE_HW_ENGINE_TIMESLICE_MIN; 412 hwe->eclass->sched_props.timeslice_max = XE_HW_ENGINE_TIMESLICE_MAX; 413 hwe->eclass->sched_props.preempt_timeout_us = XE_HW_ENGINE_PREEMPT_TIMEOUT; 414 hwe->eclass->sched_props.preempt_timeout_min = XE_HW_ENGINE_PREEMPT_TIMEOUT_MIN; 415 hwe->eclass->sched_props.preempt_timeout_max = XE_HW_ENGINE_PREEMPT_TIMEOUT_MAX; 416 /* Record default props */ 417 hwe->eclass->defaults = hwe->eclass->sched_props; 418 } 419 420 xe_reg_sr_init(&hwe->reg_sr, hwe->name, gt_to_xe(gt)); 421 xe_tuning_process_engine(hwe); 422 xe_wa_process_engine(hwe); 423 hw_engine_setup_default_state(hwe); 424 425 xe_reg_sr_init(&hwe->reg_whitelist, hwe->name, gt_to_xe(gt)); 426 xe_reg_whitelist_process_engine(hwe); 427 } 428 429 static int hw_engine_init(struct xe_gt *gt, struct xe_hw_engine *hwe, 430 enum xe_hw_engine_id id) 431 { 432 struct xe_device *xe = gt_to_xe(gt); 433 struct xe_tile *tile = gt_to_tile(gt); 434 int err; 435 436 xe_gt_assert(gt, id < ARRAY_SIZE(engine_infos) && engine_infos[id].name); 437 xe_gt_assert(gt, gt->info.engine_mask & BIT(id)); 438 439 xe_reg_sr_apply_mmio(&hwe->reg_sr, gt); 440 xe_reg_sr_apply_whitelist(hwe); 441 442 hwe->hwsp = xe_managed_bo_create_pin_map(xe, tile, SZ_4K, 443 XE_BO_CREATE_VRAM_IF_DGFX(tile) | 444 XE_BO_CREATE_GGTT_BIT); 445 if (IS_ERR(hwe->hwsp)) { 446 err = PTR_ERR(hwe->hwsp); 447 goto err_name; 448 } 449 450 err = xe_lrc_init(&hwe->kernel_lrc, hwe, NULL, NULL, SZ_16K); 451 if (err) 452 goto err_hwsp; 453 454 if (!xe_device_uc_enabled(xe)) { 455 hwe->exl_port = xe_execlist_port_create(xe, hwe); 456 if (IS_ERR(hwe->exl_port)) { 457 err = PTR_ERR(hwe->exl_port); 458 goto err_kernel_lrc; 459 } 460 } 461 462 if (xe_device_uc_enabled(xe)) 463 xe_hw_engine_enable_ring(hwe); 464 465 /* We reserve the highest BCS instance for USM */ 466 if (xe->info.has_usm && hwe->class == XE_ENGINE_CLASS_COPY) 467 gt->usm.reserved_bcs_instance = hwe->instance; 468 469 err = drmm_add_action_or_reset(&xe->drm, hw_engine_fini, hwe); 470 if (err) 471 return err; 472 473 return 0; 474 475 err_kernel_lrc: 476 xe_lrc_finish(&hwe->kernel_lrc); 477 err_hwsp: 478 xe_bo_unpin_map_no_vm(hwe->hwsp); 479 err_name: 480 hwe->name = NULL; 481 482 return err; 483 } 484 485 static void hw_engine_setup_logical_mapping(struct xe_gt *gt) 486 { 487 int class; 488 489 /* FIXME: Doing a simple logical mapping that works for most hardware */ 490 for (class = 0; class < XE_ENGINE_CLASS_MAX; ++class) { 491 struct xe_hw_engine *hwe; 492 enum xe_hw_engine_id id; 493 int logical_instance = 0; 494 495 for_each_hw_engine(hwe, gt, id) 496 if (hwe->class == class) 497 hwe->logical_instance = logical_instance++; 498 } 499 } 500 501 static void read_media_fuses(struct xe_gt *gt) 502 { 503 struct xe_device *xe = gt_to_xe(gt); 504 u32 media_fuse; 505 u16 vdbox_mask; 506 u16 vebox_mask; 507 int i, j; 508 509 xe_force_wake_assert_held(gt_to_fw(gt), XE_FW_GT); 510 511 media_fuse = xe_mmio_read32(gt, GT_VEBOX_VDBOX_DISABLE); 512 513 /* 514 * Pre-Xe_HP platforms had register bits representing absent engines, 515 * whereas Xe_HP and beyond have bits representing present engines. 516 * Invert the polarity on old platforms so that we can use common 517 * handling below. 518 */ 519 if (GRAPHICS_VERx100(xe) < 1250) 520 media_fuse = ~media_fuse; 521 522 vdbox_mask = REG_FIELD_GET(GT_VDBOX_DISABLE_MASK, media_fuse); 523 vebox_mask = REG_FIELD_GET(GT_VEBOX_DISABLE_MASK, media_fuse); 524 525 for (i = XE_HW_ENGINE_VCS0, j = 0; i <= XE_HW_ENGINE_VCS7; ++i, ++j) { 526 if (!(gt->info.engine_mask & BIT(i))) 527 continue; 528 529 if (!(BIT(j) & vdbox_mask)) { 530 gt->info.engine_mask &= ~BIT(i); 531 drm_info(&xe->drm, "vcs%u fused off\n", j); 532 } 533 } 534 535 for (i = XE_HW_ENGINE_VECS0, j = 0; i <= XE_HW_ENGINE_VECS3; ++i, ++j) { 536 if (!(gt->info.engine_mask & BIT(i))) 537 continue; 538 539 if (!(BIT(j) & vebox_mask)) { 540 gt->info.engine_mask &= ~BIT(i); 541 drm_info(&xe->drm, "vecs%u fused off\n", j); 542 } 543 } 544 } 545 546 static void read_copy_fuses(struct xe_gt *gt) 547 { 548 struct xe_device *xe = gt_to_xe(gt); 549 u32 bcs_mask; 550 551 if (GRAPHICS_VERx100(xe) < 1260 || GRAPHICS_VERx100(xe) >= 1270) 552 return; 553 554 xe_force_wake_assert_held(gt_to_fw(gt), XE_FW_GT); 555 556 bcs_mask = xe_mmio_read32(gt, MIRROR_FUSE3); 557 bcs_mask = REG_FIELD_GET(MEML3_EN_MASK, bcs_mask); 558 559 /* BCS0 is always present; only BCS1-BCS8 may be fused off */ 560 for (int i = XE_HW_ENGINE_BCS1, j = 0; i <= XE_HW_ENGINE_BCS8; ++i, ++j) { 561 if (!(gt->info.engine_mask & BIT(i))) 562 continue; 563 564 if (!(BIT(j / 2) & bcs_mask)) { 565 gt->info.engine_mask &= ~BIT(i); 566 drm_info(&xe->drm, "bcs%u fused off\n", j); 567 } 568 } 569 } 570 571 static void read_compute_fuses_from_dss(struct xe_gt *gt) 572 { 573 struct xe_device *xe = gt_to_xe(gt); 574 575 /* 576 * CCS fusing based on DSS masks only applies to platforms that can 577 * have more than one CCS. 578 */ 579 if (hweight64(gt->info.engine_mask & 580 GENMASK_ULL(XE_HW_ENGINE_CCS3, XE_HW_ENGINE_CCS0)) <= 1) 581 return; 582 583 /* 584 * CCS availability on Xe_HP is inferred from the presence of DSS in 585 * each quadrant. 586 */ 587 for (int i = XE_HW_ENGINE_CCS0, j = 0; i <= XE_HW_ENGINE_CCS3; ++i, ++j) { 588 if (!(gt->info.engine_mask & BIT(i))) 589 continue; 590 591 if (!xe_gt_topology_has_dss_in_quadrant(gt, j)) { 592 gt->info.engine_mask &= ~BIT(i); 593 drm_info(&xe->drm, "ccs%u fused off\n", j); 594 } 595 } 596 } 597 598 static void read_compute_fuses_from_reg(struct xe_gt *gt) 599 { 600 struct xe_device *xe = gt_to_xe(gt); 601 u32 ccs_mask; 602 603 ccs_mask = xe_mmio_read32(gt, XEHP_FUSE4); 604 ccs_mask = REG_FIELD_GET(CCS_EN_MASK, ccs_mask); 605 606 for (int i = XE_HW_ENGINE_CCS0, j = 0; i <= XE_HW_ENGINE_CCS3; ++i, ++j) { 607 if (!(gt->info.engine_mask & BIT(i))) 608 continue; 609 610 if ((ccs_mask & BIT(j)) == 0) { 611 gt->info.engine_mask &= ~BIT(i); 612 drm_info(&xe->drm, "ccs%u fused off\n", j); 613 } 614 } 615 } 616 617 static void read_compute_fuses(struct xe_gt *gt) 618 { 619 if (GRAPHICS_VER(gt_to_xe(gt)) >= 20) 620 read_compute_fuses_from_reg(gt); 621 else 622 read_compute_fuses_from_dss(gt); 623 } 624 625 static void check_gsc_availability(struct xe_gt *gt) 626 { 627 struct xe_device *xe = gt_to_xe(gt); 628 629 if (!(gt->info.engine_mask & BIT(XE_HW_ENGINE_GSCCS0))) 630 return; 631 632 /* 633 * The GSCCS is only used to communicate with the GSC FW, so if we don't 634 * have the FW there is nothing we need the engine for and can therefore 635 * skip its initialization. 636 */ 637 if (!xe_uc_fw_is_available(>->uc.gsc.fw)) { 638 gt->info.engine_mask &= ~BIT(XE_HW_ENGINE_GSCCS0); 639 drm_info(&xe->drm, "gsccs disabled due to lack of FW\n"); 640 } 641 } 642 643 int xe_hw_engines_init_early(struct xe_gt *gt) 644 { 645 int i; 646 647 read_media_fuses(gt); 648 read_copy_fuses(gt); 649 read_compute_fuses(gt); 650 check_gsc_availability(gt); 651 652 BUILD_BUG_ON(XE_HW_ENGINE_PREEMPT_TIMEOUT < XE_HW_ENGINE_PREEMPT_TIMEOUT_MIN); 653 BUILD_BUG_ON(XE_HW_ENGINE_PREEMPT_TIMEOUT > XE_HW_ENGINE_PREEMPT_TIMEOUT_MAX); 654 655 for (i = 0; i < ARRAY_SIZE(gt->hw_engines); i++) 656 hw_engine_init_early(gt, >->hw_engines[i], i); 657 658 return 0; 659 } 660 661 int xe_hw_engines_init(struct xe_gt *gt) 662 { 663 int err; 664 struct xe_hw_engine *hwe; 665 enum xe_hw_engine_id id; 666 667 for_each_hw_engine(hwe, gt, id) { 668 err = hw_engine_init(gt, hwe, id); 669 if (err) 670 return err; 671 } 672 673 hw_engine_setup_logical_mapping(gt); 674 675 return 0; 676 } 677 678 void xe_hw_engine_handle_irq(struct xe_hw_engine *hwe, u16 intr_vec) 679 { 680 wake_up_all(>_to_xe(hwe->gt)->ufence_wq); 681 682 if (hwe->irq_handler) 683 hwe->irq_handler(hwe, intr_vec); 684 685 if (intr_vec & GT_RENDER_USER_INTERRUPT) 686 xe_hw_fence_irq_run(hwe->fence_irq); 687 } 688 689 /** 690 * xe_hw_engine_snapshot_capture - Take a quick snapshot of the HW Engine. 691 * @hwe: Xe HW Engine. 692 * 693 * This can be printed out in a later stage like during dev_coredump 694 * analysis. 695 * 696 * Returns: a Xe HW Engine snapshot object that must be freed by the 697 * caller, using `xe_hw_engine_snapshot_free`. 698 */ 699 struct xe_hw_engine_snapshot * 700 xe_hw_engine_snapshot_capture(struct xe_hw_engine *hwe) 701 { 702 struct xe_hw_engine_snapshot *snapshot; 703 int len; 704 705 if (!xe_hw_engine_is_valid(hwe)) 706 return NULL; 707 708 snapshot = kzalloc(sizeof(*snapshot), GFP_ATOMIC); 709 710 if (!snapshot) 711 return NULL; 712 713 len = strlen(hwe->name) + 1; 714 snapshot->name = kzalloc(len, GFP_ATOMIC); 715 if (snapshot->name) 716 strscpy(snapshot->name, hwe->name, len); 717 718 snapshot->class = hwe->class; 719 snapshot->logical_instance = hwe->logical_instance; 720 snapshot->forcewake.domain = hwe->domain; 721 snapshot->forcewake.ref = xe_force_wake_ref(gt_to_fw(hwe->gt), 722 hwe->domain); 723 snapshot->mmio_base = hwe->mmio_base; 724 725 snapshot->reg.ring_hwstam = hw_engine_mmio_read32(hwe, RING_HWSTAM(0)); 726 snapshot->reg.ring_hws_pga = hw_engine_mmio_read32(hwe, 727 RING_HWS_PGA(0)); 728 snapshot->reg.ring_execlist_status_lo = 729 hw_engine_mmio_read32(hwe, RING_EXECLIST_STATUS_LO(0)); 730 snapshot->reg.ring_execlist_status_hi = 731 hw_engine_mmio_read32(hwe, RING_EXECLIST_STATUS_HI(0)); 732 snapshot->reg.ring_execlist_sq_contents_lo = 733 hw_engine_mmio_read32(hwe, 734 RING_EXECLIST_SQ_CONTENTS_LO(0)); 735 snapshot->reg.ring_execlist_sq_contents_hi = 736 hw_engine_mmio_read32(hwe, 737 RING_EXECLIST_SQ_CONTENTS_HI(0)); 738 snapshot->reg.ring_start = hw_engine_mmio_read32(hwe, RING_START(0)); 739 snapshot->reg.ring_head = 740 hw_engine_mmio_read32(hwe, RING_HEAD(0)) & HEAD_ADDR; 741 snapshot->reg.ring_tail = 742 hw_engine_mmio_read32(hwe, RING_TAIL(0)) & TAIL_ADDR; 743 snapshot->reg.ring_ctl = hw_engine_mmio_read32(hwe, RING_CTL(0)); 744 snapshot->reg.ring_mi_mode = 745 hw_engine_mmio_read32(hwe, RING_MI_MODE(0)); 746 snapshot->reg.ring_mode = hw_engine_mmio_read32(hwe, RING_MODE(0)); 747 snapshot->reg.ring_imr = hw_engine_mmio_read32(hwe, RING_IMR(0)); 748 snapshot->reg.ring_esr = hw_engine_mmio_read32(hwe, RING_ESR(0)); 749 snapshot->reg.ring_emr = hw_engine_mmio_read32(hwe, RING_EMR(0)); 750 snapshot->reg.ring_eir = hw_engine_mmio_read32(hwe, RING_EIR(0)); 751 snapshot->reg.ring_acthd_udw = 752 hw_engine_mmio_read32(hwe, RING_ACTHD_UDW(0)); 753 snapshot->reg.ring_acthd = hw_engine_mmio_read32(hwe, RING_ACTHD(0)); 754 snapshot->reg.ring_bbaddr_udw = 755 hw_engine_mmio_read32(hwe, RING_BBADDR_UDW(0)); 756 snapshot->reg.ring_bbaddr = hw_engine_mmio_read32(hwe, RING_BBADDR(0)); 757 snapshot->reg.ring_dma_fadd_udw = 758 hw_engine_mmio_read32(hwe, RING_DMA_FADD_UDW(0)); 759 snapshot->reg.ring_dma_fadd = 760 hw_engine_mmio_read32(hwe, RING_DMA_FADD(0)); 761 snapshot->reg.ipehr = hw_engine_mmio_read32(hwe, RING_IPEHR(0)); 762 763 if (snapshot->class == XE_ENGINE_CLASS_COMPUTE) 764 snapshot->reg.rcu_mode = xe_mmio_read32(hwe->gt, RCU_MODE); 765 766 return snapshot; 767 } 768 769 /** 770 * xe_hw_engine_snapshot_print - Print out a given Xe HW Engine snapshot. 771 * @snapshot: Xe HW Engine snapshot object. 772 * @p: drm_printer where it will be printed out. 773 * 774 * This function prints out a given Xe HW Engine snapshot object. 775 */ 776 void xe_hw_engine_snapshot_print(struct xe_hw_engine_snapshot *snapshot, 777 struct drm_printer *p) 778 { 779 if (!snapshot) 780 return; 781 782 drm_printf(p, "%s (physical), logical instance=%d\n", 783 snapshot->name ? snapshot->name : "", 784 snapshot->logical_instance); 785 drm_printf(p, "\tForcewake: domain 0x%x, ref %d\n", 786 snapshot->forcewake.domain, snapshot->forcewake.ref); 787 drm_printf(p, "\tHWSTAM: 0x%08x\n", snapshot->reg.ring_hwstam); 788 drm_printf(p, "\tRING_HWS_PGA: 0x%08x\n", snapshot->reg.ring_hws_pga); 789 drm_printf(p, "\tRING_EXECLIST_STATUS_LO: 0x%08x\n", 790 snapshot->reg.ring_execlist_status_lo); 791 drm_printf(p, "\tRING_EXECLIST_STATUS_HI: 0x%08x\n", 792 snapshot->reg.ring_execlist_status_hi); 793 drm_printf(p, "\tRING_EXECLIST_SQ_CONTENTS_LO: 0x%08x\n", 794 snapshot->reg.ring_execlist_sq_contents_lo); 795 drm_printf(p, "\tRING_EXECLIST_SQ_CONTENTS_HI: 0x%08x\n", 796 snapshot->reg.ring_execlist_sq_contents_hi); 797 drm_printf(p, "\tRING_START: 0x%08x\n", snapshot->reg.ring_start); 798 drm_printf(p, "\tRING_HEAD: 0x%08x\n", snapshot->reg.ring_head); 799 drm_printf(p, "\tRING_TAIL: 0x%08x\n", snapshot->reg.ring_tail); 800 drm_printf(p, "\tRING_CTL: 0x%08x\n", snapshot->reg.ring_ctl); 801 drm_printf(p, "\tRING_MI_MODE: 0x%08x\n", snapshot->reg.ring_mi_mode); 802 drm_printf(p, "\tRING_MODE: 0x%08x\n", 803 snapshot->reg.ring_mode); 804 drm_printf(p, "\tRING_IMR: 0x%08x\n", snapshot->reg.ring_imr); 805 drm_printf(p, "\tRING_ESR: 0x%08x\n", snapshot->reg.ring_esr); 806 drm_printf(p, "\tRING_EMR: 0x%08x\n", snapshot->reg.ring_emr); 807 drm_printf(p, "\tRING_EIR: 0x%08x\n", snapshot->reg.ring_eir); 808 drm_printf(p, "\tACTHD: 0x%08x_%08x\n", snapshot->reg.ring_acthd_udw, 809 snapshot->reg.ring_acthd); 810 drm_printf(p, "\tBBADDR: 0x%08x_%08x\n", snapshot->reg.ring_bbaddr_udw, 811 snapshot->reg.ring_bbaddr); 812 drm_printf(p, "\tDMA_FADDR: 0x%08x_%08x\n", 813 snapshot->reg.ring_dma_fadd_udw, 814 snapshot->reg.ring_dma_fadd); 815 drm_printf(p, "\tIPEHR: 0x%08x\n\n", snapshot->reg.ipehr); 816 if (snapshot->class == XE_ENGINE_CLASS_COMPUTE) 817 drm_printf(p, "\tRCU_MODE: 0x%08x\n", 818 snapshot->reg.rcu_mode); 819 } 820 821 /** 822 * xe_hw_engine_snapshot_free - Free all allocated objects for a given snapshot. 823 * @snapshot: Xe HW Engine snapshot object. 824 * 825 * This function free all the memory that needed to be allocated at capture 826 * time. 827 */ 828 void xe_hw_engine_snapshot_free(struct xe_hw_engine_snapshot *snapshot) 829 { 830 if (!snapshot) 831 return; 832 833 kfree(snapshot->name); 834 kfree(snapshot); 835 } 836 837 /** 838 * xe_hw_engine_print - Xe HW Engine Print. 839 * @hwe: Hardware Engine. 840 * @p: drm_printer. 841 * 842 * This function quickly capture a snapshot and immediately print it out. 843 */ 844 void xe_hw_engine_print(struct xe_hw_engine *hwe, struct drm_printer *p) 845 { 846 struct xe_hw_engine_snapshot *snapshot; 847 848 snapshot = xe_hw_engine_snapshot_capture(hwe); 849 xe_hw_engine_snapshot_print(snapshot, p); 850 xe_hw_engine_snapshot_free(snapshot); 851 } 852 853 u32 xe_hw_engine_mask_per_class(struct xe_gt *gt, 854 enum xe_engine_class engine_class) 855 { 856 u32 mask = 0; 857 enum xe_hw_engine_id id; 858 859 for (id = 0; id < XE_NUM_HW_ENGINES; ++id) { 860 if (engine_infos[id].class == engine_class && 861 gt->info.engine_mask & BIT(id)) 862 mask |= BIT(engine_infos[id].instance); 863 } 864 return mask; 865 } 866 867 bool xe_hw_engine_is_reserved(struct xe_hw_engine *hwe) 868 { 869 struct xe_gt *gt = hwe->gt; 870 struct xe_device *xe = gt_to_xe(gt); 871 872 if (hwe->class == XE_ENGINE_CLASS_OTHER) 873 return true; 874 875 /* Check for engines disabled by ccs_mode setting */ 876 if (xe_gt_ccs_mode_enabled(gt) && 877 hwe->class == XE_ENGINE_CLASS_COMPUTE && 878 hwe->logical_instance >= gt->ccs_mode) 879 return true; 880 881 return xe->info.has_usm && hwe->class == XE_ENGINE_CLASS_COPY && 882 hwe->instance == gt->usm.reserved_bcs_instance; 883 } 884