1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include "xe_hw_engine.h" 7 8 #include <linux/nospec.h> 9 10 #include <drm/drm_managed.h> 11 #include <drm/drm_print.h> 12 #include <uapi/drm/xe_drm.h> 13 #include <generated/xe_wa_oob.h> 14 15 #include "regs/xe_engine_regs.h" 16 #include "regs/xe_gt_regs.h" 17 #include "regs/xe_irq_regs.h" 18 #include "xe_assert.h" 19 #include "xe_bo.h" 20 #include "xe_configfs.h" 21 #include "xe_device.h" 22 #include "xe_execlist.h" 23 #include "xe_force_wake.h" 24 #include "xe_gsc.h" 25 #include "xe_gt.h" 26 #include "xe_gt_ccs_mode.h" 27 #include "xe_gt_clock.h" 28 #include "xe_gt_printk.h" 29 #include "xe_gt_mcr.h" 30 #include "xe_gt_topology.h" 31 #include "xe_guc_capture.h" 32 #include "xe_hw_engine_group.h" 33 #include "xe_hw_fence.h" 34 #include "xe_irq.h" 35 #include "xe_lrc.h" 36 #include "xe_mmio.h" 37 #include "xe_reg_sr.h" 38 #include "xe_reg_whitelist.h" 39 #include "xe_rtp.h" 40 #include "xe_sched_job.h" 41 #include "xe_sriov.h" 42 #include "xe_tuning.h" 43 #include "xe_uc_fw.h" 44 #include "xe_wa.h" 45 46 #define MAX_MMIO_BASES 3 47 struct engine_info { 48 const char *name; 49 unsigned int class : 8; 50 unsigned int instance : 8; 51 unsigned int irq_offset : 8; 52 enum xe_force_wake_domains domain; 53 u32 mmio_base; 54 }; 55 56 static const struct engine_info engine_infos[] = { 57 [XE_HW_ENGINE_RCS0] = { 58 .name = "rcs0", 59 .class = XE_ENGINE_CLASS_RENDER, 60 .instance = 0, 61 .irq_offset = ilog2(INTR_RCS0), 62 .domain = XE_FW_RENDER, 63 .mmio_base = RENDER_RING_BASE, 64 }, 65 [XE_HW_ENGINE_BCS0] = { 66 .name = "bcs0", 67 .class = XE_ENGINE_CLASS_COPY, 68 .instance = 0, 69 .irq_offset = ilog2(INTR_BCS(0)), 70 .domain = XE_FW_GT, 71 .mmio_base = BLT_RING_BASE, 72 }, 73 [XE_HW_ENGINE_BCS1] = { 74 .name = "bcs1", 75 .class = XE_ENGINE_CLASS_COPY, 76 .instance = 1, 77 .irq_offset = ilog2(INTR_BCS(1)), 78 .domain = XE_FW_GT, 79 .mmio_base = XEHPC_BCS1_RING_BASE, 80 }, 81 [XE_HW_ENGINE_BCS2] = { 82 .name = "bcs2", 83 .class = XE_ENGINE_CLASS_COPY, 84 .instance = 2, 85 .irq_offset = ilog2(INTR_BCS(2)), 86 .domain = XE_FW_GT, 87 .mmio_base = XEHPC_BCS2_RING_BASE, 88 }, 89 [XE_HW_ENGINE_BCS3] = { 90 .name = "bcs3", 91 .class = XE_ENGINE_CLASS_COPY, 92 .instance = 3, 93 .irq_offset = ilog2(INTR_BCS(3)), 94 .domain = XE_FW_GT, 95 .mmio_base = XEHPC_BCS3_RING_BASE, 96 }, 97 [XE_HW_ENGINE_BCS4] = { 98 .name = "bcs4", 99 .class = XE_ENGINE_CLASS_COPY, 100 .instance = 4, 101 .irq_offset = ilog2(INTR_BCS(4)), 102 .domain = XE_FW_GT, 103 .mmio_base = XEHPC_BCS4_RING_BASE, 104 }, 105 [XE_HW_ENGINE_BCS5] = { 106 .name = "bcs5", 107 .class = XE_ENGINE_CLASS_COPY, 108 .instance = 5, 109 .irq_offset = ilog2(INTR_BCS(5)), 110 .domain = XE_FW_GT, 111 .mmio_base = XEHPC_BCS5_RING_BASE, 112 }, 113 [XE_HW_ENGINE_BCS6] = { 114 .name = "bcs6", 115 .class = XE_ENGINE_CLASS_COPY, 116 .instance = 6, 117 .irq_offset = ilog2(INTR_BCS(6)), 118 .domain = XE_FW_GT, 119 .mmio_base = XEHPC_BCS6_RING_BASE, 120 }, 121 [XE_HW_ENGINE_BCS7] = { 122 .name = "bcs7", 123 .class = XE_ENGINE_CLASS_COPY, 124 .irq_offset = ilog2(INTR_BCS(7)), 125 .instance = 7, 126 .domain = XE_FW_GT, 127 .mmio_base = XEHPC_BCS7_RING_BASE, 128 }, 129 [XE_HW_ENGINE_BCS8] = { 130 .name = "bcs8", 131 .class = XE_ENGINE_CLASS_COPY, 132 .instance = 8, 133 .irq_offset = ilog2(INTR_BCS8), 134 .domain = XE_FW_GT, 135 .mmio_base = XEHPC_BCS8_RING_BASE, 136 }, 137 138 [XE_HW_ENGINE_VCS0] = { 139 .name = "vcs0", 140 .class = XE_ENGINE_CLASS_VIDEO_DECODE, 141 .instance = 0, 142 .irq_offset = 32 + ilog2(INTR_VCS(0)), 143 .domain = XE_FW_MEDIA_VDBOX0, 144 .mmio_base = BSD_RING_BASE, 145 }, 146 [XE_HW_ENGINE_VCS1] = { 147 .name = "vcs1", 148 .class = XE_ENGINE_CLASS_VIDEO_DECODE, 149 .instance = 1, 150 .irq_offset = 32 + ilog2(INTR_VCS(1)), 151 .domain = XE_FW_MEDIA_VDBOX1, 152 .mmio_base = BSD2_RING_BASE, 153 }, 154 [XE_HW_ENGINE_VCS2] = { 155 .name = "vcs2", 156 .class = XE_ENGINE_CLASS_VIDEO_DECODE, 157 .instance = 2, 158 .irq_offset = 32 + ilog2(INTR_VCS(2)), 159 .domain = XE_FW_MEDIA_VDBOX2, 160 .mmio_base = BSD3_RING_BASE, 161 }, 162 [XE_HW_ENGINE_VCS3] = { 163 .name = "vcs3", 164 .class = XE_ENGINE_CLASS_VIDEO_DECODE, 165 .instance = 3, 166 .irq_offset = 32 + ilog2(INTR_VCS(3)), 167 .domain = XE_FW_MEDIA_VDBOX3, 168 .mmio_base = BSD4_RING_BASE, 169 }, 170 [XE_HW_ENGINE_VCS4] = { 171 .name = "vcs4", 172 .class = XE_ENGINE_CLASS_VIDEO_DECODE, 173 .instance = 4, 174 .irq_offset = 32 + ilog2(INTR_VCS(4)), 175 .domain = XE_FW_MEDIA_VDBOX4, 176 .mmio_base = XEHP_BSD5_RING_BASE, 177 }, 178 [XE_HW_ENGINE_VCS5] = { 179 .name = "vcs5", 180 .class = XE_ENGINE_CLASS_VIDEO_DECODE, 181 .instance = 5, 182 .irq_offset = 32 + ilog2(INTR_VCS(5)), 183 .domain = XE_FW_MEDIA_VDBOX5, 184 .mmio_base = XEHP_BSD6_RING_BASE, 185 }, 186 [XE_HW_ENGINE_VCS6] = { 187 .name = "vcs6", 188 .class = XE_ENGINE_CLASS_VIDEO_DECODE, 189 .instance = 6, 190 .irq_offset = 32 + ilog2(INTR_VCS(6)), 191 .domain = XE_FW_MEDIA_VDBOX6, 192 .mmio_base = XEHP_BSD7_RING_BASE, 193 }, 194 [XE_HW_ENGINE_VCS7] = { 195 .name = "vcs7", 196 .class = XE_ENGINE_CLASS_VIDEO_DECODE, 197 .instance = 7, 198 .irq_offset = 32 + ilog2(INTR_VCS(7)), 199 .domain = XE_FW_MEDIA_VDBOX7, 200 .mmio_base = XEHP_BSD8_RING_BASE, 201 }, 202 [XE_HW_ENGINE_VECS0] = { 203 .name = "vecs0", 204 .class = XE_ENGINE_CLASS_VIDEO_ENHANCE, 205 .instance = 0, 206 .irq_offset = 32 + ilog2(INTR_VECS(0)), 207 .domain = XE_FW_MEDIA_VEBOX0, 208 .mmio_base = VEBOX_RING_BASE, 209 }, 210 [XE_HW_ENGINE_VECS1] = { 211 .name = "vecs1", 212 .class = XE_ENGINE_CLASS_VIDEO_ENHANCE, 213 .instance = 1, 214 .irq_offset = 32 + ilog2(INTR_VECS(1)), 215 .domain = XE_FW_MEDIA_VEBOX1, 216 .mmio_base = VEBOX2_RING_BASE, 217 }, 218 [XE_HW_ENGINE_VECS2] = { 219 .name = "vecs2", 220 .class = XE_ENGINE_CLASS_VIDEO_ENHANCE, 221 .instance = 2, 222 .irq_offset = 32 + ilog2(INTR_VECS(2)), 223 .domain = XE_FW_MEDIA_VEBOX2, 224 .mmio_base = XEHP_VEBOX3_RING_BASE, 225 }, 226 [XE_HW_ENGINE_VECS3] = { 227 .name = "vecs3", 228 .class = XE_ENGINE_CLASS_VIDEO_ENHANCE, 229 .instance = 3, 230 .irq_offset = 32 + ilog2(INTR_VECS(3)), 231 .domain = XE_FW_MEDIA_VEBOX3, 232 .mmio_base = XEHP_VEBOX4_RING_BASE, 233 }, 234 [XE_HW_ENGINE_CCS0] = { 235 .name = "ccs0", 236 .class = XE_ENGINE_CLASS_COMPUTE, 237 .instance = 0, 238 .irq_offset = ilog2(INTR_CCS(0)), 239 .domain = XE_FW_RENDER, 240 .mmio_base = COMPUTE0_RING_BASE, 241 }, 242 [XE_HW_ENGINE_CCS1] = { 243 .name = "ccs1", 244 .class = XE_ENGINE_CLASS_COMPUTE, 245 .instance = 1, 246 .irq_offset = ilog2(INTR_CCS(1)), 247 .domain = XE_FW_RENDER, 248 .mmio_base = COMPUTE1_RING_BASE, 249 }, 250 [XE_HW_ENGINE_CCS2] = { 251 .name = "ccs2", 252 .class = XE_ENGINE_CLASS_COMPUTE, 253 .instance = 2, 254 .irq_offset = ilog2(INTR_CCS(2)), 255 .domain = XE_FW_RENDER, 256 .mmio_base = COMPUTE2_RING_BASE, 257 }, 258 [XE_HW_ENGINE_CCS3] = { 259 .name = "ccs3", 260 .class = XE_ENGINE_CLASS_COMPUTE, 261 .instance = 3, 262 .irq_offset = ilog2(INTR_CCS(3)), 263 .domain = XE_FW_RENDER, 264 .mmio_base = COMPUTE3_RING_BASE, 265 }, 266 [XE_HW_ENGINE_GSCCS0] = { 267 .name = "gsccs0", 268 .class = XE_ENGINE_CLASS_OTHER, 269 .instance = OTHER_GSC_INSTANCE, 270 .domain = XE_FW_GSC, 271 .mmio_base = GSCCS_RING_BASE, 272 }, 273 }; 274 275 static void hw_engine_fini(void *arg) 276 { 277 struct xe_hw_engine *hwe = arg; 278 279 if (hwe->exl_port) 280 xe_execlist_port_destroy(hwe->exl_port); 281 282 hwe->gt = NULL; 283 } 284 285 /** 286 * xe_hw_engine_mmio_read32() - Read engine register 287 * @hwe: engine 288 * @reg: register to read from 289 * 290 * This function will read from an engine specific register. 291 * Forcewake must be held by the caller. 292 * 293 * Return: value of the 32-bit register. 294 */ 295 u32 xe_hw_engine_mmio_read32(struct xe_hw_engine *hwe, struct xe_reg reg) 296 { 297 xe_gt_assert(hwe->gt, !(reg.addr & hwe->mmio_base)); 298 xe_force_wake_assert_held(gt_to_fw(hwe->gt), hwe->domain); 299 300 reg.addr += hwe->mmio_base; 301 302 return xe_mmio_read32(&hwe->gt->mmio, reg); 303 } 304 305 void xe_hw_engine_enable_ring(struct xe_hw_engine *hwe) 306 { 307 xe_mmio_write32(&hwe->gt->mmio, RING_HWS_PGA(hwe->mmio_base), 308 xe_bo_ggtt_addr(hwe->hwsp)); 309 } 310 311 static bool xe_hw_engine_match_fixed_cslice_mode(const struct xe_device *xe, 312 const struct xe_gt *gt, 313 const struct xe_hw_engine *hwe) 314 { 315 /* 316 * Xe3p no longer supports load balance mode, so "fixed cslice" mode 317 * is automatic and no RCU_MODE programming is required. 318 */ 319 if (GRAPHICS_VER(gt_to_xe(gt)) >= 35) 320 return false; 321 322 return xe_gt_ccs_mode_enabled(gt) && 323 xe_rtp_match_first_render_or_compute(xe, gt, hwe); 324 } 325 326 static bool xe_rtp_cfeg_wmtp_disabled(const struct xe_device *xe, 327 const struct xe_gt *gt, 328 const struct xe_hw_engine *hwe) 329 { 330 if (GRAPHICS_VER(xe) < 20) 331 return false; 332 333 if (hwe->class != XE_ENGINE_CLASS_COMPUTE && 334 hwe->class != XE_ENGINE_CLASS_RENDER) 335 return false; 336 337 return xe_mmio_read32(&hwe->gt->mmio, XEHP_FUSE4) & CFEG_WMTP_DISABLE; 338 } 339 340 static u32 blit_cctl_val(struct xe_gt *gt, struct xe_hw_engine *hwe) 341 { 342 return REG_FIELD_PREP(BLIT_CCTL_DST_MOCS_MASK, gt->mocs.uc_index) | 343 REG_FIELD_PREP(BLIT_CCTL_SRC_MOCS_MASK, gt->mocs.uc_index); 344 } 345 346 static const struct xe_rtp_table_sr lrc_setup = XE_RTP_TABLE_SR( 347 /* 348 * Some blitter commands do not have a field for MOCS, those 349 * commands will use MOCS index pointed by BLIT_CCTL. 350 * BLIT_CCTL registers are needed to be programmed to un-cached. 351 */ 352 { XE_RTP_NAME("BLIT_CCTL_default_MOCS"), 353 XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1274), 354 ENGINE_CLASS(COPY)), 355 XE_RTP_ACTIONS(FIELD_SET_FUNC(BLIT_CCTL(0), 356 BLIT_CCTL_DST_MOCS_MASK | 357 BLIT_CCTL_SRC_MOCS_MASK, 358 blit_cctl_val, 359 XE_RTP_ACTION_FLAG(ENGINE_BASE))) 360 }, 361 /* Disable WMTP if HW doesn't support it */ 362 { XE_RTP_NAME("DISABLE_WMTP_ON_UNSUPPORTED_HW"), 363 XE_RTP_RULES(FUNC(xe_rtp_cfeg_wmtp_disabled)), 364 XE_RTP_ACTIONS(FIELD_SET(CS_CHICKEN1(0), 365 PREEMPT_GPGPU_LEVEL_MASK, 366 PREEMPT_GPGPU_THREAD_GROUP_LEVEL)), 367 XE_RTP_ENTRY_FLAG(FOREACH_ENGINE) 368 }, 369 ); 370 371 static void 372 hw_engine_setup_default_lrc_state(struct xe_hw_engine *hwe) 373 { 374 struct xe_rtp_process_ctx ctx = XE_RTP_PROCESS_CTX_INITIALIZER(hwe); 375 376 xe_rtp_process_to_sr(&ctx, &lrc_setup, &hwe->reg_lrc, true); 377 } 378 379 void xe_hw_engine_setup_reg_lrc(struct xe_hw_engine *hwe) 380 { 381 struct xe_gt *gt = hwe->gt; 382 struct xe_device *xe = gt_to_xe(gt); 383 384 xe_reg_sr_init(&hwe->reg_lrc, hwe->name, xe); 385 xe_wa_process_lrc(hwe); 386 hw_engine_setup_default_lrc_state(hwe); 387 xe_tuning_process_lrc(hwe); 388 } 389 390 /* 391 * RING_CMD_CCTL specifies the default MOCS entry that will be 392 * used by the command streamer when executing commands that 393 * don't have a way to explicitly specify a MOCS setting. 394 * The default should usually reference whichever MOCS entry 395 * corresponds to uncached behavior, although use of a WB cached 396 * entry is recommended by the spec in certain circumstances on 397 * specific platforms. 398 * Bspec: 72161 399 */ 400 static u32 ring_cmd_cctl_val(struct xe_gt *gt, struct xe_hw_engine *hwe) 401 { 402 struct xe_device *xe = gt_to_xe(gt); 403 u8 mocs_read_idx = gt->mocs.uc_index; 404 405 if (hwe->class == XE_ENGINE_CLASS_COMPUTE && IS_DGFX(xe) && 406 (GRAPHICS_VER(xe) >= 20 || xe->info.platform == XE_PVC)) 407 mocs_read_idx = gt->mocs.wb_index; 408 409 return REG_FIELD_PREP(CMD_CCTL_WRITE_OVERRIDE_MASK, gt->mocs.uc_index) | 410 REG_FIELD_PREP(CMD_CCTL_READ_OVERRIDE_MASK, mocs_read_idx); 411 } 412 413 static const struct xe_rtp_table_sr engine_sr = XE_RTP_TABLE_SR( 414 { XE_RTP_NAME("RING_CMD_CCTL_default_MOCS"), 415 XE_RTP_RULES(FUNC(xe_rtp_match_always)), 416 XE_RTP_ACTIONS(FIELD_SET_FUNC(RING_CMD_CCTL(0), 417 CMD_CCTL_WRITE_OVERRIDE_MASK | 418 CMD_CCTL_READ_OVERRIDE_MASK, 419 ring_cmd_cctl_val, 420 XE_RTP_ACTION_FLAG(ENGINE_BASE))) 421 }, 422 { XE_RTP_NAME("Disable HW status page updates for interrupts"), 423 XE_RTP_RULES(FUNC(xe_rtp_match_always)), 424 XE_RTP_ACTIONS(SET(RING_HWSTAM(0), ~0x0, 425 XE_RTP_ACTION_FLAG(ENGINE_BASE))) 426 }, 427 { XE_RTP_NAME("Disable engine 'legacy' mode"), 428 XE_RTP_RULES(FUNC(xe_rtp_match_always)), 429 XE_RTP_ACTIONS(SET(GFX_MODE(0), GFX_DISABLE_LEGACY_MODE, 430 XE_RTP_ACTION_FLAG(ENGINE_BASE))) 431 }, 432 /* 433 * To allow the GSC engine to go idle on MTL we need to enable 434 * idle messaging and set the hysteresis value (we use 0xA=5us 435 * as recommended in spec). On platforms after MTL this is 436 * enabled by default. 437 */ 438 { XE_RTP_NAME("MTL GSCCS IDLE MSG enable"), 439 XE_RTP_RULES(MEDIA_VERSION(1300), ENGINE_CLASS(OTHER)), 440 XE_RTP_ACTIONS(CLR(RING_PSMI_CTL(0), 441 IDLE_MSG_DISABLE, 442 XE_RTP_ACTION_FLAG(ENGINE_BASE)), 443 FIELD_SET(RING_PWRCTX_MAXCNT(0), 444 IDLE_WAIT_TIME, 445 0xA, 446 XE_RTP_ACTION_FLAG(ENGINE_BASE))) 447 }, 448 /* Enable Priority Mem Read */ 449 { XE_RTP_NAME("Priority_Mem_Read"), 450 XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, XE_RTP_END_VERSION_UNDEFINED)), 451 XE_RTP_ACTIONS(SET(CSFE_CHICKEN1(0), CS_PRIORITY_MEM_READ, 452 XE_RTP_ACTION_FLAG(ENGINE_BASE))) 453 }, 454 { XE_RTP_NAME("Enable CCS Engine(s)"), 455 XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1255, XE_RTP_END_VERSION_UNDEFINED), 456 FUNC(xe_rtp_match_first_render_or_compute)), 457 XE_RTP_ACTIONS(SET(RCU_MODE, RCU_MODE_CCS_ENABLE)) 458 }, 459 /* Use Fixed slice CCS mode */ 460 { XE_RTP_NAME("RCU_MODE_FIXED_SLICE_CCS_MODE"), 461 XE_RTP_RULES(FUNC(xe_hw_engine_match_fixed_cslice_mode)), 462 XE_RTP_ACTIONS(FIELD_SET(RCU_MODE, RCU_MODE_FIXED_SLICE_CCS_MODE, 463 RCU_MODE_FIXED_SLICE_CCS_MODE)) 464 }, 465 { XE_RTP_NAME("Enable MSI-X interrupt support"), 466 XE_RTP_RULES(FUNC(xe_rtp_match_has_msix)), 467 XE_RTP_ACTIONS(SET(GFX_MODE(0), GFX_MSIX_INTERRUPT_ENABLE, 468 XE_RTP_ACTION_FLAG(ENGINE_BASE))) 469 }, 470 ); 471 472 static void 473 hw_engine_setup_default_state(struct xe_hw_engine *hwe) 474 { 475 struct xe_rtp_process_ctx ctx = XE_RTP_PROCESS_CTX_INITIALIZER(hwe); 476 477 xe_rtp_process_to_sr(&ctx, &engine_sr, &hwe->reg_sr, false); 478 } 479 480 static const struct engine_info *find_engine_info(enum xe_engine_class class, int instance) 481 { 482 const struct engine_info *info; 483 enum xe_hw_engine_id id; 484 485 for (id = 0; id < XE_NUM_HW_ENGINES; ++id) { 486 info = &engine_infos[id]; 487 if (info->class == class && info->instance == instance) 488 return info; 489 } 490 491 return NULL; 492 } 493 494 static u16 get_msix_irq_offset(struct xe_gt *gt, enum xe_engine_class class) 495 { 496 /* For MSI-X, hw engines report to offset of engine instance zero */ 497 const struct engine_info *info = find_engine_info(class, 0); 498 499 xe_gt_assert(gt, info); 500 501 return info ? info->irq_offset : 0; 502 } 503 504 static void hw_engine_init_early(struct xe_gt *gt, struct xe_hw_engine *hwe, 505 enum xe_hw_engine_id id) 506 { 507 const struct engine_info *info; 508 509 if (WARN_ON(id >= ARRAY_SIZE(engine_infos) || !engine_infos[id].name)) 510 return; 511 512 if (!(gt->info.engine_mask & BIT(id))) 513 return; 514 515 info = &engine_infos[id]; 516 517 xe_gt_assert(gt, !hwe->gt); 518 519 hwe->gt = gt; 520 hwe->class = info->class; 521 hwe->instance = info->instance; 522 hwe->mmio_base = info->mmio_base; 523 if (xe_device_has_msix(gt_to_xe(gt))) { 524 hwe->irq_offset = get_msix_irq_offset(gt, info->class); 525 hwe->irq_page = info->instance; 526 527 } else { 528 hwe->irq_offset = info->irq_offset; 529 hwe->irq_page = 0; 530 } 531 hwe->domain = info->domain; 532 hwe->name = info->name; 533 hwe->fence_irq = >->fence_irq[info->class]; 534 hwe->engine_id = id; 535 536 hwe->eclass = >->eclass[hwe->class]; 537 if (!hwe->eclass->sched_props.job_timeout_ms) { 538 hwe->eclass->sched_props.job_timeout_ms = 5 * 1000; 539 hwe->eclass->sched_props.job_timeout_min = XE_HW_ENGINE_JOB_TIMEOUT_MIN; 540 hwe->eclass->sched_props.job_timeout_max = XE_HW_ENGINE_JOB_TIMEOUT_MAX; 541 hwe->eclass->sched_props.timeslice_us = 1 * 1000; 542 hwe->eclass->sched_props.timeslice_min = XE_HW_ENGINE_TIMESLICE_MIN; 543 hwe->eclass->sched_props.timeslice_max = XE_HW_ENGINE_TIMESLICE_MAX; 544 hwe->eclass->sched_props.preempt_timeout_us = XE_HW_ENGINE_PREEMPT_TIMEOUT; 545 hwe->eclass->sched_props.preempt_timeout_min = XE_HW_ENGINE_PREEMPT_TIMEOUT_MIN; 546 hwe->eclass->sched_props.preempt_timeout_max = XE_HW_ENGINE_PREEMPT_TIMEOUT_MAX; 547 548 /* 549 * The GSC engine can accept submissions while the GSC shim is 550 * being reset, during which time the submission is stalled. In 551 * the worst case, the shim reset can take up to the maximum GSC 552 * command execution time (250ms), so the request start can be 553 * delayed by that much; the request itself can take that long 554 * without being preemptible, which means worst case it can 555 * theoretically take up to 500ms for a preemption to go through 556 * on the GSC engine. Adding to that an extra 100ms as a safety 557 * margin, we get a minimum recommended timeout of 600ms. 558 * The preempt_timeout value can't be tuned for OTHER_CLASS 559 * because the class is reserved for kernel usage, so we just 560 * need to make sure that the starting value is above that 561 * threshold; since our default value (640ms) is greater than 562 * 600ms, the only way we can go below is via a kconfig setting. 563 * If that happens, log it in dmesg and update the value. 564 */ 565 if (hwe->class == XE_ENGINE_CLASS_OTHER) { 566 const u32 min_preempt_timeout = 600 * 1000; 567 if (hwe->eclass->sched_props.preempt_timeout_us < min_preempt_timeout) { 568 hwe->eclass->sched_props.preempt_timeout_us = min_preempt_timeout; 569 xe_gt_notice(gt, "Increasing preempt_timeout for GSC to 600ms\n"); 570 } 571 } 572 573 /* Record default props */ 574 hwe->eclass->defaults = hwe->eclass->sched_props; 575 } 576 577 xe_reg_sr_init(&hwe->reg_sr, hwe->name, gt_to_xe(gt)); 578 xe_tuning_process_engine(hwe); 579 xe_wa_process_engine(hwe); 580 hw_engine_setup_default_state(hwe); 581 582 xe_reg_sr_init(&hwe->reg_whitelist, hwe->name, gt_to_xe(gt)); 583 xe_reg_sr_init(&hwe->oa_whitelist, hwe->name, gt_to_xe(gt)); 584 xe_reg_sr_init(&hwe->oa_sr, hwe->name, gt_to_xe(gt)); 585 xe_reg_whitelist_process_engine(hwe); 586 } 587 588 static void adjust_idledly(struct xe_hw_engine *hwe) 589 { 590 struct xe_gt *gt = hwe->gt; 591 u32 idledly, maxcnt; 592 u32 idledly_units_ps = 8 * gt->info.timestamp_base; 593 u32 maxcnt_units_ns = 640; 594 bool inhibit_switch = 0; 595 596 if (!IS_SRIOV_VF(gt_to_xe(hwe->gt)) && XE_GT_WA(gt, 16023105232)) { 597 idledly = xe_mmio_read32(>->mmio, RING_IDLEDLY(hwe->mmio_base)); 598 maxcnt = xe_mmio_read32(>->mmio, RING_PWRCTX_MAXCNT(hwe->mmio_base)); 599 600 inhibit_switch = idledly & INHIBIT_SWITCH_UNTIL_PREEMPTED; 601 idledly = REG_FIELD_GET(IDLE_DELAY, idledly); 602 idledly = DIV_ROUND_CLOSEST(idledly * idledly_units_ps, 1000); 603 maxcnt = REG_FIELD_GET(IDLE_WAIT_TIME, maxcnt); 604 maxcnt *= maxcnt_units_ns; 605 606 if (xe_gt_WARN_ON(gt, idledly >= maxcnt || inhibit_switch)) { 607 idledly = DIV_ROUND_CLOSEST(((maxcnt - 1) * 1000), 608 idledly_units_ps); 609 xe_mmio_write32(>->mmio, RING_IDLEDLY(hwe->mmio_base), idledly); 610 } 611 } 612 } 613 614 static int hw_engine_init(struct xe_gt *gt, struct xe_hw_engine *hwe, 615 enum xe_hw_engine_id id) 616 { 617 struct xe_device *xe = gt_to_xe(gt); 618 struct xe_tile *tile = gt_to_tile(gt); 619 int err; 620 621 xe_gt_assert(gt, id < ARRAY_SIZE(engine_infos) && engine_infos[id].name); 622 xe_gt_assert(gt, gt->info.engine_mask & BIT(id)); 623 624 xe_reg_sr_apply_mmio(&hwe->reg_sr, gt); 625 626 hwe->hwsp = xe_managed_bo_create_pin_map(xe, tile, SZ_4K, 627 XE_BO_FLAG_VRAM_IF_DGFX(tile) | 628 XE_BO_FLAG_GGTT | 629 XE_BO_FLAG_GGTT_INVALIDATE); 630 if (IS_ERR(hwe->hwsp)) { 631 err = PTR_ERR(hwe->hwsp); 632 goto err_name; 633 } 634 635 if (!xe_device_uc_enabled(xe)) { 636 hwe->exl_port = xe_execlist_port_create(xe, hwe); 637 if (IS_ERR(hwe->exl_port)) { 638 err = PTR_ERR(hwe->exl_port); 639 goto err_name; 640 } 641 } else { 642 /* GSCCS has a special interrupt for reset */ 643 if (hwe->class == XE_ENGINE_CLASS_OTHER) 644 hwe->irq_handler = xe_gsc_hwe_irq_handler; 645 646 if (!IS_SRIOV_VF(xe)) 647 xe_hw_engine_enable_ring(hwe); 648 } 649 650 /* We reserve the highest BCS instance for USM */ 651 if (xe->info.has_usm && hwe->class == XE_ENGINE_CLASS_COPY) 652 gt->usm.reserved_bcs_instance = hwe->instance; 653 654 /* Ensure IDLEDLY is lower than MAXCNT */ 655 adjust_idledly(hwe); 656 657 return devm_add_action_or_reset(xe->drm.dev, hw_engine_fini, hwe); 658 659 err_name: 660 hwe->name = NULL; 661 662 return err; 663 } 664 665 static void hw_engine_setup_logical_mapping(struct xe_gt *gt) 666 { 667 int class; 668 669 /* FIXME: Doing a simple logical mapping that works for most hardware */ 670 for (class = 0; class < XE_ENGINE_CLASS_MAX; ++class) { 671 struct xe_hw_engine *hwe; 672 enum xe_hw_engine_id id; 673 int logical_instance = 0; 674 675 for_each_hw_engine(hwe, gt, id) 676 if (hwe->class == class) 677 hwe->logical_instance = logical_instance++; 678 } 679 } 680 681 static void read_media_fuses(struct xe_gt *gt) 682 { 683 struct xe_device *xe = gt_to_xe(gt); 684 u32 media_fuse; 685 u16 vdbox_mask; 686 u16 vebox_mask; 687 int i, j; 688 689 xe_force_wake_assert_held(gt_to_fw(gt), XE_FW_GT); 690 691 media_fuse = xe_mmio_read32(>->mmio, GT_VEBOX_VDBOX_DISABLE); 692 693 /* 694 * Pre-Xe_HP platforms had register bits representing absent engines, 695 * whereas Xe_HP and beyond have bits representing present engines. 696 * Invert the polarity on old platforms so that we can use common 697 * handling below. 698 */ 699 if (GRAPHICS_VERx100(xe) < 1250) 700 media_fuse = ~media_fuse; 701 702 vdbox_mask = REG_FIELD_GET(GT_VDBOX_DISABLE_MASK, media_fuse); 703 vebox_mask = REG_FIELD_GET(GT_VEBOX_DISABLE_MASK, media_fuse); 704 705 for (i = XE_HW_ENGINE_VCS0, j = 0; i <= XE_HW_ENGINE_VCS7; ++i, ++j) { 706 if (!(gt->info.engine_mask & BIT(i))) 707 continue; 708 709 if (!(BIT(j) & vdbox_mask)) { 710 gt->info.engine_mask &= ~BIT(i); 711 xe_gt_info(gt, "vcs%u fused off\n", j); 712 } 713 } 714 715 for (i = XE_HW_ENGINE_VECS0, j = 0; i <= XE_HW_ENGINE_VECS3; ++i, ++j) { 716 if (!(gt->info.engine_mask & BIT(i))) 717 continue; 718 719 if (!(BIT(j) & vebox_mask)) { 720 gt->info.engine_mask &= ~BIT(i); 721 xe_gt_info(gt, "vecs%u fused off\n", j); 722 } 723 } 724 } 725 726 static u32 infer_svccopy_from_meml3(struct xe_gt *gt) 727 { 728 u32 meml3 = REG_FIELD_GET(MEML3_EN_MASK, 729 xe_mmio_read32(>->mmio, MIRROR_FUSE3)); 730 u32 svccopy_mask = 0; 731 732 /* 733 * Each of the four meml3 bits determines the fusing of two service 734 * copy engines. 735 */ 736 for (int i = 0; i < 4; i++) 737 svccopy_mask |= (meml3 & BIT(i)) ? 0b11 << 2 * i : 0; 738 739 return svccopy_mask; 740 } 741 742 static u32 read_svccopy_fuses(struct xe_gt *gt) 743 { 744 return REG_FIELD_GET(FUSE_SERVICE_COPY_ENABLE_MASK, 745 xe_mmio_read32(>->mmio, SERVICE_COPY_ENABLE)); 746 } 747 748 static void read_copy_fuses(struct xe_gt *gt) 749 { 750 struct xe_device *xe = gt_to_xe(gt); 751 u32 bcs_mask; 752 753 xe_force_wake_assert_held(gt_to_fw(gt), XE_FW_GT); 754 755 if (GRAPHICS_VER(xe) >= 35) 756 bcs_mask = read_svccopy_fuses(gt); 757 else if (GRAPHICS_VERx100(xe) == 1260) 758 bcs_mask = infer_svccopy_from_meml3(gt); 759 else 760 return; 761 762 /* Only BCS1-BCS8 may be fused off */ 763 bcs_mask <<= XE_HW_ENGINE_BCS1; 764 for (int i = XE_HW_ENGINE_BCS1; i <= XE_HW_ENGINE_BCS8; ++i) { 765 if (!(gt->info.engine_mask & BIT(i))) 766 continue; 767 768 if (!(bcs_mask & BIT(i))) { 769 gt->info.engine_mask &= ~BIT(i); 770 xe_gt_info(gt, "bcs%u fused off\n", 771 i - XE_HW_ENGINE_BCS0); 772 } 773 } 774 } 775 776 static void read_compute_fuses_from_dss(struct xe_gt *gt) 777 { 778 /* 779 * CCS fusing based on DSS masks only applies to platforms that can 780 * have more than one CCS. 781 */ 782 if (hweight64(gt->info.engine_mask & 783 GENMASK_ULL(XE_HW_ENGINE_CCS3, XE_HW_ENGINE_CCS0)) <= 1) 784 return; 785 786 /* 787 * CCS availability on Xe_HP is inferred from the presence of DSS in 788 * each quadrant. 789 */ 790 for (int i = XE_HW_ENGINE_CCS0, j = 0; i <= XE_HW_ENGINE_CCS3; ++i, ++j) { 791 if (!(gt->info.engine_mask & BIT(i))) 792 continue; 793 794 if (!xe_gt_topology_has_dss_in_quadrant(gt, j)) { 795 gt->info.engine_mask &= ~BIT(i); 796 xe_gt_info(gt, "ccs%u fused off\n", j); 797 } 798 } 799 } 800 801 static void read_compute_fuses_from_reg(struct xe_gt *gt) 802 { 803 u32 ccs_mask; 804 805 ccs_mask = xe_mmio_read32(>->mmio, XEHP_FUSE4); 806 ccs_mask = REG_FIELD_GET(CCS_EN_MASK, ccs_mask); 807 808 for (int i = XE_HW_ENGINE_CCS0, j = 0; i <= XE_HW_ENGINE_CCS3; ++i, ++j) { 809 if (!(gt->info.engine_mask & BIT(i))) 810 continue; 811 812 if ((ccs_mask & BIT(j)) == 0) { 813 gt->info.engine_mask &= ~BIT(i); 814 xe_gt_info(gt, "ccs%u fused off\n", j); 815 } 816 } 817 } 818 819 static void read_compute_fuses(struct xe_gt *gt) 820 { 821 if (GRAPHICS_VER(gt_to_xe(gt)) >= 20) 822 read_compute_fuses_from_reg(gt); 823 else 824 read_compute_fuses_from_dss(gt); 825 } 826 827 static void check_gsc_availability(struct xe_gt *gt) 828 { 829 if (!(gt->info.engine_mask & BIT(XE_HW_ENGINE_GSCCS0))) 830 return; 831 832 /* 833 * The GSCCS is only used to communicate with the GSC FW, so if we don't 834 * have the FW there is nothing we need the engine for and can therefore 835 * skip its initialization. 836 */ 837 if (!xe_uc_fw_is_available(>->uc.gsc.fw)) { 838 gt->info.engine_mask &= ~BIT(XE_HW_ENGINE_GSCCS0); 839 840 /* interrupts where previously enabled, so turn them off */ 841 xe_mmio_write32(>->mmio, GUNIT_GSC_INTR_ENABLE, 0); 842 xe_mmio_write32(>->mmio, GUNIT_GSC_INTR_MASK, ~0); 843 844 xe_gt_dbg(gt, "GSC FW not used, disabling gsccs\n"); 845 } 846 } 847 848 static void check_sw_disable(struct xe_gt *gt) 849 { 850 struct xe_device *xe = gt_to_xe(gt); 851 u64 sw_allowed = xe_configfs_get_engines_allowed(to_pci_dev(xe->drm.dev)); 852 enum xe_hw_engine_id id; 853 854 for (id = 0; id < XE_NUM_HW_ENGINES; ++id) { 855 if (!(gt->info.engine_mask & BIT(id))) 856 continue; 857 858 if (!(sw_allowed & BIT(id))) { 859 gt->info.engine_mask &= ~BIT(id); 860 xe_gt_info(gt, "%s disabled via configfs\n", 861 engine_infos[id].name); 862 } 863 } 864 } 865 866 int xe_hw_engines_init_early(struct xe_gt *gt) 867 { 868 int i; 869 870 read_media_fuses(gt); 871 read_copy_fuses(gt); 872 read_compute_fuses(gt); 873 check_gsc_availability(gt); 874 check_sw_disable(gt); 875 876 BUILD_BUG_ON(XE_HW_ENGINE_PREEMPT_TIMEOUT < XE_HW_ENGINE_PREEMPT_TIMEOUT_MIN); 877 BUILD_BUG_ON(XE_HW_ENGINE_PREEMPT_TIMEOUT > XE_HW_ENGINE_PREEMPT_TIMEOUT_MAX); 878 879 for (i = 0; i < ARRAY_SIZE(gt->hw_engines); i++) 880 hw_engine_init_early(gt, >->hw_engines[i], i); 881 882 return 0; 883 } 884 885 int xe_hw_engines_init(struct xe_gt *gt) 886 { 887 int err; 888 struct xe_hw_engine *hwe; 889 enum xe_hw_engine_id id; 890 891 for_each_hw_engine(hwe, gt, id) { 892 err = hw_engine_init(gt, hwe, id); 893 if (err) 894 return err; 895 } 896 897 hw_engine_setup_logical_mapping(gt); 898 err = xe_hw_engine_setup_groups(gt); 899 if (err) 900 return err; 901 902 return 0; 903 } 904 905 void xe_hw_engine_handle_irq(struct xe_hw_engine *hwe, u16 intr_vec) 906 { 907 wake_up_all(>_to_xe(hwe->gt)->ufence_wq); 908 909 if (hwe->irq_handler) 910 hwe->irq_handler(hwe, intr_vec); 911 912 if (intr_vec & GT_MI_USER_INTERRUPT) 913 xe_hw_fence_irq_run(hwe->fence_irq); 914 } 915 916 /** 917 * xe_hw_engine_snapshot_capture - Take a quick snapshot of the HW Engine. 918 * @hwe: Xe HW Engine. 919 * @q: The exec queue object. 920 * 921 * This can be printed out in a later stage like during dev_coredump 922 * analysis. 923 * 924 * Returns: a Xe HW Engine snapshot object that must be freed by the 925 * caller, using `xe_hw_engine_snapshot_free`. 926 */ 927 struct xe_hw_engine_snapshot * 928 xe_hw_engine_snapshot_capture(struct xe_hw_engine *hwe, struct xe_exec_queue *q) 929 { 930 struct xe_hw_engine_snapshot *snapshot; 931 struct __guc_capture_parsed_output *node; 932 933 if (!xe_hw_engine_is_valid(hwe)) 934 return NULL; 935 936 snapshot = kzalloc_obj(*snapshot, GFP_ATOMIC); 937 938 if (!snapshot) 939 return NULL; 940 941 snapshot->name = kstrdup(hwe->name, GFP_ATOMIC); 942 snapshot->hwe = hwe; 943 snapshot->logical_instance = hwe->logical_instance; 944 snapshot->forcewake.domain = hwe->domain; 945 snapshot->forcewake.ref = xe_force_wake_ref(gt_to_fw(hwe->gt), 946 hwe->domain); 947 snapshot->mmio_base = hwe->mmio_base; 948 snapshot->kernel_reserved = xe_hw_engine_is_reserved(hwe); 949 950 /* no more VF accessible data below this point */ 951 if (IS_SRIOV_VF(gt_to_xe(hwe->gt))) 952 return snapshot; 953 954 if (q) { 955 /* If got guc capture, set source to GuC */ 956 node = xe_guc_capture_get_matching_and_lock(q); 957 if (node) { 958 struct xe_device *xe = gt_to_xe(hwe->gt); 959 struct xe_devcoredump *coredump = &xe->devcoredump; 960 961 coredump->snapshot.matched_node = node; 962 xe_gt_dbg(hwe->gt, "Found and locked GuC-err-capture node"); 963 return snapshot; 964 } 965 } 966 967 /* otherwise, do manual capture */ 968 xe_engine_manual_capture(hwe, snapshot); 969 xe_gt_dbg(hwe->gt, "Proceeding with manual engine snapshot"); 970 971 return snapshot; 972 } 973 974 /** 975 * xe_hw_engine_snapshot_free - Free all allocated objects for a given snapshot. 976 * @snapshot: Xe HW Engine snapshot object. 977 * 978 * This function free all the memory that needed to be allocated at capture 979 * time. 980 */ 981 void xe_hw_engine_snapshot_free(struct xe_hw_engine_snapshot *snapshot) 982 { 983 struct xe_gt *gt; 984 if (!snapshot) 985 return; 986 987 gt = snapshot->hwe->gt; 988 /* 989 * xe_guc_capture_put_matched_nodes is called here and from 990 * xe_devcoredump_snapshot_free, to cover the 2 calling paths 991 * of hw_engines - debugfs and devcoredump free. 992 */ 993 xe_guc_capture_put_matched_nodes(>->uc.guc); 994 995 kfree(snapshot->name); 996 kfree(snapshot); 997 } 998 999 /** 1000 * xe_hw_engine_print - Xe HW Engine Print. 1001 * @hwe: Hardware Engine. 1002 * @p: drm_printer. 1003 * 1004 * This function quickly capture a snapshot and immediately print it out. 1005 */ 1006 void xe_hw_engine_print(struct xe_hw_engine *hwe, struct drm_printer *p) 1007 { 1008 struct xe_hw_engine_snapshot *snapshot; 1009 1010 snapshot = xe_hw_engine_snapshot_capture(hwe, NULL); 1011 xe_engine_snapshot_print(snapshot, p); 1012 xe_hw_engine_snapshot_free(snapshot); 1013 } 1014 1015 u32 xe_hw_engine_mask_per_class(struct xe_gt *gt, 1016 enum xe_engine_class engine_class) 1017 { 1018 u32 mask = 0; 1019 enum xe_hw_engine_id id; 1020 1021 for (id = 0; id < XE_NUM_HW_ENGINES; ++id) { 1022 if (engine_infos[id].class == engine_class && 1023 gt->info.engine_mask & BIT(id)) 1024 mask |= BIT(engine_infos[id].instance); 1025 } 1026 return mask; 1027 } 1028 1029 bool xe_hw_engine_is_reserved(struct xe_hw_engine *hwe) 1030 { 1031 struct xe_gt *gt = hwe->gt; 1032 struct xe_device *xe = gt_to_xe(gt); 1033 1034 if (xe_device_is_admin_only(xe)) 1035 return true; 1036 1037 if (hwe->class == XE_ENGINE_CLASS_OTHER) 1038 return true; 1039 1040 /* Check for engines disabled by ccs_mode setting */ 1041 if (xe_gt_ccs_mode_enabled(gt) && 1042 hwe->class == XE_ENGINE_CLASS_COMPUTE && 1043 hwe->logical_instance >= gt->ccs_mode) 1044 return true; 1045 1046 return xe->info.has_usm && hwe->class == XE_ENGINE_CLASS_COPY && 1047 hwe->instance == gt->usm.reserved_bcs_instance; 1048 } 1049 1050 const char *xe_hw_engine_class_to_str(enum xe_engine_class class) 1051 { 1052 switch (class) { 1053 case XE_ENGINE_CLASS_RENDER: 1054 return "rcs"; 1055 case XE_ENGINE_CLASS_VIDEO_DECODE: 1056 return "vcs"; 1057 case XE_ENGINE_CLASS_VIDEO_ENHANCE: 1058 return "vecs"; 1059 case XE_ENGINE_CLASS_COPY: 1060 return "bcs"; 1061 case XE_ENGINE_CLASS_OTHER: 1062 return "other"; 1063 case XE_ENGINE_CLASS_COMPUTE: 1064 return "ccs"; 1065 case XE_ENGINE_CLASS_MAX: 1066 break; 1067 } 1068 1069 return NULL; 1070 } 1071 1072 u64 xe_hw_engine_read_timestamp(struct xe_hw_engine *hwe) 1073 { 1074 return xe_mmio_read64_2x32(&hwe->gt->mmio, RING_TIMESTAMP(hwe->mmio_base)); 1075 } 1076 1077 enum xe_force_wake_domains xe_hw_engine_to_fw_domain(struct xe_hw_engine *hwe) 1078 { 1079 return engine_infos[hwe->engine_id].domain; 1080 } 1081 1082 static const enum xe_engine_class user_to_xe_engine_class[] = { 1083 [DRM_XE_ENGINE_CLASS_RENDER] = XE_ENGINE_CLASS_RENDER, 1084 [DRM_XE_ENGINE_CLASS_COPY] = XE_ENGINE_CLASS_COPY, 1085 [DRM_XE_ENGINE_CLASS_VIDEO_DECODE] = XE_ENGINE_CLASS_VIDEO_DECODE, 1086 [DRM_XE_ENGINE_CLASS_VIDEO_ENHANCE] = XE_ENGINE_CLASS_VIDEO_ENHANCE, 1087 [DRM_XE_ENGINE_CLASS_COMPUTE] = XE_ENGINE_CLASS_COMPUTE, 1088 }; 1089 1090 /** 1091 * xe_hw_engine_lookup() - Lookup hardware engine for class:instance 1092 * @xe: xe device 1093 * @eci: engine class and instance 1094 * 1095 * This function will find a hardware engine for given engine 1096 * class and instance. 1097 * 1098 * Return: If found xe_hw_engine pointer, NULL otherwise. 1099 */ 1100 struct xe_hw_engine * 1101 xe_hw_engine_lookup(struct xe_device *xe, 1102 struct drm_xe_engine_class_instance eci) 1103 { 1104 struct xe_gt *gt = xe_device_get_gt(xe, eci.gt_id); 1105 unsigned int idx; 1106 1107 if (eci.engine_class >= ARRAY_SIZE(user_to_xe_engine_class)) 1108 return NULL; 1109 1110 if (!gt) 1111 return NULL; 1112 1113 idx = array_index_nospec(eci.engine_class, 1114 ARRAY_SIZE(user_to_xe_engine_class)); 1115 1116 return xe_gt_hw_engine(xe_device_get_gt(xe, eci.gt_id), 1117 user_to_xe_engine_class[idx], 1118 eci.engine_instance, true); 1119 } 1120