1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include "xe_ring_ops.h" 7 8 #include <generated/xe_wa_oob.h> 9 10 #include "instructions/xe_gpu_commands.h" 11 #include "instructions/xe_mi_commands.h" 12 #include "regs/xe_engine_regs.h" 13 #include "regs/xe_gt_regs.h" 14 #include "xe_exec_queue.h" 15 #include "xe_gt_types.h" 16 #include "xe_lrc.h" 17 #include "xe_sched_job.h" 18 #include "xe_sriov.h" 19 #include "xe_vm_types.h" 20 #include "xe_vm.h" 21 #include "xe_wa.h" 22 23 /* 24 * 3D-related flags that can't be set on _engines_ that lack access to the 3D 25 * pipeline (i.e., CCS engines). 26 */ 27 #define PIPE_CONTROL_3D_ENGINE_FLAGS (\ 28 PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH | \ 29 PIPE_CONTROL_DEPTH_CACHE_FLUSH | \ 30 PIPE_CONTROL_TILE_CACHE_FLUSH | \ 31 PIPE_CONTROL_DEPTH_STALL | \ 32 PIPE_CONTROL_STALL_AT_SCOREBOARD | \ 33 PIPE_CONTROL_PSD_SYNC | \ 34 PIPE_CONTROL_AMFS_FLUSH | \ 35 PIPE_CONTROL_VF_CACHE_INVALIDATE | \ 36 PIPE_CONTROL_GLOBAL_SNAPSHOT_RESET) 37 38 /* 3D-related flags that can't be set on _platforms_ that lack a 3D pipeline */ 39 #define PIPE_CONTROL_3D_ARCH_FLAGS ( \ 40 PIPE_CONTROL_3D_ENGINE_FLAGS | \ 41 PIPE_CONTROL_INDIRECT_STATE_DISABLE | \ 42 PIPE_CONTROL_FLUSH_ENABLE | \ 43 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE | \ 44 PIPE_CONTROL_DC_FLUSH_ENABLE) 45 46 static u32 preparser_disable(bool state) 47 { 48 return MI_ARB_CHECK | BIT(8) | state; 49 } 50 51 static u32 * 52 __emit_aux_table_inv(u32 *cmd, const struct xe_reg reg, u32 adj_offset) 53 { 54 *cmd++ = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(1) | 55 MI_LRI_MMIO_REMAP_EN; 56 *cmd++ = reg.addr + adj_offset; 57 *cmd++ = AUX_INV; 58 *cmd++ = MI_SEMAPHORE_WAIT_TOKEN | MI_SEMAPHORE_REGISTER_POLL | 59 MI_SEMAPHORE_POLL | MI_SEMAPHORE_SAD_EQ_SDD; 60 *cmd++ = 0; 61 *cmd++ = reg.addr + adj_offset; 62 *cmd++ = 0; 63 *cmd++ = 0; 64 65 return cmd; 66 } 67 68 static u32 *emit_aux_table_inv_render_compute(struct xe_gt *gt, u32 *cmd) 69 { 70 return __emit_aux_table_inv(cmd, CCS_AUX_INV, gt->mmio.adj_offset); 71 } 72 73 static u32 *emit_aux_table_inv_video_decode(struct xe_gt *gt, u32 *cmd) 74 { 75 return __emit_aux_table_inv(cmd, VD0_AUX_INV, gt->mmio.adj_offset); 76 } 77 78 static u32 *emit_aux_table_inv_video_enhance(struct xe_gt *gt, u32 *cmd) 79 { 80 return __emit_aux_table_inv(cmd, VE0_AUX_INV, gt->mmio.adj_offset); 81 } 82 83 static int emit_aux_table_inv(struct xe_hw_engine *hwe, u32 *dw, int i) 84 { 85 struct xe_gt *gt = hwe->gt; 86 u32 *(*emit)(struct xe_gt *gt, u32 *cmd) = 87 gt->ring_ops[hwe->class]->emit_aux_table_inv; 88 89 if (emit) 90 return emit(gt, dw + i) - dw; 91 else 92 return i; 93 } 94 95 static int emit_user_interrupt(u32 *dw, int i) 96 { 97 dw[i++] = MI_USER_INTERRUPT; 98 dw[i++] = MI_ARB_ON_OFF | MI_ARB_ENABLE; 99 dw[i++] = MI_ARB_CHECK; 100 101 return i; 102 } 103 104 static int emit_store_imm_ggtt(u32 addr, u32 value, u32 *dw, int i) 105 { 106 dw[i++] = MI_STORE_DATA_IMM | MI_SDI_GGTT | MI_SDI_NUM_DW(1); 107 dw[i++] = addr; 108 dw[i++] = 0; 109 dw[i++] = value; 110 111 return i; 112 } 113 114 static int emit_flush_dw(u32 *dw, int i) 115 { 116 dw[i++] = MI_FLUSH_DW | MI_FLUSH_IMM_DW; 117 dw[i++] = 0; 118 dw[i++] = 0; 119 dw[i++] = 0; 120 121 return i; 122 } 123 124 static int emit_flush_imm_ggtt(u32 addr, u32 value, u32 flags, u32 *dw, int i) 125 { 126 dw[i++] = MI_FLUSH_DW | MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_IMM_DW | 127 flags; 128 dw[i++] = addr | MI_FLUSH_DW_USE_GTT; 129 dw[i++] = 0; 130 dw[i++] = value; 131 132 return i; 133 } 134 135 static int emit_bb_start(u64 batch_addr, u32 ppgtt_flag, u32 *dw, int i) 136 { 137 dw[i++] = MI_BATCH_BUFFER_START | ppgtt_flag | XE_INSTR_NUM_DW(3); 138 dw[i++] = lower_32_bits(batch_addr); 139 dw[i++] = upper_32_bits(batch_addr); 140 141 return i; 142 } 143 144 static int emit_flush_invalidate(u32 addr, u32 val, u32 flush_flags, u32 *dw, int i) 145 { 146 dw[i++] = MI_FLUSH_DW | MI_FLUSH_DW_OP_STOREDW | 147 MI_FLUSH_IMM_DW | (flush_flags & MI_INVALIDATE_TLB) ?: 0; 148 149 dw[i++] = addr | MI_FLUSH_DW_USE_GTT; 150 dw[i++] = 0; 151 dw[i++] = val; 152 153 return i; 154 } 155 156 static int 157 emit_pipe_control(u32 *dw, int i, u32 bit_group_0, u32 bit_group_1, u32 offset, u32 value) 158 { 159 dw[i++] = GFX_OP_PIPE_CONTROL(6) | bit_group_0; 160 dw[i++] = bit_group_1; 161 dw[i++] = offset; 162 dw[i++] = 0; 163 dw[i++] = value; 164 dw[i++] = 0; 165 166 return i; 167 } 168 169 static int emit_pipe_invalidate(struct xe_exec_queue *q, u32 mask_flags, 170 bool invalidate_tlb, u32 *dw, int i) 171 { 172 u32 flags0 = 0; 173 u32 flags1 = PIPE_CONTROL_COMMAND_CACHE_INVALIDATE | 174 PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE | 175 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE | 176 PIPE_CONTROL_VF_CACHE_INVALIDATE | 177 PIPE_CONTROL_CONST_CACHE_INVALIDATE | 178 PIPE_CONTROL_STATE_CACHE_INVALIDATE | 179 PIPE_CONTROL_QW_WRITE | 180 PIPE_CONTROL_STORE_DATA_INDEX; 181 182 if (invalidate_tlb) 183 flags1 |= PIPE_CONTROL_TLB_INVALIDATE; 184 185 if (xe_exec_queue_is_multi_queue(q)) 186 flags0 |= PIPE_CONTROL0_QUEUE_DRAIN_MODE; 187 else 188 flags1 |= PIPE_CONTROL_CS_STALL; 189 190 flags1 &= ~mask_flags; 191 192 if (flags1 & PIPE_CONTROL_VF_CACHE_INVALIDATE) 193 flags0 |= PIPE_CONTROL0_L3_READ_ONLY_CACHE_INVALIDATE; 194 195 return emit_pipe_control(dw, i, flags0, flags1, 196 LRC_PPHWSP_FLUSH_INVAL_SCRATCH_ADDR, 0); 197 } 198 199 static int emit_store_imm_ppgtt_posted(u64 addr, u64 value, 200 u32 *dw, int i) 201 { 202 dw[i++] = MI_STORE_DATA_IMM | MI_SDI_NUM_QW(1); 203 dw[i++] = lower_32_bits(addr); 204 dw[i++] = upper_32_bits(addr); 205 dw[i++] = lower_32_bits(value); 206 dw[i++] = upper_32_bits(value); 207 208 return i; 209 } 210 211 static int emit_render_cache_flush(struct xe_sched_job *job, u32 *dw, int i) 212 { 213 struct xe_exec_queue *q = job->q; 214 struct xe_gt *gt = q->gt; 215 struct xe_device *xe = gt_to_xe(gt); 216 bool lacks_render = !(gt->info.engine_mask & XE_HW_ENGINE_RCS_MASK); 217 u32 flags0, flags1; 218 219 if (XE_GT_WA(gt, 14016712196)) 220 i = emit_pipe_control(dw, i, 0, PIPE_CONTROL_DEPTH_CACHE_FLUSH, 221 LRC_PPHWSP_FLUSH_INVAL_SCRATCH_ADDR, 0); 222 223 flags0 = PIPE_CONTROL0_HDC_PIPELINE_FLUSH; 224 /* 225 * Prior to MTL, HDC Pipeline Flush reliably also flushes the LSC 226 * untyped L1 dataport cache, provided HDC_CHICKEN0 is programmed 227 * correctly. Starting with MTL that coupling no longer holds 228 * regardless of how HDC_CHICKEN0 is programmed, but explicitly 229 * requesting the flush via PIPE_CONTROL is itself only reliable 230 * from Xe2 onward, so only gate it in on Xe2+. 231 */ 232 if (GRAPHICS_VERx100(xe) >= 2000) 233 flags0 |= PIPE_CONTROL0_UNTYPED_DATAPORT_CACHE_FLUSH; 234 flags1 = (PIPE_CONTROL_TILE_CACHE_FLUSH | 235 PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH | 236 PIPE_CONTROL_DEPTH_CACHE_FLUSH | 237 PIPE_CONTROL_DC_FLUSH_ENABLE | 238 PIPE_CONTROL_FLUSH_ENABLE); 239 240 if (XE_GT_WA(gt, 1409600907)) 241 flags1 |= PIPE_CONTROL_DEPTH_STALL; 242 243 if (lacks_render) 244 flags1 &= ~PIPE_CONTROL_3D_ARCH_FLAGS; 245 else if (job->q->class == XE_ENGINE_CLASS_COMPUTE) 246 flags1 &= ~PIPE_CONTROL_3D_ENGINE_FLAGS; 247 248 if (xe_exec_queue_is_multi_queue(q)) 249 flags0 |= PIPE_CONTROL0_QUEUE_DRAIN_MODE; 250 else 251 flags1 |= PIPE_CONTROL_CS_STALL; 252 253 return emit_pipe_control(dw, i, flags0, flags1, 0, 0); 254 } 255 256 static int emit_pipe_imm_ggtt(struct xe_exec_queue *q, u32 addr, u32 value, 257 bool stall_only, u32 *dw, int i) 258 { 259 u32 flags0 = 0, flags1 = PIPE_CONTROL_GLOBAL_GTT_IVB | PIPE_CONTROL_QW_WRITE; 260 261 if (!stall_only) 262 flags1 |= PIPE_CONTROL_FLUSH_ENABLE; 263 264 if (xe_exec_queue_is_multi_queue(q)) 265 flags0 |= PIPE_CONTROL0_QUEUE_DRAIN_MODE; 266 else 267 flags1 |= PIPE_CONTROL_CS_STALL; 268 269 return emit_pipe_control(dw, i, flags0, flags1, addr, value); 270 } 271 272 static u32 get_ppgtt_flag(struct xe_sched_job *job) 273 { 274 if (job->q->vm && !job->ggtt) 275 return BIT(8); 276 277 return 0; 278 } 279 280 static int emit_copy_timestamp(struct xe_device *xe, struct xe_lrc *lrc, 281 u32 *dw, int i) 282 { 283 const struct xe_reg reg = xe_lrc_is_multi_queue(lrc) ? 284 RING_QUEUE_TIMESTAMP(0) : 285 RING_CTX_TIMESTAMP(0); 286 287 dw[i++] = MI_STORE_REGISTER_MEM | MI_SRM_USE_GGTT | MI_SRM_ADD_CS_OFFSET; 288 dw[i++] = reg.addr; 289 dw[i++] = xe_lrc_ctx_job_timestamp_ggtt_addr(lrc); 290 dw[i++] = 0; 291 292 /* 293 * Ensure CTX timestamp >= Job timestamp during VF sampling to avoid 294 * arithmetic wraparound in TDR. 295 */ 296 if (IS_SRIOV_VF(xe)) { 297 dw[i++] = MI_STORE_REGISTER_MEM | MI_SRM_USE_GGTT | 298 MI_SRM_ADD_CS_OFFSET; 299 dw[i++] = reg.addr; 300 dw[i++] = xe_lrc_ctx_timestamp_ggtt_addr(lrc); 301 dw[i++] = 0; 302 } 303 304 return i; 305 } 306 307 static int emit_fake_watchdog(struct xe_lrc *lrc, u32 *dw, int i) 308 { 309 /* 310 * Setup a watchdog with impossible condition to always trigger an 311 * hardware interrupt that would force the GuC to reset the engine. 312 */ 313 314 dw[i++] = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(2) | MI_LRI_LRM_CS_MMIO; 315 dw[i++] = PR_CTR_THRSH(0).addr; 316 dw[i++] = 2; /* small threshold */ 317 dw[i++] = PR_CTR_CTRL(0).addr; 318 dw[i++] = CTR_LOGIC_OP(START); 319 320 dw[i++] = MI_SEMAPHORE_WAIT | MI_SEMW_GGTT | MI_SEMW_POLL | MI_SEMW_COMPARE(SAD_EQ_SDD); 321 dw[i++] = 0xdead; /* this should never be seen */ 322 dw[i++] = lower_32_bits(xe_lrc_ggtt_addr(lrc)); 323 dw[i++] = upper_32_bits(xe_lrc_ggtt_addr(lrc)); 324 dw[i++] = 0; /* unused token */ 325 326 dw[i++] = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(1) | MI_LRI_LRM_CS_MMIO; 327 dw[i++] = PR_CTR_CTRL(0).addr; 328 dw[i++] = CTR_LOGIC_OP(STOP); 329 330 return i; 331 } 332 333 /* for engines that don't require any special HW handling (no EUs, no aux inval, etc) */ 334 static void __emit_job_gen12_simple(struct xe_sched_job *job, struct xe_lrc *lrc, 335 u64 batch_addr, u32 *head, u32 seqno) 336 { 337 u32 dw[MAX_JOB_SIZE_DW], i = 0; 338 u32 ppgtt_flag = get_ppgtt_flag(job); 339 struct xe_gt *gt = job->q->gt; 340 341 *head = lrc->ring.tail; 342 343 if (job->ring_ops_force_reset) 344 i = emit_fake_watchdog(lrc, dw, i); 345 346 i = emit_copy_timestamp(gt_to_xe(gt), lrc, dw, i); 347 348 if (job->ring_ops_flush_tlb) { 349 dw[i++] = preparser_disable(true); 350 i = emit_flush_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc), 351 seqno, MI_INVALIDATE_TLB, dw, i); 352 dw[i++] = preparser_disable(false); 353 } else { 354 i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc), 355 seqno, dw, i); 356 } 357 358 i = emit_bb_start(batch_addr, ppgtt_flag, dw, i); 359 360 /* Don't preempt fence signaling */ 361 dw[i++] = MI_ARB_ON_OFF | MI_ARB_DISABLE; 362 363 if (job->user_fence.used) { 364 i = emit_flush_dw(dw, i); 365 i = emit_store_imm_ppgtt_posted(job->user_fence.addr, 366 job->user_fence.value, 367 dw, i); 368 } 369 370 i = emit_flush_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, 0, dw, i); 371 372 i = emit_user_interrupt(dw, i); 373 374 xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW); 375 376 xe_lrc_write_ring(lrc, dw, i * sizeof(*dw)); 377 } 378 379 static bool has_aux_ccs(struct xe_device *xe) 380 { 381 /* 382 * PVC is a special case that has no compression of either type 383 * (FlatCCS or AuxCCS). Also, AuxCCS is no longer used from Xe2 384 * onward, so any future platforms with no FlatCCS will not have 385 * AuxCCS, and we explicitly do not want to support it on MTL. 386 */ 387 if (GRAPHICS_VERx100(xe) >= 1270 || xe->info.platform == XE_PVC) 388 return false; 389 390 return !xe->info.has_flat_ccs; 391 } 392 393 static void __emit_job_gen12_video(struct xe_sched_job *job, struct xe_lrc *lrc, 394 u64 batch_addr, u32 *head, u32 seqno) 395 { 396 u32 dw[MAX_JOB_SIZE_DW], i = 0; 397 u32 ppgtt_flag = get_ppgtt_flag(job); 398 struct xe_gt *gt = job->q->gt; 399 struct xe_device *xe = gt_to_xe(gt); 400 401 *head = lrc->ring.tail; 402 403 if (job->ring_ops_force_reset) 404 i = emit_fake_watchdog(lrc, dw, i); 405 406 i = emit_copy_timestamp(xe, lrc, dw, i); 407 408 dw[i++] = preparser_disable(true); 409 410 /* hsdes: 1809175790 */ 411 i = emit_aux_table_inv(job->q->hwe, dw, i); 412 413 if (job->ring_ops_flush_tlb) 414 i = emit_flush_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc), 415 seqno, MI_INVALIDATE_TLB, dw, i); 416 417 dw[i++] = preparser_disable(false); 418 419 if (!job->ring_ops_flush_tlb) 420 i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc), 421 seqno, dw, i); 422 423 i = emit_bb_start(batch_addr, ppgtt_flag, dw, i); 424 425 /* Don't preempt fence signaling */ 426 dw[i++] = MI_ARB_ON_OFF | MI_ARB_DISABLE; 427 428 if (job->user_fence.used) { 429 i = emit_flush_dw(dw, i); 430 i = emit_store_imm_ppgtt_posted(job->user_fence.addr, 431 job->user_fence.value, 432 dw, i); 433 } 434 435 i = emit_flush_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, 0, dw, i); 436 437 i = emit_user_interrupt(dw, i); 438 439 xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW); 440 441 xe_lrc_write_ring(lrc, dw, i * sizeof(*dw)); 442 } 443 444 static void __emit_job_gen12_render_compute(struct xe_sched_job *job, 445 struct xe_lrc *lrc, 446 u64 batch_addr, u32 *head, 447 u32 seqno) 448 { 449 u32 dw[MAX_JOB_SIZE_DW], i = 0; 450 u32 ppgtt_flag = get_ppgtt_flag(job); 451 struct xe_gt *gt = job->q->gt; 452 struct xe_device *xe = gt_to_xe(gt); 453 bool lacks_render = !(gt->info.engine_mask & XE_HW_ENGINE_RCS_MASK); 454 u32 mask_flags = 0; 455 456 *head = lrc->ring.tail; 457 458 if (job->ring_ops_force_reset) 459 i = emit_fake_watchdog(lrc, dw, i); 460 461 i = emit_copy_timestamp(xe, lrc, dw, i); 462 463 /* 464 * On AuxCCS platforms the invalidation of the Aux table requires 465 * quiescing the memory traffic beforehand. 466 */ 467 if (has_aux_ccs(xe)) 468 i = emit_render_cache_flush(job, dw, i); 469 470 dw[i++] = preparser_disable(true); 471 if (lacks_render) 472 mask_flags = PIPE_CONTROL_3D_ARCH_FLAGS; 473 else if (job->q->class == XE_ENGINE_CLASS_COMPUTE) 474 mask_flags = PIPE_CONTROL_3D_ENGINE_FLAGS; 475 476 /* See __xe_pt_bind_vma() for a discussion on TLB invalidations. */ 477 i = emit_pipe_invalidate(job->q, mask_flags, job->ring_ops_flush_tlb, dw, i); 478 479 /* hsdes: 1809175790 */ 480 i = emit_aux_table_inv(job->q->hwe, dw, i); 481 482 dw[i++] = preparser_disable(false); 483 484 i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc), 485 seqno, dw, i); 486 487 i = emit_bb_start(batch_addr, ppgtt_flag, dw, i); 488 489 /* Don't preempt fence signaling */ 490 dw[i++] = MI_ARB_ON_OFF | MI_ARB_DISABLE; 491 492 i = emit_render_cache_flush(job, dw, i); 493 494 if (job->user_fence.used) 495 i = emit_store_imm_ppgtt_posted(job->user_fence.addr, 496 job->user_fence.value, 497 dw, i); 498 499 i = emit_pipe_imm_ggtt(job->q, xe_lrc_seqno_ggtt_addr(lrc), seqno, lacks_render, dw, i); 500 501 i = emit_user_interrupt(dw, i); 502 503 xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW); 504 505 xe_lrc_write_ring(lrc, dw, i * sizeof(*dw)); 506 } 507 508 static void emit_migration_job_gen12(struct xe_sched_job *job, 509 struct xe_lrc *lrc, u32 *head, 510 u32 seqno) 511 { 512 struct xe_gt *gt = job->q->gt; 513 struct xe_device *xe = gt_to_xe(gt); 514 u32 saddr = xe_lrc_start_seqno_ggtt_addr(lrc); 515 u32 dw[MAX_JOB_SIZE_DW], i = 0; 516 517 *head = lrc->ring.tail; 518 519 xe_gt_assert(gt, !job->ring_ops_force_reset); 520 521 i = emit_copy_timestamp(xe, lrc, dw, i); 522 523 i = emit_store_imm_ggtt(saddr, seqno, dw, i); 524 525 dw[i++] = MI_ARB_ON_OFF | MI_ARB_DISABLE; /* Enabled again below */ 526 527 i = emit_bb_start(job->ptrs[0].batch_addr, BIT(8), dw, i); 528 529 dw[i++] = preparser_disable(true); 530 i = emit_flush_invalidate(saddr, seqno, job->migrate_flush_flags, dw, i); 531 dw[i++] = preparser_disable(false); 532 533 i = emit_bb_start(job->ptrs[1].batch_addr, BIT(8), dw, i); 534 535 i = emit_flush_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, 536 job->migrate_flush_flags, 537 dw, i); 538 539 i = emit_user_interrupt(dw, i); 540 541 xe_gt_assert(job->q->gt, i <= MAX_JOB_SIZE_DW); 542 543 xe_lrc_write_ring(lrc, dw, i * sizeof(*dw)); 544 } 545 546 static void emit_job_gen12_gsc(struct xe_sched_job *job) 547 { 548 struct xe_gt *gt = job->q->gt; 549 550 xe_gt_assert(gt, job->q->width <= 1); /* no parallel submission for GSCCS */ 551 552 __emit_job_gen12_simple(job, job->q->lrc[0], 553 job->ptrs[0].batch_addr, 554 &job->ptrs[0].head, 555 xe_sched_job_lrc_seqno(job)); 556 } 557 558 static void emit_job_gen12_copy(struct xe_sched_job *job) 559 { 560 int i; 561 562 if (xe_sched_job_is_migration(job->q)) { 563 emit_migration_job_gen12(job, job->q->lrc[0], 564 &job->ptrs[0].head, 565 xe_sched_job_lrc_seqno(job)); 566 return; 567 } 568 569 for (i = 0; i < job->q->width; ++i) 570 __emit_job_gen12_simple(job, job->q->lrc[i], 571 job->ptrs[i].batch_addr, 572 &job->ptrs[i].head, 573 xe_sched_job_lrc_seqno(job)); 574 } 575 576 static void emit_job_gen12_video(struct xe_sched_job *job) 577 { 578 int i; 579 580 /* FIXME: Not doing parallel handshake for now */ 581 for (i = 0; i < job->q->width; ++i) 582 __emit_job_gen12_video(job, job->q->lrc[i], 583 job->ptrs[i].batch_addr, 584 &job->ptrs[i].head, 585 xe_sched_job_lrc_seqno(job)); 586 } 587 588 static void emit_job_gen12_render_compute(struct xe_sched_job *job) 589 { 590 int i; 591 592 for (i = 0; i < job->q->width; ++i) 593 __emit_job_gen12_render_compute(job, job->q->lrc[i], 594 job->ptrs[i].batch_addr, 595 &job->ptrs[i].head, 596 xe_sched_job_lrc_seqno(job)); 597 } 598 599 static const struct xe_ring_ops ring_ops_gen12_gsc = { 600 .emit_job = emit_job_gen12_gsc, 601 }; 602 603 static const struct xe_ring_ops ring_ops_gen12_copy = { 604 .emit_job = emit_job_gen12_copy, 605 }; 606 607 static const struct xe_ring_ops ring_ops_gen12_video_decode = { 608 .emit_job = emit_job_gen12_video, 609 }; 610 611 static const struct xe_ring_ops ring_ops_gen12_video_enhance = { 612 .emit_job = emit_job_gen12_video, 613 }; 614 615 static const struct xe_ring_ops ring_ops_gen12_render_compute = { 616 .emit_job = emit_job_gen12_render_compute, 617 }; 618 619 static const struct xe_ring_ops auxccs_ring_ops_gen12_video_decode = { 620 .emit_job = emit_job_gen12_video, 621 .emit_aux_table_inv = emit_aux_table_inv_video_decode, 622 }; 623 624 static const struct xe_ring_ops auxccs_ring_ops_gen12_video_enhance = { 625 .emit_job = emit_job_gen12_video, 626 .emit_aux_table_inv = emit_aux_table_inv_video_enhance, 627 }; 628 629 static const struct xe_ring_ops auxccs_ring_ops_gen12_render_compute = { 630 .emit_job = emit_job_gen12_render_compute, 631 .emit_aux_table_inv = emit_aux_table_inv_render_compute, 632 }; 633 634 const struct xe_ring_ops * 635 xe_ring_ops_get(struct xe_gt *gt, enum xe_engine_class class) 636 { 637 struct xe_device *xe = gt_to_xe(gt); 638 639 switch (class) { 640 case XE_ENGINE_CLASS_OTHER: 641 return &ring_ops_gen12_gsc; 642 case XE_ENGINE_CLASS_COPY: 643 return &ring_ops_gen12_copy; 644 case XE_ENGINE_CLASS_VIDEO_DECODE: 645 if (has_aux_ccs(xe)) 646 return &auxccs_ring_ops_gen12_video_decode; 647 else 648 return &ring_ops_gen12_video_decode; 649 case XE_ENGINE_CLASS_VIDEO_ENHANCE: 650 if (has_aux_ccs(xe)) 651 return &auxccs_ring_ops_gen12_video_enhance; 652 else 653 return &ring_ops_gen12_video_enhance; 654 case XE_ENGINE_CLASS_RENDER: 655 case XE_ENGINE_CLASS_COMPUTE: 656 if (has_aux_ccs(xe)) 657 return &auxccs_ring_ops_gen12_render_compute; 658 else 659 return &ring_ops_gen12_render_compute; 660 default: 661 return NULL; 662 } 663 } 664