1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include "xe_ring_ops.h" 7 8 #include <generated/xe_wa_oob.h> 9 10 #include "instructions/xe_gpu_commands.h" 11 #include "instructions/xe_mi_commands.h" 12 #include "regs/xe_engine_regs.h" 13 #include "regs/xe_gt_regs.h" 14 #include "xe_exec_queue.h" 15 #include "xe_gt_types.h" 16 #include "xe_lrc.h" 17 #include "xe_sched_job.h" 18 #include "xe_sriov.h" 19 #include "xe_vm_types.h" 20 #include "xe_vm.h" 21 #include "xe_wa.h" 22 23 /* 24 * 3D-related flags that can't be set on _engines_ that lack access to the 3D 25 * pipeline (i.e., CCS engines). 26 */ 27 #define PIPE_CONTROL_3D_ENGINE_FLAGS (\ 28 PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH | \ 29 PIPE_CONTROL_DEPTH_CACHE_FLUSH | \ 30 PIPE_CONTROL_TILE_CACHE_FLUSH | \ 31 PIPE_CONTROL_DEPTH_STALL | \ 32 PIPE_CONTROL_STALL_AT_SCOREBOARD | \ 33 PIPE_CONTROL_PSD_SYNC | \ 34 PIPE_CONTROL_AMFS_FLUSH | \ 35 PIPE_CONTROL_VF_CACHE_INVALIDATE | \ 36 PIPE_CONTROL_GLOBAL_SNAPSHOT_RESET) 37 38 /* 3D-related flags that can't be set on _platforms_ that lack a 3D pipeline */ 39 #define PIPE_CONTROL_3D_ARCH_FLAGS ( \ 40 PIPE_CONTROL_3D_ENGINE_FLAGS | \ 41 PIPE_CONTROL_INDIRECT_STATE_DISABLE | \ 42 PIPE_CONTROL_FLUSH_ENABLE | \ 43 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE | \ 44 PIPE_CONTROL_DC_FLUSH_ENABLE) 45 46 static u32 preparser_disable(bool state) 47 { 48 return MI_ARB_CHECK | BIT(8) | state; 49 } 50 51 static u32 * 52 __emit_aux_table_inv(u32 *cmd, const struct xe_reg reg, u32 adj_offset) 53 { 54 *cmd++ = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(1) | 55 MI_LRI_MMIO_REMAP_EN; 56 *cmd++ = reg.addr + adj_offset; 57 *cmd++ = AUX_INV; 58 *cmd++ = MI_SEMAPHORE_WAIT_TOKEN | MI_SEMAPHORE_REGISTER_POLL | 59 MI_SEMAPHORE_POLL | MI_SEMAPHORE_SAD_EQ_SDD; 60 *cmd++ = 0; 61 *cmd++ = reg.addr + adj_offset; 62 *cmd++ = 0; 63 *cmd++ = 0; 64 65 return cmd; 66 } 67 68 static u32 *emit_aux_table_inv_render_compute(struct xe_gt *gt, u32 *cmd) 69 { 70 return __emit_aux_table_inv(cmd, CCS_AUX_INV, gt->mmio.adj_offset); 71 } 72 73 static u32 *emit_aux_table_inv_video_decode(struct xe_gt *gt, u32 *cmd) 74 { 75 return __emit_aux_table_inv(cmd, VD0_AUX_INV, gt->mmio.adj_offset); 76 } 77 78 static u32 *emit_aux_table_inv_video_enhance(struct xe_gt *gt, u32 *cmd) 79 { 80 return __emit_aux_table_inv(cmd, VE0_AUX_INV, gt->mmio.adj_offset); 81 } 82 83 static int emit_aux_table_inv(struct xe_hw_engine *hwe, u32 *dw, int i) 84 { 85 struct xe_gt *gt = hwe->gt; 86 u32 *(*emit)(struct xe_gt *gt, u32 *cmd) = 87 gt->ring_ops[hwe->class]->emit_aux_table_inv; 88 89 if (emit) 90 return emit(gt, dw + i) - dw; 91 else 92 return i; 93 } 94 95 static int emit_user_interrupt(u32 *dw, int i) 96 { 97 dw[i++] = MI_USER_INTERRUPT; 98 dw[i++] = MI_ARB_ON_OFF | MI_ARB_ENABLE; 99 dw[i++] = MI_ARB_CHECK; 100 101 return i; 102 } 103 104 static int emit_store_imm_ggtt(u32 addr, u32 value, u32 *dw, int i) 105 { 106 dw[i++] = MI_STORE_DATA_IMM | MI_SDI_GGTT | MI_SDI_NUM_DW(1); 107 dw[i++] = addr; 108 dw[i++] = 0; 109 dw[i++] = value; 110 111 return i; 112 } 113 114 static int emit_flush_dw(u32 *dw, int i) 115 { 116 dw[i++] = MI_FLUSH_DW | MI_FLUSH_IMM_DW; 117 dw[i++] = 0; 118 dw[i++] = 0; 119 dw[i++] = 0; 120 121 return i; 122 } 123 124 static int emit_flush_imm_ggtt(u32 addr, u32 value, u32 flags, u32 *dw, int i) 125 { 126 dw[i++] = MI_FLUSH_DW | MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_IMM_DW | 127 flags; 128 dw[i++] = addr | MI_FLUSH_DW_USE_GTT; 129 dw[i++] = 0; 130 dw[i++] = value; 131 132 return i; 133 } 134 135 static int emit_bb_start(u64 batch_addr, u32 ppgtt_flag, u32 *dw, int i) 136 { 137 dw[i++] = MI_BATCH_BUFFER_START | ppgtt_flag | XE_INSTR_NUM_DW(3); 138 dw[i++] = lower_32_bits(batch_addr); 139 dw[i++] = upper_32_bits(batch_addr); 140 141 return i; 142 } 143 144 static int emit_flush_invalidate(u32 addr, u32 val, u32 flush_flags, u32 *dw, int i) 145 { 146 dw[i++] = MI_FLUSH_DW | MI_FLUSH_DW_OP_STOREDW | 147 MI_FLUSH_IMM_DW | (flush_flags & MI_INVALIDATE_TLB) ?: 0; 148 149 dw[i++] = addr | MI_FLUSH_DW_USE_GTT; 150 dw[i++] = 0; 151 dw[i++] = val; 152 153 return i; 154 } 155 156 static int 157 emit_pipe_control(u32 *dw, int i, u32 bit_group_0, u32 bit_group_1, u32 offset, u32 value) 158 { 159 dw[i++] = GFX_OP_PIPE_CONTROL(6) | bit_group_0; 160 dw[i++] = bit_group_1; 161 dw[i++] = offset; 162 dw[i++] = 0; 163 dw[i++] = value; 164 dw[i++] = 0; 165 166 return i; 167 } 168 169 static int emit_pipe_invalidate(struct xe_exec_queue *q, u32 mask_flags, 170 bool invalidate_tlb, u32 *dw, int i) 171 { 172 u32 flags0 = 0; 173 u32 flags1 = PIPE_CONTROL_COMMAND_CACHE_INVALIDATE | 174 PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE | 175 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE | 176 PIPE_CONTROL_VF_CACHE_INVALIDATE | 177 PIPE_CONTROL_CONST_CACHE_INVALIDATE | 178 PIPE_CONTROL_STATE_CACHE_INVALIDATE | 179 PIPE_CONTROL_QW_WRITE | 180 PIPE_CONTROL_STORE_DATA_INDEX; 181 182 if (invalidate_tlb) 183 flags1 |= PIPE_CONTROL_TLB_INVALIDATE; 184 185 if (xe_exec_queue_is_multi_queue(q)) 186 flags0 |= PIPE_CONTROL0_QUEUE_DRAIN_MODE; 187 else 188 flags1 |= PIPE_CONTROL_CS_STALL; 189 190 flags1 &= ~mask_flags; 191 192 if (flags1 & PIPE_CONTROL_VF_CACHE_INVALIDATE) 193 flags0 |= PIPE_CONTROL0_L3_READ_ONLY_CACHE_INVALIDATE; 194 195 return emit_pipe_control(dw, i, flags0, flags1, 196 LRC_PPHWSP_FLUSH_INVAL_SCRATCH_ADDR, 0); 197 } 198 199 static int emit_store_imm_ppgtt_posted(u64 addr, u64 value, 200 u32 *dw, int i) 201 { 202 dw[i++] = MI_STORE_DATA_IMM | MI_SDI_NUM_QW(1); 203 dw[i++] = lower_32_bits(addr); 204 dw[i++] = upper_32_bits(addr); 205 dw[i++] = lower_32_bits(value); 206 dw[i++] = upper_32_bits(value); 207 208 return i; 209 } 210 211 static int emit_render_cache_flush(struct xe_sched_job *job, u32 *dw, int i) 212 { 213 struct xe_exec_queue *q = job->q; 214 struct xe_gt *gt = q->gt; 215 bool lacks_render = !(gt->info.engine_mask & XE_HW_ENGINE_RCS_MASK); 216 u32 flags0, flags1; 217 218 if (XE_GT_WA(gt, 14016712196)) 219 i = emit_pipe_control(dw, i, 0, PIPE_CONTROL_DEPTH_CACHE_FLUSH, 220 LRC_PPHWSP_FLUSH_INVAL_SCRATCH_ADDR, 0); 221 222 flags0 = PIPE_CONTROL0_HDC_PIPELINE_FLUSH; 223 flags1 = (PIPE_CONTROL_TILE_CACHE_FLUSH | 224 PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH | 225 PIPE_CONTROL_DEPTH_CACHE_FLUSH | 226 PIPE_CONTROL_DC_FLUSH_ENABLE | 227 PIPE_CONTROL_FLUSH_ENABLE); 228 229 if (XE_GT_WA(gt, 1409600907)) 230 flags1 |= PIPE_CONTROL_DEPTH_STALL; 231 232 if (lacks_render) 233 flags1 &= ~PIPE_CONTROL_3D_ARCH_FLAGS; 234 else if (job->q->class == XE_ENGINE_CLASS_COMPUTE) 235 flags1 &= ~PIPE_CONTROL_3D_ENGINE_FLAGS; 236 237 if (xe_exec_queue_is_multi_queue(q)) 238 flags0 |= PIPE_CONTROL0_QUEUE_DRAIN_MODE; 239 else 240 flags1 |= PIPE_CONTROL_CS_STALL; 241 242 return emit_pipe_control(dw, i, flags0, flags1, 0, 0); 243 } 244 245 static int emit_pipe_imm_ggtt(struct xe_exec_queue *q, u32 addr, u32 value, 246 bool stall_only, u32 *dw, int i) 247 { 248 u32 flags0 = 0, flags1 = PIPE_CONTROL_GLOBAL_GTT_IVB | PIPE_CONTROL_QW_WRITE; 249 250 if (!stall_only) 251 flags1 |= PIPE_CONTROL_FLUSH_ENABLE; 252 253 if (xe_exec_queue_is_multi_queue(q)) 254 flags0 |= PIPE_CONTROL0_QUEUE_DRAIN_MODE; 255 else 256 flags1 |= PIPE_CONTROL_CS_STALL; 257 258 return emit_pipe_control(dw, i, flags0, flags1, addr, value); 259 } 260 261 static u32 get_ppgtt_flag(struct xe_sched_job *job) 262 { 263 if (job->q->vm && !job->ggtt) 264 return BIT(8); 265 266 return 0; 267 } 268 269 static int emit_copy_timestamp(struct xe_device *xe, struct xe_lrc *lrc, 270 u32 *dw, int i) 271 { 272 const struct xe_reg reg = xe_lrc_is_multi_queue(lrc) ? 273 RING_QUEUE_TIMESTAMP(0) : 274 RING_CTX_TIMESTAMP(0); 275 276 dw[i++] = MI_STORE_REGISTER_MEM | MI_SRM_USE_GGTT | MI_SRM_ADD_CS_OFFSET; 277 dw[i++] = reg.addr; 278 dw[i++] = xe_lrc_ctx_job_timestamp_ggtt_addr(lrc); 279 dw[i++] = 0; 280 281 /* 282 * Ensure CTX timestamp >= Job timestamp during VF sampling to avoid 283 * arithmetic wraparound in TDR. 284 */ 285 if (IS_SRIOV_VF(xe)) { 286 dw[i++] = MI_STORE_REGISTER_MEM | MI_SRM_USE_GGTT | 287 MI_SRM_ADD_CS_OFFSET; 288 dw[i++] = reg.addr; 289 dw[i++] = xe_lrc_ctx_timestamp_ggtt_addr(lrc); 290 dw[i++] = 0; 291 } 292 293 return i; 294 } 295 296 static int emit_fake_watchdog(struct xe_lrc *lrc, u32 *dw, int i) 297 { 298 /* 299 * Setup a watchdog with impossible condition to always trigger an 300 * hardware interrupt that would force the GuC to reset the engine. 301 */ 302 303 dw[i++] = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(2) | MI_LRI_LRM_CS_MMIO; 304 dw[i++] = PR_CTR_THRSH(0).addr; 305 dw[i++] = 2; /* small threshold */ 306 dw[i++] = PR_CTR_CTRL(0).addr; 307 dw[i++] = CTR_LOGIC_OP(START); 308 309 dw[i++] = MI_SEMAPHORE_WAIT | MI_SEMW_GGTT | MI_SEMW_POLL | MI_SEMW_COMPARE(SAD_EQ_SDD); 310 dw[i++] = 0xdead; /* this should never be seen */ 311 dw[i++] = lower_32_bits(xe_lrc_ggtt_addr(lrc)); 312 dw[i++] = upper_32_bits(xe_lrc_ggtt_addr(lrc)); 313 dw[i++] = 0; /* unused token */ 314 315 dw[i++] = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(1) | MI_LRI_LRM_CS_MMIO; 316 dw[i++] = PR_CTR_CTRL(0).addr; 317 dw[i++] = CTR_LOGIC_OP(STOP); 318 319 return i; 320 } 321 322 /* for engines that don't require any special HW handling (no EUs, no aux inval, etc) */ 323 static void __emit_job_gen12_simple(struct xe_sched_job *job, struct xe_lrc *lrc, 324 u64 batch_addr, u32 *head, u32 seqno) 325 { 326 u32 dw[MAX_JOB_SIZE_DW], i = 0; 327 u32 ppgtt_flag = get_ppgtt_flag(job); 328 struct xe_gt *gt = job->q->gt; 329 330 *head = lrc->ring.tail; 331 332 if (job->ring_ops_force_reset) 333 i = emit_fake_watchdog(lrc, dw, i); 334 335 i = emit_copy_timestamp(gt_to_xe(gt), lrc, dw, i); 336 337 if (job->ring_ops_flush_tlb) { 338 dw[i++] = preparser_disable(true); 339 i = emit_flush_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc), 340 seqno, MI_INVALIDATE_TLB, dw, i); 341 dw[i++] = preparser_disable(false); 342 } else { 343 i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc), 344 seqno, dw, i); 345 } 346 347 i = emit_bb_start(batch_addr, ppgtt_flag, dw, i); 348 349 /* Don't preempt fence signaling */ 350 dw[i++] = MI_ARB_ON_OFF | MI_ARB_DISABLE; 351 352 if (job->user_fence.used) { 353 i = emit_flush_dw(dw, i); 354 i = emit_store_imm_ppgtt_posted(job->user_fence.addr, 355 job->user_fence.value, 356 dw, i); 357 } 358 359 i = emit_flush_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, 0, dw, i); 360 361 i = emit_user_interrupt(dw, i); 362 363 xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW); 364 365 xe_lrc_write_ring(lrc, dw, i * sizeof(*dw)); 366 } 367 368 static bool has_aux_ccs(struct xe_device *xe) 369 { 370 /* 371 * PVC is a special case that has no compression of either type 372 * (FlatCCS or AuxCCS). Also, AuxCCS is no longer used from Xe2 373 * onward, so any future platforms with no FlatCCS will not have 374 * AuxCCS, and we explicitly do not want to support it on MTL. 375 */ 376 if (GRAPHICS_VERx100(xe) >= 1270 || xe->info.platform == XE_PVC) 377 return false; 378 379 return !xe->info.has_flat_ccs; 380 } 381 382 static void __emit_job_gen12_video(struct xe_sched_job *job, struct xe_lrc *lrc, 383 u64 batch_addr, u32 *head, u32 seqno) 384 { 385 u32 dw[MAX_JOB_SIZE_DW], i = 0; 386 u32 ppgtt_flag = get_ppgtt_flag(job); 387 struct xe_gt *gt = job->q->gt; 388 struct xe_device *xe = gt_to_xe(gt); 389 390 *head = lrc->ring.tail; 391 392 if (job->ring_ops_force_reset) 393 i = emit_fake_watchdog(lrc, dw, i); 394 395 i = emit_copy_timestamp(xe, lrc, dw, i); 396 397 dw[i++] = preparser_disable(true); 398 399 /* hsdes: 1809175790 */ 400 i = emit_aux_table_inv(job->q->hwe, dw, i); 401 402 if (job->ring_ops_flush_tlb) 403 i = emit_flush_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc), 404 seqno, MI_INVALIDATE_TLB, dw, i); 405 406 dw[i++] = preparser_disable(false); 407 408 if (!job->ring_ops_flush_tlb) 409 i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc), 410 seqno, dw, i); 411 412 i = emit_bb_start(batch_addr, ppgtt_flag, dw, i); 413 414 /* Don't preempt fence signaling */ 415 dw[i++] = MI_ARB_ON_OFF | MI_ARB_DISABLE; 416 417 if (job->user_fence.used) { 418 i = emit_flush_dw(dw, i); 419 i = emit_store_imm_ppgtt_posted(job->user_fence.addr, 420 job->user_fence.value, 421 dw, i); 422 } 423 424 i = emit_flush_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, 0, dw, i); 425 426 i = emit_user_interrupt(dw, i); 427 428 xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW); 429 430 xe_lrc_write_ring(lrc, dw, i * sizeof(*dw)); 431 } 432 433 static void __emit_job_gen12_render_compute(struct xe_sched_job *job, 434 struct xe_lrc *lrc, 435 u64 batch_addr, u32 *head, 436 u32 seqno) 437 { 438 u32 dw[MAX_JOB_SIZE_DW], i = 0; 439 u32 ppgtt_flag = get_ppgtt_flag(job); 440 struct xe_gt *gt = job->q->gt; 441 struct xe_device *xe = gt_to_xe(gt); 442 bool lacks_render = !(gt->info.engine_mask & XE_HW_ENGINE_RCS_MASK); 443 u32 mask_flags = 0; 444 445 *head = lrc->ring.tail; 446 447 if (job->ring_ops_force_reset) 448 i = emit_fake_watchdog(lrc, dw, i); 449 450 i = emit_copy_timestamp(xe, lrc, dw, i); 451 452 /* 453 * On AuxCCS platforms the invalidation of the Aux table requires 454 * quiescing the memory traffic beforehand. 455 */ 456 if (has_aux_ccs(xe)) 457 i = emit_render_cache_flush(job, dw, i); 458 459 dw[i++] = preparser_disable(true); 460 if (lacks_render) 461 mask_flags = PIPE_CONTROL_3D_ARCH_FLAGS; 462 else if (job->q->class == XE_ENGINE_CLASS_COMPUTE) 463 mask_flags = PIPE_CONTROL_3D_ENGINE_FLAGS; 464 465 /* See __xe_pt_bind_vma() for a discussion on TLB invalidations. */ 466 i = emit_pipe_invalidate(job->q, mask_flags, job->ring_ops_flush_tlb, dw, i); 467 468 /* hsdes: 1809175790 */ 469 i = emit_aux_table_inv(job->q->hwe, dw, i); 470 471 dw[i++] = preparser_disable(false); 472 473 i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc), 474 seqno, dw, i); 475 476 i = emit_bb_start(batch_addr, ppgtt_flag, dw, i); 477 478 /* Don't preempt fence signaling */ 479 dw[i++] = MI_ARB_ON_OFF | MI_ARB_DISABLE; 480 481 i = emit_render_cache_flush(job, dw, i); 482 483 if (job->user_fence.used) 484 i = emit_store_imm_ppgtt_posted(job->user_fence.addr, 485 job->user_fence.value, 486 dw, i); 487 488 i = emit_pipe_imm_ggtt(job->q, xe_lrc_seqno_ggtt_addr(lrc), seqno, lacks_render, dw, i); 489 490 i = emit_user_interrupt(dw, i); 491 492 xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW); 493 494 xe_lrc_write_ring(lrc, dw, i * sizeof(*dw)); 495 } 496 497 static void emit_migration_job_gen12(struct xe_sched_job *job, 498 struct xe_lrc *lrc, u32 *head, 499 u32 seqno) 500 { 501 struct xe_gt *gt = job->q->gt; 502 struct xe_device *xe = gt_to_xe(gt); 503 u32 saddr = xe_lrc_start_seqno_ggtt_addr(lrc); 504 u32 dw[MAX_JOB_SIZE_DW], i = 0; 505 506 *head = lrc->ring.tail; 507 508 xe_gt_assert(gt, !job->ring_ops_force_reset); 509 510 i = emit_copy_timestamp(xe, lrc, dw, i); 511 512 i = emit_store_imm_ggtt(saddr, seqno, dw, i); 513 514 dw[i++] = MI_ARB_ON_OFF | MI_ARB_DISABLE; /* Enabled again below */ 515 516 i = emit_bb_start(job->ptrs[0].batch_addr, BIT(8), dw, i); 517 518 dw[i++] = preparser_disable(true); 519 i = emit_flush_invalidate(saddr, seqno, job->migrate_flush_flags, dw, i); 520 dw[i++] = preparser_disable(false); 521 522 i = emit_bb_start(job->ptrs[1].batch_addr, BIT(8), dw, i); 523 524 i = emit_flush_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, 525 job->migrate_flush_flags, 526 dw, i); 527 528 i = emit_user_interrupt(dw, i); 529 530 xe_gt_assert(job->q->gt, i <= MAX_JOB_SIZE_DW); 531 532 xe_lrc_write_ring(lrc, dw, i * sizeof(*dw)); 533 } 534 535 static void emit_job_gen12_gsc(struct xe_sched_job *job) 536 { 537 struct xe_gt *gt = job->q->gt; 538 539 xe_gt_assert(gt, job->q->width <= 1); /* no parallel submission for GSCCS */ 540 541 __emit_job_gen12_simple(job, job->q->lrc[0], 542 job->ptrs[0].batch_addr, 543 &job->ptrs[0].head, 544 xe_sched_job_lrc_seqno(job)); 545 } 546 547 static void emit_job_gen12_copy(struct xe_sched_job *job) 548 { 549 int i; 550 551 if (xe_sched_job_is_migration(job->q)) { 552 emit_migration_job_gen12(job, job->q->lrc[0], 553 &job->ptrs[0].head, 554 xe_sched_job_lrc_seqno(job)); 555 return; 556 } 557 558 for (i = 0; i < job->q->width; ++i) 559 __emit_job_gen12_simple(job, job->q->lrc[i], 560 job->ptrs[i].batch_addr, 561 &job->ptrs[i].head, 562 xe_sched_job_lrc_seqno(job)); 563 } 564 565 static void emit_job_gen12_video(struct xe_sched_job *job) 566 { 567 int i; 568 569 /* FIXME: Not doing parallel handshake for now */ 570 for (i = 0; i < job->q->width; ++i) 571 __emit_job_gen12_video(job, job->q->lrc[i], 572 job->ptrs[i].batch_addr, 573 &job->ptrs[i].head, 574 xe_sched_job_lrc_seqno(job)); 575 } 576 577 static void emit_job_gen12_render_compute(struct xe_sched_job *job) 578 { 579 int i; 580 581 for (i = 0; i < job->q->width; ++i) 582 __emit_job_gen12_render_compute(job, job->q->lrc[i], 583 job->ptrs[i].batch_addr, 584 &job->ptrs[i].head, 585 xe_sched_job_lrc_seqno(job)); 586 } 587 588 static const struct xe_ring_ops ring_ops_gen12_gsc = { 589 .emit_job = emit_job_gen12_gsc, 590 }; 591 592 static const struct xe_ring_ops ring_ops_gen12_copy = { 593 .emit_job = emit_job_gen12_copy, 594 }; 595 596 static const struct xe_ring_ops ring_ops_gen12_video_decode = { 597 .emit_job = emit_job_gen12_video, 598 }; 599 600 static const struct xe_ring_ops ring_ops_gen12_video_enhance = { 601 .emit_job = emit_job_gen12_video, 602 }; 603 604 static const struct xe_ring_ops ring_ops_gen12_render_compute = { 605 .emit_job = emit_job_gen12_render_compute, 606 }; 607 608 static const struct xe_ring_ops auxccs_ring_ops_gen12_video_decode = { 609 .emit_job = emit_job_gen12_video, 610 .emit_aux_table_inv = emit_aux_table_inv_video_decode, 611 }; 612 613 static const struct xe_ring_ops auxccs_ring_ops_gen12_video_enhance = { 614 .emit_job = emit_job_gen12_video, 615 .emit_aux_table_inv = emit_aux_table_inv_video_enhance, 616 }; 617 618 static const struct xe_ring_ops auxccs_ring_ops_gen12_render_compute = { 619 .emit_job = emit_job_gen12_render_compute, 620 .emit_aux_table_inv = emit_aux_table_inv_render_compute, 621 }; 622 623 const struct xe_ring_ops * 624 xe_ring_ops_get(struct xe_gt *gt, enum xe_engine_class class) 625 { 626 struct xe_device *xe = gt_to_xe(gt); 627 628 switch (class) { 629 case XE_ENGINE_CLASS_OTHER: 630 return &ring_ops_gen12_gsc; 631 case XE_ENGINE_CLASS_COPY: 632 return &ring_ops_gen12_copy; 633 case XE_ENGINE_CLASS_VIDEO_DECODE: 634 if (has_aux_ccs(xe)) 635 return &auxccs_ring_ops_gen12_video_decode; 636 else 637 return &ring_ops_gen12_video_decode; 638 case XE_ENGINE_CLASS_VIDEO_ENHANCE: 639 if (has_aux_ccs(xe)) 640 return &auxccs_ring_ops_gen12_video_enhance; 641 else 642 return &ring_ops_gen12_video_enhance; 643 case XE_ENGINE_CLASS_RENDER: 644 case XE_ENGINE_CLASS_COMPUTE: 645 if (has_aux_ccs(xe)) 646 return &auxccs_ring_ops_gen12_render_compute; 647 else 648 return &ring_ops_gen12_render_compute; 649 default: 650 return NULL; 651 } 652 } 653