1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include "xe_ring_ops.h" 7 8 #include <generated/xe_wa_oob.h> 9 10 #include "instructions/xe_gpu_commands.h" 11 #include "instructions/xe_mi_commands.h" 12 #include "regs/xe_engine_regs.h" 13 #include "regs/xe_gt_regs.h" 14 #include "regs/xe_lrc_layout.h" 15 #include "xe_exec_queue.h" 16 #include "xe_gt.h" 17 #include "xe_lrc.h" 18 #include "xe_macros.h" 19 #include "xe_sched_job.h" 20 #include "xe_sriov.h" 21 #include "xe_vm_types.h" 22 #include "xe_vm.h" 23 #include "xe_wa.h" 24 25 /* 26 * 3D-related flags that can't be set on _engines_ that lack access to the 3D 27 * pipeline (i.e., CCS engines). 28 */ 29 #define PIPE_CONTROL_3D_ENGINE_FLAGS (\ 30 PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH | \ 31 PIPE_CONTROL_DEPTH_CACHE_FLUSH | \ 32 PIPE_CONTROL_TILE_CACHE_FLUSH | \ 33 PIPE_CONTROL_DEPTH_STALL | \ 34 PIPE_CONTROL_STALL_AT_SCOREBOARD | \ 35 PIPE_CONTROL_PSD_SYNC | \ 36 PIPE_CONTROL_AMFS_FLUSH | \ 37 PIPE_CONTROL_VF_CACHE_INVALIDATE | \ 38 PIPE_CONTROL_GLOBAL_SNAPSHOT_RESET) 39 40 /* 3D-related flags that can't be set on _platforms_ that lack a 3D pipeline */ 41 #define PIPE_CONTROL_3D_ARCH_FLAGS ( \ 42 PIPE_CONTROL_3D_ENGINE_FLAGS | \ 43 PIPE_CONTROL_INDIRECT_STATE_DISABLE | \ 44 PIPE_CONTROL_FLUSH_ENABLE | \ 45 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE | \ 46 PIPE_CONTROL_DC_FLUSH_ENABLE) 47 48 static u32 preparser_disable(bool state) 49 { 50 return MI_ARB_CHECK | BIT(8) | state; 51 } 52 53 static int emit_aux_table_inv(struct xe_gt *gt, struct xe_reg reg, 54 u32 *dw, int i) 55 { 56 dw[i++] = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(1) | MI_LRI_MMIO_REMAP_EN; 57 dw[i++] = reg.addr + gt->mmio.adj_offset; 58 dw[i++] = AUX_INV; 59 dw[i++] = MI_NOOP; 60 61 return i; 62 } 63 64 static int emit_user_interrupt(u32 *dw, int i) 65 { 66 dw[i++] = MI_USER_INTERRUPT; 67 dw[i++] = MI_ARB_ON_OFF | MI_ARB_ENABLE; 68 dw[i++] = MI_ARB_CHECK; 69 70 return i; 71 } 72 73 static int emit_store_imm_ggtt(u32 addr, u32 value, u32 *dw, int i) 74 { 75 dw[i++] = MI_STORE_DATA_IMM | MI_SDI_GGTT | MI_SDI_NUM_DW(1); 76 dw[i++] = addr; 77 dw[i++] = 0; 78 dw[i++] = value; 79 80 return i; 81 } 82 83 static int emit_flush_dw(u32 *dw, int i) 84 { 85 dw[i++] = MI_FLUSH_DW | MI_FLUSH_IMM_DW; 86 dw[i++] = 0; 87 dw[i++] = 0; 88 dw[i++] = 0; 89 90 return i; 91 } 92 93 static int emit_flush_imm_ggtt(u32 addr, u32 value, u32 flags, u32 *dw, int i) 94 { 95 dw[i++] = MI_FLUSH_DW | MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_IMM_DW | 96 flags; 97 dw[i++] = addr | MI_FLUSH_DW_USE_GTT; 98 dw[i++] = 0; 99 dw[i++] = value; 100 101 return i; 102 } 103 104 static int emit_bb_start(u64 batch_addr, u32 ppgtt_flag, u32 *dw, int i) 105 { 106 dw[i++] = MI_BATCH_BUFFER_START | ppgtt_flag | XE_INSTR_NUM_DW(3); 107 dw[i++] = lower_32_bits(batch_addr); 108 dw[i++] = upper_32_bits(batch_addr); 109 110 return i; 111 } 112 113 static int emit_flush_invalidate(u32 addr, u32 val, u32 flush_flags, u32 *dw, int i) 114 { 115 dw[i++] = MI_FLUSH_DW | MI_FLUSH_DW_OP_STOREDW | 116 MI_FLUSH_IMM_DW | (flush_flags & MI_INVALIDATE_TLB) ?: 0; 117 118 dw[i++] = addr | MI_FLUSH_DW_USE_GTT; 119 dw[i++] = 0; 120 dw[i++] = val; 121 122 return i; 123 } 124 125 static int 126 emit_pipe_control(u32 *dw, int i, u32 bit_group_0, u32 bit_group_1, u32 offset, u32 value) 127 { 128 dw[i++] = GFX_OP_PIPE_CONTROL(6) | bit_group_0; 129 dw[i++] = bit_group_1; 130 dw[i++] = offset; 131 dw[i++] = 0; 132 dw[i++] = value; 133 dw[i++] = 0; 134 135 return i; 136 } 137 138 static int emit_pipe_invalidate(struct xe_exec_queue *q, u32 mask_flags, 139 bool invalidate_tlb, u32 *dw, int i) 140 { 141 u32 flags0 = 0; 142 u32 flags1 = PIPE_CONTROL_COMMAND_CACHE_INVALIDATE | 143 PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE | 144 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE | 145 PIPE_CONTROL_VF_CACHE_INVALIDATE | 146 PIPE_CONTROL_CONST_CACHE_INVALIDATE | 147 PIPE_CONTROL_STATE_CACHE_INVALIDATE | 148 PIPE_CONTROL_QW_WRITE | 149 PIPE_CONTROL_STORE_DATA_INDEX; 150 151 if (invalidate_tlb) 152 flags1 |= PIPE_CONTROL_TLB_INVALIDATE; 153 154 if (xe_exec_queue_is_multi_queue(q)) 155 flags0 |= PIPE_CONTROL0_QUEUE_DRAIN_MODE; 156 else 157 flags1 |= PIPE_CONTROL_CS_STALL; 158 159 flags1 &= ~mask_flags; 160 161 if (flags1 & PIPE_CONTROL_VF_CACHE_INVALIDATE) 162 flags0 |= PIPE_CONTROL0_L3_READ_ONLY_CACHE_INVALIDATE; 163 164 return emit_pipe_control(dw, i, flags0, flags1, 165 LRC_PPHWSP_FLUSH_INVAL_SCRATCH_ADDR, 0); 166 } 167 168 static int emit_store_imm_ppgtt_posted(u64 addr, u64 value, 169 u32 *dw, int i) 170 { 171 dw[i++] = MI_STORE_DATA_IMM | MI_SDI_NUM_QW(1); 172 dw[i++] = lower_32_bits(addr); 173 dw[i++] = upper_32_bits(addr); 174 dw[i++] = lower_32_bits(value); 175 dw[i++] = upper_32_bits(value); 176 177 return i; 178 } 179 180 static int emit_render_cache_flush(struct xe_sched_job *job, u32 *dw, int i) 181 { 182 struct xe_exec_queue *q = job->q; 183 struct xe_gt *gt = q->gt; 184 bool lacks_render = !(gt->info.engine_mask & XE_HW_ENGINE_RCS_MASK); 185 u32 flags0, flags1; 186 187 if (XE_GT_WA(gt, 14016712196)) 188 i = emit_pipe_control(dw, i, 0, PIPE_CONTROL_DEPTH_CACHE_FLUSH, 189 LRC_PPHWSP_FLUSH_INVAL_SCRATCH_ADDR, 0); 190 191 flags0 = PIPE_CONTROL0_HDC_PIPELINE_FLUSH; 192 flags1 = (PIPE_CONTROL_TILE_CACHE_FLUSH | 193 PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH | 194 PIPE_CONTROL_DEPTH_CACHE_FLUSH | 195 PIPE_CONTROL_DC_FLUSH_ENABLE | 196 PIPE_CONTROL_FLUSH_ENABLE); 197 198 if (XE_GT_WA(gt, 1409600907)) 199 flags1 |= PIPE_CONTROL_DEPTH_STALL; 200 201 if (lacks_render) 202 flags1 &= ~PIPE_CONTROL_3D_ARCH_FLAGS; 203 else if (job->q->class == XE_ENGINE_CLASS_COMPUTE) 204 flags1 &= ~PIPE_CONTROL_3D_ENGINE_FLAGS; 205 206 if (xe_exec_queue_is_multi_queue(q)) 207 flags0 |= PIPE_CONTROL0_QUEUE_DRAIN_MODE; 208 else 209 flags1 |= PIPE_CONTROL_CS_STALL; 210 211 return emit_pipe_control(dw, i, flags0, flags1, 0, 0); 212 } 213 214 static int emit_pipe_control_to_ring_end(struct xe_exec_queue *q, u32 *dw, int i) 215 { 216 struct xe_hw_engine *hwe = q->hwe; 217 218 if (hwe->class != XE_ENGINE_CLASS_RENDER) 219 return i; 220 221 xe_gt_assert(q->gt, !xe_exec_queue_is_multi_queue(q)); 222 223 if (XE_GT_WA(hwe->gt, 16020292621)) 224 i = emit_pipe_control(dw, i, 0, PIPE_CONTROL_LRI_POST_SYNC, 225 RING_NOPID(hwe->mmio_base).addr, 0); 226 227 return i; 228 } 229 230 static int emit_pipe_imm_ggtt(struct xe_exec_queue *q, u32 addr, u32 value, 231 bool stall_only, u32 *dw, int i) 232 { 233 u32 flags0 = 0, flags1 = PIPE_CONTROL_GLOBAL_GTT_IVB | PIPE_CONTROL_QW_WRITE; 234 235 if (!stall_only) 236 flags1 |= PIPE_CONTROL_FLUSH_ENABLE; 237 238 if (xe_exec_queue_is_multi_queue(q)) 239 flags0 |= PIPE_CONTROL0_QUEUE_DRAIN_MODE; 240 else 241 flags1 |= PIPE_CONTROL_CS_STALL; 242 243 return emit_pipe_control(dw, i, flags0, flags1, addr, value); 244 } 245 246 static u32 get_ppgtt_flag(struct xe_sched_job *job) 247 { 248 if (job->q->vm && !job->ggtt) 249 return BIT(8); 250 251 return 0; 252 } 253 254 static int emit_copy_timestamp(struct xe_lrc *lrc, u32 *dw, int i) 255 { 256 dw[i++] = MI_STORE_REGISTER_MEM | MI_SRM_USE_GGTT | MI_SRM_ADD_CS_OFFSET; 257 dw[i++] = RING_CTX_TIMESTAMP(0).addr; 258 dw[i++] = xe_lrc_ctx_job_timestamp_ggtt_addr(lrc); 259 dw[i++] = 0; 260 261 return i; 262 } 263 264 /* for engines that don't require any special HW handling (no EUs, no aux inval, etc) */ 265 static void __emit_job_gen12_simple(struct xe_sched_job *job, struct xe_lrc *lrc, 266 u64 batch_addr, u32 *head, u32 seqno) 267 { 268 u32 dw[MAX_JOB_SIZE_DW], i = 0; 269 u32 ppgtt_flag = get_ppgtt_flag(job); 270 struct xe_gt *gt = job->q->gt; 271 272 *head = lrc->ring.tail; 273 274 i = emit_copy_timestamp(lrc, dw, i); 275 276 if (job->ring_ops_flush_tlb) { 277 dw[i++] = preparser_disable(true); 278 i = emit_flush_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc), 279 seqno, MI_INVALIDATE_TLB, dw, i); 280 dw[i++] = preparser_disable(false); 281 } else { 282 i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc), 283 seqno, dw, i); 284 } 285 286 i = emit_bb_start(batch_addr, ppgtt_flag, dw, i); 287 288 if (job->user_fence.used) { 289 i = emit_flush_dw(dw, i); 290 i = emit_store_imm_ppgtt_posted(job->user_fence.addr, 291 job->user_fence.value, 292 dw, i); 293 } 294 295 i = emit_flush_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, 0, dw, i); 296 297 i = emit_user_interrupt(dw, i); 298 299 xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW); 300 301 xe_lrc_write_ring(lrc, dw, i * sizeof(*dw)); 302 } 303 304 static bool has_aux_ccs(struct xe_device *xe) 305 { 306 /* 307 * PVC is a special case that has no compression of either type 308 * (FlatCCS or AuxCCS). Also, AuxCCS is no longer used from Xe2 309 * onward, so any future platforms with no FlatCCS will not have 310 * AuxCCS either. 311 */ 312 if (GRAPHICS_VER(xe) >= 20 || xe->info.platform == XE_PVC) 313 return false; 314 315 return !xe->info.has_flat_ccs; 316 } 317 318 static void __emit_job_gen12_video(struct xe_sched_job *job, struct xe_lrc *lrc, 319 u64 batch_addr, u32 *head, u32 seqno) 320 { 321 u32 dw[MAX_JOB_SIZE_DW], i = 0; 322 u32 ppgtt_flag = get_ppgtt_flag(job); 323 struct xe_gt *gt = job->q->gt; 324 struct xe_device *xe = gt_to_xe(gt); 325 bool decode = job->q->class == XE_ENGINE_CLASS_VIDEO_DECODE; 326 327 *head = lrc->ring.tail; 328 329 i = emit_copy_timestamp(lrc, dw, i); 330 331 dw[i++] = preparser_disable(true); 332 333 /* hsdes: 1809175790 */ 334 if (has_aux_ccs(xe)) { 335 if (decode) 336 i = emit_aux_table_inv(gt, VD0_AUX_INV, dw, i); 337 else 338 i = emit_aux_table_inv(gt, VE0_AUX_INV, dw, i); 339 } 340 341 if (job->ring_ops_flush_tlb) 342 i = emit_flush_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc), 343 seqno, MI_INVALIDATE_TLB, dw, i); 344 345 dw[i++] = preparser_disable(false); 346 347 if (!job->ring_ops_flush_tlb) 348 i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc), 349 seqno, dw, i); 350 351 i = emit_bb_start(batch_addr, ppgtt_flag, dw, i); 352 353 if (job->user_fence.used) { 354 i = emit_flush_dw(dw, i); 355 i = emit_store_imm_ppgtt_posted(job->user_fence.addr, 356 job->user_fence.value, 357 dw, i); 358 } 359 360 i = emit_flush_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, 0, dw, i); 361 362 i = emit_user_interrupt(dw, i); 363 364 xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW); 365 366 xe_lrc_write_ring(lrc, dw, i * sizeof(*dw)); 367 } 368 369 static void __emit_job_gen12_render_compute(struct xe_sched_job *job, 370 struct xe_lrc *lrc, 371 u64 batch_addr, u32 *head, 372 u32 seqno) 373 { 374 u32 dw[MAX_JOB_SIZE_DW], i = 0; 375 u32 ppgtt_flag = get_ppgtt_flag(job); 376 struct xe_gt *gt = job->q->gt; 377 struct xe_device *xe = gt_to_xe(gt); 378 bool lacks_render = !(gt->info.engine_mask & XE_HW_ENGINE_RCS_MASK); 379 u32 mask_flags = 0; 380 381 *head = lrc->ring.tail; 382 383 i = emit_copy_timestamp(lrc, dw, i); 384 385 dw[i++] = preparser_disable(true); 386 if (lacks_render) 387 mask_flags = PIPE_CONTROL_3D_ARCH_FLAGS; 388 else if (job->q->class == XE_ENGINE_CLASS_COMPUTE) 389 mask_flags = PIPE_CONTROL_3D_ENGINE_FLAGS; 390 391 /* See __xe_pt_bind_vma() for a discussion on TLB invalidations. */ 392 i = emit_pipe_invalidate(job->q, mask_flags, job->ring_ops_flush_tlb, dw, i); 393 394 /* hsdes: 1809175790 */ 395 if (has_aux_ccs(xe)) 396 i = emit_aux_table_inv(gt, CCS_AUX_INV, dw, i); 397 398 dw[i++] = preparser_disable(false); 399 400 i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc), 401 seqno, dw, i); 402 403 i = emit_bb_start(batch_addr, ppgtt_flag, dw, i); 404 405 i = emit_render_cache_flush(job, dw, i); 406 407 if (job->user_fence.used) 408 i = emit_store_imm_ppgtt_posted(job->user_fence.addr, 409 job->user_fence.value, 410 dw, i); 411 412 i = emit_pipe_imm_ggtt(job->q, xe_lrc_seqno_ggtt_addr(lrc), seqno, lacks_render, dw, i); 413 414 i = emit_user_interrupt(dw, i); 415 416 i = emit_pipe_control_to_ring_end(job->q, dw, i); 417 418 xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW); 419 420 xe_lrc_write_ring(lrc, dw, i * sizeof(*dw)); 421 } 422 423 static void emit_migration_job_gen12(struct xe_sched_job *job, 424 struct xe_lrc *lrc, u32 *head, 425 u32 seqno) 426 { 427 u32 saddr = xe_lrc_start_seqno_ggtt_addr(lrc); 428 u32 dw[MAX_JOB_SIZE_DW], i = 0; 429 430 *head = lrc->ring.tail; 431 432 i = emit_copy_timestamp(lrc, dw, i); 433 434 i = emit_store_imm_ggtt(saddr, seqno, dw, i); 435 436 dw[i++] = MI_ARB_ON_OFF | MI_ARB_DISABLE; /* Enabled again below */ 437 438 i = emit_bb_start(job->ptrs[0].batch_addr, BIT(8), dw, i); 439 440 dw[i++] = preparser_disable(true); 441 i = emit_flush_invalidate(saddr, seqno, job->migrate_flush_flags, dw, i); 442 dw[i++] = preparser_disable(false); 443 444 i = emit_bb_start(job->ptrs[1].batch_addr, BIT(8), dw, i); 445 446 i = emit_flush_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, 447 job->migrate_flush_flags, 448 dw, i); 449 450 i = emit_user_interrupt(dw, i); 451 452 xe_gt_assert(job->q->gt, i <= MAX_JOB_SIZE_DW); 453 454 xe_lrc_write_ring(lrc, dw, i * sizeof(*dw)); 455 } 456 457 static void emit_job_gen12_gsc(struct xe_sched_job *job) 458 { 459 struct xe_gt *gt = job->q->gt; 460 461 xe_gt_assert(gt, job->q->width <= 1); /* no parallel submission for GSCCS */ 462 463 __emit_job_gen12_simple(job, job->q->lrc[0], 464 job->ptrs[0].batch_addr, 465 &job->ptrs[0].head, 466 xe_sched_job_lrc_seqno(job)); 467 } 468 469 static void emit_job_gen12_copy(struct xe_sched_job *job) 470 { 471 int i; 472 473 if (xe_sched_job_is_migration(job->q)) { 474 emit_migration_job_gen12(job, job->q->lrc[0], 475 &job->ptrs[0].head, 476 xe_sched_job_lrc_seqno(job)); 477 return; 478 } 479 480 for (i = 0; i < job->q->width; ++i) 481 __emit_job_gen12_simple(job, job->q->lrc[i], 482 job->ptrs[i].batch_addr, 483 &job->ptrs[i].head, 484 xe_sched_job_lrc_seqno(job)); 485 } 486 487 static void emit_job_gen12_video(struct xe_sched_job *job) 488 { 489 int i; 490 491 /* FIXME: Not doing parallel handshake for now */ 492 for (i = 0; i < job->q->width; ++i) 493 __emit_job_gen12_video(job, job->q->lrc[i], 494 job->ptrs[i].batch_addr, 495 &job->ptrs[i].head, 496 xe_sched_job_lrc_seqno(job)); 497 } 498 499 static void emit_job_gen12_render_compute(struct xe_sched_job *job) 500 { 501 int i; 502 503 for (i = 0; i < job->q->width; ++i) 504 __emit_job_gen12_render_compute(job, job->q->lrc[i], 505 job->ptrs[i].batch_addr, 506 &job->ptrs[i].head, 507 xe_sched_job_lrc_seqno(job)); 508 } 509 510 static const struct xe_ring_ops ring_ops_gen12_gsc = { 511 .emit_job = emit_job_gen12_gsc, 512 }; 513 514 static const struct xe_ring_ops ring_ops_gen12_copy = { 515 .emit_job = emit_job_gen12_copy, 516 }; 517 518 static const struct xe_ring_ops ring_ops_gen12_video = { 519 .emit_job = emit_job_gen12_video, 520 }; 521 522 static const struct xe_ring_ops ring_ops_gen12_render_compute = { 523 .emit_job = emit_job_gen12_render_compute, 524 }; 525 526 const struct xe_ring_ops * 527 xe_ring_ops_get(struct xe_gt *gt, enum xe_engine_class class) 528 { 529 switch (class) { 530 case XE_ENGINE_CLASS_OTHER: 531 return &ring_ops_gen12_gsc; 532 case XE_ENGINE_CLASS_COPY: 533 return &ring_ops_gen12_copy; 534 case XE_ENGINE_CLASS_VIDEO_DECODE: 535 case XE_ENGINE_CLASS_VIDEO_ENHANCE: 536 return &ring_ops_gen12_video; 537 case XE_ENGINE_CLASS_RENDER: 538 case XE_ENGINE_CLASS_COMPUTE: 539 return &ring_ops_gen12_render_compute; 540 default: 541 return NULL; 542 } 543 } 544