1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include "xe_ring_ops.h" 7 8 #include <generated/xe_wa_oob.h> 9 10 #include "instructions/xe_gpu_commands.h" 11 #include "instructions/xe_mi_commands.h" 12 #include "regs/xe_engine_regs.h" 13 #include "regs/xe_gt_regs.h" 14 #include "regs/xe_lrc_layout.h" 15 #include "xe_exec_queue_types.h" 16 #include "xe_gt.h" 17 #include "xe_lrc.h" 18 #include "xe_macros.h" 19 #include "xe_sched_job.h" 20 #include "xe_sriov.h" 21 #include "xe_vm_types.h" 22 #include "xe_vm.h" 23 #include "xe_wa.h" 24 25 /* 26 * 3D-related flags that can't be set on _engines_ that lack access to the 3D 27 * pipeline (i.e., CCS engines). 28 */ 29 #define PIPE_CONTROL_3D_ENGINE_FLAGS (\ 30 PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH | \ 31 PIPE_CONTROL_DEPTH_CACHE_FLUSH | \ 32 PIPE_CONTROL_TILE_CACHE_FLUSH | \ 33 PIPE_CONTROL_DEPTH_STALL | \ 34 PIPE_CONTROL_STALL_AT_SCOREBOARD | \ 35 PIPE_CONTROL_PSD_SYNC | \ 36 PIPE_CONTROL_AMFS_FLUSH | \ 37 PIPE_CONTROL_VF_CACHE_INVALIDATE | \ 38 PIPE_CONTROL_GLOBAL_SNAPSHOT_RESET) 39 40 /* 3D-related flags that can't be set on _platforms_ that lack a 3D pipeline */ 41 #define PIPE_CONTROL_3D_ARCH_FLAGS ( \ 42 PIPE_CONTROL_3D_ENGINE_FLAGS | \ 43 PIPE_CONTROL_INDIRECT_STATE_DISABLE | \ 44 PIPE_CONTROL_FLUSH_ENABLE | \ 45 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE | \ 46 PIPE_CONTROL_DC_FLUSH_ENABLE) 47 48 static u32 preparser_disable(bool state) 49 { 50 return MI_ARB_CHECK | BIT(8) | state; 51 } 52 53 static int emit_aux_table_inv(struct xe_gt *gt, struct xe_reg reg, 54 u32 *dw, int i) 55 { 56 dw[i++] = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(1) | MI_LRI_MMIO_REMAP_EN; 57 dw[i++] = reg.addr + gt->mmio.adj_offset; 58 dw[i++] = AUX_INV; 59 dw[i++] = MI_NOOP; 60 61 return i; 62 } 63 64 static int emit_user_interrupt(u32 *dw, int i) 65 { 66 dw[i++] = MI_USER_INTERRUPT; 67 dw[i++] = MI_ARB_ON_OFF | MI_ARB_ENABLE; 68 dw[i++] = MI_ARB_CHECK; 69 70 return i; 71 } 72 73 static int emit_store_imm_ggtt(u32 addr, u32 value, u32 *dw, int i) 74 { 75 dw[i++] = MI_STORE_DATA_IMM | MI_SDI_GGTT | MI_SDI_NUM_DW(1); 76 dw[i++] = addr; 77 dw[i++] = 0; 78 dw[i++] = value; 79 80 return i; 81 } 82 83 static int emit_flush_dw(u32 *dw, int i) 84 { 85 dw[i++] = MI_FLUSH_DW | MI_FLUSH_IMM_DW; 86 dw[i++] = 0; 87 dw[i++] = 0; 88 dw[i++] = 0; 89 90 return i; 91 } 92 93 static int emit_flush_imm_ggtt(u32 addr, u32 value, bool invalidate_tlb, 94 u32 *dw, int i) 95 { 96 dw[i++] = MI_FLUSH_DW | MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_IMM_DW | 97 (invalidate_tlb ? MI_INVALIDATE_TLB : 0); 98 dw[i++] = addr | MI_FLUSH_DW_USE_GTT; 99 dw[i++] = 0; 100 dw[i++] = value; 101 102 return i; 103 } 104 105 static int emit_bb_start(u64 batch_addr, u32 ppgtt_flag, u32 *dw, int i) 106 { 107 dw[i++] = MI_BATCH_BUFFER_START | ppgtt_flag | XE_INSTR_NUM_DW(3); 108 dw[i++] = lower_32_bits(batch_addr); 109 dw[i++] = upper_32_bits(batch_addr); 110 111 return i; 112 } 113 114 static int emit_flush_invalidate(u32 flag, u32 *dw, int i) 115 { 116 dw[i] = MI_FLUSH_DW; 117 dw[i] |= flag; 118 dw[i++] |= MI_INVALIDATE_TLB | MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_IMM_DW | 119 MI_FLUSH_DW_STORE_INDEX; 120 121 dw[i++] = LRC_PPHWSP_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT; 122 dw[i++] = 0; 123 dw[i++] = ~0U; 124 125 return i; 126 } 127 128 static int 129 emit_pipe_control(u32 *dw, int i, u32 bit_group_0, u32 bit_group_1, u32 offset, u32 value) 130 { 131 dw[i++] = GFX_OP_PIPE_CONTROL(6) | bit_group_0; 132 dw[i++] = bit_group_1; 133 dw[i++] = offset; 134 dw[i++] = 0; 135 dw[i++] = value; 136 dw[i++] = 0; 137 138 return i; 139 } 140 141 static int emit_pipe_invalidate(u32 mask_flags, bool invalidate_tlb, u32 *dw, 142 int i) 143 { 144 u32 flags = PIPE_CONTROL_CS_STALL | 145 PIPE_CONTROL_COMMAND_CACHE_INVALIDATE | 146 PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE | 147 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE | 148 PIPE_CONTROL_VF_CACHE_INVALIDATE | 149 PIPE_CONTROL_CONST_CACHE_INVALIDATE | 150 PIPE_CONTROL_STATE_CACHE_INVALIDATE | 151 PIPE_CONTROL_QW_WRITE | 152 PIPE_CONTROL_STORE_DATA_INDEX; 153 154 if (invalidate_tlb) 155 flags |= PIPE_CONTROL_TLB_INVALIDATE; 156 157 flags &= ~mask_flags; 158 159 return emit_pipe_control(dw, i, 0, flags, LRC_PPHWSP_SCRATCH_ADDR, 0); 160 } 161 162 static int emit_store_imm_ppgtt_posted(u64 addr, u64 value, 163 u32 *dw, int i) 164 { 165 dw[i++] = MI_STORE_DATA_IMM | MI_SDI_NUM_QW(1); 166 dw[i++] = lower_32_bits(addr); 167 dw[i++] = upper_32_bits(addr); 168 dw[i++] = lower_32_bits(value); 169 dw[i++] = upper_32_bits(value); 170 171 return i; 172 } 173 174 static int emit_render_cache_flush(struct xe_sched_job *job, u32 *dw, int i) 175 { 176 struct xe_gt *gt = job->q->gt; 177 bool lacks_render = !(gt->info.engine_mask & XE_HW_ENGINE_RCS_MASK); 178 u32 flags; 179 180 flags = (PIPE_CONTROL_CS_STALL | 181 PIPE_CONTROL_TILE_CACHE_FLUSH | 182 PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH | 183 PIPE_CONTROL_DEPTH_CACHE_FLUSH | 184 PIPE_CONTROL_DC_FLUSH_ENABLE | 185 PIPE_CONTROL_FLUSH_ENABLE); 186 187 if (XE_WA(gt, 1409600907)) 188 flags |= PIPE_CONTROL_DEPTH_STALL; 189 190 if (lacks_render) 191 flags &= ~PIPE_CONTROL_3D_ARCH_FLAGS; 192 else if (job->q->class == XE_ENGINE_CLASS_COMPUTE) 193 flags &= ~PIPE_CONTROL_3D_ENGINE_FLAGS; 194 195 return emit_pipe_control(dw, i, PIPE_CONTROL0_HDC_PIPELINE_FLUSH, flags, 0, 0); 196 } 197 198 static int emit_pipe_control_to_ring_end(struct xe_hw_engine *hwe, u32 *dw, int i) 199 { 200 if (hwe->class != XE_ENGINE_CLASS_RENDER) 201 return i; 202 203 if (XE_WA(hwe->gt, 16020292621)) 204 i = emit_pipe_control(dw, i, 0, PIPE_CONTROL_LRI_POST_SYNC, 205 RING_NOPID(hwe->mmio_base).addr, 0); 206 207 return i; 208 } 209 210 static int emit_pipe_imm_ggtt(u32 addr, u32 value, bool stall_only, u32 *dw, 211 int i) 212 { 213 u32 flags = PIPE_CONTROL_CS_STALL | PIPE_CONTROL_GLOBAL_GTT_IVB | 214 PIPE_CONTROL_QW_WRITE; 215 216 if (!stall_only) 217 flags |= PIPE_CONTROL_FLUSH_ENABLE; 218 219 return emit_pipe_control(dw, i, 0, flags, addr, value); 220 } 221 222 static u32 get_ppgtt_flag(struct xe_sched_job *job) 223 { 224 return job->q->vm ? BIT(8) : 0; 225 } 226 227 static int emit_copy_timestamp(struct xe_lrc *lrc, u32 *dw, int i) 228 { 229 dw[i++] = MI_COPY_MEM_MEM | MI_COPY_MEM_MEM_SRC_GGTT | 230 MI_COPY_MEM_MEM_DST_GGTT; 231 dw[i++] = xe_lrc_ctx_job_timestamp_ggtt_addr(lrc); 232 dw[i++] = 0; 233 dw[i++] = xe_lrc_ctx_timestamp_ggtt_addr(lrc); 234 dw[i++] = 0; 235 dw[i++] = MI_NOOP; 236 237 return i; 238 } 239 240 /* for engines that don't require any special HW handling (no EUs, no aux inval, etc) */ 241 static void __emit_job_gen12_simple(struct xe_sched_job *job, struct xe_lrc *lrc, 242 u64 batch_addr, u32 seqno) 243 { 244 u32 dw[MAX_JOB_SIZE_DW], i = 0; 245 u32 ppgtt_flag = get_ppgtt_flag(job); 246 struct xe_gt *gt = job->q->gt; 247 248 i = emit_copy_timestamp(lrc, dw, i); 249 250 if (job->ring_ops_flush_tlb) { 251 dw[i++] = preparser_disable(true); 252 i = emit_flush_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc), 253 seqno, true, dw, i); 254 dw[i++] = preparser_disable(false); 255 } else { 256 i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc), 257 seqno, dw, i); 258 } 259 260 i = emit_bb_start(batch_addr, ppgtt_flag, dw, i); 261 262 if (job->user_fence.used) { 263 i = emit_flush_dw(dw, i); 264 i = emit_store_imm_ppgtt_posted(job->user_fence.addr, 265 job->user_fence.value, 266 dw, i); 267 } 268 269 i = emit_flush_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, false, dw, i); 270 271 i = emit_user_interrupt(dw, i); 272 273 xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW); 274 275 xe_lrc_write_ring(lrc, dw, i * sizeof(*dw)); 276 } 277 278 static bool has_aux_ccs(struct xe_device *xe) 279 { 280 /* 281 * PVC is a special case that has no compression of either type 282 * (FlatCCS or AuxCCS). Also, AuxCCS is no longer used from Xe2 283 * onward, so any future platforms with no FlatCCS will not have 284 * AuxCCS either. 285 */ 286 if (GRAPHICS_VER(xe) >= 20 || xe->info.platform == XE_PVC) 287 return false; 288 289 return !xe->info.has_flat_ccs; 290 } 291 292 static void __emit_job_gen12_video(struct xe_sched_job *job, struct xe_lrc *lrc, 293 u64 batch_addr, u32 seqno) 294 { 295 u32 dw[MAX_JOB_SIZE_DW], i = 0; 296 u32 ppgtt_flag = get_ppgtt_flag(job); 297 struct xe_gt *gt = job->q->gt; 298 struct xe_device *xe = gt_to_xe(gt); 299 bool decode = job->q->class == XE_ENGINE_CLASS_VIDEO_DECODE; 300 301 i = emit_copy_timestamp(lrc, dw, i); 302 303 dw[i++] = preparser_disable(true); 304 305 /* hsdes: 1809175790 */ 306 if (has_aux_ccs(xe)) { 307 if (decode) 308 i = emit_aux_table_inv(gt, VD0_AUX_INV, dw, i); 309 else 310 i = emit_aux_table_inv(gt, VE0_AUX_INV, dw, i); 311 } 312 313 if (job->ring_ops_flush_tlb) 314 i = emit_flush_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc), 315 seqno, true, dw, i); 316 317 dw[i++] = preparser_disable(false); 318 319 if (!job->ring_ops_flush_tlb) 320 i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc), 321 seqno, dw, i); 322 323 i = emit_bb_start(batch_addr, ppgtt_flag, dw, i); 324 325 if (job->user_fence.used) { 326 i = emit_flush_dw(dw, i); 327 i = emit_store_imm_ppgtt_posted(job->user_fence.addr, 328 job->user_fence.value, 329 dw, i); 330 } 331 332 i = emit_flush_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, false, dw, i); 333 334 i = emit_user_interrupt(dw, i); 335 336 xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW); 337 338 xe_lrc_write_ring(lrc, dw, i * sizeof(*dw)); 339 } 340 341 static void __emit_job_gen12_render_compute(struct xe_sched_job *job, 342 struct xe_lrc *lrc, 343 u64 batch_addr, u32 seqno) 344 { 345 u32 dw[MAX_JOB_SIZE_DW], i = 0; 346 u32 ppgtt_flag = get_ppgtt_flag(job); 347 struct xe_gt *gt = job->q->gt; 348 struct xe_device *xe = gt_to_xe(gt); 349 bool lacks_render = !(gt->info.engine_mask & XE_HW_ENGINE_RCS_MASK); 350 u32 mask_flags = 0; 351 352 i = emit_copy_timestamp(lrc, dw, i); 353 354 dw[i++] = preparser_disable(true); 355 if (lacks_render) 356 mask_flags = PIPE_CONTROL_3D_ARCH_FLAGS; 357 else if (job->q->class == XE_ENGINE_CLASS_COMPUTE) 358 mask_flags = PIPE_CONTROL_3D_ENGINE_FLAGS; 359 360 /* See __xe_pt_bind_vma() for a discussion on TLB invalidations. */ 361 i = emit_pipe_invalidate(mask_flags, job->ring_ops_flush_tlb, dw, i); 362 363 /* hsdes: 1809175790 */ 364 if (has_aux_ccs(xe)) 365 i = emit_aux_table_inv(gt, CCS_AUX_INV, dw, i); 366 367 dw[i++] = preparser_disable(false); 368 369 i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc), 370 seqno, dw, i); 371 372 i = emit_bb_start(batch_addr, ppgtt_flag, dw, i); 373 374 i = emit_render_cache_flush(job, dw, i); 375 376 if (job->user_fence.used) 377 i = emit_store_imm_ppgtt_posted(job->user_fence.addr, 378 job->user_fence.value, 379 dw, i); 380 381 i = emit_pipe_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, lacks_render, dw, i); 382 383 i = emit_user_interrupt(dw, i); 384 385 i = emit_pipe_control_to_ring_end(job->q->hwe, dw, i); 386 387 xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW); 388 389 xe_lrc_write_ring(lrc, dw, i * sizeof(*dw)); 390 } 391 392 static void emit_migration_job_gen12(struct xe_sched_job *job, 393 struct xe_lrc *lrc, u32 seqno) 394 { 395 u32 dw[MAX_JOB_SIZE_DW], i = 0; 396 397 i = emit_copy_timestamp(lrc, dw, i); 398 399 i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc), 400 seqno, dw, i); 401 402 dw[i++] = MI_ARB_ON_OFF | MI_ARB_DISABLE; /* Enabled again below */ 403 404 i = emit_bb_start(job->ptrs[0].batch_addr, BIT(8), dw, i); 405 406 if (!IS_SRIOV_VF(gt_to_xe(job->q->gt))) { 407 /* XXX: Do we need this? Leaving for now. */ 408 dw[i++] = preparser_disable(true); 409 i = emit_flush_invalidate(0, dw, i); 410 dw[i++] = preparser_disable(false); 411 } 412 413 i = emit_bb_start(job->ptrs[1].batch_addr, BIT(8), dw, i); 414 415 dw[i++] = MI_FLUSH_DW | MI_INVALIDATE_TLB | job->migrate_flush_flags | 416 MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_IMM_DW; 417 dw[i++] = xe_lrc_seqno_ggtt_addr(lrc) | MI_FLUSH_DW_USE_GTT; 418 dw[i++] = 0; 419 dw[i++] = seqno; /* value */ 420 421 i = emit_user_interrupt(dw, i); 422 423 xe_gt_assert(job->q->gt, i <= MAX_JOB_SIZE_DW); 424 425 xe_lrc_write_ring(lrc, dw, i * sizeof(*dw)); 426 } 427 428 static void emit_job_gen12_gsc(struct xe_sched_job *job) 429 { 430 struct xe_gt *gt = job->q->gt; 431 432 xe_gt_assert(gt, job->q->width <= 1); /* no parallel submission for GSCCS */ 433 434 __emit_job_gen12_simple(job, job->q->lrc[0], 435 job->ptrs[0].batch_addr, 436 xe_sched_job_lrc_seqno(job)); 437 } 438 439 static void emit_job_gen12_copy(struct xe_sched_job *job) 440 { 441 int i; 442 443 if (xe_sched_job_is_migration(job->q)) { 444 emit_migration_job_gen12(job, job->q->lrc[0], 445 xe_sched_job_lrc_seqno(job)); 446 return; 447 } 448 449 for (i = 0; i < job->q->width; ++i) 450 __emit_job_gen12_simple(job, job->q->lrc[i], 451 job->ptrs[i].batch_addr, 452 xe_sched_job_lrc_seqno(job)); 453 } 454 455 static void emit_job_gen12_video(struct xe_sched_job *job) 456 { 457 int i; 458 459 /* FIXME: Not doing parallel handshake for now */ 460 for (i = 0; i < job->q->width; ++i) 461 __emit_job_gen12_video(job, job->q->lrc[i], 462 job->ptrs[i].batch_addr, 463 xe_sched_job_lrc_seqno(job)); 464 } 465 466 static void emit_job_gen12_render_compute(struct xe_sched_job *job) 467 { 468 int i; 469 470 for (i = 0; i < job->q->width; ++i) 471 __emit_job_gen12_render_compute(job, job->q->lrc[i], 472 job->ptrs[i].batch_addr, 473 xe_sched_job_lrc_seqno(job)); 474 } 475 476 static const struct xe_ring_ops ring_ops_gen12_gsc = { 477 .emit_job = emit_job_gen12_gsc, 478 }; 479 480 static const struct xe_ring_ops ring_ops_gen12_copy = { 481 .emit_job = emit_job_gen12_copy, 482 }; 483 484 static const struct xe_ring_ops ring_ops_gen12_video = { 485 .emit_job = emit_job_gen12_video, 486 }; 487 488 static const struct xe_ring_ops ring_ops_gen12_render_compute = { 489 .emit_job = emit_job_gen12_render_compute, 490 }; 491 492 const struct xe_ring_ops * 493 xe_ring_ops_get(struct xe_gt *gt, enum xe_engine_class class) 494 { 495 switch (class) { 496 case XE_ENGINE_CLASS_OTHER: 497 return &ring_ops_gen12_gsc; 498 case XE_ENGINE_CLASS_COPY: 499 return &ring_ops_gen12_copy; 500 case XE_ENGINE_CLASS_VIDEO_DECODE: 501 case XE_ENGINE_CLASS_VIDEO_ENHANCE: 502 return &ring_ops_gen12_video; 503 case XE_ENGINE_CLASS_RENDER: 504 case XE_ENGINE_CLASS_COMPUTE: 505 return &ring_ops_gen12_render_compute; 506 default: 507 return NULL; 508 } 509 } 510