1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include "xe_ring_ops.h" 7 8 #include "generated/xe_wa_oob.h" 9 #include "instructions/xe_mi_commands.h" 10 #include "regs/xe_gpu_commands.h" 11 #include "regs/xe_gt_regs.h" 12 #include "regs/xe_lrc_layout.h" 13 #include "regs/xe_regs.h" 14 #include "xe_exec_queue_types.h" 15 #include "xe_gt.h" 16 #include "xe_lrc.h" 17 #include "xe_macros.h" 18 #include "xe_sched_job.h" 19 #include "xe_vm_types.h" 20 #include "xe_vm.h" 21 #include "xe_wa.h" 22 23 /* 24 * 3D-related flags that can't be set on _engines_ that lack access to the 3D 25 * pipeline (i.e., CCS engines). 26 */ 27 #define PIPE_CONTROL_3D_ENGINE_FLAGS (\ 28 PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH | \ 29 PIPE_CONTROL_DEPTH_CACHE_FLUSH | \ 30 PIPE_CONTROL_TILE_CACHE_FLUSH | \ 31 PIPE_CONTROL_DEPTH_STALL | \ 32 PIPE_CONTROL_STALL_AT_SCOREBOARD | \ 33 PIPE_CONTROL_PSD_SYNC | \ 34 PIPE_CONTROL_AMFS_FLUSH | \ 35 PIPE_CONTROL_VF_CACHE_INVALIDATE | \ 36 PIPE_CONTROL_GLOBAL_SNAPSHOT_RESET) 37 38 /* 3D-related flags that can't be set on _platforms_ that lack a 3D pipeline */ 39 #define PIPE_CONTROL_3D_ARCH_FLAGS ( \ 40 PIPE_CONTROL_3D_ENGINE_FLAGS | \ 41 PIPE_CONTROL_INDIRECT_STATE_DISABLE | \ 42 PIPE_CONTROL_FLUSH_ENABLE | \ 43 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE | \ 44 PIPE_CONTROL_DC_FLUSH_ENABLE) 45 46 static u32 preparser_disable(bool state) 47 { 48 return MI_ARB_CHECK | BIT(8) | state; 49 } 50 51 static int emit_aux_table_inv(struct xe_gt *gt, struct xe_reg reg, 52 u32 *dw, int i) 53 { 54 dw[i++] = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(1) | MI_LRI_MMIO_REMAP_EN; 55 dw[i++] = reg.addr + gt->mmio.adj_offset; 56 dw[i++] = AUX_INV; 57 dw[i++] = MI_NOOP; 58 59 return i; 60 } 61 62 static int emit_user_interrupt(u32 *dw, int i) 63 { 64 dw[i++] = MI_USER_INTERRUPT; 65 dw[i++] = MI_ARB_ON_OFF | MI_ARB_ENABLE; 66 dw[i++] = MI_ARB_CHECK; 67 68 return i; 69 } 70 71 static int emit_store_imm_ggtt(u32 addr, u32 value, u32 *dw, int i) 72 { 73 dw[i++] = MI_STORE_DATA_IMM | MI_SDI_GGTT | MI_SDI_NUM_DW(1); 74 dw[i++] = addr; 75 dw[i++] = 0; 76 dw[i++] = value; 77 78 return i; 79 } 80 81 static int emit_flush_imm_ggtt(u32 addr, u32 value, bool invalidate_tlb, 82 u32 *dw, int i) 83 { 84 dw[i++] = MI_FLUSH_DW | MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_IMM_DW | 85 (invalidate_tlb ? MI_INVALIDATE_TLB : 0); 86 dw[i++] = addr | MI_FLUSH_DW_USE_GTT; 87 dw[i++] = 0; 88 dw[i++] = value; 89 90 return i; 91 } 92 93 static int emit_bb_start(u64 batch_addr, u32 ppgtt_flag, u32 *dw, int i) 94 { 95 dw[i++] = MI_BATCH_BUFFER_START | ppgtt_flag | XE_INSTR_NUM_DW(3); 96 dw[i++] = lower_32_bits(batch_addr); 97 dw[i++] = upper_32_bits(batch_addr); 98 99 return i; 100 } 101 102 static int emit_flush_invalidate(u32 flag, u32 *dw, int i) 103 { 104 dw[i] = MI_FLUSH_DW; 105 dw[i] |= flag; 106 dw[i++] |= MI_INVALIDATE_TLB | MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_IMM_DW | 107 MI_FLUSH_DW_STORE_INDEX; 108 109 dw[i++] = LRC_PPHWSP_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT; 110 dw[i++] = 0; 111 dw[i++] = ~0U; 112 113 return i; 114 } 115 116 static int emit_pipe_invalidate(u32 mask_flags, bool invalidate_tlb, u32 *dw, 117 int i) 118 { 119 u32 flags = PIPE_CONTROL_CS_STALL | 120 PIPE_CONTROL_COMMAND_CACHE_INVALIDATE | 121 PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE | 122 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE | 123 PIPE_CONTROL_VF_CACHE_INVALIDATE | 124 PIPE_CONTROL_CONST_CACHE_INVALIDATE | 125 PIPE_CONTROL_STATE_CACHE_INVALIDATE | 126 PIPE_CONTROL_QW_WRITE | 127 PIPE_CONTROL_STORE_DATA_INDEX; 128 129 if (invalidate_tlb) 130 flags |= PIPE_CONTROL_TLB_INVALIDATE; 131 132 flags &= ~mask_flags; 133 134 dw[i++] = GFX_OP_PIPE_CONTROL(6); 135 dw[i++] = flags; 136 dw[i++] = LRC_PPHWSP_SCRATCH_ADDR; 137 dw[i++] = 0; 138 dw[i++] = 0; 139 dw[i++] = 0; 140 141 return i; 142 } 143 144 static int emit_store_imm_ppgtt_posted(u64 addr, u64 value, 145 u32 *dw, int i) 146 { 147 dw[i++] = MI_STORE_DATA_IMM | MI_SDI_NUM_QW(1); 148 dw[i++] = lower_32_bits(addr); 149 dw[i++] = upper_32_bits(addr); 150 dw[i++] = lower_32_bits(value); 151 dw[i++] = upper_32_bits(value); 152 153 return i; 154 } 155 156 static int emit_render_cache_flush(struct xe_sched_job *job, u32 *dw, int i) 157 { 158 struct xe_gt *gt = job->q->gt; 159 bool lacks_render = !(gt->info.engine_mask & XE_HW_ENGINE_RCS_MASK); 160 u32 flags; 161 162 flags = (PIPE_CONTROL_CS_STALL | 163 PIPE_CONTROL_TILE_CACHE_FLUSH | 164 PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH | 165 PIPE_CONTROL_DEPTH_CACHE_FLUSH | 166 PIPE_CONTROL_DC_FLUSH_ENABLE | 167 PIPE_CONTROL_FLUSH_ENABLE); 168 169 if (XE_WA(gt, 1409600907)) 170 flags |= PIPE_CONTROL_DEPTH_STALL; 171 172 if (lacks_render) 173 flags &= ~PIPE_CONTROL_3D_ARCH_FLAGS; 174 else if (job->q->class == XE_ENGINE_CLASS_COMPUTE) 175 flags &= ~PIPE_CONTROL_3D_ENGINE_FLAGS; 176 177 dw[i++] = GFX_OP_PIPE_CONTROL(6) | PIPE_CONTROL0_HDC_PIPELINE_FLUSH; 178 dw[i++] = flags; 179 dw[i++] = 0; 180 dw[i++] = 0; 181 dw[i++] = 0; 182 dw[i++] = 0; 183 184 return i; 185 } 186 187 static int emit_pipe_imm_ggtt(u32 addr, u32 value, bool stall_only, u32 *dw, 188 int i) 189 { 190 dw[i++] = GFX_OP_PIPE_CONTROL(6); 191 dw[i++] = (stall_only ? PIPE_CONTROL_CS_STALL : 192 PIPE_CONTROL_FLUSH_ENABLE | PIPE_CONTROL_CS_STALL) | 193 PIPE_CONTROL_GLOBAL_GTT_IVB | PIPE_CONTROL_QW_WRITE; 194 dw[i++] = addr; 195 dw[i++] = 0; 196 dw[i++] = value; 197 dw[i++] = 0; /* We're thrashing one extra dword. */ 198 199 return i; 200 } 201 202 static u32 get_ppgtt_flag(struct xe_sched_job *job) 203 { 204 return job->q->vm ? BIT(8) : 0; 205 } 206 207 /* for engines that don't require any special HW handling (no EUs, no aux inval, etc) */ 208 static void __emit_job_gen12_simple(struct xe_sched_job *job, struct xe_lrc *lrc, 209 u64 batch_addr, u32 seqno) 210 { 211 u32 dw[MAX_JOB_SIZE_DW], i = 0; 212 u32 ppgtt_flag = get_ppgtt_flag(job); 213 struct xe_vm *vm = job->q->vm; 214 struct xe_gt *gt = job->q->gt; 215 216 if (vm && vm->batch_invalidate_tlb) { 217 dw[i++] = preparser_disable(true); 218 i = emit_flush_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc), 219 seqno, true, dw, i); 220 dw[i++] = preparser_disable(false); 221 } else { 222 i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc), 223 seqno, dw, i); 224 } 225 226 i = emit_bb_start(batch_addr, ppgtt_flag, dw, i); 227 228 if (job->user_fence.used) 229 i = emit_store_imm_ppgtt_posted(job->user_fence.addr, 230 job->user_fence.value, 231 dw, i); 232 233 i = emit_flush_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, false, dw, i); 234 235 i = emit_user_interrupt(dw, i); 236 237 xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW); 238 239 xe_lrc_write_ring(lrc, dw, i * sizeof(*dw)); 240 } 241 242 static bool has_aux_ccs(struct xe_device *xe) 243 { 244 /* 245 * PVC is a special case that has no compression of either type 246 * (FlatCCS or AuxCCS). Also, AuxCCS is no longer used from Xe2 247 * onward, so any future platforms with no FlatCCS will not have 248 * AuxCCS either. 249 */ 250 if (GRAPHICS_VER(xe) >= 20 || xe->info.platform == XE_PVC) 251 return false; 252 253 return !xe->info.has_flat_ccs; 254 } 255 256 static void __emit_job_gen12_video(struct xe_sched_job *job, struct xe_lrc *lrc, 257 u64 batch_addr, u32 seqno) 258 { 259 u32 dw[MAX_JOB_SIZE_DW], i = 0; 260 u32 ppgtt_flag = get_ppgtt_flag(job); 261 struct xe_gt *gt = job->q->gt; 262 struct xe_device *xe = gt_to_xe(gt); 263 bool decode = job->q->class == XE_ENGINE_CLASS_VIDEO_DECODE; 264 struct xe_vm *vm = job->q->vm; 265 266 dw[i++] = preparser_disable(true); 267 268 /* hsdes: 1809175790 */ 269 if (has_aux_ccs(xe)) { 270 if (decode) 271 i = emit_aux_table_inv(gt, VD0_AUX_INV, dw, i); 272 else 273 i = emit_aux_table_inv(gt, VE0_AUX_INV, dw, i); 274 } 275 276 if (vm && vm->batch_invalidate_tlb) 277 i = emit_flush_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc), 278 seqno, true, dw, i); 279 280 dw[i++] = preparser_disable(false); 281 282 if (!vm || !vm->batch_invalidate_tlb) 283 i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc), 284 seqno, dw, i); 285 286 i = emit_bb_start(batch_addr, ppgtt_flag, dw, i); 287 288 if (job->user_fence.used) 289 i = emit_store_imm_ppgtt_posted(job->user_fence.addr, 290 job->user_fence.value, 291 dw, i); 292 293 i = emit_flush_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, false, dw, i); 294 295 i = emit_user_interrupt(dw, i); 296 297 xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW); 298 299 xe_lrc_write_ring(lrc, dw, i * sizeof(*dw)); 300 } 301 302 static void __emit_job_gen12_render_compute(struct xe_sched_job *job, 303 struct xe_lrc *lrc, 304 u64 batch_addr, u32 seqno) 305 { 306 u32 dw[MAX_JOB_SIZE_DW], i = 0; 307 u32 ppgtt_flag = get_ppgtt_flag(job); 308 struct xe_gt *gt = job->q->gt; 309 struct xe_device *xe = gt_to_xe(gt); 310 bool lacks_render = !(gt->info.engine_mask & XE_HW_ENGINE_RCS_MASK); 311 struct xe_vm *vm = job->q->vm; 312 u32 mask_flags = 0; 313 314 dw[i++] = preparser_disable(true); 315 if (lacks_render) 316 mask_flags = PIPE_CONTROL_3D_ARCH_FLAGS; 317 else if (job->q->class == XE_ENGINE_CLASS_COMPUTE) 318 mask_flags = PIPE_CONTROL_3D_ENGINE_FLAGS; 319 320 /* See __xe_pt_bind_vma() for a discussion on TLB invalidations. */ 321 i = emit_pipe_invalidate(mask_flags, vm && vm->batch_invalidate_tlb, dw, i); 322 323 /* hsdes: 1809175790 */ 324 if (has_aux_ccs(xe)) 325 i = emit_aux_table_inv(gt, CCS_AUX_INV, dw, i); 326 327 dw[i++] = preparser_disable(false); 328 329 i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc), 330 seqno, dw, i); 331 332 i = emit_bb_start(batch_addr, ppgtt_flag, dw, i); 333 334 i = emit_render_cache_flush(job, dw, i); 335 336 if (job->user_fence.used) 337 i = emit_store_imm_ppgtt_posted(job->user_fence.addr, 338 job->user_fence.value, 339 dw, i); 340 341 i = emit_pipe_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, lacks_render, dw, i); 342 343 i = emit_user_interrupt(dw, i); 344 345 xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW); 346 347 xe_lrc_write_ring(lrc, dw, i * sizeof(*dw)); 348 } 349 350 static void emit_migration_job_gen12(struct xe_sched_job *job, 351 struct xe_lrc *lrc, u32 seqno) 352 { 353 u32 dw[MAX_JOB_SIZE_DW], i = 0; 354 355 i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc), 356 seqno, dw, i); 357 358 dw[i++] = MI_ARB_ON_OFF | MI_ARB_DISABLE; /* Enabled again below */ 359 360 i = emit_bb_start(job->batch_addr[0], BIT(8), dw, i); 361 362 /* XXX: Do we need this? Leaving for now. */ 363 dw[i++] = preparser_disable(true); 364 i = emit_flush_invalidate(0, dw, i); 365 dw[i++] = preparser_disable(false); 366 367 i = emit_bb_start(job->batch_addr[1], BIT(8), dw, i); 368 369 dw[i++] = MI_FLUSH_DW | MI_INVALIDATE_TLB | job->migrate_flush_flags | 370 MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_IMM_DW; 371 dw[i++] = xe_lrc_seqno_ggtt_addr(lrc) | MI_FLUSH_DW_USE_GTT; 372 dw[i++] = 0; 373 dw[i++] = seqno; /* value */ 374 375 i = emit_user_interrupt(dw, i); 376 377 xe_gt_assert(job->q->gt, i <= MAX_JOB_SIZE_DW); 378 379 xe_lrc_write_ring(lrc, dw, i * sizeof(*dw)); 380 } 381 382 static void emit_job_gen12_gsc(struct xe_sched_job *job) 383 { 384 struct xe_gt *gt = job->q->gt; 385 386 xe_gt_assert(gt, job->q->width <= 1); /* no parallel submission for GSCCS */ 387 388 __emit_job_gen12_simple(job, job->q->lrc, 389 job->batch_addr[0], 390 xe_sched_job_seqno(job)); 391 } 392 393 static void emit_job_gen12_copy(struct xe_sched_job *job) 394 { 395 int i; 396 397 if (xe_sched_job_is_migration(job->q)) { 398 emit_migration_job_gen12(job, job->q->lrc, 399 xe_sched_job_seqno(job)); 400 return; 401 } 402 403 for (i = 0; i < job->q->width; ++i) 404 __emit_job_gen12_simple(job, job->q->lrc + i, 405 job->batch_addr[i], 406 xe_sched_job_seqno(job)); 407 } 408 409 static void emit_job_gen12_video(struct xe_sched_job *job) 410 { 411 int i; 412 413 /* FIXME: Not doing parallel handshake for now */ 414 for (i = 0; i < job->q->width; ++i) 415 __emit_job_gen12_video(job, job->q->lrc + i, 416 job->batch_addr[i], 417 xe_sched_job_seqno(job)); 418 } 419 420 static void emit_job_gen12_render_compute(struct xe_sched_job *job) 421 { 422 int i; 423 424 for (i = 0; i < job->q->width; ++i) 425 __emit_job_gen12_render_compute(job, job->q->lrc + i, 426 job->batch_addr[i], 427 xe_sched_job_seqno(job)); 428 } 429 430 static const struct xe_ring_ops ring_ops_gen12_gsc = { 431 .emit_job = emit_job_gen12_gsc, 432 }; 433 434 static const struct xe_ring_ops ring_ops_gen12_copy = { 435 .emit_job = emit_job_gen12_copy, 436 }; 437 438 static const struct xe_ring_ops ring_ops_gen12_video = { 439 .emit_job = emit_job_gen12_video, 440 }; 441 442 static const struct xe_ring_ops ring_ops_gen12_render_compute = { 443 .emit_job = emit_job_gen12_render_compute, 444 }; 445 446 const struct xe_ring_ops * 447 xe_ring_ops_get(struct xe_gt *gt, enum xe_engine_class class) 448 { 449 switch (class) { 450 case XE_ENGINE_CLASS_OTHER: 451 return &ring_ops_gen12_gsc; 452 case XE_ENGINE_CLASS_COPY: 453 return &ring_ops_gen12_copy; 454 case XE_ENGINE_CLASS_VIDEO_DECODE: 455 case XE_ENGINE_CLASS_VIDEO_ENHANCE: 456 return &ring_ops_gen12_video; 457 case XE_ENGINE_CLASS_RENDER: 458 case XE_ENGINE_CLASS_COMPUTE: 459 return &ring_ops_gen12_render_compute; 460 default: 461 return NULL; 462 } 463 } 464