54 			      u32 *dw, int i)  in emit_aux_table_inv()  argument
56 	dw[i++] = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(1) | MI_LRI_MMIO_REMAP_EN;  in emit_aux_table_inv()
57 	dw[i++] = reg.addr + gt->mmio.adj_offset;  in emit_aux_table_inv()
58 	dw[i++] = AUX_INV;  in emit_aux_table_inv()
59 	dw[i++] = MI_NOOP;  in emit_aux_table_inv()
64 static int emit_user_interrupt(u32 *dw, int i)  in emit_user_interrupt()  argument
66 	dw[i++] = MI_USER_INTERRUPT;  in emit_user_interrupt()
67 	dw[i++] = MI_ARB_ON_OFF | MI_ARB_ENABLE;  in emit_user_interrupt()
68 	dw[i++] = MI_ARB_CHECK;  in emit_user_interrupt()
73 static int emit_store_imm_ggtt(u32 addr, u32 value, u32 *dw, int i)  in emit_store_imm_ggtt()  argument
75 	dw[i++] = MI_STORE_DATA_IMM | MI_SDI_GGTT | MI_SDI_NUM_DW(1);  in emit_store_imm_ggtt()
76 	dw[i++] = addr;  in emit_store_imm_ggtt()
77 	dw[i++] = 0;  in emit_store_imm_ggtt()
78 	dw[i++] = value;  in emit_store_imm_ggtt()
83 static int emit_flush_dw(u32 *dw, int i)  in emit_flush_dw()  argument
85 	dw[i++] = MI_FLUSH_DW | MI_FLUSH_IMM_DW;  in emit_flush_dw()
86 	dw[i++] = 0;  in emit_flush_dw()
87 	dw[i++] = 0;  in emit_flush_dw()
88 	dw[i++] = 0;  in emit_flush_dw()
94 			       u32 *dw, int i)  in emit_flush_imm_ggtt()  argument
96 	dw[i++] = MI_FLUSH_DW | MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_IMM_DW |  in emit_flush_imm_ggtt()
98 	dw[i++] = addr | MI_FLUSH_DW_USE_GTT;  in emit_flush_imm_ggtt()
99 	dw[i++] = 0;  in emit_flush_imm_ggtt()
100 	dw[i++] = value;  in emit_flush_imm_ggtt()
105 static int emit_bb_start(u64 batch_addr, u32 ppgtt_flag, u32 *dw, int i)  in emit_bb_start()  argument
107 	dw[i++] = MI_BATCH_BUFFER_START | ppgtt_flag | XE_INSTR_NUM_DW(3);  in emit_bb_start()
108 	dw[i++] = lower_32_bits(batch_addr);  in emit_bb_start()
109 	dw[i++] = upper_32_bits(batch_addr);  in emit_bb_start()
114 static int emit_flush_invalidate(u32 flag, u32 *dw, int i)  in emit_flush_invalidate()  argument
116 	dw[i] = MI_FLUSH_DW;  in emit_flush_invalidate()
117 	dw[i] |= flag;  in emit_flush_invalidate()
118 	dw[i++] |= MI_INVALIDATE_TLB | MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_IMM_DW |  in emit_flush_invalidate()
121 	dw[i++] = LRC_PPHWSP_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT;  in emit_flush_invalidate()
122 	dw[i++] = 0;  in emit_flush_invalidate()
123 	dw[i++] = ~0U;  in emit_flush_invalidate()
129 emit_pipe_control(u32 *dw, int i, u32 bit_group_0, u32 bit_group_1, u32 offset, u32 value)  in emit_pipe_control()  argument
131 	dw[i++] = GFX_OP_PIPE_CONTROL(6) | bit_group_0;  in emit_pipe_control()
132 	dw[i++] = bit_group_1;  in emit_pipe_control()
133 	dw[i++] = offset;  in emit_pipe_control()
134 	dw[i++] = 0;  in emit_pipe_control()
135 	dw[i++] = value;  in emit_pipe_control()
136 	dw[i++] = 0;  in emit_pipe_control()
141 static int emit_pipe_invalidate(u32 mask_flags, bool invalidate_tlb, u32 *dw,  in emit_pipe_invalidate()  argument
159 	return emit_pipe_control(dw, i, 0, flags, LRC_PPHWSP_SCRATCH_ADDR, 0);  in emit_pipe_invalidate()
163 				       u32 *dw, int i)  in emit_store_imm_ppgtt_posted()  argument
165 	dw[i++] = MI_STORE_DATA_IMM | MI_SDI_NUM_QW(1);  in emit_store_imm_ppgtt_posted()
166 	dw[i++] = lower_32_bits(addr);  in emit_store_imm_ppgtt_posted()
167 	dw[i++] = upper_32_bits(addr);  in emit_store_imm_ppgtt_posted()
168 	dw[i++] = lower_32_bits(value);  in emit_store_imm_ppgtt_posted()
169 	dw[i++] = upper_32_bits(value);  in emit_store_imm_ppgtt_posted()
174 static int emit_render_cache_flush(struct xe_sched_job *job, u32 *dw, int i)  in emit_render_cache_flush()  argument
195 	return emit_pipe_control(dw, i, PIPE_CONTROL0_HDC_PIPELINE_FLUSH, flags, 0, 0);  in emit_render_cache_flush()
198 static int emit_pipe_control_to_ring_end(struct xe_hw_engine *hwe, u32 *dw, int i)  in emit_pipe_control_to_ring_end()  argument
204 		i = emit_pipe_control(dw, i, 0, PIPE_CONTROL_LRI_POST_SYNC,  in emit_pipe_control_to_ring_end()
210 static int emit_pipe_imm_ggtt(u32 addr, u32 value, bool stall_only, u32 *dw,  in emit_pipe_imm_ggtt()  argument
219 	return emit_pipe_control(dw, i, 0, flags, addr, value);  in emit_pipe_imm_ggtt()
230 static int emit_copy_timestamp(struct xe_lrc *lrc, u32 *dw, int i)  in emit_copy_timestamp()  argument
232 	dw[i++] = MI_COPY_MEM_MEM | MI_COPY_MEM_MEM_SRC_GGTT |  in emit_copy_timestamp()
234 	dw[i++] = xe_lrc_ctx_job_timestamp_ggtt_addr(lrc);  in emit_copy_timestamp()
235 	dw[i++] = 0;  in emit_copy_timestamp()
236 	dw[i++] = xe_lrc_ctx_timestamp_ggtt_addr(lrc);  in emit_copy_timestamp()
237 	dw[i++] = 0;  in emit_copy_timestamp()
238 	dw[i++] = MI_NOOP;  in emit_copy_timestamp()
247 	u32 dw[MAX_JOB_SIZE_DW], i = 0;  in __emit_job_gen12_simple()  local
251 	i = emit_copy_timestamp(lrc, dw, i);  in __emit_job_gen12_simple()
254 		dw[i++] = preparser_disable(true);  in __emit_job_gen12_simple()
256 					seqno, true, dw, i);  in __emit_job_gen12_simple()
257 		dw[i++] = preparser_disable(false);  in __emit_job_gen12_simple()
260 					seqno, dw, i);  in __emit_job_gen12_simple()
263 	i = emit_bb_start(batch_addr, ppgtt_flag, dw, i);  in __emit_job_gen12_simple()
266 		i = emit_flush_dw(dw, i);  in __emit_job_gen12_simple()
269 						dw, i);  in __emit_job_gen12_simple()
272 	i = emit_flush_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, false, dw, i);  in __emit_job_gen12_simple()
274 	i = emit_user_interrupt(dw, i);  in __emit_job_gen12_simple()
278 	xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));  in __emit_job_gen12_simple()
298 	u32 dw[MAX_JOB_SIZE_DW], i = 0;  in __emit_job_gen12_video()  local
304 	i = emit_copy_timestamp(lrc, dw, i);  in __emit_job_gen12_video()
306 	dw[i++] = preparser_disable(true);  in __emit_job_gen12_video()
311 			i = emit_aux_table_inv(gt, VD0_AUX_INV, dw, i);  in __emit_job_gen12_video()
313 			i = emit_aux_table_inv(gt, VE0_AUX_INV, dw, i);  in __emit_job_gen12_video()
318 					seqno, true, dw, i);  in __emit_job_gen12_video()
320 	dw[i++] = preparser_disable(false);  in __emit_job_gen12_video()
324 					seqno, dw, i);  in __emit_job_gen12_video()
326 	i = emit_bb_start(batch_addr, ppgtt_flag, dw, i);  in __emit_job_gen12_video()
329 		i = emit_flush_dw(dw, i);  in __emit_job_gen12_video()
332 						dw, i);  in __emit_job_gen12_video()
335 	i = emit_flush_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, false, dw, i);  in __emit_job_gen12_video()
337 	i = emit_user_interrupt(dw, i);  in __emit_job_gen12_video()
341 	xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));  in __emit_job_gen12_video()
348 	u32 dw[MAX_JOB_SIZE_DW], i = 0;  in __emit_job_gen12_render_compute()  local
355 	i = emit_copy_timestamp(lrc, dw, i);  in __emit_job_gen12_render_compute()
357 	dw[i++] = preparser_disable(true);  in __emit_job_gen12_render_compute()
364 	i = emit_pipe_invalidate(mask_flags, job->ring_ops_flush_tlb, dw, i);  in __emit_job_gen12_render_compute()
368 		i = emit_aux_table_inv(gt, CCS_AUX_INV, dw, i);  in __emit_job_gen12_render_compute()
370 	dw[i++] = preparser_disable(false);  in __emit_job_gen12_render_compute()
373 				seqno, dw, i);  in __emit_job_gen12_render_compute()
375 	i = emit_bb_start(batch_addr, ppgtt_flag, dw, i);  in __emit_job_gen12_render_compute()
377 	i = emit_render_cache_flush(job, dw, i);  in __emit_job_gen12_render_compute()
382 						dw, i);  in __emit_job_gen12_render_compute()
384 	i = emit_pipe_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, lacks_render, dw, i);  in __emit_job_gen12_render_compute()
386 	i = emit_user_interrupt(dw, i);  in __emit_job_gen12_render_compute()
388 	i = emit_pipe_control_to_ring_end(job->q->hwe, dw, i);  in __emit_job_gen12_render_compute()
392 	xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));  in __emit_job_gen12_render_compute()
398 	u32 dw[MAX_JOB_SIZE_DW], i = 0;  in emit_migration_job_gen12()  local
400 	i = emit_copy_timestamp(lrc, dw, i);  in emit_migration_job_gen12()
403 				seqno, dw, i);  in emit_migration_job_gen12()
405 	dw[i++] = MI_ARB_ON_OFF | MI_ARB_DISABLE; /* Enabled again below */  in emit_migration_job_gen12()
407 	i = emit_bb_start(job->ptrs[0].batch_addr, BIT(8), dw, i);  in emit_migration_job_gen12()
411 		dw[i++] = preparser_disable(true);  in emit_migration_job_gen12()
412 		i = emit_flush_invalidate(0, dw, i);  in emit_migration_job_gen12()
413 		dw[i++] = preparser_disable(false);  in emit_migration_job_gen12()
416 	i = emit_bb_start(job->ptrs[1].batch_addr, BIT(8), dw, i);  in emit_migration_job_gen12()
418 	dw[i++] = MI_FLUSH_DW | MI_INVALIDATE_TLB | job->migrate_flush_flags |  in emit_migration_job_gen12()
420 	dw[i++] = xe_lrc_seqno_ggtt_addr(lrc) | MI_FLUSH_DW_USE_GTT;  in emit_migration_job_gen12()
421 	dw[i++] = 0;  in emit_migration_job_gen12()
422 	dw[i++] = seqno; /* value */  in emit_migration_job_gen12()
424 	i = emit_user_interrupt(dw, i);  in emit_migration_job_gen12()
428 	xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));  in emit_migration_job_gen12()