xref: /linux/drivers/gpu/drm/xe/xe_ring_ops.c (revision 173b0b5b0e865348684c02bd9cb1d22b5d46e458)
1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2022 Intel Corporation
4  */
5 
6 #include "xe_ring_ops.h"
7 
8 #include <generated/xe_wa_oob.h>
9 
10 #include "instructions/xe_mi_commands.h"
11 #include "regs/xe_engine_regs.h"
12 #include "regs/xe_gpu_commands.h"
13 #include "regs/xe_gt_regs.h"
14 #include "regs/xe_lrc_layout.h"
15 #include "xe_exec_queue_types.h"
16 #include "xe_gt.h"
17 #include "xe_lrc.h"
18 #include "xe_macros.h"
19 #include "xe_sched_job.h"
20 #include "xe_vm_types.h"
21 #include "xe_vm.h"
22 #include "xe_wa.h"
23 
24 /*
25  * 3D-related flags that can't be set on _engines_ that lack access to the 3D
26  * pipeline (i.e., CCS engines).
27  */
28 #define PIPE_CONTROL_3D_ENGINE_FLAGS (\
29 		PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH | \
30 		PIPE_CONTROL_DEPTH_CACHE_FLUSH | \
31 		PIPE_CONTROL_TILE_CACHE_FLUSH | \
32 		PIPE_CONTROL_DEPTH_STALL | \
33 		PIPE_CONTROL_STALL_AT_SCOREBOARD | \
34 		PIPE_CONTROL_PSD_SYNC | \
35 		PIPE_CONTROL_AMFS_FLUSH | \
36 		PIPE_CONTROL_VF_CACHE_INVALIDATE | \
37 		PIPE_CONTROL_GLOBAL_SNAPSHOT_RESET)
38 
39 /* 3D-related flags that can't be set on _platforms_ that lack a 3D pipeline */
40 #define PIPE_CONTROL_3D_ARCH_FLAGS ( \
41 		PIPE_CONTROL_3D_ENGINE_FLAGS | \
42 		PIPE_CONTROL_INDIRECT_STATE_DISABLE | \
43 		PIPE_CONTROL_FLUSH_ENABLE | \
44 		PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE | \
45 		PIPE_CONTROL_DC_FLUSH_ENABLE)
46 
47 static u32 preparser_disable(bool state)
48 {
49 	return MI_ARB_CHECK | BIT(8) | state;
50 }
51 
52 static int emit_aux_table_inv(struct xe_gt *gt, struct xe_reg reg,
53 			      u32 *dw, int i)
54 {
55 	dw[i++] = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(1) | MI_LRI_MMIO_REMAP_EN;
56 	dw[i++] = reg.addr + gt->mmio.adj_offset;
57 	dw[i++] = AUX_INV;
58 	dw[i++] = MI_NOOP;
59 
60 	return i;
61 }
62 
63 static int emit_user_interrupt(u32 *dw, int i)
64 {
65 	dw[i++] = MI_USER_INTERRUPT;
66 	dw[i++] = MI_ARB_ON_OFF | MI_ARB_ENABLE;
67 	dw[i++] = MI_ARB_CHECK;
68 
69 	return i;
70 }
71 
72 static int emit_store_imm_ggtt(u32 addr, u32 value, u32 *dw, int i)
73 {
74 	dw[i++] = MI_STORE_DATA_IMM | MI_SDI_GGTT | MI_SDI_NUM_DW(1);
75 	dw[i++] = addr;
76 	dw[i++] = 0;
77 	dw[i++] = value;
78 
79 	return i;
80 }
81 
82 static int emit_flush_imm_ggtt(u32 addr, u32 value, bool invalidate_tlb,
83 			       u32 *dw, int i)
84 {
85 	dw[i++] = MI_FLUSH_DW | MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_IMM_DW |
86 		(invalidate_tlb ? MI_INVALIDATE_TLB : 0);
87 	dw[i++] = addr | MI_FLUSH_DW_USE_GTT;
88 	dw[i++] = 0;
89 	dw[i++] = value;
90 
91 	return i;
92 }
93 
94 static int emit_bb_start(u64 batch_addr, u32 ppgtt_flag, u32 *dw, int i)
95 {
96 	dw[i++] = MI_BATCH_BUFFER_START | ppgtt_flag | XE_INSTR_NUM_DW(3);
97 	dw[i++] = lower_32_bits(batch_addr);
98 	dw[i++] = upper_32_bits(batch_addr);
99 
100 	return i;
101 }
102 
103 static int emit_flush_invalidate(u32 flag, u32 *dw, int i)
104 {
105 	dw[i] = MI_FLUSH_DW;
106 	dw[i] |= flag;
107 	dw[i++] |= MI_INVALIDATE_TLB | MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_IMM_DW |
108 		MI_FLUSH_DW_STORE_INDEX;
109 
110 	dw[i++] = LRC_PPHWSP_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT;
111 	dw[i++] = 0;
112 	dw[i++] = ~0U;
113 
114 	return i;
115 }
116 
117 static int
118 emit_pipe_control(u32 *dw, int i, u32 bit_group_0, u32 bit_group_1, u32 offset, u32 value)
119 {
120 	dw[i++] = GFX_OP_PIPE_CONTROL(6) | bit_group_0;
121 	dw[i++] = bit_group_1;
122 	dw[i++] = offset;
123 	dw[i++] = 0;
124 	dw[i++] = value;
125 	dw[i++] = 0;
126 
127 	return i;
128 }
129 
130 static int emit_pipe_invalidate(u32 mask_flags, bool invalidate_tlb, u32 *dw,
131 				int i)
132 {
133 	u32 flags = PIPE_CONTROL_CS_STALL |
134 		PIPE_CONTROL_COMMAND_CACHE_INVALIDATE |
135 		PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE |
136 		PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
137 		PIPE_CONTROL_VF_CACHE_INVALIDATE |
138 		PIPE_CONTROL_CONST_CACHE_INVALIDATE |
139 		PIPE_CONTROL_STATE_CACHE_INVALIDATE |
140 		PIPE_CONTROL_QW_WRITE |
141 		PIPE_CONTROL_STORE_DATA_INDEX;
142 
143 	if (invalidate_tlb)
144 		flags |= PIPE_CONTROL_TLB_INVALIDATE;
145 
146 	flags &= ~mask_flags;
147 
148 	return emit_pipe_control(dw, i, 0, flags, LRC_PPHWSP_SCRATCH_ADDR, 0);
149 }
150 
151 static int emit_store_imm_ppgtt_posted(u64 addr, u64 value,
152 				       u32 *dw, int i)
153 {
154 	dw[i++] = MI_STORE_DATA_IMM | MI_SDI_NUM_QW(1);
155 	dw[i++] = lower_32_bits(addr);
156 	dw[i++] = upper_32_bits(addr);
157 	dw[i++] = lower_32_bits(value);
158 	dw[i++] = upper_32_bits(value);
159 
160 	return i;
161 }
162 
163 static int emit_render_cache_flush(struct xe_sched_job *job, u32 *dw, int i)
164 {
165 	struct xe_gt *gt = job->q->gt;
166 	bool lacks_render = !(gt->info.engine_mask & XE_HW_ENGINE_RCS_MASK);
167 	u32 flags;
168 
169 	flags = (PIPE_CONTROL_CS_STALL |
170 		 PIPE_CONTROL_TILE_CACHE_FLUSH |
171 		 PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH |
172 		 PIPE_CONTROL_DEPTH_CACHE_FLUSH |
173 		 PIPE_CONTROL_DC_FLUSH_ENABLE |
174 		 PIPE_CONTROL_FLUSH_ENABLE);
175 
176 	if (XE_WA(gt, 1409600907))
177 		flags |= PIPE_CONTROL_DEPTH_STALL;
178 
179 	if (lacks_render)
180 		flags &= ~PIPE_CONTROL_3D_ARCH_FLAGS;
181 	else if (job->q->class == XE_ENGINE_CLASS_COMPUTE)
182 		flags &= ~PIPE_CONTROL_3D_ENGINE_FLAGS;
183 
184 	return emit_pipe_control(dw, i, PIPE_CONTROL0_HDC_PIPELINE_FLUSH, flags, 0, 0);
185 }
186 
187 static int emit_pipe_control_to_ring_end(struct xe_hw_engine *hwe, u32 *dw, int i)
188 {
189 	if (hwe->class != XE_ENGINE_CLASS_RENDER)
190 		return i;
191 
192 	if (XE_WA(hwe->gt, 16020292621))
193 		i = emit_pipe_control(dw, i, 0, PIPE_CONTROL_LRI_POST_SYNC,
194 				      RING_NOPID(hwe->mmio_base).addr, 0);
195 
196 	return i;
197 }
198 
199 static int emit_pipe_imm_ggtt(u32 addr, u32 value, bool stall_only, u32 *dw,
200 			      int i)
201 {
202 	u32 flags = PIPE_CONTROL_CS_STALL | PIPE_CONTROL_GLOBAL_GTT_IVB |
203 		    PIPE_CONTROL_QW_WRITE;
204 
205 	if (!stall_only)
206 		flags |= PIPE_CONTROL_FLUSH_ENABLE;
207 
208 	return emit_pipe_control(dw, i, 0, flags, addr, value);
209 }
210 
211 static u32 get_ppgtt_flag(struct xe_sched_job *job)
212 {
213 	return job->q->vm ? BIT(8) : 0;
214 }
215 
216 /* for engines that don't require any special HW handling (no EUs, no aux inval, etc) */
217 static void __emit_job_gen12_simple(struct xe_sched_job *job, struct xe_lrc *lrc,
218 				    u64 batch_addr, u32 seqno)
219 {
220 	u32 dw[MAX_JOB_SIZE_DW], i = 0;
221 	u32 ppgtt_flag = get_ppgtt_flag(job);
222 	struct xe_gt *gt = job->q->gt;
223 
224 	if (job->ring_ops_flush_tlb) {
225 		dw[i++] = preparser_disable(true);
226 		i = emit_flush_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),
227 					seqno, true, dw, i);
228 		dw[i++] = preparser_disable(false);
229 	} else {
230 		i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),
231 					seqno, dw, i);
232 	}
233 
234 	i = emit_bb_start(batch_addr, ppgtt_flag, dw, i);
235 
236 	if (job->user_fence.used)
237 		i = emit_store_imm_ppgtt_posted(job->user_fence.addr,
238 						job->user_fence.value,
239 						dw, i);
240 
241 	i = emit_flush_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, false, dw, i);
242 
243 	i = emit_user_interrupt(dw, i);
244 
245 	xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW);
246 
247 	xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));
248 }
249 
250 static bool has_aux_ccs(struct xe_device *xe)
251 {
252 	/*
253 	 * PVC is a special case that has no compression of either type
254 	 * (FlatCCS or AuxCCS).  Also, AuxCCS is no longer used from Xe2
255 	 * onward, so any future platforms with no FlatCCS will not have
256 	 * AuxCCS either.
257 	 */
258 	if (GRAPHICS_VER(xe) >= 20 || xe->info.platform == XE_PVC)
259 		return false;
260 
261 	return !xe->info.has_flat_ccs;
262 }
263 
264 static void __emit_job_gen12_video(struct xe_sched_job *job, struct xe_lrc *lrc,
265 				   u64 batch_addr, u32 seqno)
266 {
267 	u32 dw[MAX_JOB_SIZE_DW], i = 0;
268 	u32 ppgtt_flag = get_ppgtt_flag(job);
269 	struct xe_gt *gt = job->q->gt;
270 	struct xe_device *xe = gt_to_xe(gt);
271 	bool decode = job->q->class == XE_ENGINE_CLASS_VIDEO_DECODE;
272 
273 	dw[i++] = preparser_disable(true);
274 
275 	/* hsdes: 1809175790 */
276 	if (has_aux_ccs(xe)) {
277 		if (decode)
278 			i = emit_aux_table_inv(gt, VD0_AUX_INV, dw, i);
279 		else
280 			i = emit_aux_table_inv(gt, VE0_AUX_INV, dw, i);
281 	}
282 
283 	if (job->ring_ops_flush_tlb)
284 		i = emit_flush_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),
285 					seqno, true, dw, i);
286 
287 	dw[i++] = preparser_disable(false);
288 
289 	if (!job->ring_ops_flush_tlb)
290 		i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),
291 					seqno, dw, i);
292 
293 	i = emit_bb_start(batch_addr, ppgtt_flag, dw, i);
294 
295 	if (job->user_fence.used)
296 		i = emit_store_imm_ppgtt_posted(job->user_fence.addr,
297 						job->user_fence.value,
298 						dw, i);
299 
300 	i = emit_flush_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, false, dw, i);
301 
302 	i = emit_user_interrupt(dw, i);
303 
304 	xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW);
305 
306 	xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));
307 }
308 
309 static void __emit_job_gen12_render_compute(struct xe_sched_job *job,
310 					    struct xe_lrc *lrc,
311 					    u64 batch_addr, u32 seqno)
312 {
313 	u32 dw[MAX_JOB_SIZE_DW], i = 0;
314 	u32 ppgtt_flag = get_ppgtt_flag(job);
315 	struct xe_gt *gt = job->q->gt;
316 	struct xe_device *xe = gt_to_xe(gt);
317 	bool lacks_render = !(gt->info.engine_mask & XE_HW_ENGINE_RCS_MASK);
318 	u32 mask_flags = 0;
319 
320 	dw[i++] = preparser_disable(true);
321 	if (lacks_render)
322 		mask_flags = PIPE_CONTROL_3D_ARCH_FLAGS;
323 	else if (job->q->class == XE_ENGINE_CLASS_COMPUTE)
324 		mask_flags = PIPE_CONTROL_3D_ENGINE_FLAGS;
325 
326 	/* See __xe_pt_bind_vma() for a discussion on TLB invalidations. */
327 	i = emit_pipe_invalidate(mask_flags, job->ring_ops_flush_tlb, dw, i);
328 
329 	/* hsdes: 1809175790 */
330 	if (has_aux_ccs(xe))
331 		i = emit_aux_table_inv(gt, CCS_AUX_INV, dw, i);
332 
333 	dw[i++] = preparser_disable(false);
334 
335 	i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),
336 				seqno, dw, i);
337 
338 	i = emit_bb_start(batch_addr, ppgtt_flag, dw, i);
339 
340 	i = emit_render_cache_flush(job, dw, i);
341 
342 	if (job->user_fence.used)
343 		i = emit_store_imm_ppgtt_posted(job->user_fence.addr,
344 						job->user_fence.value,
345 						dw, i);
346 
347 	i = emit_pipe_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, lacks_render, dw, i);
348 
349 	i = emit_user_interrupt(dw, i);
350 
351 	i = emit_pipe_control_to_ring_end(job->q->hwe, dw, i);
352 
353 	xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW);
354 
355 	xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));
356 }
357 
358 static void emit_migration_job_gen12(struct xe_sched_job *job,
359 				     struct xe_lrc *lrc, u32 seqno)
360 {
361 	u32 dw[MAX_JOB_SIZE_DW], i = 0;
362 
363 	i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),
364 				seqno, dw, i);
365 
366 	dw[i++] = MI_ARB_ON_OFF | MI_ARB_DISABLE; /* Enabled again below */
367 
368 	i = emit_bb_start(job->batch_addr[0], BIT(8), dw, i);
369 
370 	/* XXX: Do we need this? Leaving for now. */
371 	dw[i++] = preparser_disable(true);
372 	i = emit_flush_invalidate(0, dw, i);
373 	dw[i++] = preparser_disable(false);
374 
375 	i = emit_bb_start(job->batch_addr[1], BIT(8), dw, i);
376 
377 	dw[i++] = MI_FLUSH_DW | MI_INVALIDATE_TLB | job->migrate_flush_flags |
378 		MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_IMM_DW;
379 	dw[i++] = xe_lrc_seqno_ggtt_addr(lrc) | MI_FLUSH_DW_USE_GTT;
380 	dw[i++] = 0;
381 	dw[i++] = seqno; /* value */
382 
383 	i = emit_user_interrupt(dw, i);
384 
385 	xe_gt_assert(job->q->gt, i <= MAX_JOB_SIZE_DW);
386 
387 	xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));
388 }
389 
390 static void emit_job_gen12_gsc(struct xe_sched_job *job)
391 {
392 	struct xe_gt *gt = job->q->gt;
393 
394 	xe_gt_assert(gt, job->q->width <= 1); /* no parallel submission for GSCCS */
395 
396 	__emit_job_gen12_simple(job, job->q->lrc,
397 				job->batch_addr[0],
398 				xe_sched_job_seqno(job));
399 }
400 
401 static void emit_job_gen12_copy(struct xe_sched_job *job)
402 {
403 	int i;
404 
405 	if (xe_sched_job_is_migration(job->q)) {
406 		emit_migration_job_gen12(job, job->q->lrc,
407 					 xe_sched_job_seqno(job));
408 		return;
409 	}
410 
411 	for (i = 0; i < job->q->width; ++i)
412 		__emit_job_gen12_simple(job, job->q->lrc + i,
413 				        job->batch_addr[i],
414 				        xe_sched_job_seqno(job));
415 }
416 
417 static void emit_job_gen12_video(struct xe_sched_job *job)
418 {
419 	int i;
420 
421 	/* FIXME: Not doing parallel handshake for now */
422 	for (i = 0; i < job->q->width; ++i)
423 		__emit_job_gen12_video(job, job->q->lrc + i,
424 				       job->batch_addr[i],
425 				       xe_sched_job_seqno(job));
426 }
427 
428 static void emit_job_gen12_render_compute(struct xe_sched_job *job)
429 {
430 	int i;
431 
432 	for (i = 0; i < job->q->width; ++i)
433 		__emit_job_gen12_render_compute(job, job->q->lrc + i,
434 						job->batch_addr[i],
435 						xe_sched_job_seqno(job));
436 }
437 
438 static const struct xe_ring_ops ring_ops_gen12_gsc = {
439 	.emit_job = emit_job_gen12_gsc,
440 };
441 
442 static const struct xe_ring_ops ring_ops_gen12_copy = {
443 	.emit_job = emit_job_gen12_copy,
444 };
445 
446 static const struct xe_ring_ops ring_ops_gen12_video = {
447 	.emit_job = emit_job_gen12_video,
448 };
449 
450 static const struct xe_ring_ops ring_ops_gen12_render_compute = {
451 	.emit_job = emit_job_gen12_render_compute,
452 };
453 
454 const struct xe_ring_ops *
455 xe_ring_ops_get(struct xe_gt *gt, enum xe_engine_class class)
456 {
457 	switch (class) {
458 	case XE_ENGINE_CLASS_OTHER:
459 		return &ring_ops_gen12_gsc;
460 	case XE_ENGINE_CLASS_COPY:
461 		return &ring_ops_gen12_copy;
462 	case XE_ENGINE_CLASS_VIDEO_DECODE:
463 	case XE_ENGINE_CLASS_VIDEO_ENHANCE:
464 		return &ring_ops_gen12_video;
465 	case XE_ENGINE_CLASS_RENDER:
466 	case XE_ENGINE_CLASS_COMPUTE:
467 		return &ring_ops_gen12_render_compute;
468 	default:
469 		return NULL;
470 	}
471 }
472