xref: /linux/drivers/gpu/drm/xe/xe_ring_ops.c (revision 5488bec96bccbd87335921338f8dc38b87db7d2c) 
1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2022 Intel Corporation
4  */
5 
6 #include "xe_ring_ops.h"
7 
8 #include <generated/xe_wa_oob.h>
9 
10 #include "instructions/xe_gpu_commands.h"
11 #include "instructions/xe_mi_commands.h"
12 #include "regs/xe_engine_regs.h"
13 #include "regs/xe_gt_regs.h"
14 #include "regs/xe_lrc_layout.h"
15 #include "xe_exec_queue_types.h"
16 #include "xe_gt.h"
17 #include "xe_lrc.h"
18 #include "xe_macros.h"
19 #include "xe_sched_job.h"
20 #include "xe_sriov.h"
21 #include "xe_vm_types.h"
22 #include "xe_vm.h"
23 #include "xe_wa.h"
24 
25 /*
26  * 3D-related flags that can't be set on _engines_ that lack access to the 3D
27  * pipeline (i.e., CCS engines).
28  */
29 #define PIPE_CONTROL_3D_ENGINE_FLAGS (\
30 		PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH | \
31 		PIPE_CONTROL_DEPTH_CACHE_FLUSH | \
32 		PIPE_CONTROL_TILE_CACHE_FLUSH | \
33 		PIPE_CONTROL_DEPTH_STALL | \
34 		PIPE_CONTROL_STALL_AT_SCOREBOARD | \
35 		PIPE_CONTROL_PSD_SYNC | \
36 		PIPE_CONTROL_AMFS_FLUSH | \
37 		PIPE_CONTROL_VF_CACHE_INVALIDATE | \
38 		PIPE_CONTROL_GLOBAL_SNAPSHOT_RESET)
39 
40 /* 3D-related flags that can't be set on _platforms_ that lack a 3D pipeline */
41 #define PIPE_CONTROL_3D_ARCH_FLAGS ( \
42 		PIPE_CONTROL_3D_ENGINE_FLAGS | \
43 		PIPE_CONTROL_INDIRECT_STATE_DISABLE | \
44 		PIPE_CONTROL_FLUSH_ENABLE | \
45 		PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE | \
46 		PIPE_CONTROL_DC_FLUSH_ENABLE)
47 
48 static u32 preparser_disable(bool state)
49 {
50 	return MI_ARB_CHECK | BIT(8) | state;
51 }
52 
53 static int emit_aux_table_inv(struct xe_gt *gt, struct xe_reg reg,
54 			      u32 *dw, int i)
55 {
56 	dw[i++] = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(1) | MI_LRI_MMIO_REMAP_EN;
57 	dw[i++] = reg.addr + gt->mmio.adj_offset;
58 	dw[i++] = AUX_INV;
59 	dw[i++] = MI_NOOP;
60 
61 	return i;
62 }
63 
64 static int emit_user_interrupt(u32 *dw, int i)
65 {
66 	dw[i++] = MI_USER_INTERRUPT;
67 	dw[i++] = MI_ARB_ON_OFF | MI_ARB_ENABLE;
68 	dw[i++] = MI_ARB_CHECK;
69 
70 	return i;
71 }
72 
73 static int emit_store_imm_ggtt(u32 addr, u32 value, u32 *dw, int i)
74 {
75 	dw[i++] = MI_STORE_DATA_IMM | MI_SDI_GGTT | MI_SDI_NUM_DW(1);
76 	dw[i++] = addr;
77 	dw[i++] = 0;
78 	dw[i++] = value;
79 
80 	return i;
81 }
82 
83 static int emit_flush_dw(u32 *dw, int i)
84 {
85 	dw[i++] = MI_FLUSH_DW | MI_FLUSH_IMM_DW;
86 	dw[i++] = 0;
87 	dw[i++] = 0;
88 	dw[i++] = 0;
89 
90 	return i;
91 }
92 
93 static int emit_flush_imm_ggtt(u32 addr, u32 value, bool invalidate_tlb,
94 			       u32 *dw, int i)
95 {
96 	dw[i++] = MI_FLUSH_DW | MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_IMM_DW |
97 		(invalidate_tlb ? MI_INVALIDATE_TLB : 0);
98 	dw[i++] = addr | MI_FLUSH_DW_USE_GTT;
99 	dw[i++] = 0;
100 	dw[i++] = value;
101 
102 	return i;
103 }
104 
105 static int emit_bb_start(u64 batch_addr, u32 ppgtt_flag, u32 *dw, int i)
106 {
107 	dw[i++] = MI_BATCH_BUFFER_START | ppgtt_flag | XE_INSTR_NUM_DW(3);
108 	dw[i++] = lower_32_bits(batch_addr);
109 	dw[i++] = upper_32_bits(batch_addr);
110 
111 	return i;
112 }
113 
114 static int emit_flush_invalidate(u32 flag, u32 *dw, int i)
115 {
116 	dw[i] = MI_FLUSH_DW;
117 	dw[i] |= flag;
118 	dw[i++] |= MI_INVALIDATE_TLB | MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_IMM_DW |
119 		MI_FLUSH_DW_STORE_INDEX;
120 
121 	dw[i++] = LRC_PPHWSP_FLUSH_INVAL_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT;
122 	dw[i++] = 0;
123 	dw[i++] = ~0U;
124 
125 	return i;
126 }
127 
128 static int
129 emit_pipe_control(u32 *dw, int i, u32 bit_group_0, u32 bit_group_1, u32 offset, u32 value)
130 {
131 	dw[i++] = GFX_OP_PIPE_CONTROL(6) | bit_group_0;
132 	dw[i++] = bit_group_1;
133 	dw[i++] = offset;
134 	dw[i++] = 0;
135 	dw[i++] = value;
136 	dw[i++] = 0;
137 
138 	return i;
139 }
140 
141 static int emit_pipe_invalidate(u32 mask_flags, bool invalidate_tlb, u32 *dw,
142 				int i)
143 {
144 	u32 flags = PIPE_CONTROL_CS_STALL |
145 		PIPE_CONTROL_COMMAND_CACHE_INVALIDATE |
146 		PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE |
147 		PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
148 		PIPE_CONTROL_VF_CACHE_INVALIDATE |
149 		PIPE_CONTROL_CONST_CACHE_INVALIDATE |
150 		PIPE_CONTROL_STATE_CACHE_INVALIDATE |
151 		PIPE_CONTROL_QW_WRITE |
152 		PIPE_CONTROL_STORE_DATA_INDEX;
153 
154 	if (invalidate_tlb)
155 		flags |= PIPE_CONTROL_TLB_INVALIDATE;
156 
157 	flags &= ~mask_flags;
158 
159 	return emit_pipe_control(dw, i, 0, flags, LRC_PPHWSP_FLUSH_INVAL_SCRATCH_ADDR, 0);
160 }
161 
162 static int emit_store_imm_ppgtt_posted(u64 addr, u64 value,
163 				       u32 *dw, int i)
164 {
165 	dw[i++] = MI_STORE_DATA_IMM | MI_SDI_NUM_QW(1);
166 	dw[i++] = lower_32_bits(addr);
167 	dw[i++] = upper_32_bits(addr);
168 	dw[i++] = lower_32_bits(value);
169 	dw[i++] = upper_32_bits(value);
170 
171 	return i;
172 }
173 
174 static int emit_render_cache_flush(struct xe_sched_job *job, u32 *dw, int i)
175 {
176 	struct xe_gt *gt = job->q->gt;
177 	bool lacks_render = !(gt->info.engine_mask & XE_HW_ENGINE_RCS_MASK);
178 	u32 flags;
179 
180 	if (XE_WA(gt, 14016712196))
181 		i = emit_pipe_control(dw, i, 0, PIPE_CONTROL_DEPTH_CACHE_FLUSH,
182 				      LRC_PPHWSP_FLUSH_INVAL_SCRATCH_ADDR, 0);
183 
184 	flags = (PIPE_CONTROL_CS_STALL |
185 		 PIPE_CONTROL_TILE_CACHE_FLUSH |
186 		 PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH |
187 		 PIPE_CONTROL_DEPTH_CACHE_FLUSH |
188 		 PIPE_CONTROL_DC_FLUSH_ENABLE |
189 		 PIPE_CONTROL_FLUSH_ENABLE);
190 
191 	if (XE_WA(gt, 1409600907))
192 		flags |= PIPE_CONTROL_DEPTH_STALL;
193 
194 	if (lacks_render)
195 		flags &= ~PIPE_CONTROL_3D_ARCH_FLAGS;
196 	else if (job->q->class == XE_ENGINE_CLASS_COMPUTE)
197 		flags &= ~PIPE_CONTROL_3D_ENGINE_FLAGS;
198 
199 	return emit_pipe_control(dw, i, PIPE_CONTROL0_HDC_PIPELINE_FLUSH, flags, 0, 0);
200 }
201 
202 static int emit_pipe_control_to_ring_end(struct xe_hw_engine *hwe, u32 *dw, int i)
203 {
204 	if (hwe->class != XE_ENGINE_CLASS_RENDER)
205 		return i;
206 
207 	if (XE_WA(hwe->gt, 16020292621))
208 		i = emit_pipe_control(dw, i, 0, PIPE_CONTROL_LRI_POST_SYNC,
209 				      RING_NOPID(hwe->mmio_base).addr, 0);
210 
211 	return i;
212 }
213 
214 static int emit_pipe_imm_ggtt(u32 addr, u32 value, bool stall_only, u32 *dw,
215 			      int i)
216 {
217 	u32 flags = PIPE_CONTROL_CS_STALL | PIPE_CONTROL_GLOBAL_GTT_IVB |
218 		    PIPE_CONTROL_QW_WRITE;
219 
220 	if (!stall_only)
221 		flags |= PIPE_CONTROL_FLUSH_ENABLE;
222 
223 	return emit_pipe_control(dw, i, 0, flags, addr, value);
224 }
225 
226 static u32 get_ppgtt_flag(struct xe_sched_job *job)
227 {
228 	if (job->q->vm && !job->ggtt)
229 		return BIT(8);
230 
231 	return 0;
232 }
233 
234 static int emit_copy_timestamp(struct xe_lrc *lrc, u32 *dw, int i)
235 {
236 	dw[i++] = MI_COPY_MEM_MEM | MI_COPY_MEM_MEM_SRC_GGTT |
237 		MI_COPY_MEM_MEM_DST_GGTT;
238 	dw[i++] = xe_lrc_ctx_job_timestamp_ggtt_addr(lrc);
239 	dw[i++] = 0;
240 	dw[i++] = xe_lrc_ctx_timestamp_ggtt_addr(lrc);
241 	dw[i++] = 0;
242 	dw[i++] = MI_NOOP;
243 
244 	return i;
245 }
246 
247 /* for engines that don't require any special HW handling (no EUs, no aux inval, etc) */
248 static void __emit_job_gen12_simple(struct xe_sched_job *job, struct xe_lrc *lrc,
249 				    u64 batch_addr, u32 seqno)
250 {
251 	u32 dw[MAX_JOB_SIZE_DW], i = 0;
252 	u32 ppgtt_flag = get_ppgtt_flag(job);
253 	struct xe_gt *gt = job->q->gt;
254 
255 	i = emit_copy_timestamp(lrc, dw, i);
256 
257 	if (job->ring_ops_flush_tlb) {
258 		dw[i++] = preparser_disable(true);
259 		i = emit_flush_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),
260 					seqno, true, dw, i);
261 		dw[i++] = preparser_disable(false);
262 	} else {
263 		i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),
264 					seqno, dw, i);
265 	}
266 
267 	i = emit_bb_start(batch_addr, ppgtt_flag, dw, i);
268 
269 	if (job->user_fence.used) {
270 		i = emit_flush_dw(dw, i);
271 		i = emit_store_imm_ppgtt_posted(job->user_fence.addr,
272 						job->user_fence.value,
273 						dw, i);
274 	}
275 
276 	i = emit_flush_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, false, dw, i);
277 
278 	i = emit_user_interrupt(dw, i);
279 
280 	xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW);
281 
282 	xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));
283 }
284 
285 static bool has_aux_ccs(struct xe_device *xe)
286 {
287 	/*
288 	 * PVC is a special case that has no compression of either type
289 	 * (FlatCCS or AuxCCS).  Also, AuxCCS is no longer used from Xe2
290 	 * onward, so any future platforms with no FlatCCS will not have
291 	 * AuxCCS either.
292 	 */
293 	if (GRAPHICS_VER(xe) >= 20 || xe->info.platform == XE_PVC)
294 		return false;
295 
296 	return !xe->info.has_flat_ccs;
297 }
298 
299 static void __emit_job_gen12_video(struct xe_sched_job *job, struct xe_lrc *lrc,
300 				   u64 batch_addr, u32 seqno)
301 {
302 	u32 dw[MAX_JOB_SIZE_DW], i = 0;
303 	u32 ppgtt_flag = get_ppgtt_flag(job);
304 	struct xe_gt *gt = job->q->gt;
305 	struct xe_device *xe = gt_to_xe(gt);
306 	bool decode = job->q->class == XE_ENGINE_CLASS_VIDEO_DECODE;
307 
308 	i = emit_copy_timestamp(lrc, dw, i);
309 
310 	dw[i++] = preparser_disable(true);
311 
312 	/* hsdes: 1809175790 */
313 	if (has_aux_ccs(xe)) {
314 		if (decode)
315 			i = emit_aux_table_inv(gt, VD0_AUX_INV, dw, i);
316 		else
317 			i = emit_aux_table_inv(gt, VE0_AUX_INV, dw, i);
318 	}
319 
320 	if (job->ring_ops_flush_tlb)
321 		i = emit_flush_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),
322 					seqno, true, dw, i);
323 
324 	dw[i++] = preparser_disable(false);
325 
326 	if (!job->ring_ops_flush_tlb)
327 		i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),
328 					seqno, dw, i);
329 
330 	i = emit_bb_start(batch_addr, ppgtt_flag, dw, i);
331 
332 	if (job->user_fence.used) {
333 		i = emit_flush_dw(dw, i);
334 		i = emit_store_imm_ppgtt_posted(job->user_fence.addr,
335 						job->user_fence.value,
336 						dw, i);
337 	}
338 
339 	i = emit_flush_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, false, dw, i);
340 
341 	i = emit_user_interrupt(dw, i);
342 
343 	xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW);
344 
345 	xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));
346 }
347 
348 static void __emit_job_gen12_render_compute(struct xe_sched_job *job,
349 					    struct xe_lrc *lrc,
350 					    u64 batch_addr, u32 seqno)
351 {
352 	u32 dw[MAX_JOB_SIZE_DW], i = 0;
353 	u32 ppgtt_flag = get_ppgtt_flag(job);
354 	struct xe_gt *gt = job->q->gt;
355 	struct xe_device *xe = gt_to_xe(gt);
356 	bool lacks_render = !(gt->info.engine_mask & XE_HW_ENGINE_RCS_MASK);
357 	u32 mask_flags = 0;
358 
359 	i = emit_copy_timestamp(lrc, dw, i);
360 
361 	dw[i++] = preparser_disable(true);
362 	if (lacks_render)
363 		mask_flags = PIPE_CONTROL_3D_ARCH_FLAGS;
364 	else if (job->q->class == XE_ENGINE_CLASS_COMPUTE)
365 		mask_flags = PIPE_CONTROL_3D_ENGINE_FLAGS;
366 
367 	/* See __xe_pt_bind_vma() for a discussion on TLB invalidations. */
368 	i = emit_pipe_invalidate(mask_flags, job->ring_ops_flush_tlb, dw, i);
369 
370 	/* hsdes: 1809175790 */
371 	if (has_aux_ccs(xe))
372 		i = emit_aux_table_inv(gt, CCS_AUX_INV, dw, i);
373 
374 	dw[i++] = preparser_disable(false);
375 
376 	i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),
377 				seqno, dw, i);
378 
379 	i = emit_bb_start(batch_addr, ppgtt_flag, dw, i);
380 
381 	i = emit_render_cache_flush(job, dw, i);
382 
383 	if (job->user_fence.used)
384 		i = emit_store_imm_ppgtt_posted(job->user_fence.addr,
385 						job->user_fence.value,
386 						dw, i);
387 
388 	i = emit_pipe_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, lacks_render, dw, i);
389 
390 	i = emit_user_interrupt(dw, i);
391 
392 	i = emit_pipe_control_to_ring_end(job->q->hwe, dw, i);
393 
394 	xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW);
395 
396 	xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));
397 }
398 
399 static void emit_migration_job_gen12(struct xe_sched_job *job,
400 				     struct xe_lrc *lrc, u32 seqno)
401 {
402 	u32 dw[MAX_JOB_SIZE_DW], i = 0;
403 
404 	i = emit_copy_timestamp(lrc, dw, i);
405 
406 	i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),
407 				seqno, dw, i);
408 
409 	dw[i++] = MI_ARB_ON_OFF | MI_ARB_DISABLE; /* Enabled again below */
410 
411 	i = emit_bb_start(job->ptrs[0].batch_addr, BIT(8), dw, i);
412 
413 	if (!IS_SRIOV_VF(gt_to_xe(job->q->gt))) {
414 		/* XXX: Do we need this? Leaving for now. */
415 		dw[i++] = preparser_disable(true);
416 		i = emit_flush_invalidate(0, dw, i);
417 		dw[i++] = preparser_disable(false);
418 	}
419 
420 	i = emit_bb_start(job->ptrs[1].batch_addr, BIT(8), dw, i);
421 
422 	dw[i++] = MI_FLUSH_DW | MI_INVALIDATE_TLB | job->migrate_flush_flags |
423 		MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_IMM_DW;
424 	dw[i++] = xe_lrc_seqno_ggtt_addr(lrc) | MI_FLUSH_DW_USE_GTT;
425 	dw[i++] = 0;
426 	dw[i++] = seqno; /* value */
427 
428 	i = emit_user_interrupt(dw, i);
429 
430 	xe_gt_assert(job->q->gt, i <= MAX_JOB_SIZE_DW);
431 
432 	xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));
433 }
434 
435 static void emit_job_gen12_gsc(struct xe_sched_job *job)
436 {
437 	struct xe_gt *gt = job->q->gt;
438 
439 	xe_gt_assert(gt, job->q->width <= 1); /* no parallel submission for GSCCS */
440 
441 	__emit_job_gen12_simple(job, job->q->lrc[0],
442 				job->ptrs[0].batch_addr,
443 				xe_sched_job_lrc_seqno(job));
444 }
445 
446 static void emit_job_gen12_copy(struct xe_sched_job *job)
447 {
448 	int i;
449 
450 	if (xe_sched_job_is_migration(job->q)) {
451 		emit_migration_job_gen12(job, job->q->lrc[0],
452 					 xe_sched_job_lrc_seqno(job));
453 		return;
454 	}
455 
456 	for (i = 0; i < job->q->width; ++i)
457 		__emit_job_gen12_simple(job, job->q->lrc[i],
458 					job->ptrs[i].batch_addr,
459 					xe_sched_job_lrc_seqno(job));
460 }
461 
462 static void emit_job_gen12_video(struct xe_sched_job *job)
463 {
464 	int i;
465 
466 	/* FIXME: Not doing parallel handshake for now */
467 	for (i = 0; i < job->q->width; ++i)
468 		__emit_job_gen12_video(job, job->q->lrc[i],
469 				       job->ptrs[i].batch_addr,
470 				       xe_sched_job_lrc_seqno(job));
471 }
472 
473 static void emit_job_gen12_render_compute(struct xe_sched_job *job)
474 {
475 	int i;
476 
477 	for (i = 0; i < job->q->width; ++i)
478 		__emit_job_gen12_render_compute(job, job->q->lrc[i],
479 						job->ptrs[i].batch_addr,
480 						xe_sched_job_lrc_seqno(job));
481 }
482 
483 static const struct xe_ring_ops ring_ops_gen12_gsc = {
484 	.emit_job = emit_job_gen12_gsc,
485 };
486 
487 static const struct xe_ring_ops ring_ops_gen12_copy = {
488 	.emit_job = emit_job_gen12_copy,
489 };
490 
491 static const struct xe_ring_ops ring_ops_gen12_video = {
492 	.emit_job = emit_job_gen12_video,
493 };
494 
495 static const struct xe_ring_ops ring_ops_gen12_render_compute = {
496 	.emit_job = emit_job_gen12_render_compute,
497 };
498 
499 const struct xe_ring_ops *
500 xe_ring_ops_get(struct xe_gt *gt, enum xe_engine_class class)
501 {
502 	switch (class) {
503 	case XE_ENGINE_CLASS_OTHER:
504 		return &ring_ops_gen12_gsc;
505 	case XE_ENGINE_CLASS_COPY:
506 		return &ring_ops_gen12_copy;
507 	case XE_ENGINE_CLASS_VIDEO_DECODE:
508 	case XE_ENGINE_CLASS_VIDEO_ENHANCE:
509 		return &ring_ops_gen12_video;
510 	case XE_ENGINE_CLASS_RENDER:
511 	case XE_ENGINE_CLASS_COMPUTE:
512 		return &ring_ops_gen12_render_compute;
513 	default:
514 		return NULL;
515 	}
516 }
517