xref: /linux/drivers/gpu/drm/xe/xe_ring_ops.c (revision d9fef76e89498bf99cdb03f77b7091d7e95d7edd) 
1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2022 Intel Corporation
4  */
5 
6 #include "xe_ring_ops.h"
7 
8 #include <generated/xe_wa_oob.h>
9 
10 #include "instructions/xe_mi_commands.h"
11 #include "regs/xe_engine_regs.h"
12 #include "regs/xe_gpu_commands.h"
13 #include "regs/xe_gt_regs.h"
14 #include "regs/xe_lrc_layout.h"
15 #include "xe_exec_queue_types.h"
16 #include "xe_gt.h"
17 #include "xe_lrc.h"
18 #include "xe_macros.h"
19 #include "xe_sched_job.h"
20 #include "xe_sriov.h"
21 #include "xe_vm_types.h"
22 #include "xe_vm.h"
23 #include "xe_wa.h"
24 
25 /*
26  * 3D-related flags that can't be set on _engines_ that lack access to the 3D
27  * pipeline (i.e., CCS engines).
28  */
29 #define PIPE_CONTROL_3D_ENGINE_FLAGS (\
30 		PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH | \
31 		PIPE_CONTROL_DEPTH_CACHE_FLUSH | \
32 		PIPE_CONTROL_TILE_CACHE_FLUSH | \
33 		PIPE_CONTROL_DEPTH_STALL | \
34 		PIPE_CONTROL_STALL_AT_SCOREBOARD | \
35 		PIPE_CONTROL_PSD_SYNC | \
36 		PIPE_CONTROL_AMFS_FLUSH | \
37 		PIPE_CONTROL_VF_CACHE_INVALIDATE | \
38 		PIPE_CONTROL_GLOBAL_SNAPSHOT_RESET)
39 
40 /* 3D-related flags that can't be set on _platforms_ that lack a 3D pipeline */
41 #define PIPE_CONTROL_3D_ARCH_FLAGS ( \
42 		PIPE_CONTROL_3D_ENGINE_FLAGS | \
43 		PIPE_CONTROL_INDIRECT_STATE_DISABLE | \
44 		PIPE_CONTROL_FLUSH_ENABLE | \
45 		PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE | \
46 		PIPE_CONTROL_DC_FLUSH_ENABLE)
47 
48 static u32 preparser_disable(bool state)
49 {
50 	return MI_ARB_CHECK | BIT(8) | state;
51 }
52 
53 static int emit_aux_table_inv(struct xe_gt *gt, struct xe_reg reg,
54 			      u32 *dw, int i)
55 {
56 	dw[i++] = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(1) | MI_LRI_MMIO_REMAP_EN;
57 	dw[i++] = reg.addr + gt->mmio.adj_offset;
58 	dw[i++] = AUX_INV;
59 	dw[i++] = MI_NOOP;
60 
61 	return i;
62 }
63 
64 static int emit_user_interrupt(u32 *dw, int i)
65 {
66 	dw[i++] = MI_USER_INTERRUPT;
67 	dw[i++] = MI_ARB_ON_OFF | MI_ARB_ENABLE;
68 	dw[i++] = MI_ARB_CHECK;
69 
70 	return i;
71 }
72 
73 static int emit_store_imm_ggtt(u32 addr, u32 value, u32 *dw, int i)
74 {
75 	dw[i++] = MI_STORE_DATA_IMM | MI_SDI_GGTT | MI_SDI_NUM_DW(1);
76 	dw[i++] = addr;
77 	dw[i++] = 0;
78 	dw[i++] = value;
79 
80 	return i;
81 }
82 
83 static int emit_flush_imm_ggtt(u32 addr, u32 value, bool invalidate_tlb,
84 			       u32 *dw, int i)
85 {
86 	dw[i++] = MI_FLUSH_DW | MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_IMM_DW |
87 		(invalidate_tlb ? MI_INVALIDATE_TLB : 0);
88 	dw[i++] = addr | MI_FLUSH_DW_USE_GTT;
89 	dw[i++] = 0;
90 	dw[i++] = value;
91 
92 	return i;
93 }
94 
95 static int emit_bb_start(u64 batch_addr, u32 ppgtt_flag, u32 *dw, int i)
96 {
97 	dw[i++] = MI_BATCH_BUFFER_START | ppgtt_flag | XE_INSTR_NUM_DW(3);
98 	dw[i++] = lower_32_bits(batch_addr);
99 	dw[i++] = upper_32_bits(batch_addr);
100 
101 	return i;
102 }
103 
104 static int emit_flush_invalidate(u32 flag, u32 *dw, int i)
105 {
106 	dw[i] = MI_FLUSH_DW;
107 	dw[i] |= flag;
108 	dw[i++] |= MI_INVALIDATE_TLB | MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_IMM_DW |
109 		MI_FLUSH_DW_STORE_INDEX;
110 
111 	dw[i++] = LRC_PPHWSP_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT;
112 	dw[i++] = 0;
113 	dw[i++] = ~0U;
114 
115 	return i;
116 }
117 
118 static int
119 emit_pipe_control(u32 *dw, int i, u32 bit_group_0, u32 bit_group_1, u32 offset, u32 value)
120 {
121 	dw[i++] = GFX_OP_PIPE_CONTROL(6) | bit_group_0;
122 	dw[i++] = bit_group_1;
123 	dw[i++] = offset;
124 	dw[i++] = 0;
125 	dw[i++] = value;
126 	dw[i++] = 0;
127 
128 	return i;
129 }
130 
131 static int emit_pipe_invalidate(u32 mask_flags, bool invalidate_tlb, u32 *dw,
132 				int i)
133 {
134 	u32 flags = PIPE_CONTROL_CS_STALL |
135 		PIPE_CONTROL_COMMAND_CACHE_INVALIDATE |
136 		PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE |
137 		PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
138 		PIPE_CONTROL_VF_CACHE_INVALIDATE |
139 		PIPE_CONTROL_CONST_CACHE_INVALIDATE |
140 		PIPE_CONTROL_STATE_CACHE_INVALIDATE |
141 		PIPE_CONTROL_QW_WRITE |
142 		PIPE_CONTROL_STORE_DATA_INDEX;
143 
144 	if (invalidate_tlb)
145 		flags |= PIPE_CONTROL_TLB_INVALIDATE;
146 
147 	flags &= ~mask_flags;
148 
149 	return emit_pipe_control(dw, i, 0, flags, LRC_PPHWSP_SCRATCH_ADDR, 0);
150 }
151 
152 static int emit_store_imm_ppgtt_posted(u64 addr, u64 value,
153 				       u32 *dw, int i)
154 {
155 	dw[i++] = MI_STORE_DATA_IMM | MI_SDI_NUM_QW(1);
156 	dw[i++] = lower_32_bits(addr);
157 	dw[i++] = upper_32_bits(addr);
158 	dw[i++] = lower_32_bits(value);
159 	dw[i++] = upper_32_bits(value);
160 
161 	return i;
162 }
163 
164 static int emit_render_cache_flush(struct xe_sched_job *job, u32 *dw, int i)
165 {
166 	struct xe_gt *gt = job->q->gt;
167 	bool lacks_render = !(gt->info.engine_mask & XE_HW_ENGINE_RCS_MASK);
168 	u32 flags;
169 
170 	flags = (PIPE_CONTROL_CS_STALL |
171 		 PIPE_CONTROL_TILE_CACHE_FLUSH |
172 		 PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH |
173 		 PIPE_CONTROL_DEPTH_CACHE_FLUSH |
174 		 PIPE_CONTROL_DC_FLUSH_ENABLE |
175 		 PIPE_CONTROL_FLUSH_ENABLE);
176 
177 	if (XE_WA(gt, 1409600907))
178 		flags |= PIPE_CONTROL_DEPTH_STALL;
179 
180 	if (lacks_render)
181 		flags &= ~PIPE_CONTROL_3D_ARCH_FLAGS;
182 	else if (job->q->class == XE_ENGINE_CLASS_COMPUTE)
183 		flags &= ~PIPE_CONTROL_3D_ENGINE_FLAGS;
184 
185 	return emit_pipe_control(dw, i, PIPE_CONTROL0_HDC_PIPELINE_FLUSH, flags, 0, 0);
186 }
187 
188 static int emit_pipe_control_to_ring_end(struct xe_hw_engine *hwe, u32 *dw, int i)
189 {
190 	if (hwe->class != XE_ENGINE_CLASS_RENDER)
191 		return i;
192 
193 	if (XE_WA(hwe->gt, 16020292621))
194 		i = emit_pipe_control(dw, i, 0, PIPE_CONTROL_LRI_POST_SYNC,
195 				      RING_NOPID(hwe->mmio_base).addr, 0);
196 
197 	return i;
198 }
199 
200 static int emit_pipe_imm_ggtt(u32 addr, u32 value, bool stall_only, u32 *dw,
201 			      int i)
202 {
203 	u32 flags = PIPE_CONTROL_CS_STALL | PIPE_CONTROL_GLOBAL_GTT_IVB |
204 		    PIPE_CONTROL_QW_WRITE;
205 
206 	if (!stall_only)
207 		flags |= PIPE_CONTROL_FLUSH_ENABLE;
208 
209 	return emit_pipe_control(dw, i, 0, flags, addr, value);
210 }
211 
212 static u32 get_ppgtt_flag(struct xe_sched_job *job)
213 {
214 	return job->q->vm ? BIT(8) : 0;
215 }
216 
217 /* for engines that don't require any special HW handling (no EUs, no aux inval, etc) */
218 static void __emit_job_gen12_simple(struct xe_sched_job *job, struct xe_lrc *lrc,
219 				    u64 batch_addr, u32 seqno)
220 {
221 	u32 dw[MAX_JOB_SIZE_DW], i = 0;
222 	u32 ppgtt_flag = get_ppgtt_flag(job);
223 	struct xe_gt *gt = job->q->gt;
224 
225 	if (job->ring_ops_flush_tlb) {
226 		dw[i++] = preparser_disable(true);
227 		i = emit_flush_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),
228 					seqno, true, dw, i);
229 		dw[i++] = preparser_disable(false);
230 	} else {
231 		i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),
232 					seqno, dw, i);
233 	}
234 
235 	i = emit_bb_start(batch_addr, ppgtt_flag, dw, i);
236 
237 	if (job->user_fence.used)
238 		i = emit_store_imm_ppgtt_posted(job->user_fence.addr,
239 						job->user_fence.value,
240 						dw, i);
241 
242 	i = emit_flush_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, false, dw, i);
243 
244 	i = emit_user_interrupt(dw, i);
245 
246 	xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW);
247 
248 	xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));
249 }
250 
251 static bool has_aux_ccs(struct xe_device *xe)
252 {
253 	/*
254 	 * PVC is a special case that has no compression of either type
255 	 * (FlatCCS or AuxCCS).  Also, AuxCCS is no longer used from Xe2
256 	 * onward, so any future platforms with no FlatCCS will not have
257 	 * AuxCCS either.
258 	 */
259 	if (GRAPHICS_VER(xe) >= 20 || xe->info.platform == XE_PVC)
260 		return false;
261 
262 	return !xe->info.has_flat_ccs;
263 }
264 
265 static void __emit_job_gen12_video(struct xe_sched_job *job, struct xe_lrc *lrc,
266 				   u64 batch_addr, u32 seqno)
267 {
268 	u32 dw[MAX_JOB_SIZE_DW], i = 0;
269 	u32 ppgtt_flag = get_ppgtt_flag(job);
270 	struct xe_gt *gt = job->q->gt;
271 	struct xe_device *xe = gt_to_xe(gt);
272 	bool decode = job->q->class == XE_ENGINE_CLASS_VIDEO_DECODE;
273 
274 	dw[i++] = preparser_disable(true);
275 
276 	/* hsdes: 1809175790 */
277 	if (has_aux_ccs(xe)) {
278 		if (decode)
279 			i = emit_aux_table_inv(gt, VD0_AUX_INV, dw, i);
280 		else
281 			i = emit_aux_table_inv(gt, VE0_AUX_INV, dw, i);
282 	}
283 
284 	if (job->ring_ops_flush_tlb)
285 		i = emit_flush_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),
286 					seqno, true, dw, i);
287 
288 	dw[i++] = preparser_disable(false);
289 
290 	if (!job->ring_ops_flush_tlb)
291 		i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),
292 					seqno, dw, i);
293 
294 	i = emit_bb_start(batch_addr, ppgtt_flag, dw, i);
295 
296 	if (job->user_fence.used)
297 		i = emit_store_imm_ppgtt_posted(job->user_fence.addr,
298 						job->user_fence.value,
299 						dw, i);
300 
301 	i = emit_flush_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, false, dw, i);
302 
303 	i = emit_user_interrupt(dw, i);
304 
305 	xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW);
306 
307 	xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));
308 }
309 
310 static void __emit_job_gen12_render_compute(struct xe_sched_job *job,
311 					    struct xe_lrc *lrc,
312 					    u64 batch_addr, u32 seqno)
313 {
314 	u32 dw[MAX_JOB_SIZE_DW], i = 0;
315 	u32 ppgtt_flag = get_ppgtt_flag(job);
316 	struct xe_gt *gt = job->q->gt;
317 	struct xe_device *xe = gt_to_xe(gt);
318 	bool lacks_render = !(gt->info.engine_mask & XE_HW_ENGINE_RCS_MASK);
319 	u32 mask_flags = 0;
320 
321 	dw[i++] = preparser_disable(true);
322 	if (lacks_render)
323 		mask_flags = PIPE_CONTROL_3D_ARCH_FLAGS;
324 	else if (job->q->class == XE_ENGINE_CLASS_COMPUTE)
325 		mask_flags = PIPE_CONTROL_3D_ENGINE_FLAGS;
326 
327 	/* See __xe_pt_bind_vma() for a discussion on TLB invalidations. */
328 	i = emit_pipe_invalidate(mask_flags, job->ring_ops_flush_tlb, dw, i);
329 
330 	/* hsdes: 1809175790 */
331 	if (has_aux_ccs(xe))
332 		i = emit_aux_table_inv(gt, CCS_AUX_INV, dw, i);
333 
334 	dw[i++] = preparser_disable(false);
335 
336 	i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),
337 				seqno, dw, i);
338 
339 	i = emit_bb_start(batch_addr, ppgtt_flag, dw, i);
340 
341 	i = emit_render_cache_flush(job, dw, i);
342 
343 	if (job->user_fence.used)
344 		i = emit_store_imm_ppgtt_posted(job->user_fence.addr,
345 						job->user_fence.value,
346 						dw, i);
347 
348 	i = emit_pipe_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, lacks_render, dw, i);
349 
350 	i = emit_user_interrupt(dw, i);
351 
352 	i = emit_pipe_control_to_ring_end(job->q->hwe, dw, i);
353 
354 	xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW);
355 
356 	xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));
357 }
358 
359 static void emit_migration_job_gen12(struct xe_sched_job *job,
360 				     struct xe_lrc *lrc, u32 seqno)
361 {
362 	u32 dw[MAX_JOB_SIZE_DW], i = 0;
363 
364 	i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),
365 				seqno, dw, i);
366 
367 	dw[i++] = MI_ARB_ON_OFF | MI_ARB_DISABLE; /* Enabled again below */
368 
369 	i = emit_bb_start(job->batch_addr[0], BIT(8), dw, i);
370 
371 	if (!IS_SRIOV_VF(gt_to_xe(job->q->gt))) {
372 		/* XXX: Do we need this? Leaving for now. */
373 		dw[i++] = preparser_disable(true);
374 		i = emit_flush_invalidate(0, dw, i);
375 		dw[i++] = preparser_disable(false);
376 	}
377 
378 	i = emit_bb_start(job->batch_addr[1], BIT(8), dw, i);
379 
380 	dw[i++] = MI_FLUSH_DW | MI_INVALIDATE_TLB | job->migrate_flush_flags |
381 		MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_IMM_DW;
382 	dw[i++] = xe_lrc_seqno_ggtt_addr(lrc) | MI_FLUSH_DW_USE_GTT;
383 	dw[i++] = 0;
384 	dw[i++] = seqno; /* value */
385 
386 	i = emit_user_interrupt(dw, i);
387 
388 	xe_gt_assert(job->q->gt, i <= MAX_JOB_SIZE_DW);
389 
390 	xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));
391 }
392 
393 static void emit_job_gen12_gsc(struct xe_sched_job *job)
394 {
395 	struct xe_gt *gt = job->q->gt;
396 
397 	xe_gt_assert(gt, job->q->width <= 1); /* no parallel submission for GSCCS */
398 
399 	__emit_job_gen12_simple(job, job->q->lrc,
400 				job->batch_addr[0],
401 				xe_sched_job_seqno(job));
402 }
403 
404 static void emit_job_gen12_copy(struct xe_sched_job *job)
405 {
406 	int i;
407 
408 	if (xe_sched_job_is_migration(job->q)) {
409 		emit_migration_job_gen12(job, job->q->lrc,
410 					 xe_sched_job_seqno(job));
411 		return;
412 	}
413 
414 	for (i = 0; i < job->q->width; ++i)
415 		__emit_job_gen12_simple(job, job->q->lrc + i,
416 				        job->batch_addr[i],
417 				        xe_sched_job_seqno(job));
418 }
419 
420 static void emit_job_gen12_video(struct xe_sched_job *job)
421 {
422 	int i;
423 
424 	/* FIXME: Not doing parallel handshake for now */
425 	for (i = 0; i < job->q->width; ++i)
426 		__emit_job_gen12_video(job, job->q->lrc + i,
427 				       job->batch_addr[i],
428 				       xe_sched_job_seqno(job));
429 }
430 
431 static void emit_job_gen12_render_compute(struct xe_sched_job *job)
432 {
433 	int i;
434 
435 	for (i = 0; i < job->q->width; ++i)
436 		__emit_job_gen12_render_compute(job, job->q->lrc + i,
437 						job->batch_addr[i],
438 						xe_sched_job_seqno(job));
439 }
440 
441 static const struct xe_ring_ops ring_ops_gen12_gsc = {
442 	.emit_job = emit_job_gen12_gsc,
443 };
444 
445 static const struct xe_ring_ops ring_ops_gen12_copy = {
446 	.emit_job = emit_job_gen12_copy,
447 };
448 
449 static const struct xe_ring_ops ring_ops_gen12_video = {
450 	.emit_job = emit_job_gen12_video,
451 };
452 
453 static const struct xe_ring_ops ring_ops_gen12_render_compute = {
454 	.emit_job = emit_job_gen12_render_compute,
455 };
456 
457 const struct xe_ring_ops *
458 xe_ring_ops_get(struct xe_gt *gt, enum xe_engine_class class)
459 {
460 	switch (class) {
461 	case XE_ENGINE_CLASS_OTHER:
462 		return &ring_ops_gen12_gsc;
463 	case XE_ENGINE_CLASS_COPY:
464 		return &ring_ops_gen12_copy;
465 	case XE_ENGINE_CLASS_VIDEO_DECODE:
466 	case XE_ENGINE_CLASS_VIDEO_ENHANCE:
467 		return &ring_ops_gen12_video;
468 	case XE_ENGINE_CLASS_RENDER:
469 	case XE_ENGINE_CLASS_COMPUTE:
470 		return &ring_ops_gen12_render_compute;
471 	default:
472 		return NULL;
473 	}
474 }
475