xref: /linux/drivers/gpu/drm/xe/xe_pat.c (revision 4b660dbd9ee2059850fd30e0df420ca7a38a1856)
1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2023 Intel Corporation
4  */
5 
6 #include "xe_pat.h"
7 
8 #include <drm/xe_drm.h>
9 
10 #include "regs/xe_reg_defs.h"
11 #include "xe_assert.h"
12 #include "xe_device.h"
13 #include "xe_gt.h"
14 #include "xe_gt_mcr.h"
15 #include "xe_mmio.h"
16 #include "xe_sriov.h"
17 
18 #define _PAT_ATS				0x47fc
19 #define _PAT_INDEX(index)			_PICK_EVEN_2RANGES(index, 8, \
20 								   0x4800, 0x4804, \
21 								   0x4848, 0x484c)
22 #define _PAT_PTA				0x4820
23 
24 #define XE2_NO_PROMOTE				REG_BIT(10)
25 #define XE2_COMP_EN				REG_BIT(9)
26 #define XE2_L3_CLOS				REG_GENMASK(7, 6)
27 #define XE2_L3_POLICY				REG_GENMASK(5, 4)
28 #define XE2_L4_POLICY				REG_GENMASK(3, 2)
29 #define XE2_COH_MODE				REG_GENMASK(1, 0)
30 
31 #define XELPG_L4_POLICY_MASK			REG_GENMASK(3, 2)
32 #define XELPG_PAT_3_UC				REG_FIELD_PREP(XELPG_L4_POLICY_MASK, 3)
33 #define XELPG_PAT_1_WT				REG_FIELD_PREP(XELPG_L4_POLICY_MASK, 1)
34 #define XELPG_PAT_0_WB				REG_FIELD_PREP(XELPG_L4_POLICY_MASK, 0)
35 #define XELPG_INDEX_COH_MODE_MASK		REG_GENMASK(1, 0)
36 #define XELPG_3_COH_2W				REG_FIELD_PREP(XELPG_INDEX_COH_MODE_MASK, 3)
37 #define XELPG_2_COH_1W				REG_FIELD_PREP(XELPG_INDEX_COH_MODE_MASK, 2)
38 #define XELPG_0_COH_NON				REG_FIELD_PREP(XELPG_INDEX_COH_MODE_MASK, 0)
39 
40 #define XEHPC_CLOS_LEVEL_MASK			REG_GENMASK(3, 2)
41 #define XEHPC_PAT_CLOS(x)			REG_FIELD_PREP(XEHPC_CLOS_LEVEL_MASK, x)
42 
43 #define XELP_MEM_TYPE_MASK			REG_GENMASK(1, 0)
44 #define XELP_PAT_WB				REG_FIELD_PREP(XELP_MEM_TYPE_MASK, 3)
45 #define XELP_PAT_WT				REG_FIELD_PREP(XELP_MEM_TYPE_MASK, 2)
46 #define XELP_PAT_WC				REG_FIELD_PREP(XELP_MEM_TYPE_MASK, 1)
47 #define XELP_PAT_UC				REG_FIELD_PREP(XELP_MEM_TYPE_MASK, 0)
48 
49 static const char *XELP_MEM_TYPE_STR_MAP[] = { "UC", "WC", "WT", "WB" };
50 
51 struct xe_pat_ops {
52 	void (*program_graphics)(struct xe_gt *gt, const struct xe_pat_table_entry table[],
53 				 int n_entries);
54 	void (*program_media)(struct xe_gt *gt, const struct xe_pat_table_entry table[],
55 			      int n_entries);
56 	void (*dump)(struct xe_gt *gt, struct drm_printer *p);
57 };
58 
59 static const struct xe_pat_table_entry xelp_pat_table[] = {
60 	[0] = { XELP_PAT_WB, XE_COH_AT_LEAST_1WAY },
61 	[1] = { XELP_PAT_WC, XE_COH_NONE },
62 	[2] = { XELP_PAT_WT, XE_COH_NONE },
63 	[3] = { XELP_PAT_UC, XE_COH_NONE },
64 };
65 
66 static const struct xe_pat_table_entry xehpc_pat_table[] = {
67 	[0] = { XELP_PAT_UC, XE_COH_NONE },
68 	[1] = { XELP_PAT_WC, XE_COH_NONE },
69 	[2] = { XELP_PAT_WT, XE_COH_NONE },
70 	[3] = { XELP_PAT_WB, XE_COH_AT_LEAST_1WAY },
71 	[4] = { XEHPC_PAT_CLOS(1) | XELP_PAT_WT, XE_COH_NONE },
72 	[5] = { XEHPC_PAT_CLOS(1) | XELP_PAT_WB, XE_COH_AT_LEAST_1WAY },
73 	[6] = { XEHPC_PAT_CLOS(2) | XELP_PAT_WT, XE_COH_NONE },
74 	[7] = { XEHPC_PAT_CLOS(2) | XELP_PAT_WB, XE_COH_AT_LEAST_1WAY },
75 };
76 
77 static const struct xe_pat_table_entry xelpg_pat_table[] = {
78 	[0] = { XELPG_PAT_0_WB, XE_COH_NONE },
79 	[1] = { XELPG_PAT_1_WT, XE_COH_NONE },
80 	[2] = { XELPG_PAT_3_UC, XE_COH_NONE },
81 	[3] = { XELPG_PAT_0_WB | XELPG_2_COH_1W, XE_COH_AT_LEAST_1WAY },
82 	[4] = { XELPG_PAT_0_WB | XELPG_3_COH_2W, XE_COH_AT_LEAST_1WAY },
83 };
84 
85 /*
86  * The Xe2 table is getting large/complicated so it's easier to review if
87  * provided in a form that exactly matches the bspec's formatting.  The meaning
88  * of the fields here are:
89  *   - no_promote:  0=promotable, 1=no promote
90  *   - comp_en:     0=disable, 1=enable
91  *   - l3clos:      L3 class of service (0-3)
92  *   - l3_policy:   0=WB, 1=XD ("WB - Transient Display"), 3=UC
93  *   - l4_policy:   0=WB, 1=WT, 3=UC
94  *   - coh_mode:    0=no snoop, 2=1-way coherent, 3=2-way coherent
95  *
96  * Reserved entries should be programmed with the maximum caching, minimum
97  * coherency (which matches an all-0's encoding), so we can just omit them
98  * in the table.
99  */
100 #define XE2_PAT(no_promote, comp_en, l3clos, l3_policy, l4_policy, __coh_mode) \
101 	{ \
102 		.value = (no_promote ? XE2_NO_PROMOTE : 0) | \
103 			(comp_en ? XE2_COMP_EN : 0) | \
104 			REG_FIELD_PREP(XE2_L3_CLOS, l3clos) | \
105 			REG_FIELD_PREP(XE2_L3_POLICY, l3_policy) | \
106 			REG_FIELD_PREP(XE2_L4_POLICY, l4_policy) | \
107 			REG_FIELD_PREP(XE2_COH_MODE, __coh_mode), \
108 		.coh_mode = __coh_mode ? XE_COH_AT_LEAST_1WAY : XE_COH_NONE \
109 	}
110 
111 static const struct xe_pat_table_entry xe2_pat_table[] = {
112 	[ 0] = XE2_PAT( 0, 0, 0, 0, 3, 0 ),
113 	[ 1] = XE2_PAT( 0, 0, 0, 0, 3, 2 ),
114 	[ 2] = XE2_PAT( 0, 0, 0, 0, 3, 3 ),
115 	[ 3] = XE2_PAT( 0, 0, 0, 3, 3, 0 ),
116 	[ 4] = XE2_PAT( 0, 0, 0, 3, 0, 2 ),
117 	[ 5] = XE2_PAT( 0, 0, 0, 3, 3, 2 ),
118 	[ 6] = XE2_PAT( 1, 0, 0, 1, 3, 0 ),
119 	[ 7] = XE2_PAT( 0, 0, 0, 3, 0, 3 ),
120 	[ 8] = XE2_PAT( 0, 0, 0, 3, 0, 0 ),
121 	[ 9] = XE2_PAT( 0, 1, 0, 0, 3, 0 ),
122 	[10] = XE2_PAT( 0, 1, 0, 3, 0, 0 ),
123 	[11] = XE2_PAT( 1, 1, 0, 1, 3, 0 ),
124 	[12] = XE2_PAT( 0, 1, 0, 3, 3, 0 ),
125 	[13] = XE2_PAT( 0, 0, 0, 0, 0, 0 ),
126 	[14] = XE2_PAT( 0, 1, 0, 0, 0, 0 ),
127 	[15] = XE2_PAT( 1, 1, 0, 1, 1, 0 ),
128 	/* 16..19 are reserved; leave set to all 0's */
129 	[20] = XE2_PAT( 0, 0, 1, 0, 3, 0 ),
130 	[21] = XE2_PAT( 0, 1, 1, 0, 3, 0 ),
131 	[22] = XE2_PAT( 0, 0, 1, 0, 3, 2 ),
132 	[23] = XE2_PAT( 0, 0, 1, 0, 3, 3 ),
133 	[24] = XE2_PAT( 0, 0, 2, 0, 3, 0 ),
134 	[25] = XE2_PAT( 0, 1, 2, 0, 3, 0 ),
135 	[26] = XE2_PAT( 0, 0, 2, 0, 3, 2 ),
136 	[27] = XE2_PAT( 0, 0, 2, 0, 3, 3 ),
137 	[28] = XE2_PAT( 0, 0, 3, 0, 3, 0 ),
138 	[29] = XE2_PAT( 0, 1, 3, 0, 3, 0 ),
139 	[30] = XE2_PAT( 0, 0, 3, 0, 3, 2 ),
140 	[31] = XE2_PAT( 0, 0, 3, 0, 3, 3 ),
141 };
142 
143 /* Special PAT values programmed outside the main table */
144 static const struct xe_pat_table_entry xe2_pat_ats = XE2_PAT( 0, 0, 0, 0, 3, 3 );
145 
146 u16 xe_pat_index_get_coh_mode(struct xe_device *xe, u16 pat_index)
147 {
148 	WARN_ON(pat_index >= xe->pat.n_entries);
149 	return xe->pat.table[pat_index].coh_mode;
150 }
151 
152 static void program_pat(struct xe_gt *gt, const struct xe_pat_table_entry table[],
153 			int n_entries)
154 {
155 	for (int i = 0; i < n_entries; i++) {
156 		struct xe_reg reg = XE_REG(_PAT_INDEX(i));
157 
158 		xe_mmio_write32(gt, reg, table[i].value);
159 	}
160 }
161 
162 static void program_pat_mcr(struct xe_gt *gt, const struct xe_pat_table_entry table[],
163 			    int n_entries)
164 {
165 	for (int i = 0; i < n_entries; i++) {
166 		struct xe_reg_mcr reg_mcr = XE_REG_MCR(_PAT_INDEX(i));
167 
168 		xe_gt_mcr_multicast_write(gt, reg_mcr, table[i].value);
169 	}
170 }
171 
172 static void xelp_dump(struct xe_gt *gt, struct drm_printer *p)
173 {
174 	struct xe_device *xe = gt_to_xe(gt);
175 	int i, err;
176 
177 	xe_device_mem_access_get(xe);
178 	err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
179 	if (err)
180 		goto err_fw;
181 
182 	drm_printf(p, "PAT table:\n");
183 
184 	for (i = 0; i < xe->pat.n_entries; i++) {
185 		u32 pat = xe_mmio_read32(gt, XE_REG(_PAT_INDEX(i)));
186 		u8 mem_type = REG_FIELD_GET(XELP_MEM_TYPE_MASK, pat);
187 
188 		drm_printf(p, "PAT[%2d] = %s (%#8x)\n", i,
189 			   XELP_MEM_TYPE_STR_MAP[mem_type], pat);
190 	}
191 
192 	err = xe_force_wake_put(gt_to_fw(gt), XE_FW_GT);
193 err_fw:
194 	xe_assert(xe, !err);
195 	xe_device_mem_access_put(xe);
196 }
197 
198 static const struct xe_pat_ops xelp_pat_ops = {
199 	.program_graphics = program_pat,
200 	.dump = xelp_dump,
201 };
202 
203 static void xehp_dump(struct xe_gt *gt, struct drm_printer *p)
204 {
205 	struct xe_device *xe = gt_to_xe(gt);
206 	int i, err;
207 
208 	xe_device_mem_access_get(xe);
209 	err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
210 	if (err)
211 		goto err_fw;
212 
213 	drm_printf(p, "PAT table:\n");
214 
215 	for (i = 0; i < xe->pat.n_entries; i++) {
216 		u32 pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i)));
217 		u8 mem_type;
218 
219 		mem_type = REG_FIELD_GET(XELP_MEM_TYPE_MASK, pat);
220 
221 		drm_printf(p, "PAT[%2d] = %s (%#8x)\n", i,
222 			   XELP_MEM_TYPE_STR_MAP[mem_type], pat);
223 	}
224 
225 	err = xe_force_wake_put(gt_to_fw(gt), XE_FW_GT);
226 err_fw:
227 	xe_assert(xe, !err);
228 	xe_device_mem_access_put(xe);
229 }
230 
231 static const struct xe_pat_ops xehp_pat_ops = {
232 	.program_graphics = program_pat_mcr,
233 	.dump = xehp_dump,
234 };
235 
236 static void xehpc_dump(struct xe_gt *gt, struct drm_printer *p)
237 {
238 	struct xe_device *xe = gt_to_xe(gt);
239 	int i, err;
240 
241 	xe_device_mem_access_get(xe);
242 	err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
243 	if (err)
244 		goto err_fw;
245 
246 	drm_printf(p, "PAT table:\n");
247 
248 	for (i = 0; i < xe->pat.n_entries; i++) {
249 		u32 pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i)));
250 
251 		drm_printf(p, "PAT[%2d] = [ %u, %u ] (%#8x)\n", i,
252 			   REG_FIELD_GET(XELP_MEM_TYPE_MASK, pat),
253 			   REG_FIELD_GET(XEHPC_CLOS_LEVEL_MASK, pat), pat);
254 	}
255 
256 	err = xe_force_wake_put(gt_to_fw(gt), XE_FW_GT);
257 err_fw:
258 	xe_assert(xe, !err);
259 	xe_device_mem_access_put(xe);
260 }
261 
262 static const struct xe_pat_ops xehpc_pat_ops = {
263 	.program_graphics = program_pat_mcr,
264 	.dump = xehpc_dump,
265 };
266 
267 static void xelpg_dump(struct xe_gt *gt, struct drm_printer *p)
268 {
269 	struct xe_device *xe = gt_to_xe(gt);
270 	int i, err;
271 
272 	xe_device_mem_access_get(xe);
273 	err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
274 	if (err)
275 		goto err_fw;
276 
277 	drm_printf(p, "PAT table:\n");
278 
279 	for (i = 0; i < xe->pat.n_entries; i++) {
280 		u32 pat;
281 
282 		if (xe_gt_is_media_type(gt))
283 			pat = xe_mmio_read32(gt, XE_REG(_PAT_INDEX(i)));
284 		else
285 			pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i)));
286 
287 		drm_printf(p, "PAT[%2d] = [ %u, %u ] (%#8x)\n", i,
288 			   REG_FIELD_GET(XELPG_L4_POLICY_MASK, pat),
289 			   REG_FIELD_GET(XELPG_INDEX_COH_MODE_MASK, pat), pat);
290 	}
291 
292 	err = xe_force_wake_put(gt_to_fw(gt), XE_FW_GT);
293 err_fw:
294 	xe_assert(xe, !err);
295 	xe_device_mem_access_put(xe);
296 }
297 
298 /*
299  * SAMedia register offsets are adjusted by the write methods and they target
300  * registers that are not MCR, while for normal GT they are MCR
301  */
302 static const struct xe_pat_ops xelpg_pat_ops = {
303 	.program_graphics = program_pat,
304 	.program_media = program_pat_mcr,
305 	.dump = xelpg_dump,
306 };
307 
308 static void xe2lpg_program_pat(struct xe_gt *gt, const struct xe_pat_table_entry table[],
309 			       int n_entries)
310 {
311 	program_pat_mcr(gt, table, n_entries);
312 	xe_gt_mcr_multicast_write(gt, XE_REG_MCR(_PAT_ATS), xe2_pat_ats.value);
313 }
314 
315 static void xe2lpm_program_pat(struct xe_gt *gt, const struct xe_pat_table_entry table[],
316 			       int n_entries)
317 {
318 	program_pat(gt, table, n_entries);
319 	xe_mmio_write32(gt, XE_REG(_PAT_ATS), xe2_pat_ats.value);
320 }
321 
322 static void xe2_dump(struct xe_gt *gt, struct drm_printer *p)
323 {
324 	struct xe_device *xe = gt_to_xe(gt);
325 	int i, err;
326 	u32 pat;
327 
328 	xe_device_mem_access_get(xe);
329 	err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
330 	if (err)
331 		goto err_fw;
332 
333 	drm_printf(p, "PAT table:\n");
334 
335 	for (i = 0; i < xe->pat.n_entries; i++) {
336 		if (xe_gt_is_media_type(gt))
337 			pat = xe_mmio_read32(gt, XE_REG(_PAT_INDEX(i)));
338 		else
339 			pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i)));
340 
341 		drm_printf(p, "PAT[%2d] = [ %u, %u, %u, %u, %u, %u ]  (%#8x)\n", i,
342 			   !!(pat & XE2_NO_PROMOTE),
343 			   !!(pat & XE2_COMP_EN),
344 			   REG_FIELD_GET(XE2_L3_CLOS, pat),
345 			   REG_FIELD_GET(XE2_L3_POLICY, pat),
346 			   REG_FIELD_GET(XE2_L4_POLICY, pat),
347 			   REG_FIELD_GET(XE2_COH_MODE, pat),
348 			   pat);
349 	}
350 
351 	/*
352 	 * Also print PTA_MODE, which describes how the hardware accesses
353 	 * PPGTT entries.
354 	 */
355 	if (xe_gt_is_media_type(gt))
356 		pat = xe_mmio_read32(gt, XE_REG(_PAT_PTA));
357 	else
358 		pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_PTA));
359 
360 	drm_printf(p, "Page Table Access:\n");
361 	drm_printf(p, "PTA_MODE= [ %u, %u, %u, %u, %u, %u ]  (%#8x)\n",
362 		   !!(pat & XE2_NO_PROMOTE),
363 		   !!(pat & XE2_COMP_EN),
364 		   REG_FIELD_GET(XE2_L3_CLOS, pat),
365 		   REG_FIELD_GET(XE2_L3_POLICY, pat),
366 		   REG_FIELD_GET(XE2_L4_POLICY, pat),
367 		   REG_FIELD_GET(XE2_COH_MODE, pat),
368 		   pat);
369 
370 	err = xe_force_wake_put(gt_to_fw(gt), XE_FW_GT);
371 err_fw:
372 	xe_assert(xe, !err);
373 	xe_device_mem_access_put(xe);
374 }
375 
376 static const struct xe_pat_ops xe2_pat_ops = {
377 	.program_graphics = xe2lpg_program_pat,
378 	.program_media = xe2lpm_program_pat,
379 	.dump = xe2_dump,
380 };
381 
382 void xe_pat_init_early(struct xe_device *xe)
383 {
384 	if (GRAPHICS_VER(xe) == 20) {
385 		xe->pat.ops = &xe2_pat_ops;
386 		xe->pat.table = xe2_pat_table;
387 		xe->pat.n_entries = ARRAY_SIZE(xe2_pat_table);
388 		xe->pat.idx[XE_CACHE_NONE] = 3;
389 		xe->pat.idx[XE_CACHE_WT] = 15;
390 		xe->pat.idx[XE_CACHE_WB] = 2;
391 		xe->pat.idx[XE_CACHE_NONE_COMPRESSION] = 12; /*Applicable on xe2 and beyond */
392 	} else if (xe->info.platform == XE_METEORLAKE) {
393 		xe->pat.ops = &xelpg_pat_ops;
394 		xe->pat.table = xelpg_pat_table;
395 		xe->pat.n_entries = ARRAY_SIZE(xelpg_pat_table);
396 		xe->pat.idx[XE_CACHE_NONE] = 2;
397 		xe->pat.idx[XE_CACHE_WT] = 1;
398 		xe->pat.idx[XE_CACHE_WB] = 3;
399 	} else if (xe->info.platform == XE_PVC) {
400 		xe->pat.ops = &xehpc_pat_ops;
401 		xe->pat.table = xehpc_pat_table;
402 		xe->pat.n_entries = ARRAY_SIZE(xehpc_pat_table);
403 		xe->pat.idx[XE_CACHE_NONE] = 0;
404 		xe->pat.idx[XE_CACHE_WT] = 2;
405 		xe->pat.idx[XE_CACHE_WB] = 3;
406 	} else if (xe->info.platform == XE_DG2) {
407 		/*
408 		 * Table is the same as previous platforms, but programming
409 		 * method has changed.
410 		 */
411 		xe->pat.ops = &xehp_pat_ops;
412 		xe->pat.table = xelp_pat_table;
413 		xe->pat.n_entries = ARRAY_SIZE(xelp_pat_table);
414 		xe->pat.idx[XE_CACHE_NONE] = 3;
415 		xe->pat.idx[XE_CACHE_WT] = 2;
416 		xe->pat.idx[XE_CACHE_WB] = 0;
417 	} else if (GRAPHICS_VERx100(xe) <= 1210) {
418 		WARN_ON_ONCE(!IS_DGFX(xe) && !xe->info.has_llc);
419 		xe->pat.ops = &xelp_pat_ops;
420 		xe->pat.table = xelp_pat_table;
421 		xe->pat.n_entries = ARRAY_SIZE(xelp_pat_table);
422 		xe->pat.idx[XE_CACHE_NONE] = 3;
423 		xe->pat.idx[XE_CACHE_WT] = 2;
424 		xe->pat.idx[XE_CACHE_WB] = 0;
425 	} else {
426 		/*
427 		 * Going forward we expect to need new PAT settings for most
428 		 * new platforms; failure to provide a new table can easily
429 		 * lead to subtle, hard-to-debug problems.  If none of the
430 		 * conditions above match the platform we're running on we'll
431 		 * raise an error rather than trying to silently inherit the
432 		 * most recent platform's behavior.
433 		 */
434 		drm_err(&xe->drm, "Missing PAT table for platform with graphics version %d.%02d!\n",
435 			GRAPHICS_VER(xe), GRAPHICS_VERx100(xe) % 100);
436 	}
437 
438 	/* VFs can't program nor dump PAT settings */
439 	if (IS_SRIOV_VF(xe))
440 		xe->pat.ops = NULL;
441 }
442 
443 void xe_pat_init(struct xe_gt *gt)
444 {
445 	struct xe_device *xe = gt_to_xe(gt);
446 
447 	if (!xe->pat.ops)
448 		return;
449 
450 	if (xe_gt_is_media_type(gt))
451 		xe->pat.ops->program_media(gt, xe->pat.table, xe->pat.n_entries);
452 	else
453 		xe->pat.ops->program_graphics(gt, xe->pat.table, xe->pat.n_entries);
454 }
455 
456 void xe_pat_dump(struct xe_gt *gt, struct drm_printer *p)
457 {
458 	struct xe_device *xe = gt_to_xe(gt);
459 
460 	if (!xe->pat.ops->dump)
461 		return;
462 
463 	xe->pat.ops->dump(gt, p);
464 }
465