1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2023 Intel Corporation 4 */ 5 6 #include "xe_pat.h" 7 8 #include <uapi/drm/xe_drm.h> 9 10 #include <generated/xe_wa_oob.h> 11 12 #include "regs/xe_gt_regs.h" 13 #include "regs/xe_reg_defs.h" 14 #include "xe_assert.h" 15 #include "xe_device.h" 16 #include "xe_force_wake.h" 17 #include "xe_gt.h" 18 #include "xe_gt_mcr.h" 19 #include "xe_mmio.h" 20 #include "xe_sriov.h" 21 #include "xe_wa.h" 22 23 #define _PAT_ATS 0x47fc 24 #define _PAT_INDEX(index) _PICK_EVEN_2RANGES(index, 8, \ 25 0x4800, 0x4804, \ 26 0x4848, 0x484c) 27 #define _PAT_PTA 0x4820 28 29 #define XE2_NO_PROMOTE REG_BIT(10) 30 #define XE2_COMP_EN REG_BIT(9) 31 #define XE2_L3_CLOS REG_GENMASK(7, 6) 32 #define XE2_L3_POLICY REG_GENMASK(5, 4) 33 #define XE2_L4_POLICY REG_GENMASK(3, 2) 34 #define XE2_COH_MODE REG_GENMASK(1, 0) 35 36 #define XELPG_L4_POLICY_MASK REG_GENMASK(3, 2) 37 #define XELPG_PAT_3_UC REG_FIELD_PREP(XELPG_L4_POLICY_MASK, 3) 38 #define XELPG_PAT_1_WT REG_FIELD_PREP(XELPG_L4_POLICY_MASK, 1) 39 #define XELPG_PAT_0_WB REG_FIELD_PREP(XELPG_L4_POLICY_MASK, 0) 40 #define XELPG_INDEX_COH_MODE_MASK REG_GENMASK(1, 0) 41 #define XELPG_3_COH_2W REG_FIELD_PREP(XELPG_INDEX_COH_MODE_MASK, 3) 42 #define XELPG_2_COH_1W REG_FIELD_PREP(XELPG_INDEX_COH_MODE_MASK, 2) 43 #define XELPG_0_COH_NON REG_FIELD_PREP(XELPG_INDEX_COH_MODE_MASK, 0) 44 45 #define XEHPC_CLOS_LEVEL_MASK REG_GENMASK(3, 2) 46 #define XEHPC_PAT_CLOS(x) REG_FIELD_PREP(XEHPC_CLOS_LEVEL_MASK, x) 47 48 #define XELP_MEM_TYPE_MASK REG_GENMASK(1, 0) 49 #define XELP_PAT_WB REG_FIELD_PREP(XELP_MEM_TYPE_MASK, 3) 50 #define XELP_PAT_WT REG_FIELD_PREP(XELP_MEM_TYPE_MASK, 2) 51 #define XELP_PAT_WC REG_FIELD_PREP(XELP_MEM_TYPE_MASK, 1) 52 #define XELP_PAT_UC REG_FIELD_PREP(XELP_MEM_TYPE_MASK, 0) 53 54 #define PAT_LABEL_LEN 20 55 56 static const char *XELP_MEM_TYPE_STR_MAP[] = { "UC", "WC", "WT", "WB" }; 57 58 static void xe_pat_index_label(char *label, size_t len, int index) 59 { 60 snprintf(label, len, "PAT[%2d] ", index); 61 } 62 63 static void xelp_pat_entry_dump(struct drm_printer *p, int index, u32 pat) 64 { 65 u8 mem_type = REG_FIELD_GET(XELP_MEM_TYPE_MASK, pat); 66 67 drm_printf(p, "PAT[%2d] = %s (%#8x)\n", index, 68 XELP_MEM_TYPE_STR_MAP[mem_type], pat); 69 } 70 71 static void xehpc_pat_entry_dump(struct drm_printer *p, int index, u32 pat) 72 { 73 drm_printf(p, "PAT[%2d] = [ %u, %u ] (%#8x)\n", index, 74 REG_FIELD_GET(XELP_MEM_TYPE_MASK, pat), 75 REG_FIELD_GET(XEHPC_CLOS_LEVEL_MASK, pat), pat); 76 } 77 78 static void xelpg_pat_entry_dump(struct drm_printer *p, int index, u32 pat) 79 { 80 drm_printf(p, "PAT[%2d] = [ %u, %u ] (%#8x)\n", index, 81 REG_FIELD_GET(XELPG_L4_POLICY_MASK, pat), 82 REG_FIELD_GET(XELPG_INDEX_COH_MODE_MASK, pat), pat); 83 } 84 85 struct xe_pat_ops { 86 void (*program_graphics)(struct xe_gt *gt, const struct xe_pat_table_entry table[], 87 int n_entries); 88 void (*program_media)(struct xe_gt *gt, const struct xe_pat_table_entry table[], 89 int n_entries); 90 int (*dump)(struct xe_gt *gt, struct drm_printer *p); 91 void (*entry_dump)(struct drm_printer *p, const char *label, u32 pat, bool rsvd); 92 }; 93 94 static const struct xe_pat_table_entry xelp_pat_table[] = { 95 [0] = { XELP_PAT_WB, XE_COH_AT_LEAST_1WAY }, 96 [1] = { XELP_PAT_WC, XE_COH_NONE }, 97 [2] = { XELP_PAT_WT, XE_COH_NONE }, 98 [3] = { XELP_PAT_UC, XE_COH_NONE }, 99 }; 100 101 static const struct xe_pat_table_entry xehpc_pat_table[] = { 102 [0] = { XELP_PAT_UC, XE_COH_NONE }, 103 [1] = { XELP_PAT_WC, XE_COH_NONE }, 104 [2] = { XELP_PAT_WT, XE_COH_NONE }, 105 [3] = { XELP_PAT_WB, XE_COH_AT_LEAST_1WAY }, 106 [4] = { XEHPC_PAT_CLOS(1) | XELP_PAT_WT, XE_COH_NONE }, 107 [5] = { XEHPC_PAT_CLOS(1) | XELP_PAT_WB, XE_COH_AT_LEAST_1WAY }, 108 [6] = { XEHPC_PAT_CLOS(2) | XELP_PAT_WT, XE_COH_NONE }, 109 [7] = { XEHPC_PAT_CLOS(2) | XELP_PAT_WB, XE_COH_AT_LEAST_1WAY }, 110 }; 111 112 static const struct xe_pat_table_entry xelpg_pat_table[] = { 113 [0] = { XELPG_PAT_0_WB, XE_COH_NONE }, 114 [1] = { XELPG_PAT_1_WT, XE_COH_NONE }, 115 [2] = { XELPG_PAT_3_UC, XE_COH_NONE }, 116 [3] = { XELPG_PAT_0_WB | XELPG_2_COH_1W, XE_COH_AT_LEAST_1WAY }, 117 [4] = { XELPG_PAT_0_WB | XELPG_3_COH_2W, XE_COH_AT_LEAST_1WAY }, 118 }; 119 120 /* 121 * The Xe2 table is getting large/complicated so it's easier to review if 122 * provided in a form that exactly matches the bspec's formatting. The meaning 123 * of the fields here are: 124 * - no_promote: 0=promotable, 1=no promote 125 * - comp_en: 0=disable, 1=enable 126 * - l3clos: L3 class of service (0-3) 127 * - l3_policy: 0=WB, 1=XD ("WB - Transient Display"), 128 * 2=XA ("WB - Transient App" for Xe3p), 3=UC 129 * - l4_policy: 0=WB, 1=WT, 3=UC 130 * - coh_mode: 0=no snoop, 2=1-way coherent, 3=2-way coherent 131 * 132 * Reserved entries should be programmed with the maximum caching, minimum 133 * coherency (which matches an all-0's encoding), so we can just omit them 134 * in the table. 135 * 136 * Note: There is an implicit assumption in the driver that compression and 137 * coh_1way+ are mutually exclusive for platforms prior to Xe3. Starting 138 * with Xe3, compression can be combined with coherency. If using compression 139 * with coherency, userptr and imported dma-buf from external device will 140 * have uncleared ccs state. See also xe_bo_needs_ccs_pages(). 141 */ 142 #define XE2_PAT(no_promote, comp_en, l3clos, l3_policy, l4_policy, __coh_mode) \ 143 { \ 144 .value = (no_promote ? XE2_NO_PROMOTE : 0) | \ 145 (comp_en ? XE2_COMP_EN : 0) | \ 146 REG_FIELD_PREP(XE2_L3_CLOS, l3clos) | \ 147 REG_FIELD_PREP(XE2_L3_POLICY, l3_policy) | \ 148 REG_FIELD_PREP(XE2_L4_POLICY, l4_policy) | \ 149 REG_FIELD_PREP(XE2_COH_MODE, __coh_mode), \ 150 .coh_mode = __coh_mode ? XE_COH_AT_LEAST_1WAY : XE_COH_NONE, \ 151 .valid = 1 \ 152 } 153 154 static const struct xe_pat_table_entry xe2_pat_table[] = { 155 [ 0] = XE2_PAT( 0, 0, 0, 0, 3, 0 ), 156 [ 1] = XE2_PAT( 0, 0, 0, 0, 3, 2 ), 157 [ 2] = XE2_PAT( 0, 0, 0, 0, 3, 3 ), 158 [ 3] = XE2_PAT( 0, 0, 0, 3, 3, 0 ), 159 [ 4] = XE2_PAT( 0, 0, 0, 3, 0, 2 ), 160 [ 5] = XE2_PAT( 0, 0, 0, 3, 3, 2 ), 161 [ 6] = XE2_PAT( 1, 0, 0, 1, 3, 0 ), 162 [ 7] = XE2_PAT( 0, 0, 0, 3, 0, 3 ), 163 [ 8] = XE2_PAT( 0, 0, 0, 3, 0, 0 ), 164 [ 9] = XE2_PAT( 0, 1, 0, 0, 3, 0 ), 165 [10] = XE2_PAT( 0, 1, 0, 3, 0, 0 ), 166 [11] = XE2_PAT( 1, 1, 0, 1, 3, 0 ), 167 [12] = XE2_PAT( 0, 1, 0, 3, 3, 0 ), 168 [13] = XE2_PAT( 0, 0, 0, 0, 0, 0 ), 169 [14] = XE2_PAT( 0, 1, 0, 0, 0, 0 ), 170 [15] = XE2_PAT( 1, 1, 0, 1, 1, 0 ), 171 /* 16..19 are reserved; leave set to all 0's */ 172 [20] = XE2_PAT( 0, 0, 1, 0, 3, 0 ), 173 [21] = XE2_PAT( 0, 1, 1, 0, 3, 0 ), 174 [22] = XE2_PAT( 0, 0, 1, 0, 3, 2 ), 175 [23] = XE2_PAT( 0, 0, 1, 0, 3, 3 ), 176 [24] = XE2_PAT( 0, 0, 2, 0, 3, 0 ), 177 [25] = XE2_PAT( 0, 1, 2, 0, 3, 0 ), 178 [26] = XE2_PAT( 0, 0, 2, 0, 3, 2 ), 179 [27] = XE2_PAT( 0, 0, 2, 0, 3, 3 ), 180 [28] = XE2_PAT( 0, 0, 3, 0, 3, 0 ), 181 [29] = XE2_PAT( 0, 1, 3, 0, 3, 0 ), 182 [30] = XE2_PAT( 0, 0, 3, 0, 3, 2 ), 183 [31] = XE2_PAT( 0, 0, 3, 0, 3, 3 ), 184 }; 185 186 static const struct xe_pat_table_entry xe3_lpg_pat_table[] = { 187 [ 0] = XE2_PAT( 0, 0, 0, 0, 3, 0 ), 188 [ 1] = XE2_PAT( 0, 0, 0, 0, 3, 2 ), 189 [ 2] = XE2_PAT( 0, 0, 0, 0, 3, 3 ), 190 [ 3] = XE2_PAT( 0, 0, 0, 3, 3, 0 ), 191 [ 4] = XE2_PAT( 0, 0, 0, 3, 0, 2 ), 192 [ 5] = XE2_PAT( 0, 0, 0, 3, 3, 2 ), 193 [ 6] = XE2_PAT( 1, 0, 0, 1, 3, 0 ), 194 [ 7] = XE2_PAT( 0, 0, 0, 3, 0, 3 ), 195 [ 8] = XE2_PAT( 0, 0, 0, 3, 0, 0 ), 196 [ 9] = XE2_PAT( 0, 1, 0, 0, 3, 0 ), 197 [10] = XE2_PAT( 0, 1, 0, 3, 0, 0 ), 198 [11] = XE2_PAT( 1, 1, 0, 1, 3, 0 ), 199 [12] = XE2_PAT( 0, 1, 0, 3, 3, 0 ), 200 [13] = XE2_PAT( 0, 0, 0, 0, 0, 0 ), 201 [14] = XE2_PAT( 0, 1, 0, 0, 0, 0 ), 202 [15] = XE2_PAT( 1, 1, 0, 1, 1, 0 ), 203 [16] = XE2_PAT( 0, 1, 0, 0, 3, 2 ), 204 /* 17..19 are reserved; leave set to all 0's */ 205 [20] = XE2_PAT( 0, 0, 1, 0, 3, 0 ), 206 [21] = XE2_PAT( 0, 1, 1, 0, 3, 0 ), 207 [22] = XE2_PAT( 0, 0, 1, 0, 3, 2 ), 208 [23] = XE2_PAT( 0, 0, 1, 0, 3, 3 ), 209 [24] = XE2_PAT( 0, 0, 2, 0, 3, 0 ), 210 [25] = XE2_PAT( 0, 1, 2, 0, 3, 0 ), 211 [26] = XE2_PAT( 0, 0, 2, 0, 3, 2 ), 212 [27] = XE2_PAT( 0, 0, 2, 0, 3, 3 ), 213 [28] = XE2_PAT( 0, 0, 3, 0, 3, 0 ), 214 [29] = XE2_PAT( 0, 1, 3, 0, 3, 0 ), 215 [30] = XE2_PAT( 0, 0, 3, 0, 3, 2 ), 216 [31] = XE2_PAT( 0, 0, 3, 0, 3, 3 ), 217 }; 218 /* Special PAT values programmed outside the main table */ 219 static const struct xe_pat_table_entry xe2_pat_ats = XE2_PAT( 0, 0, 0, 0, 3, 3 ); 220 static const struct xe_pat_table_entry xe2_pat_pta = XE2_PAT( 0, 0, 0, 0, 3, 0 ); 221 222 /* 223 * Xe3p_XPC PAT table uses the same layout as Xe2/Xe3, except that there's no 224 * option for compression. Also note that the "L3" and "L4" register fields 225 * actually control L2 and L3 cache respectively on this platform. 226 */ 227 #define XE3P_XPC_PAT(no_promote, l3clos, l3_policy, l4_policy, __coh_mode) \ 228 XE2_PAT(no_promote, 0, l3clos, l3_policy, l4_policy, __coh_mode) 229 230 static const struct xe_pat_table_entry xe3p_xpc_pat_ats = XE3P_XPC_PAT( 0, 0, 0, 0, 3 ); 231 static const struct xe_pat_table_entry xe3p_xpc_pat_pta = XE3P_XPC_PAT( 0, 0, 0, 0, 0 ); 232 233 static const struct xe_pat_table_entry xe3p_xpc_pat_table[] = { 234 [ 0] = XE3P_XPC_PAT( 0, 0, 0, 0, 0 ), 235 [ 1] = XE3P_XPC_PAT( 0, 0, 0, 0, 2 ), 236 [ 2] = XE3P_XPC_PAT( 0, 0, 0, 0, 3 ), 237 [ 3] = XE3P_XPC_PAT( 0, 0, 3, 3, 0 ), 238 [ 4] = XE3P_XPC_PAT( 0, 0, 3, 3, 2 ), 239 [ 5] = XE3P_XPC_PAT( 0, 0, 3, 0, 0 ), 240 [ 6] = XE3P_XPC_PAT( 0, 0, 3, 0, 2 ), 241 [ 7] = XE3P_XPC_PAT( 0, 0, 3, 0, 3 ), 242 [ 8] = XE3P_XPC_PAT( 0, 0, 0, 3, 0 ), 243 [ 9] = XE3P_XPC_PAT( 0, 0, 0, 3, 2 ), 244 [10] = XE3P_XPC_PAT( 0, 0, 0, 3, 3 ), 245 /* 11..22 are reserved; leave set to all 0's */ 246 [23] = XE3P_XPC_PAT( 0, 1, 0, 0, 0 ), 247 [24] = XE3P_XPC_PAT( 0, 1, 0, 0, 2 ), 248 [25] = XE3P_XPC_PAT( 0, 1, 0, 0, 3 ), 249 [26] = XE3P_XPC_PAT( 0, 2, 0, 0, 0 ), 250 [27] = XE3P_XPC_PAT( 0, 2, 0, 0, 2 ), 251 [28] = XE3P_XPC_PAT( 0, 2, 0, 0, 3 ), 252 [29] = XE3P_XPC_PAT( 0, 3, 0, 0, 0 ), 253 [30] = XE3P_XPC_PAT( 0, 3, 0, 0, 2 ), 254 [31] = XE3P_XPC_PAT( 0, 3, 0, 0, 3 ), 255 }; 256 257 static const struct xe_pat_table_entry xe3p_primary_pat_pta = XE2_PAT(0, 0, 0, 0, 0, 3); 258 static const struct xe_pat_table_entry xe3p_media_pat_pta = XE2_PAT(0, 0, 0, 0, 0, 2); 259 260 static const struct xe_pat_table_entry xe3p_lpg_pat_table[] = { 261 [ 0] = XE2_PAT( 0, 0, 0, 0, 3, 0 ), 262 [ 1] = XE2_PAT( 0, 0, 0, 0, 3, 2 ), 263 [ 2] = XE2_PAT( 0, 0, 0, 0, 3, 3 ), 264 [ 3] = XE2_PAT( 0, 0, 0, 3, 3, 0 ), 265 [ 4] = XE2_PAT( 0, 0, 0, 3, 0, 2 ), 266 [ 5] = XE2_PAT( 0, 0, 0, 3, 3, 2 ), 267 [ 6] = XE2_PAT( 1, 0, 0, 1, 3, 0 ), 268 [ 7] = XE2_PAT( 0, 0, 0, 3, 0, 3 ), 269 [ 8] = XE2_PAT( 0, 0, 0, 3, 0, 0 ), 270 [ 9] = XE2_PAT( 0, 1, 0, 0, 3, 0 ), 271 [10] = XE2_PAT( 0, 1, 0, 3, 0, 0 ), 272 [11] = XE2_PAT( 1, 1, 0, 1, 3, 0 ), 273 [12] = XE2_PAT( 0, 1, 0, 3, 3, 0 ), 274 [13] = XE2_PAT( 0, 0, 0, 0, 0, 0 ), 275 [14] = XE2_PAT( 0, 1, 0, 0, 0, 0 ), 276 [15] = XE2_PAT( 1, 1, 0, 1, 1, 0 ), 277 [16] = XE2_PAT( 0, 1, 0, 0, 3, 2 ), 278 /* 17 is reserved; leave set to all 0's */ 279 [18] = XE2_PAT( 1, 0, 0, 2, 3, 0 ), 280 [19] = XE2_PAT( 1, 0, 0, 2, 3, 2 ), 281 [20] = XE2_PAT( 0, 0, 1, 0, 3, 0 ), 282 [21] = XE2_PAT( 0, 1, 1, 0, 3, 0 ), 283 [22] = XE2_PAT( 0, 0, 1, 0, 3, 2 ), 284 [23] = XE2_PAT( 0, 0, 1, 0, 3, 3 ), 285 [24] = XE2_PAT( 0, 0, 2, 0, 3, 0 ), 286 [25] = XE2_PAT( 0, 1, 2, 0, 3, 0 ), 287 [26] = XE2_PAT( 0, 0, 2, 0, 3, 2 ), 288 [27] = XE2_PAT( 0, 0, 2, 0, 3, 3 ), 289 [28] = XE2_PAT( 0, 0, 3, 0, 3, 0 ), 290 [29] = XE2_PAT( 0, 1, 3, 0, 3, 0 ), 291 [30] = XE2_PAT( 0, 0, 3, 0, 3, 2 ), 292 [31] = XE2_PAT( 0, 0, 3, 0, 3, 3 ), 293 }; 294 295 u16 xe_pat_index_get_coh_mode(struct xe_device *xe, u16 pat_index) 296 { 297 WARN_ON(pat_index >= xe->pat.n_entries); 298 return xe->pat.table[pat_index].coh_mode; 299 } 300 301 bool xe_pat_index_get_comp_en(struct xe_device *xe, u16 pat_index) 302 { 303 WARN_ON(pat_index >= xe->pat.n_entries); 304 return !!(xe->pat.table[pat_index].value & XE2_COMP_EN); 305 } 306 307 u16 xe_pat_index_get_l3_policy(struct xe_device *xe, u16 pat_index) 308 { 309 WARN_ON(pat_index >= xe->pat.n_entries); 310 311 return REG_FIELD_GET(XE2_L3_POLICY, xe->pat.table[pat_index].value); 312 } 313 314 static void program_pat(struct xe_gt *gt, const struct xe_pat_table_entry table[], 315 int n_entries) 316 { 317 struct xe_device *xe = gt_to_xe(gt); 318 319 for (int i = 0; i < n_entries; i++) { 320 struct xe_reg reg = XE_REG(_PAT_INDEX(i)); 321 322 xe_mmio_write32(>->mmio, reg, table[i].value); 323 } 324 325 if (xe->pat.pat_ats) 326 xe_mmio_write32(>->mmio, XE_REG(_PAT_ATS), xe->pat.pat_ats->value); 327 if (xe->pat.pat_primary_pta && xe_gt_is_main_type(gt)) 328 xe_mmio_write32(>->mmio, XE_REG(_PAT_PTA), xe->pat.pat_primary_pta->value); 329 if (xe->pat.pat_media_pta && xe_gt_is_media_type(gt)) 330 xe_mmio_write32(>->mmio, XE_REG(_PAT_PTA), xe->pat.pat_media_pta->value); 331 } 332 333 static void program_pat_mcr(struct xe_gt *gt, const struct xe_pat_table_entry table[], 334 int n_entries) 335 { 336 struct xe_device *xe = gt_to_xe(gt); 337 338 for (int i = 0; i < n_entries; i++) { 339 struct xe_reg_mcr reg_mcr = XE_REG_MCR(_PAT_INDEX(i)); 340 341 xe_gt_mcr_multicast_write(gt, reg_mcr, table[i].value); 342 } 343 344 if (xe->pat.pat_ats) 345 xe_gt_mcr_multicast_write(gt, XE_REG_MCR(_PAT_ATS), xe->pat.pat_ats->value); 346 if (xe->pat.pat_primary_pta && xe_gt_is_main_type(gt)) 347 xe_gt_mcr_multicast_write(gt, XE_REG_MCR(_PAT_PTA), xe->pat.pat_primary_pta->value); 348 if (xe->pat.pat_media_pta && xe_gt_is_media_type(gt)) 349 xe_gt_mcr_multicast_write(gt, XE_REG_MCR(_PAT_PTA), xe->pat.pat_media_pta->value); 350 } 351 352 static int xelp_dump(struct xe_gt *gt, struct drm_printer *p) 353 { 354 struct xe_device *xe = gt_to_xe(gt); 355 int i; 356 357 CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); 358 if (!fw_ref.domains) 359 return -ETIMEDOUT; 360 361 drm_printf(p, "PAT table:\n"); 362 363 for (i = 0; i < xe->pat.n_entries; i++) { 364 u32 pat = xe_mmio_read32(>->mmio, XE_REG(_PAT_INDEX(i))); 365 366 xelp_pat_entry_dump(p, i, pat); 367 } 368 369 return 0; 370 } 371 372 static const struct xe_pat_ops xelp_pat_ops = { 373 .program_graphics = program_pat, 374 .dump = xelp_dump, 375 }; 376 377 static int xehp_dump(struct xe_gt *gt, struct drm_printer *p) 378 { 379 struct xe_device *xe = gt_to_xe(gt); 380 int i; 381 382 CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); 383 if (!fw_ref.domains) 384 return -ETIMEDOUT; 385 386 drm_printf(p, "PAT table:\n"); 387 388 for (i = 0; i < xe->pat.n_entries; i++) { 389 u32 pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i))); 390 391 xelp_pat_entry_dump(p, i, pat); 392 } 393 394 return 0; 395 } 396 397 static const struct xe_pat_ops xehp_pat_ops = { 398 .program_graphics = program_pat_mcr, 399 .dump = xehp_dump, 400 }; 401 402 static int xehpc_dump(struct xe_gt *gt, struct drm_printer *p) 403 { 404 struct xe_device *xe = gt_to_xe(gt); 405 int i; 406 407 CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); 408 if (!fw_ref.domains) 409 return -ETIMEDOUT; 410 411 drm_printf(p, "PAT table:\n"); 412 413 for (i = 0; i < xe->pat.n_entries; i++) { 414 u32 pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i))); 415 416 xehpc_pat_entry_dump(p, i, pat); 417 } 418 419 return 0; 420 } 421 422 static const struct xe_pat_ops xehpc_pat_ops = { 423 .program_graphics = program_pat_mcr, 424 .dump = xehpc_dump, 425 }; 426 427 static int xelpg_dump(struct xe_gt *gt, struct drm_printer *p) 428 { 429 struct xe_device *xe = gt_to_xe(gt); 430 int i; 431 432 CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); 433 if (!fw_ref.domains) 434 return -ETIMEDOUT; 435 436 drm_printf(p, "PAT table:\n"); 437 438 for (i = 0; i < xe->pat.n_entries; i++) { 439 u32 pat; 440 441 if (xe_gt_is_media_type(gt)) 442 pat = xe_mmio_read32(>->mmio, XE_REG(_PAT_INDEX(i))); 443 else 444 pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i))); 445 446 xelpg_pat_entry_dump(p, i, pat); 447 } 448 449 return 0; 450 } 451 452 /* 453 * SAMedia register offsets are adjusted by the write methods and they target 454 * registers that are not MCR, while for normal GT they are MCR 455 */ 456 static const struct xe_pat_ops xelpg_pat_ops = { 457 .program_graphics = program_pat, 458 .program_media = program_pat_mcr, 459 .dump = xelpg_dump, 460 }; 461 462 static void xe2_pat_entry_dump(struct drm_printer *p, const char *label, u32 pat, bool rsvd) 463 { 464 drm_printf(p, "%s= [ %u, %u, %u, %u, %u, %u ] (%#8x)%s\n", label, 465 !!(pat & XE2_NO_PROMOTE), 466 !!(pat & XE2_COMP_EN), 467 REG_FIELD_GET(XE2_L3_CLOS, pat), 468 REG_FIELD_GET(XE2_L3_POLICY, pat), 469 REG_FIELD_GET(XE2_L4_POLICY, pat), 470 REG_FIELD_GET(XE2_COH_MODE, pat), 471 pat, rsvd ? " *" : ""); 472 } 473 474 static void xe3p_xpc_pat_entry_dump(struct drm_printer *p, const char *label, u32 pat, bool rsvd) 475 { 476 drm_printf(p, "%s= [ %u, %u, %u, %u, %u ] (%#8x)%s\n", label, 477 !!(pat & XE2_NO_PROMOTE), 478 REG_FIELD_GET(XE2_L3_CLOS, pat), 479 REG_FIELD_GET(XE2_L3_POLICY, pat), 480 REG_FIELD_GET(XE2_L4_POLICY, pat), 481 REG_FIELD_GET(XE2_COH_MODE, pat), 482 pat, rsvd ? " *" : ""); 483 } 484 485 static int xe2_dump(struct xe_gt *gt, struct drm_printer *p) 486 { 487 struct xe_device *xe = gt_to_xe(gt); 488 u32 pat; 489 int i; 490 char label[PAT_LABEL_LEN]; 491 492 CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); 493 if (!fw_ref.domains) 494 return -ETIMEDOUT; 495 496 drm_printf(p, "PAT table: (* = reserved entry)\n"); 497 498 for (i = 0; i < xe->pat.n_entries; i++) { 499 if (xe_gt_is_media_type(gt)) 500 pat = xe_mmio_read32(>->mmio, XE_REG(_PAT_INDEX(i))); 501 else 502 pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i))); 503 504 xe_pat_index_label(label, sizeof(label), i); 505 xe->pat.ops->entry_dump(p, label, pat, !xe->pat.table[i].valid); 506 } 507 508 /* 509 * Also print PTA_MODE, which describes how the hardware accesses 510 * PPGTT entries. 511 */ 512 if (xe_gt_is_media_type(gt)) 513 pat = xe_mmio_read32(>->mmio, XE_REG(_PAT_PTA)); 514 else 515 pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_PTA)); 516 517 drm_printf(p, "Page Table Access:\n"); 518 xe->pat.ops->entry_dump(p, "PTA_MODE", pat, false); 519 520 return 0; 521 } 522 523 static const struct xe_pat_ops xe2_pat_ops = { 524 .program_graphics = program_pat_mcr, 525 .program_media = program_pat, 526 .dump = xe2_dump, 527 .entry_dump = xe2_pat_entry_dump, 528 }; 529 530 static const struct xe_pat_ops xe3p_xpc_pat_ops = { 531 .program_graphics = program_pat_mcr, 532 .program_media = program_pat, 533 .dump = xe2_dump, 534 .entry_dump = xe3p_xpc_pat_entry_dump, 535 }; 536 537 void xe_pat_init_early(struct xe_device *xe) 538 { 539 xe->pat.idx[XE_CACHE_WB_COMPRESSION] = XE_PAT_INVALID_IDX; 540 if (GRAPHICS_VERx100(xe) == 3511) { 541 xe->pat.ops = &xe3p_xpc_pat_ops; 542 xe->pat.table = xe3p_xpc_pat_table; 543 xe->pat.pat_ats = &xe3p_xpc_pat_ats; 544 xe->pat.pat_primary_pta = &xe3p_xpc_pat_pta; 545 xe->pat.pat_media_pta = &xe3p_xpc_pat_pta; 546 xe->pat.n_entries = ARRAY_SIZE(xe3p_xpc_pat_table); 547 xe->pat.idx[XE_CACHE_NONE] = 3; 548 xe->pat.idx[XE_CACHE_WT] = 3; /* N/A (no display); use UC */ 549 xe->pat.idx[XE_CACHE_WB] = 2; 550 } else if (GRAPHICS_VER(xe) == 35) { 551 xe->pat.ops = &xe2_pat_ops; 552 xe->pat.table = xe3p_lpg_pat_table; 553 xe->pat.pat_ats = &xe2_pat_ats; 554 if (!IS_DGFX(xe)) { 555 xe->pat.pat_primary_pta = &xe3p_primary_pat_pta; 556 xe->pat.pat_media_pta = &xe3p_media_pat_pta; 557 } 558 xe->pat.n_entries = ARRAY_SIZE(xe3p_lpg_pat_table); 559 xe->pat.idx[XE_CACHE_NONE] = 3; 560 xe->pat.idx[XE_CACHE_WT] = 15; 561 xe->pat.idx[XE_CACHE_WB] = 2; 562 xe->pat.idx[XE_CACHE_NONE_COMPRESSION] = 12; 563 xe->pat.idx[XE_CACHE_WB_COMPRESSION] = 16; 564 } else if (GRAPHICS_VER(xe) == 30 || GRAPHICS_VER(xe) == 20) { 565 xe->pat.ops = &xe2_pat_ops; 566 if (GRAPHICS_VER(xe) == 30) { 567 xe->pat.table = xe3_lpg_pat_table; 568 xe->pat.idx[XE_CACHE_WB_COMPRESSION] = 16; 569 } else { 570 xe->pat.table = xe2_pat_table; 571 } 572 xe->pat.pat_ats = &xe2_pat_ats; 573 if (IS_DGFX(xe)) { 574 xe->pat.pat_primary_pta = &xe2_pat_pta; 575 xe->pat.pat_media_pta = &xe2_pat_pta; 576 } 577 578 /* Wa_16023588340. XXX: Should use XE_WA */ 579 if (GRAPHICS_VERx100(xe) == 2001) 580 xe->pat.n_entries = 28; /* Disable CLOS3 */ 581 else 582 xe->pat.n_entries = ARRAY_SIZE(xe2_pat_table); 583 584 xe->pat.idx[XE_CACHE_NONE] = 3; 585 xe->pat.idx[XE_CACHE_WT] = 15; 586 xe->pat.idx[XE_CACHE_WB] = 2; 587 xe->pat.idx[XE_CACHE_NONE_COMPRESSION] = 12; /*Applicable on xe2 and beyond */ 588 } else if (xe->info.platform == XE_METEORLAKE) { 589 xe->pat.ops = &xelpg_pat_ops; 590 xe->pat.table = xelpg_pat_table; 591 xe->pat.n_entries = ARRAY_SIZE(xelpg_pat_table); 592 xe->pat.idx[XE_CACHE_NONE] = 2; 593 xe->pat.idx[XE_CACHE_WT] = 1; 594 xe->pat.idx[XE_CACHE_WB] = 3; 595 } else if (xe->info.platform == XE_PVC) { 596 xe->pat.ops = &xehpc_pat_ops; 597 xe->pat.table = xehpc_pat_table; 598 xe->pat.n_entries = ARRAY_SIZE(xehpc_pat_table); 599 xe->pat.idx[XE_CACHE_NONE] = 0; 600 xe->pat.idx[XE_CACHE_WT] = 2; 601 xe->pat.idx[XE_CACHE_WB] = 3; 602 } else if (xe->info.platform == XE_DG2) { 603 /* 604 * Table is the same as previous platforms, but programming 605 * method has changed. 606 */ 607 xe->pat.ops = &xehp_pat_ops; 608 xe->pat.table = xelp_pat_table; 609 xe->pat.n_entries = ARRAY_SIZE(xelp_pat_table); 610 xe->pat.idx[XE_CACHE_NONE] = 3; 611 xe->pat.idx[XE_CACHE_WT] = 2; 612 xe->pat.idx[XE_CACHE_WB] = 0; 613 } else if (GRAPHICS_VERx100(xe) <= 1210) { 614 WARN_ON_ONCE(!IS_DGFX(xe) && !xe->info.has_llc); 615 xe->pat.ops = &xelp_pat_ops; 616 xe->pat.table = xelp_pat_table; 617 xe->pat.n_entries = ARRAY_SIZE(xelp_pat_table); 618 xe->pat.idx[XE_CACHE_NONE] = 3; 619 xe->pat.idx[XE_CACHE_WT] = 2; 620 xe->pat.idx[XE_CACHE_WB] = 0; 621 } else { 622 /* 623 * Going forward we expect to need new PAT settings for most 624 * new platforms; failure to provide a new table can easily 625 * lead to subtle, hard-to-debug problems. If none of the 626 * conditions above match the platform we're running on we'll 627 * raise an error rather than trying to silently inherit the 628 * most recent platform's behavior. 629 */ 630 drm_err(&xe->drm, "Missing PAT table for platform with graphics version %d.%02d!\n", 631 GRAPHICS_VER(xe), GRAPHICS_VERx100(xe) % 100); 632 } 633 634 xe_assert(xe, xe->pat.ops->dump); 635 xe_assert(xe, xe->pat.ops->program_graphics); 636 xe_assert(xe, MEDIA_VER(xe) < 13 || xe->pat.ops->program_media); 637 xe_assert(xe, GRAPHICS_VER(xe) < 20 || xe->pat.ops->entry_dump); 638 } 639 640 void xe_pat_init(struct xe_gt *gt) 641 { 642 struct xe_device *xe = gt_to_xe(gt); 643 644 if (IS_SRIOV_VF(xe)) 645 return; 646 647 if (xe_gt_is_media_type(gt)) 648 xe->pat.ops->program_media(gt, xe->pat.table, xe->pat.n_entries); 649 else 650 xe->pat.ops->program_graphics(gt, xe->pat.table, xe->pat.n_entries); 651 } 652 653 /** 654 * xe_pat_dump() - Dump GT PAT table into a drm printer. 655 * @gt: the &xe_gt 656 * @p: the &drm_printer 657 * 658 * Return: 0 on success or a negative error code on failure. 659 */ 660 int xe_pat_dump(struct xe_gt *gt, struct drm_printer *p) 661 { 662 struct xe_device *xe = gt_to_xe(gt); 663 664 if (IS_SRIOV_VF(xe)) 665 return -EOPNOTSUPP; 666 667 return xe->pat.ops->dump(gt, p); 668 } 669 670 /** 671 * xe_pat_dump_sw_config() - Dump the software-configured GT PAT table into a drm printer. 672 * @gt: the &xe_gt 673 * @p: the &drm_printer 674 * 675 * Return: 0 on success or a negative error code on failure. 676 */ 677 int xe_pat_dump_sw_config(struct xe_gt *gt, struct drm_printer *p) 678 { 679 struct xe_device *xe = gt_to_xe(gt); 680 const struct xe_pat_table_entry *pta_entry = xe_gt_is_main_type(gt) ? 681 xe->pat.pat_primary_pta : xe->pat.pat_media_pta; 682 char label[PAT_LABEL_LEN]; 683 684 if (!xe->pat.table || !xe->pat.n_entries) 685 return -EOPNOTSUPP; 686 687 drm_printf(p, "PAT table:%s\n", GRAPHICS_VER(xe) >= 20 ? " (* = reserved entry)" : ""); 688 for (u32 i = 0; i < xe->pat.n_entries; i++) { 689 u32 pat = xe->pat.table[i].value; 690 691 if (GRAPHICS_VER(xe) >= 20) { 692 xe_pat_index_label(label, sizeof(label), i); 693 xe->pat.ops->entry_dump(p, label, pat, !xe->pat.table[i].valid); 694 } else if (xe->info.platform == XE_METEORLAKE) { 695 xelpg_pat_entry_dump(p, i, pat); 696 } else if (xe->info.platform == XE_PVC) { 697 xehpc_pat_entry_dump(p, i, pat); 698 } else if (xe->info.platform == XE_DG2 || GRAPHICS_VERx100(xe) <= 1210) { 699 xelp_pat_entry_dump(p, i, pat); 700 } else { 701 return -EOPNOTSUPP; 702 } 703 } 704 705 if (pta_entry) { 706 u32 pat = pta_entry->value; 707 708 drm_printf(p, "Page Table Access:\n"); 709 xe->pat.ops->entry_dump(p, "PTA_MODE", pat, false); 710 } 711 712 if (xe->pat.pat_ats) { 713 u32 pat = xe->pat.pat_ats->value; 714 715 drm_printf(p, "PCIe ATS/PASID:\n"); 716 xe->pat.ops->entry_dump(p, "PAT_ATS ", pat, false); 717 } 718 719 drm_printf(p, "Cache Level:\n"); 720 drm_printf(p, "IDX[XE_CACHE_NONE] = %d\n", xe->pat.idx[XE_CACHE_NONE]); 721 drm_printf(p, "IDX[XE_CACHE_WT] = %d\n", xe->pat.idx[XE_CACHE_WT]); 722 drm_printf(p, "IDX[XE_CACHE_WB] = %d\n", xe->pat.idx[XE_CACHE_WB]); 723 if (GRAPHICS_VER(xe) >= 20) { 724 drm_printf(p, "IDX[XE_CACHE_NONE_COMPRESSION] = %d\n", 725 xe->pat.idx[XE_CACHE_NONE_COMPRESSION]); 726 drm_printf(p, "IDX[XE_CACHE_WB_COMPRESSION] = %d\n", 727 xe->pat.idx[XE_CACHE_WB_COMPRESSION]); 728 } 729 730 return 0; 731 } 732