1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2023 Intel Corporation 4 */ 5 6 #include "xe_pat.h" 7 8 #include <uapi/drm/xe_drm.h> 9 10 #include <generated/xe_wa_oob.h> 11 12 #include "regs/xe_gt_regs.h" 13 #include "regs/xe_reg_defs.h" 14 #include "xe_assert.h" 15 #include "xe_device.h" 16 #include "xe_force_wake.h" 17 #include "xe_gt.h" 18 #include "xe_gt_mcr.h" 19 #include "xe_mmio.h" 20 #include "xe_sriov.h" 21 #include "xe_wa.h" 22 23 #define _PAT_ATS 0x47fc 24 #define _PAT_INDEX(index) _PICK_EVEN_2RANGES(index, 8, \ 25 0x4800, 0x4804, \ 26 0x4848, 0x484c) 27 #define _PAT_PTA 0x4820 28 29 #define XE2_NO_PROMOTE REG_BIT(10) 30 #define XE2_COMP_EN REG_BIT(9) 31 #define XE2_L3_CLOS REG_GENMASK(7, 6) 32 #define XE2_L3_POLICY REG_GENMASK(5, 4) 33 #define XE2_L4_POLICY REG_GENMASK(3, 2) 34 #define XE2_COH_MODE REG_GENMASK(1, 0) 35 36 #define XELPG_L4_POLICY_MASK REG_GENMASK(3, 2) 37 #define XELPG_PAT_3_UC REG_FIELD_PREP(XELPG_L4_POLICY_MASK, 3) 38 #define XELPG_PAT_1_WT REG_FIELD_PREP(XELPG_L4_POLICY_MASK, 1) 39 #define XELPG_PAT_0_WB REG_FIELD_PREP(XELPG_L4_POLICY_MASK, 0) 40 #define XELPG_INDEX_COH_MODE_MASK REG_GENMASK(1, 0) 41 #define XELPG_3_COH_2W REG_FIELD_PREP(XELPG_INDEX_COH_MODE_MASK, 3) 42 #define XELPG_2_COH_1W REG_FIELD_PREP(XELPG_INDEX_COH_MODE_MASK, 2) 43 #define XELPG_0_COH_NON REG_FIELD_PREP(XELPG_INDEX_COH_MODE_MASK, 0) 44 45 #define XEHPC_CLOS_LEVEL_MASK REG_GENMASK(3, 2) 46 #define XEHPC_PAT_CLOS(x) REG_FIELD_PREP(XEHPC_CLOS_LEVEL_MASK, x) 47 48 #define XELP_MEM_TYPE_MASK REG_GENMASK(1, 0) 49 #define XELP_PAT_WB REG_FIELD_PREP(XELP_MEM_TYPE_MASK, 3) 50 #define XELP_PAT_WT REG_FIELD_PREP(XELP_MEM_TYPE_MASK, 2) 51 #define XELP_PAT_WC REG_FIELD_PREP(XELP_MEM_TYPE_MASK, 1) 52 #define XELP_PAT_UC REG_FIELD_PREP(XELP_MEM_TYPE_MASK, 0) 53 54 #define PAT_LABEL_LEN 20 55 56 static const char *XELP_MEM_TYPE_STR_MAP[] = { "UC", "WC", "WT", "WB" }; 57 58 static void xe_pat_index_label(char *label, size_t len, int index) 59 { 60 snprintf(label, len, "PAT[%2d] ", index); 61 } 62 63 static void xelp_pat_entry_dump(struct drm_printer *p, int index, u32 pat) 64 { 65 u8 mem_type = REG_FIELD_GET(XELP_MEM_TYPE_MASK, pat); 66 67 drm_printf(p, "PAT[%2d] = %s (%#8x)\n", index, 68 XELP_MEM_TYPE_STR_MAP[mem_type], pat); 69 } 70 71 static void xehpc_pat_entry_dump(struct drm_printer *p, int index, u32 pat) 72 { 73 drm_printf(p, "PAT[%2d] = [ %u, %u ] (%#8x)\n", index, 74 REG_FIELD_GET(XELP_MEM_TYPE_MASK, pat), 75 REG_FIELD_GET(XEHPC_CLOS_LEVEL_MASK, pat), pat); 76 } 77 78 static void xelpg_pat_entry_dump(struct drm_printer *p, int index, u32 pat) 79 { 80 drm_printf(p, "PAT[%2d] = [ %u, %u ] (%#8x)\n", index, 81 REG_FIELD_GET(XELPG_L4_POLICY_MASK, pat), 82 REG_FIELD_GET(XELPG_INDEX_COH_MODE_MASK, pat), pat); 83 } 84 85 struct xe_pat_ops { 86 void (*program_graphics)(struct xe_gt *gt, const struct xe_pat_table_entry table[], 87 int n_entries); 88 void (*program_media)(struct xe_gt *gt, const struct xe_pat_table_entry table[], 89 int n_entries); 90 int (*dump)(struct xe_gt *gt, struct drm_printer *p); 91 void (*entry_dump)(struct drm_printer *p, const char *label, u32 pat, bool rsvd); 92 }; 93 94 static const struct xe_pat_table_entry xelp_pat_table[] = { 95 [0] = { XELP_PAT_WB, XE_COH_1WAY }, 96 [1] = { XELP_PAT_WC, XE_COH_NONE }, 97 [2] = { XELP_PAT_WT, XE_COH_NONE }, 98 [3] = { XELP_PAT_UC, XE_COH_NONE }, 99 }; 100 101 static const struct xe_pat_table_entry xehpc_pat_table[] = { 102 [0] = { XELP_PAT_UC, XE_COH_NONE }, 103 [1] = { XELP_PAT_WC, XE_COH_NONE }, 104 [2] = { XELP_PAT_WT, XE_COH_NONE }, 105 [3] = { XELP_PAT_WB, XE_COH_1WAY }, 106 [4] = { XEHPC_PAT_CLOS(1) | XELP_PAT_WT, XE_COH_NONE }, 107 [5] = { XEHPC_PAT_CLOS(1) | XELP_PAT_WB, XE_COH_1WAY }, 108 [6] = { XEHPC_PAT_CLOS(2) | XELP_PAT_WT, XE_COH_NONE }, 109 [7] = { XEHPC_PAT_CLOS(2) | XELP_PAT_WB, XE_COH_1WAY }, 110 }; 111 112 static const struct xe_pat_table_entry xelpg_pat_table[] = { 113 [0] = { XELPG_PAT_0_WB, XE_COH_NONE }, 114 [1] = { XELPG_PAT_1_WT, XE_COH_NONE }, 115 [2] = { XELPG_PAT_3_UC, XE_COH_NONE }, 116 [3] = { XELPG_PAT_0_WB | XELPG_2_COH_1W, XE_COH_1WAY }, 117 [4] = { XELPG_PAT_0_WB | XELPG_3_COH_2W, XE_COH_2WAY }, 118 }; 119 120 /* 121 * The Xe2 table is getting large/complicated so it's easier to review if 122 * provided in a form that exactly matches the bspec's formatting. The meaning 123 * of the fields here are: 124 * - no_promote: 0=promotable, 1=no promote 125 * - comp_en: 0=disable, 1=enable 126 * - l3clos: L3 class of service (0-3) 127 * - l3_policy: 0=WB, 1=XD ("WB - Transient Display"), 128 * 2=XA ("WB - Transient App" for Xe3p), 3=UC 129 * - l4_policy: 0=WB, 1=WT, 3=UC 130 * - coh_mode: 0=no snoop, 2=1-way coherent, 3=2-way coherent 131 * 132 * Reserved entries should be programmed with the maximum caching, minimum 133 * coherency (which matches an all-0's encoding), so we can just omit them 134 * in the table. 135 * 136 * Note: There is an implicit assumption in the driver that compression and 137 * coh_1way+ are mutually exclusive for platforms prior to Xe3. Starting 138 * with Xe3, compression can be combined with coherency. If using compression 139 * with coherency, userptr and imported dma-buf from external device will 140 * have uncleared ccs state. See also xe_bo_needs_ccs_pages(). 141 */ 142 #define XE2_PAT(no_promote, comp_en, l3clos, l3_policy, l4_policy, __coh_mode) \ 143 { \ 144 .value = (no_promote ? XE2_NO_PROMOTE : 0) | \ 145 (comp_en ? XE2_COMP_EN : 0) | \ 146 REG_FIELD_PREP(XE2_L3_CLOS, l3clos) | \ 147 REG_FIELD_PREP(XE2_L3_POLICY, l3_policy) | \ 148 REG_FIELD_PREP(XE2_L4_POLICY, l4_policy) | \ 149 REG_FIELD_PREP(XE2_COH_MODE, __coh_mode), \ 150 .coh_mode = __coh_mode ? __coh_mode : XE_COH_NONE, \ 151 .valid = 1 \ 152 } 153 154 static const struct xe_pat_table_entry xe2_pat_table[] = { 155 [ 0] = XE2_PAT( 0, 0, 0, 0, 3, 0 ), 156 [ 1] = XE2_PAT( 0, 0, 0, 0, 3, 2 ), 157 [ 2] = XE2_PAT( 0, 0, 0, 0, 3, 3 ), 158 [ 3] = XE2_PAT( 0, 0, 0, 3, 3, 0 ), 159 [ 4] = XE2_PAT( 0, 0, 0, 3, 0, 2 ), 160 [ 5] = XE2_PAT( 0, 0, 0, 3, 3, 2 ), 161 [ 6] = XE2_PAT( 1, 0, 0, 1, 3, 0 ), 162 [ 7] = XE2_PAT( 0, 0, 0, 3, 0, 3 ), 163 [ 8] = XE2_PAT( 0, 0, 0, 3, 0, 0 ), 164 [ 9] = XE2_PAT( 0, 1, 0, 0, 3, 0 ), 165 [10] = XE2_PAT( 0, 1, 0, 3, 0, 0 ), 166 [11] = XE2_PAT( 1, 1, 0, 1, 3, 0 ), 167 [12] = XE2_PAT( 0, 1, 0, 3, 3, 0 ), 168 [13] = XE2_PAT( 0, 0, 0, 0, 0, 0 ), 169 [14] = XE2_PAT( 0, 1, 0, 0, 0, 0 ), 170 [15] = XE2_PAT( 1, 1, 0, 1, 1, 0 ), 171 /* 16..19 are reserved; leave set to all 0's */ 172 [20] = XE2_PAT( 0, 0, 1, 0, 3, 0 ), 173 [21] = XE2_PAT( 0, 1, 1, 0, 3, 0 ), 174 [22] = XE2_PAT( 0, 0, 1, 0, 3, 2 ), 175 [23] = XE2_PAT( 0, 0, 1, 0, 3, 3 ), 176 [24] = XE2_PAT( 0, 0, 2, 0, 3, 0 ), 177 [25] = XE2_PAT( 0, 1, 2, 0, 3, 0 ), 178 [26] = XE2_PAT( 0, 0, 2, 0, 3, 2 ), 179 [27] = XE2_PAT( 0, 0, 2, 0, 3, 3 ), 180 [28] = XE2_PAT( 0, 0, 3, 0, 3, 0 ), 181 [29] = XE2_PAT( 0, 1, 3, 0, 3, 0 ), 182 [30] = XE2_PAT( 0, 0, 3, 0, 3, 2 ), 183 [31] = XE2_PAT( 0, 0, 3, 0, 3, 3 ), 184 }; 185 186 static const struct xe_pat_table_entry xe3_lpg_pat_table[] = { 187 [ 0] = XE2_PAT( 0, 0, 0, 0, 3, 0 ), 188 [ 1] = XE2_PAT( 0, 0, 0, 0, 3, 2 ), 189 [ 2] = XE2_PAT( 0, 0, 0, 0, 3, 3 ), 190 [ 3] = XE2_PAT( 0, 0, 0, 3, 3, 0 ), 191 [ 4] = XE2_PAT( 0, 0, 0, 3, 0, 2 ), 192 [ 5] = XE2_PAT( 0, 0, 0, 3, 3, 2 ), 193 [ 6] = XE2_PAT( 1, 0, 0, 1, 3, 0 ), 194 [ 7] = XE2_PAT( 0, 0, 0, 3, 0, 3 ), 195 [ 8] = XE2_PAT( 0, 0, 0, 3, 0, 0 ), 196 [ 9] = XE2_PAT( 0, 1, 0, 0, 3, 0 ), 197 [10] = XE2_PAT( 0, 1, 0, 3, 0, 0 ), 198 [11] = XE2_PAT( 1, 1, 0, 1, 3, 0 ), 199 [12] = XE2_PAT( 0, 1, 0, 3, 3, 0 ), 200 [13] = XE2_PAT( 0, 0, 0, 0, 0, 0 ), 201 [14] = XE2_PAT( 0, 1, 0, 0, 0, 0 ), 202 [15] = XE2_PAT( 1, 1, 0, 1, 1, 0 ), 203 [16] = XE2_PAT( 0, 1, 0, 0, 3, 2 ), 204 /* 17..19 are reserved; leave set to all 0's */ 205 [20] = XE2_PAT( 0, 0, 1, 0, 3, 0 ), 206 [21] = XE2_PAT( 0, 1, 1, 0, 3, 0 ), 207 [22] = XE2_PAT( 0, 0, 1, 0, 3, 2 ), 208 [23] = XE2_PAT( 0, 0, 1, 0, 3, 3 ), 209 [24] = XE2_PAT( 0, 0, 2, 0, 3, 0 ), 210 [25] = XE2_PAT( 0, 1, 2, 0, 3, 0 ), 211 [26] = XE2_PAT( 0, 0, 2, 0, 3, 2 ), 212 [27] = XE2_PAT( 0, 0, 2, 0, 3, 3 ), 213 [28] = XE2_PAT( 0, 0, 3, 0, 3, 0 ), 214 [29] = XE2_PAT( 0, 1, 3, 0, 3, 0 ), 215 [30] = XE2_PAT( 0, 0, 3, 0, 3, 2 ), 216 [31] = XE2_PAT( 0, 0, 3, 0, 3, 3 ), 217 }; 218 /* Special PAT values programmed outside the main table */ 219 static const struct xe_pat_table_entry xe2_pat_ats = XE2_PAT( 0, 0, 0, 0, 3, 3 ); 220 static const struct xe_pat_table_entry xe2_pat_pta = XE2_PAT( 0, 0, 0, 0, 3, 0 ); 221 222 /* 223 * Xe3p_XPC PAT table uses the same layout as Xe2/Xe3, except that there's no 224 * option for compression. Also note that the "L3" and "L4" register fields 225 * actually control L2 and L3 cache respectively on this platform. 226 */ 227 #define XE3P_XPC_PAT(no_promote, l3clos, l3_policy, l4_policy, __coh_mode) \ 228 XE2_PAT(no_promote, 0, l3clos, l3_policy, l4_policy, __coh_mode) 229 230 static const struct xe_pat_table_entry xe3p_xpc_pat_ats = XE3P_XPC_PAT( 0, 0, 0, 0, 3 ); 231 static const struct xe_pat_table_entry xe3p_xpc_pat_pta = XE3P_XPC_PAT( 0, 0, 0, 0, 0 ); 232 233 static const struct xe_pat_table_entry xe3p_xpc_pat_table[] = { 234 [ 0] = XE3P_XPC_PAT( 0, 0, 0, 0, 0 ), 235 [ 1] = XE3P_XPC_PAT( 0, 0, 0, 0, 2 ), 236 [ 2] = XE3P_XPC_PAT( 0, 0, 0, 0, 3 ), 237 [ 3] = XE3P_XPC_PAT( 0, 0, 3, 3, 0 ), 238 [ 4] = XE3P_XPC_PAT( 0, 0, 3, 3, 2 ), 239 [ 5] = XE3P_XPC_PAT( 0, 0, 3, 0, 0 ), 240 [ 6] = XE3P_XPC_PAT( 0, 0, 3, 0, 2 ), 241 [ 7] = XE3P_XPC_PAT( 0, 0, 3, 0, 3 ), 242 [ 8] = XE3P_XPC_PAT( 0, 0, 0, 3, 0 ), 243 [ 9] = XE3P_XPC_PAT( 0, 0, 0, 3, 2 ), 244 [10] = XE3P_XPC_PAT( 0, 0, 0, 3, 3 ), 245 /* 11..22 are reserved; leave set to all 0's */ 246 [23] = XE3P_XPC_PAT( 0, 1, 0, 0, 0 ), 247 [24] = XE3P_XPC_PAT( 0, 1, 0, 0, 2 ), 248 [25] = XE3P_XPC_PAT( 0, 1, 0, 0, 3 ), 249 [26] = XE3P_XPC_PAT( 0, 2, 0, 0, 0 ), 250 [27] = XE3P_XPC_PAT( 0, 2, 0, 0, 2 ), 251 [28] = XE3P_XPC_PAT( 0, 2, 0, 0, 3 ), 252 [29] = XE3P_XPC_PAT( 0, 3, 0, 0, 0 ), 253 [30] = XE3P_XPC_PAT( 0, 3, 0, 0, 2 ), 254 [31] = XE3P_XPC_PAT( 0, 3, 0, 0, 3 ), 255 }; 256 257 static const struct xe_pat_table_entry xe3p_primary_pat_pta = XE2_PAT(0, 0, 0, 0, 0, 3); 258 static const struct xe_pat_table_entry xe3p_media_pat_pta = XE2_PAT(0, 0, 0, 0, 0, 2); 259 260 static const struct xe_pat_table_entry xe3p_lpg_pat_table[] = { 261 [ 0] = XE2_PAT( 0, 0, 0, 0, 3, 0 ), 262 [ 1] = XE2_PAT( 0, 0, 0, 0, 3, 2 ), 263 [ 2] = XE2_PAT( 0, 0, 0, 0, 3, 3 ), 264 [ 3] = XE2_PAT( 0, 0, 0, 3, 3, 0 ), 265 [ 4] = XE2_PAT( 0, 0, 0, 3, 0, 2 ), 266 [ 5] = XE2_PAT( 0, 0, 0, 3, 3, 2 ), 267 [ 6] = XE2_PAT( 1, 0, 0, 1, 3, 0 ), 268 [ 7] = XE2_PAT( 0, 0, 0, 3, 0, 3 ), 269 [ 8] = XE2_PAT( 0, 0, 0, 3, 0, 0 ), 270 [ 9] = XE2_PAT( 0, 1, 0, 0, 3, 0 ), 271 [10] = XE2_PAT( 0, 1, 0, 3, 0, 0 ), 272 [11] = XE2_PAT( 1, 1, 0, 1, 3, 0 ), 273 [12] = XE2_PAT( 0, 1, 0, 3, 3, 0 ), 274 [13] = XE2_PAT( 0, 0, 0, 0, 0, 0 ), 275 [14] = XE2_PAT( 0, 1, 0, 0, 0, 0 ), 276 [15] = XE2_PAT( 1, 1, 0, 1, 1, 0 ), 277 [16] = XE2_PAT( 0, 1, 0, 0, 3, 2 ), 278 /* 17 is reserved; leave set to all 0's */ 279 [18] = XE2_PAT( 1, 0, 0, 2, 3, 0 ), 280 [19] = XE2_PAT( 1, 0, 0, 2, 3, 2 ), 281 [20] = XE2_PAT( 0, 0, 1, 0, 3, 0 ), 282 [21] = XE2_PAT( 0, 1, 1, 0, 3, 0 ), 283 [22] = XE2_PAT( 0, 0, 1, 0, 3, 2 ), 284 [23] = XE2_PAT( 0, 0, 1, 0, 3, 3 ), 285 [24] = XE2_PAT( 0, 0, 2, 0, 3, 0 ), 286 [25] = XE2_PAT( 0, 1, 2, 0, 3, 0 ), 287 [26] = XE2_PAT( 0, 0, 2, 0, 3, 2 ), 288 [27] = XE2_PAT( 0, 0, 2, 0, 3, 3 ), 289 [28] = XE2_PAT( 0, 0, 3, 0, 3, 0 ), 290 [29] = XE2_PAT( 0, 1, 3, 0, 3, 0 ), 291 [30] = XE2_PAT( 0, 0, 3, 0, 3, 2 ), 292 [31] = XE2_PAT( 0, 0, 3, 0, 3, 3 ), 293 }; 294 295 u16 xe_pat_index_get_coh_mode(struct xe_device *xe, u16 pat_index) 296 { 297 WARN_ON(pat_index >= xe->pat.n_entries); 298 return xe->pat.table[pat_index].coh_mode; 299 } 300 301 bool xe_pat_index_get_comp_en(struct xe_device *xe, u16 pat_index) 302 { 303 WARN_ON(pat_index >= xe->pat.n_entries); 304 return !!(xe->pat.table[pat_index].value & XE2_COMP_EN); 305 } 306 307 u16 xe_pat_index_get_l3_policy(struct xe_device *xe, u16 pat_index) 308 { 309 WARN_ON(pat_index >= xe->pat.n_entries); 310 311 return REG_FIELD_GET(XE2_L3_POLICY, xe->pat.table[pat_index].value); 312 } 313 314 static const struct xe_pat_table_entry *gt_pta_entry(struct xe_gt *gt) 315 { 316 struct xe_device *xe = gt_to_xe(gt); 317 318 if (xe_gt_is_main_type(gt)) 319 return xe->pat.pat_primary_pta; 320 321 if (xe_gt_is_media_type(gt)) 322 return xe->pat.pat_media_pta; 323 324 xe_assert(xe, false); 325 return NULL; 326 } 327 328 static void program_pat(struct xe_gt *gt, const struct xe_pat_table_entry table[], 329 int n_entries) 330 { 331 struct xe_device *xe = gt_to_xe(gt); 332 const struct xe_pat_table_entry *pta_entry = gt_pta_entry(gt); 333 334 for (int i = 0; i < n_entries; i++) { 335 struct xe_reg reg = XE_REG(_PAT_INDEX(i)); 336 337 xe_mmio_write32(>->mmio, reg, table[i].value); 338 } 339 340 if (xe->pat.pat_ats) 341 xe_mmio_write32(>->mmio, XE_REG(_PAT_ATS), xe->pat.pat_ats->value); 342 343 if (pta_entry) 344 xe_mmio_write32(>->mmio, XE_REG(_PAT_PTA), pta_entry->value); 345 } 346 347 static void program_pat_mcr(struct xe_gt *gt, const struct xe_pat_table_entry table[], 348 int n_entries) 349 { 350 struct xe_device *xe = gt_to_xe(gt); 351 const struct xe_pat_table_entry *pta_entry = gt_pta_entry(gt); 352 353 for (int i = 0; i < n_entries; i++) { 354 struct xe_reg_mcr reg_mcr = XE_REG_MCR(_PAT_INDEX(i)); 355 356 xe_gt_mcr_multicast_write(gt, reg_mcr, table[i].value); 357 } 358 359 if (xe->pat.pat_ats) 360 xe_gt_mcr_multicast_write(gt, XE_REG_MCR(_PAT_ATS), xe->pat.pat_ats->value); 361 362 if (pta_entry) 363 xe_gt_mcr_multicast_write(gt, XE_REG_MCR(_PAT_PTA), pta_entry->value); 364 } 365 366 static int xelp_dump(struct xe_gt *gt, struct drm_printer *p) 367 { 368 struct xe_device *xe = gt_to_xe(gt); 369 int i; 370 371 CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); 372 if (!fw_ref.domains) 373 return -ETIMEDOUT; 374 375 drm_printf(p, "PAT table:\n"); 376 377 for (i = 0; i < xe->pat.n_entries; i++) { 378 u32 pat = xe_mmio_read32(>->mmio, XE_REG(_PAT_INDEX(i))); 379 380 xelp_pat_entry_dump(p, i, pat); 381 } 382 383 return 0; 384 } 385 386 static const struct xe_pat_ops xelp_pat_ops = { 387 .program_graphics = program_pat, 388 .dump = xelp_dump, 389 }; 390 391 static int xehp_dump(struct xe_gt *gt, struct drm_printer *p) 392 { 393 struct xe_device *xe = gt_to_xe(gt); 394 int i; 395 396 CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); 397 if (!fw_ref.domains) 398 return -ETIMEDOUT; 399 400 drm_printf(p, "PAT table:\n"); 401 402 for (i = 0; i < xe->pat.n_entries; i++) { 403 u32 pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i))); 404 405 xelp_pat_entry_dump(p, i, pat); 406 } 407 408 return 0; 409 } 410 411 static const struct xe_pat_ops xehp_pat_ops = { 412 .program_graphics = program_pat_mcr, 413 .dump = xehp_dump, 414 }; 415 416 static int xehpc_dump(struct xe_gt *gt, struct drm_printer *p) 417 { 418 struct xe_device *xe = gt_to_xe(gt); 419 int i; 420 421 CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); 422 if (!fw_ref.domains) 423 return -ETIMEDOUT; 424 425 drm_printf(p, "PAT table:\n"); 426 427 for (i = 0; i < xe->pat.n_entries; i++) { 428 u32 pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i))); 429 430 xehpc_pat_entry_dump(p, i, pat); 431 } 432 433 return 0; 434 } 435 436 static const struct xe_pat_ops xehpc_pat_ops = { 437 .program_graphics = program_pat_mcr, 438 .dump = xehpc_dump, 439 }; 440 441 static int xelpg_dump(struct xe_gt *gt, struct drm_printer *p) 442 { 443 struct xe_device *xe = gt_to_xe(gt); 444 int i; 445 446 CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); 447 if (!fw_ref.domains) 448 return -ETIMEDOUT; 449 450 drm_printf(p, "PAT table:\n"); 451 452 for (i = 0; i < xe->pat.n_entries; i++) { 453 u32 pat; 454 455 if (xe_gt_is_media_type(gt)) 456 pat = xe_mmio_read32(>->mmio, XE_REG(_PAT_INDEX(i))); 457 else 458 pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i))); 459 460 xelpg_pat_entry_dump(p, i, pat); 461 } 462 463 return 0; 464 } 465 466 /* 467 * SAMedia register offsets are adjusted by the write methods and they target 468 * registers that are not MCR, while for normal GT they are MCR 469 */ 470 static const struct xe_pat_ops xelpg_pat_ops = { 471 .program_graphics = program_pat, 472 .program_media = program_pat_mcr, 473 .dump = xelpg_dump, 474 }; 475 476 static void xe2_pat_entry_dump(struct drm_printer *p, const char *label, u32 pat, bool rsvd) 477 { 478 drm_printf(p, "%s= [ %u, %u, %u, %u, %u, %u ] (%#8x)%s\n", label, 479 !!(pat & XE2_NO_PROMOTE), 480 !!(pat & XE2_COMP_EN), 481 REG_FIELD_GET(XE2_L3_CLOS, pat), 482 REG_FIELD_GET(XE2_L3_POLICY, pat), 483 REG_FIELD_GET(XE2_L4_POLICY, pat), 484 REG_FIELD_GET(XE2_COH_MODE, pat), 485 pat, rsvd ? " *" : ""); 486 } 487 488 static void xe3p_xpc_pat_entry_dump(struct drm_printer *p, const char *label, u32 pat, bool rsvd) 489 { 490 drm_printf(p, "%s= [ %u, %u, %u, %u, %u ] (%#8x)%s\n", label, 491 !!(pat & XE2_NO_PROMOTE), 492 REG_FIELD_GET(XE2_L3_CLOS, pat), 493 REG_FIELD_GET(XE2_L3_POLICY, pat), 494 REG_FIELD_GET(XE2_L4_POLICY, pat), 495 REG_FIELD_GET(XE2_COH_MODE, pat), 496 pat, rsvd ? " *" : ""); 497 } 498 499 static int xe2_dump(struct xe_gt *gt, struct drm_printer *p) 500 { 501 struct xe_device *xe = gt_to_xe(gt); 502 u32 pat; 503 int i; 504 char label[PAT_LABEL_LEN]; 505 506 CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); 507 if (!fw_ref.domains) 508 return -ETIMEDOUT; 509 510 drm_printf(p, "PAT table: (* = reserved entry)\n"); 511 512 for (i = 0; i < xe->pat.n_entries; i++) { 513 if (xe_gt_is_media_type(gt)) 514 pat = xe_mmio_read32(>->mmio, XE_REG(_PAT_INDEX(i))); 515 else 516 pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i))); 517 518 xe_pat_index_label(label, sizeof(label), i); 519 xe->pat.ops->entry_dump(p, label, pat, !xe->pat.table[i].valid); 520 } 521 522 /* 523 * Also print PTA_MODE, which describes how the hardware accesses 524 * PPGTT entries. 525 */ 526 if (xe_gt_is_media_type(gt)) 527 pat = xe_mmio_read32(>->mmio, XE_REG(_PAT_PTA)); 528 else 529 pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_PTA)); 530 531 drm_printf(p, "Page Table Access:\n"); 532 xe->pat.ops->entry_dump(p, "PTA_MODE", pat, false); 533 534 if (xe_gt_is_media_type(gt)) 535 pat = xe_mmio_read32(>->mmio, XE_REG(_PAT_ATS)); 536 else 537 pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_ATS)); 538 539 drm_printf(p, "PCIe ATS/PASID:\n"); 540 xe->pat.ops->entry_dump(p, "PAT_ATS ", pat, false); 541 542 return 0; 543 } 544 545 static const struct xe_pat_ops xe2_pat_ops = { 546 .program_graphics = program_pat_mcr, 547 .program_media = program_pat, 548 .dump = xe2_dump, 549 .entry_dump = xe2_pat_entry_dump, 550 }; 551 552 static const struct xe_pat_ops xe3p_xpc_pat_ops = { 553 .program_graphics = program_pat_mcr, 554 .program_media = program_pat, 555 .dump = xe2_dump, 556 .entry_dump = xe3p_xpc_pat_entry_dump, 557 }; 558 559 void xe_pat_init_early(struct xe_device *xe) 560 { 561 xe->pat.idx[XE_CACHE_WB_COMPRESSION] = XE_PAT_INVALID_IDX; 562 xe->pat.idx[XE_CACHE_NONE_COMPRESSION] = XE_PAT_INVALID_IDX; 563 if (GRAPHICS_VERx100(xe) == 3511) { 564 xe->pat.ops = &xe3p_xpc_pat_ops; 565 xe->pat.table = xe3p_xpc_pat_table; 566 xe->pat.pat_ats = &xe3p_xpc_pat_ats; 567 xe->pat.pat_primary_pta = &xe3p_xpc_pat_pta; 568 xe->pat.pat_media_pta = &xe3p_xpc_pat_pta; 569 xe->pat.n_entries = ARRAY_SIZE(xe3p_xpc_pat_table); 570 xe->pat.idx[XE_CACHE_NONE] = 3; 571 xe->pat.idx[XE_CACHE_WT] = 3; /* N/A (no display); use UC */ 572 xe->pat.idx[XE_CACHE_WB] = 2; 573 } else if (GRAPHICS_VER(xe) == 35) { 574 xe->pat.ops = &xe2_pat_ops; 575 xe->pat.table = xe3p_lpg_pat_table; 576 xe->pat.pat_ats = &xe2_pat_ats; 577 if (!IS_DGFX(xe)) { 578 xe->pat.pat_primary_pta = &xe3p_primary_pat_pta; 579 xe->pat.pat_media_pta = &xe3p_media_pat_pta; 580 } 581 xe->pat.n_entries = ARRAY_SIZE(xe3p_lpg_pat_table); 582 xe->pat.idx[XE_CACHE_NONE] = 3; 583 xe->pat.idx[XE_CACHE_WT] = 15; 584 xe->pat.idx[XE_CACHE_WB] = 2; 585 xe->pat.idx[XE_CACHE_NONE_COMPRESSION] = 12; 586 xe->pat.idx[XE_CACHE_WB_COMPRESSION] = 16; 587 } else if (GRAPHICS_VER(xe) == 30 || GRAPHICS_VER(xe) == 20) { 588 xe->pat.ops = &xe2_pat_ops; 589 if (GRAPHICS_VER(xe) == 30) { 590 xe->pat.table = xe3_lpg_pat_table; 591 xe->pat.idx[XE_CACHE_WB_COMPRESSION] = 16; 592 } else { 593 xe->pat.table = xe2_pat_table; 594 } 595 xe->pat.pat_ats = &xe2_pat_ats; 596 if (IS_DGFX(xe)) { 597 xe->pat.pat_primary_pta = &xe2_pat_pta; 598 xe->pat.pat_media_pta = &xe2_pat_pta; 599 } 600 601 /* Wa_16023588340. XXX: Should use XE_WA */ 602 if (GRAPHICS_VERx100(xe) == 2001) 603 xe->pat.n_entries = 28; /* Disable CLOS3 */ 604 else 605 xe->pat.n_entries = ARRAY_SIZE(xe2_pat_table); 606 607 xe->pat.idx[XE_CACHE_NONE] = 3; 608 xe->pat.idx[XE_CACHE_WT] = 15; 609 xe->pat.idx[XE_CACHE_WB] = 2; 610 xe->pat.idx[XE_CACHE_NONE_COMPRESSION] = 12; /*Applicable on xe2 and beyond */ 611 } else if (xe->info.platform == XE_METEORLAKE) { 612 xe->pat.ops = &xelpg_pat_ops; 613 xe->pat.table = xelpg_pat_table; 614 xe->pat.n_entries = ARRAY_SIZE(xelpg_pat_table); 615 xe->pat.idx[XE_CACHE_NONE] = 2; 616 xe->pat.idx[XE_CACHE_WT] = 1; 617 xe->pat.idx[XE_CACHE_WB] = 3; 618 } else if (xe->info.platform == XE_PVC) { 619 xe->pat.ops = &xehpc_pat_ops; 620 xe->pat.table = xehpc_pat_table; 621 xe->pat.n_entries = ARRAY_SIZE(xehpc_pat_table); 622 xe->pat.idx[XE_CACHE_NONE] = 0; 623 xe->pat.idx[XE_CACHE_WT] = 2; 624 xe->pat.idx[XE_CACHE_WB] = 3; 625 } else if (xe->info.platform == XE_DG2) { 626 /* 627 * Table is the same as previous platforms, but programming 628 * method has changed. 629 */ 630 xe->pat.ops = &xehp_pat_ops; 631 xe->pat.table = xelp_pat_table; 632 xe->pat.n_entries = ARRAY_SIZE(xelp_pat_table); 633 xe->pat.idx[XE_CACHE_NONE] = 3; 634 xe->pat.idx[XE_CACHE_WT] = 2; 635 xe->pat.idx[XE_CACHE_WB] = 0; 636 } else if (GRAPHICS_VERx100(xe) <= 1210) { 637 WARN_ON_ONCE(!IS_DGFX(xe) && !xe->info.has_llc); 638 xe->pat.ops = &xelp_pat_ops; 639 xe->pat.table = xelp_pat_table; 640 xe->pat.n_entries = ARRAY_SIZE(xelp_pat_table); 641 xe->pat.idx[XE_CACHE_NONE] = 3; 642 xe->pat.idx[XE_CACHE_WT] = 2; 643 xe->pat.idx[XE_CACHE_WB] = 0; 644 } else { 645 /* 646 * Going forward we expect to need new PAT settings for most 647 * new platforms; failure to provide a new table can easily 648 * lead to subtle, hard-to-debug problems. If none of the 649 * conditions above match the platform we're running on we'll 650 * raise an error rather than trying to silently inherit the 651 * most recent platform's behavior. 652 */ 653 drm_err(&xe->drm, "Missing PAT table for platform with graphics version %d.%02d!\n", 654 GRAPHICS_VER(xe), GRAPHICS_VERx100(xe) % 100); 655 } 656 657 xe_assert(xe, xe->pat.ops->dump); 658 xe_assert(xe, xe->pat.ops->program_graphics); 659 xe_assert(xe, MEDIA_VER(xe) < 13 || xe->pat.ops->program_media); 660 xe_assert(xe, GRAPHICS_VER(xe) < 20 || xe->pat.ops->entry_dump); 661 } 662 663 void xe_pat_init(struct xe_gt *gt) 664 { 665 struct xe_device *xe = gt_to_xe(gt); 666 667 if (IS_SRIOV_VF(xe)) 668 return; 669 670 if (xe_gt_is_media_type(gt)) 671 xe->pat.ops->program_media(gt, xe->pat.table, xe->pat.n_entries); 672 else 673 xe->pat.ops->program_graphics(gt, xe->pat.table, xe->pat.n_entries); 674 } 675 676 /** 677 * xe_pat_dump() - Dump GT PAT table into a drm printer. 678 * @gt: the &xe_gt 679 * @p: the &drm_printer 680 * 681 * Return: 0 on success or a negative error code on failure. 682 */ 683 int xe_pat_dump(struct xe_gt *gt, struct drm_printer *p) 684 { 685 struct xe_device *xe = gt_to_xe(gt); 686 687 if (IS_SRIOV_VF(xe)) 688 return -EOPNOTSUPP; 689 690 return xe->pat.ops->dump(gt, p); 691 } 692 693 /** 694 * xe_pat_dump_sw_config() - Dump the software-configured GT PAT table into a drm printer. 695 * @gt: the &xe_gt 696 * @p: the &drm_printer 697 * 698 * Return: 0 on success or a negative error code on failure. 699 */ 700 int xe_pat_dump_sw_config(struct xe_gt *gt, struct drm_printer *p) 701 { 702 struct xe_device *xe = gt_to_xe(gt); 703 const struct xe_pat_table_entry *pta_entry = gt_pta_entry(gt); 704 char label[PAT_LABEL_LEN]; 705 706 if (!xe->pat.table || !xe->pat.n_entries) 707 return -EOPNOTSUPP; 708 709 drm_printf(p, "PAT table:%s\n", GRAPHICS_VER(xe) >= 20 ? " (* = reserved entry)" : ""); 710 for (u32 i = 0; i < xe->pat.n_entries; i++) { 711 u32 pat = xe->pat.table[i].value; 712 713 if (GRAPHICS_VER(xe) >= 20) { 714 xe_pat_index_label(label, sizeof(label), i); 715 xe->pat.ops->entry_dump(p, label, pat, !xe->pat.table[i].valid); 716 } else if (xe->info.platform == XE_METEORLAKE) { 717 xelpg_pat_entry_dump(p, i, pat); 718 } else if (xe->info.platform == XE_PVC) { 719 xehpc_pat_entry_dump(p, i, pat); 720 } else if (xe->info.platform == XE_DG2 || GRAPHICS_VERx100(xe) <= 1210) { 721 xelp_pat_entry_dump(p, i, pat); 722 } else { 723 return -EOPNOTSUPP; 724 } 725 } 726 727 if (pta_entry) { 728 u32 pat = pta_entry->value; 729 730 drm_printf(p, "Page Table Access:\n"); 731 xe->pat.ops->entry_dump(p, "PTA_MODE", pat, false); 732 } 733 734 if (xe->pat.pat_ats) { 735 u32 pat = xe->pat.pat_ats->value; 736 737 drm_printf(p, "PCIe ATS/PASID:\n"); 738 xe->pat.ops->entry_dump(p, "PAT_ATS ", pat, false); 739 } 740 741 drm_printf(p, "Cache Level:\n"); 742 drm_printf(p, "IDX[XE_CACHE_NONE] = %d\n", xe->pat.idx[XE_CACHE_NONE]); 743 drm_printf(p, "IDX[XE_CACHE_WT] = %d\n", xe->pat.idx[XE_CACHE_WT]); 744 drm_printf(p, "IDX[XE_CACHE_WB] = %d\n", xe->pat.idx[XE_CACHE_WB]); 745 if (GRAPHICS_VER(xe) >= 20) { 746 drm_printf(p, "IDX[XE_CACHE_NONE_COMPRESSION] = %d\n", 747 xe->pat.idx[XE_CACHE_NONE_COMPRESSION]); 748 drm_printf(p, "IDX[XE_CACHE_WB_COMPRESSION] = %d\n", 749 xe->pat.idx[XE_CACHE_WB_COMPRESSION]); 750 } 751 752 return 0; 753 } 754