1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2023 Intel Corporation 4 */ 5 6 #include "xe_pat.h" 7 8 #include <uapi/drm/xe_drm.h> 9 10 #include <generated/xe_wa_oob.h> 11 12 #include "regs/xe_gt_regs.h" 13 #include "regs/xe_reg_defs.h" 14 #include "xe_assert.h" 15 #include "xe_device.h" 16 #include "xe_force_wake.h" 17 #include "xe_gt.h" 18 #include "xe_gt_mcr.h" 19 #include "xe_mmio.h" 20 #include "xe_sriov.h" 21 #include "xe_wa.h" 22 23 #define _PAT_ATS 0x47fc 24 #define _PAT_INDEX(index) _PICK_EVEN_2RANGES(index, 8, \ 25 0x4800, 0x4804, \ 26 0x4848, 0x484c) 27 #define _PAT_PTA 0x4820 28 #define _PAT_TR_PTA 0x48cc 29 30 #define XE2_NO_PROMOTE REG_BIT(10) 31 #define XE2_COMP_EN REG_BIT(9) 32 #define XE2_L3_CLOS REG_GENMASK(7, 6) 33 #define XE2_L3_POLICY REG_GENMASK(5, 4) 34 #define XE2_L4_POLICY REG_GENMASK(3, 2) 35 #define XE2_COH_MODE REG_GENMASK(1, 0) 36 37 #define XELPG_L4_POLICY_MASK REG_GENMASK(3, 2) 38 #define XELPG_PAT_3_UC REG_FIELD_PREP(XELPG_L4_POLICY_MASK, 3) 39 #define XELPG_PAT_1_WT REG_FIELD_PREP(XELPG_L4_POLICY_MASK, 1) 40 #define XELPG_PAT_0_WB REG_FIELD_PREP(XELPG_L4_POLICY_MASK, 0) 41 #define XELPG_INDEX_COH_MODE_MASK REG_GENMASK(1, 0) 42 #define XELPG_3_COH_2W REG_FIELD_PREP(XELPG_INDEX_COH_MODE_MASK, 3) 43 #define XELPG_2_COH_1W REG_FIELD_PREP(XELPG_INDEX_COH_MODE_MASK, 2) 44 #define XELPG_0_COH_NON REG_FIELD_PREP(XELPG_INDEX_COH_MODE_MASK, 0) 45 46 #define XEHPC_CLOS_LEVEL_MASK REG_GENMASK(3, 2) 47 #define XEHPC_PAT_CLOS(x) REG_FIELD_PREP(XEHPC_CLOS_LEVEL_MASK, x) 48 49 #define XELP_MEM_TYPE_MASK REG_GENMASK(1, 0) 50 #define XELP_PAT_WB REG_FIELD_PREP(XELP_MEM_TYPE_MASK, 3) 51 #define XELP_PAT_WT REG_FIELD_PREP(XELP_MEM_TYPE_MASK, 2) 52 #define XELP_PAT_WC REG_FIELD_PREP(XELP_MEM_TYPE_MASK, 1) 53 #define XELP_PAT_UC REG_FIELD_PREP(XELP_MEM_TYPE_MASK, 0) 54 55 #define PAT_LABEL_LEN 20 56 57 static const char *XELP_MEM_TYPE_STR_MAP[] = { "UC", "WC", "WT", "WB" }; 58 59 static void xe_pat_index_label(char *label, size_t len, int index) 60 { 61 snprintf(label, len, "PAT[%2d] ", index); 62 } 63 64 static void xelp_pat_entry_dump(struct drm_printer *p, int index, u32 pat) 65 { 66 u8 mem_type = REG_FIELD_GET(XELP_MEM_TYPE_MASK, pat); 67 68 drm_printf(p, "PAT[%2d] = %s (%#8x)\n", index, 69 XELP_MEM_TYPE_STR_MAP[mem_type], pat); 70 } 71 72 static void xehpc_pat_entry_dump(struct drm_printer *p, int index, u32 pat) 73 { 74 drm_printf(p, "PAT[%2d] = [ %u, %u ] (%#8x)\n", index, 75 REG_FIELD_GET(XELP_MEM_TYPE_MASK, pat), 76 REG_FIELD_GET(XEHPC_CLOS_LEVEL_MASK, pat), pat); 77 } 78 79 static void xelpg_pat_entry_dump(struct drm_printer *p, int index, u32 pat) 80 { 81 drm_printf(p, "PAT[%2d] = [ %u, %u ] (%#8x)\n", index, 82 REG_FIELD_GET(XELPG_L4_POLICY_MASK, pat), 83 REG_FIELD_GET(XELPG_INDEX_COH_MODE_MASK, pat), pat); 84 } 85 86 struct xe_pat_ops { 87 void (*program_graphics)(struct xe_gt *gt, const struct xe_pat_table_entry table[], 88 int n_entries); 89 void (*program_media)(struct xe_gt *gt, const struct xe_pat_table_entry table[], 90 int n_entries); 91 int (*dump)(struct xe_gt *gt, struct drm_printer *p); 92 void (*entry_dump)(struct drm_printer *p, const char *label, u32 pat, bool rsvd); 93 }; 94 95 static const struct xe_pat_table_entry xelp_pat_table[] = { 96 [0] = { XELP_PAT_WB, XE_COH_1WAY }, 97 [1] = { XELP_PAT_WC, XE_COH_NONE }, 98 [2] = { XELP_PAT_WT, XE_COH_NONE }, 99 [3] = { XELP_PAT_UC, XE_COH_NONE }, 100 }; 101 102 static const struct xe_pat_table_entry xehpc_pat_table[] = { 103 [0] = { XELP_PAT_UC, XE_COH_NONE }, 104 [1] = { XELP_PAT_WC, XE_COH_NONE }, 105 [2] = { XELP_PAT_WT, XE_COH_NONE }, 106 [3] = { XELP_PAT_WB, XE_COH_1WAY }, 107 [4] = { XEHPC_PAT_CLOS(1) | XELP_PAT_WT, XE_COH_NONE }, 108 [5] = { XEHPC_PAT_CLOS(1) | XELP_PAT_WB, XE_COH_1WAY }, 109 [6] = { XEHPC_PAT_CLOS(2) | XELP_PAT_WT, XE_COH_NONE }, 110 [7] = { XEHPC_PAT_CLOS(2) | XELP_PAT_WB, XE_COH_1WAY }, 111 }; 112 113 static const struct xe_pat_table_entry xelpg_pat_table[] = { 114 [0] = { XELPG_PAT_0_WB, XE_COH_NONE }, 115 [1] = { XELPG_PAT_1_WT, XE_COH_NONE }, 116 [2] = { XELPG_PAT_3_UC, XE_COH_NONE }, 117 [3] = { XELPG_PAT_0_WB | XELPG_2_COH_1W, XE_COH_1WAY }, 118 [4] = { XELPG_PAT_0_WB | XELPG_3_COH_2W, XE_COH_2WAY }, 119 }; 120 121 /* 122 * The Xe2 table is getting large/complicated so it's easier to review if 123 * provided in a form that exactly matches the bspec's formatting. The meaning 124 * of the fields here are: 125 * - no_promote: 0=promotable, 1=no promote 126 * - comp_en: 0=disable, 1=enable 127 * - l3clos: L3 class of service (0-3) 128 * - l3_policy: 0=WB, 1=XD ("WB - Transient Display"), 129 * 2=XA ("WB - Transient App" for Xe3p), 3=UC 130 * - l4_policy: 0=WB, 1=WT, 3=UC 131 * - coh_mode: 0=no snoop, 2=1-way coherent, 3=2-way coherent 132 * 133 * Reserved entries should be programmed with the maximum caching, minimum 134 * coherency (which matches an all-0's encoding), so we can just omit them 135 * in the table. 136 * 137 * Note: There is an implicit assumption in the driver that compression and 138 * coh_1way+ are mutually exclusive for platforms prior to Xe3. Starting 139 * with Xe3, compression can be combined with coherency. If using compression 140 * with coherency, userptr and imported dma-buf from external device will 141 * have uncleared ccs state. See also xe_bo_needs_ccs_pages(). 142 */ 143 #define XE2_PAT(no_promote, comp_en, l3clos, l3_policy, l4_policy, __coh_mode) \ 144 { \ 145 .value = (no_promote ? XE2_NO_PROMOTE : 0) | \ 146 (comp_en ? XE2_COMP_EN : 0) | \ 147 REG_FIELD_PREP(XE2_L3_CLOS, l3clos) | \ 148 REG_FIELD_PREP(XE2_L3_POLICY, l3_policy) | \ 149 REG_FIELD_PREP(XE2_L4_POLICY, l4_policy) | \ 150 REG_FIELD_PREP(XE2_COH_MODE, __coh_mode), \ 151 .coh_mode = __coh_mode ? __coh_mode : XE_COH_NONE, \ 152 .valid = 1 \ 153 } 154 155 static const struct xe_pat_table_entry xe2_pat_table[] = { 156 [ 0] = XE2_PAT( 0, 0, 0, 0, 3, 0 ), 157 [ 1] = XE2_PAT( 0, 0, 0, 0, 3, 2 ), 158 [ 2] = XE2_PAT( 0, 0, 0, 0, 3, 3 ), 159 [ 3] = XE2_PAT( 0, 0, 0, 3, 3, 0 ), 160 [ 4] = XE2_PAT( 0, 0, 0, 3, 0, 2 ), 161 [ 5] = XE2_PAT( 0, 0, 0, 3, 3, 2 ), 162 [ 6] = XE2_PAT( 1, 0, 0, 1, 3, 0 ), 163 [ 7] = XE2_PAT( 0, 0, 0, 3, 0, 3 ), 164 [ 8] = XE2_PAT( 0, 0, 0, 3, 0, 0 ), 165 [ 9] = XE2_PAT( 0, 1, 0, 0, 3, 0 ), 166 [10] = XE2_PAT( 0, 1, 0, 3, 0, 0 ), 167 [11] = XE2_PAT( 1, 1, 0, 1, 3, 0 ), 168 [12] = XE2_PAT( 0, 1, 0, 3, 3, 0 ), 169 [13] = XE2_PAT( 0, 0, 0, 0, 0, 0 ), 170 [14] = XE2_PAT( 0, 1, 0, 0, 0, 0 ), 171 [15] = XE2_PAT( 1, 1, 0, 1, 1, 0 ), 172 /* 16..19 are reserved; leave set to all 0's */ 173 [20] = XE2_PAT( 0, 0, 1, 0, 3, 0 ), 174 [21] = XE2_PAT( 0, 1, 1, 0, 3, 0 ), 175 [22] = XE2_PAT( 0, 0, 1, 0, 3, 2 ), 176 [23] = XE2_PAT( 0, 0, 1, 0, 3, 3 ), 177 [24] = XE2_PAT( 0, 0, 2, 0, 3, 0 ), 178 [25] = XE2_PAT( 0, 1, 2, 0, 3, 0 ), 179 [26] = XE2_PAT( 0, 0, 2, 0, 3, 2 ), 180 [27] = XE2_PAT( 0, 0, 2, 0, 3, 3 ), 181 [28] = XE2_PAT( 0, 0, 3, 0, 3, 0 ), 182 [29] = XE2_PAT( 0, 1, 3, 0, 3, 0 ), 183 [30] = XE2_PAT( 0, 0, 3, 0, 3, 2 ), 184 [31] = XE2_PAT( 0, 0, 3, 0, 3, 3 ), 185 }; 186 187 static const struct xe_pat_table_entry xe3_lpg_pat_table[] = { 188 [ 0] = XE2_PAT( 0, 0, 0, 0, 3, 0 ), 189 [ 1] = XE2_PAT( 0, 0, 0, 0, 3, 2 ), 190 [ 2] = XE2_PAT( 0, 0, 0, 0, 3, 3 ), 191 [ 3] = XE2_PAT( 0, 0, 0, 3, 3, 0 ), 192 [ 4] = XE2_PAT( 0, 0, 0, 3, 0, 2 ), 193 [ 5] = XE2_PAT( 0, 0, 0, 3, 3, 2 ), 194 [ 6] = XE2_PAT( 1, 0, 0, 1, 3, 0 ), 195 [ 7] = XE2_PAT( 0, 0, 0, 3, 0, 3 ), 196 [ 8] = XE2_PAT( 0, 0, 0, 3, 0, 0 ), 197 [ 9] = XE2_PAT( 0, 1, 0, 0, 3, 0 ), 198 [10] = XE2_PAT( 0, 1, 0, 3, 0, 0 ), 199 [11] = XE2_PAT( 1, 1, 0, 1, 3, 0 ), 200 [12] = XE2_PAT( 0, 1, 0, 3, 3, 0 ), 201 [13] = XE2_PAT( 0, 0, 0, 0, 0, 0 ), 202 [14] = XE2_PAT( 0, 1, 0, 0, 0, 0 ), 203 [15] = XE2_PAT( 1, 1, 0, 1, 1, 0 ), 204 [16] = XE2_PAT( 0, 1, 0, 0, 3, 2 ), 205 /* 17..19 are reserved; leave set to all 0's */ 206 [20] = XE2_PAT( 0, 0, 1, 0, 3, 0 ), 207 [21] = XE2_PAT( 0, 1, 1, 0, 3, 0 ), 208 [22] = XE2_PAT( 0, 0, 1, 0, 3, 2 ), 209 [23] = XE2_PAT( 0, 0, 1, 0, 3, 3 ), 210 [24] = XE2_PAT( 0, 0, 2, 0, 3, 0 ), 211 [25] = XE2_PAT( 0, 1, 2, 0, 3, 0 ), 212 [26] = XE2_PAT( 0, 0, 2, 0, 3, 2 ), 213 [27] = XE2_PAT( 0, 0, 2, 0, 3, 3 ), 214 [28] = XE2_PAT( 0, 0, 3, 0, 3, 0 ), 215 [29] = XE2_PAT( 0, 1, 3, 0, 3, 0 ), 216 [30] = XE2_PAT( 0, 0, 3, 0, 3, 2 ), 217 [31] = XE2_PAT( 0, 0, 3, 0, 3, 3 ), 218 }; 219 /* Special PAT values programmed outside the main table */ 220 static const struct xe_pat_table_entry xe2_pat_ats = XE2_PAT( 0, 0, 0, 0, 3, 3 ); 221 static const struct xe_pat_table_entry xe2_pat_pta = XE2_PAT( 0, 0, 0, 0, 3, 0 ); 222 223 /* 224 * Xe3p_XPC PAT table uses the same layout as Xe2/Xe3, except that there's no 225 * option for compression. Also note that the "L3" and "L4" register fields 226 * actually control L2 and L3 cache respectively on this platform. 227 */ 228 #define XE3P_XPC_PAT(no_promote, l3clos, l3_policy, l4_policy, __coh_mode) \ 229 XE2_PAT(no_promote, 0, l3clos, l3_policy, l4_policy, __coh_mode) 230 231 static const struct xe_pat_table_entry xe3p_xpc_pat_ats = XE3P_XPC_PAT( 0, 0, 0, 0, 3 ); 232 static const struct xe_pat_table_entry xe3p_xpc_pat_pta = XE3P_XPC_PAT( 0, 0, 0, 0, 0 ); 233 234 static const struct xe_pat_table_entry xe3p_xpc_pat_table[] = { 235 [ 0] = XE3P_XPC_PAT( 0, 0, 0, 0, 0 ), 236 [ 1] = XE3P_XPC_PAT( 0, 0, 0, 0, 2 ), 237 [ 2] = XE3P_XPC_PAT( 0, 0, 0, 0, 3 ), 238 [ 3] = XE3P_XPC_PAT( 0, 0, 3, 3, 0 ), 239 [ 4] = XE3P_XPC_PAT( 0, 0, 3, 3, 2 ), 240 [ 5] = XE3P_XPC_PAT( 0, 0, 3, 0, 0 ), 241 [ 6] = XE3P_XPC_PAT( 0, 0, 3, 0, 2 ), 242 [ 7] = XE3P_XPC_PAT( 0, 0, 3, 0, 3 ), 243 [ 8] = XE3P_XPC_PAT( 0, 0, 0, 3, 0 ), 244 [ 9] = XE3P_XPC_PAT( 0, 0, 0, 3, 2 ), 245 [10] = XE3P_XPC_PAT( 0, 0, 0, 3, 3 ), 246 /* 11..22 are reserved; leave set to all 0's */ 247 [23] = XE3P_XPC_PAT( 0, 1, 0, 0, 0 ), 248 [24] = XE3P_XPC_PAT( 0, 1, 0, 0, 2 ), 249 [25] = XE3P_XPC_PAT( 0, 1, 0, 0, 3 ), 250 [26] = XE3P_XPC_PAT( 0, 2, 0, 0, 0 ), 251 [27] = XE3P_XPC_PAT( 0, 2, 0, 0, 2 ), 252 [28] = XE3P_XPC_PAT( 0, 2, 0, 0, 3 ), 253 [29] = XE3P_XPC_PAT( 0, 3, 0, 0, 0 ), 254 [30] = XE3P_XPC_PAT( 0, 3, 0, 0, 2 ), 255 [31] = XE3P_XPC_PAT( 0, 3, 0, 0, 3 ), 256 }; 257 258 static const struct xe_pat_table_entry xe3p_primary_pat_pta = XE2_PAT(0, 0, 0, 0, 0, 3); 259 static const struct xe_pat_table_entry xe3p_media_pat_pta = XE2_PAT(0, 0, 0, 0, 0, 2); 260 static const struct xe_pat_table_entry xe3p_pat_tr_pta = XE2_PAT(0, 0, 0, 0, 0, 0); 261 262 static const struct xe_pat_table_entry xe3p_lpg_pat_table[] = { 263 [ 0] = XE2_PAT( 0, 0, 0, 0, 3, 0 ), 264 [ 1] = XE2_PAT( 0, 0, 0, 0, 3, 2 ), 265 [ 2] = XE2_PAT( 0, 0, 0, 0, 3, 3 ), 266 [ 3] = XE2_PAT( 0, 0, 0, 3, 3, 0 ), 267 [ 4] = XE2_PAT( 0, 0, 0, 3, 0, 2 ), 268 [ 5] = XE2_PAT( 0, 0, 0, 3, 3, 2 ), 269 [ 6] = XE2_PAT( 1, 0, 0, 1, 3, 0 ), 270 [ 7] = XE2_PAT( 0, 0, 0, 3, 0, 3 ), 271 [ 8] = XE2_PAT( 0, 0, 0, 3, 0, 0 ), 272 [ 9] = XE2_PAT( 0, 1, 0, 0, 3, 0 ), 273 [10] = XE2_PAT( 0, 1, 0, 3, 0, 0 ), 274 [11] = XE2_PAT( 1, 1, 0, 1, 3, 0 ), 275 [12] = XE2_PAT( 0, 1, 0, 3, 3, 0 ), 276 [13] = XE2_PAT( 0, 0, 0, 0, 0, 0 ), 277 [14] = XE2_PAT( 0, 1, 0, 0, 0, 0 ), 278 [15] = XE2_PAT( 1, 1, 0, 1, 1, 0 ), 279 [16] = XE2_PAT( 0, 1, 0, 0, 3, 2 ), 280 /* 17 is reserved; leave set to all 0's */ 281 [18] = XE2_PAT( 1, 0, 0, 2, 3, 0 ), 282 [19] = XE2_PAT( 1, 0, 0, 2, 3, 2 ), 283 [20] = XE2_PAT( 0, 0, 1, 0, 3, 0 ), 284 [21] = XE2_PAT( 0, 1, 1, 0, 3, 0 ), 285 [22] = XE2_PAT( 0, 0, 1, 0, 3, 2 ), 286 [23] = XE2_PAT( 0, 0, 1, 0, 3, 3 ), 287 [24] = XE2_PAT( 0, 0, 2, 0, 3, 0 ), 288 [25] = XE2_PAT( 0, 1, 2, 0, 3, 0 ), 289 [26] = XE2_PAT( 0, 0, 2, 0, 3, 2 ), 290 [27] = XE2_PAT( 0, 0, 2, 0, 3, 3 ), 291 [28] = XE2_PAT( 0, 0, 3, 0, 3, 0 ), 292 [29] = XE2_PAT( 0, 1, 3, 0, 3, 0 ), 293 [30] = XE2_PAT( 0, 0, 3, 0, 3, 2 ), 294 [31] = XE2_PAT( 0, 0, 3, 0, 3, 3 ), 295 }; 296 297 u16 xe_pat_index_get_coh_mode(struct xe_device *xe, u16 pat_index) 298 { 299 WARN_ON(pat_index >= xe->pat.n_entries); 300 return xe->pat.table[pat_index].coh_mode; 301 } 302 303 bool xe_pat_index_get_comp_en(struct xe_device *xe, u16 pat_index) 304 { 305 WARN_ON(pat_index >= xe->pat.n_entries); 306 return !!(xe->pat.table[pat_index].value & XE2_COMP_EN); 307 } 308 309 u16 xe_pat_index_get_l3_policy(struct xe_device *xe, u16 pat_index) 310 { 311 WARN_ON(pat_index >= xe->pat.n_entries); 312 313 return REG_FIELD_GET(XE2_L3_POLICY, xe->pat.table[pat_index].value); 314 } 315 316 static const struct xe_pat_table_entry *gt_pta_entry(struct xe_gt *gt) 317 { 318 struct xe_device *xe = gt_to_xe(gt); 319 320 if (xe_gt_is_main_type(gt)) 321 return xe->pat.pat_primary_pta; 322 323 if (xe_gt_is_media_type(gt)) 324 return xe->pat.pat_media_pta; 325 326 xe_assert(xe, false); 327 return NULL; 328 } 329 330 static const struct xe_pat_table_entry *gt_tr_pta_entry(struct xe_gt *gt) 331 { 332 struct xe_device *xe = gt_to_xe(gt); 333 334 if (xe_gt_is_main_type(gt)) 335 return xe->pat.pat_primary_tr_pta; 336 337 if (xe_gt_is_media_type(gt)) 338 return xe->pat.pat_media_tr_pta; 339 340 xe_assert(xe, false); 341 return NULL; 342 } 343 344 static void program_pat(struct xe_gt *gt, const struct xe_pat_table_entry table[], 345 int n_entries) 346 { 347 struct xe_device *xe = gt_to_xe(gt); 348 const struct xe_pat_table_entry *pta_entry = gt_pta_entry(gt); 349 const struct xe_pat_table_entry *tr_pta_entry = gt_tr_pta_entry(gt); 350 351 for (int i = 0; i < n_entries; i++) { 352 struct xe_reg reg = XE_REG(_PAT_INDEX(i)); 353 354 xe_mmio_write32(>->mmio, reg, table[i].value); 355 } 356 357 if (xe->pat.pat_ats) 358 xe_mmio_write32(>->mmio, XE_REG(_PAT_ATS), xe->pat.pat_ats->value); 359 360 if (pta_entry) 361 xe_mmio_write32(>->mmio, XE_REG(_PAT_PTA), pta_entry->value); 362 363 if (tr_pta_entry) 364 xe_mmio_write32(>->mmio, XE_REG(_PAT_TR_PTA), tr_pta_entry->value); 365 } 366 367 static void program_pat_mcr(struct xe_gt *gt, const struct xe_pat_table_entry table[], 368 int n_entries) 369 { 370 struct xe_device *xe = gt_to_xe(gt); 371 const struct xe_pat_table_entry *pta_entry = gt_pta_entry(gt); 372 const struct xe_pat_table_entry *tr_pta_entry = gt_tr_pta_entry(gt); 373 374 for (int i = 0; i < n_entries; i++) { 375 struct xe_reg_mcr reg_mcr = XE_REG_MCR(_PAT_INDEX(i)); 376 377 xe_gt_mcr_multicast_write(gt, reg_mcr, table[i].value); 378 } 379 380 if (xe->pat.pat_ats) 381 xe_gt_mcr_multicast_write(gt, XE_REG_MCR(_PAT_ATS), xe->pat.pat_ats->value); 382 383 if (pta_entry) 384 xe_gt_mcr_multicast_write(gt, XE_REG_MCR(_PAT_PTA), pta_entry->value); 385 386 if (tr_pta_entry) 387 xe_gt_mcr_multicast_write(gt, XE_REG_MCR(_PAT_TR_PTA), tr_pta_entry->value); 388 } 389 390 static int xelp_dump(struct xe_gt *gt, struct drm_printer *p) 391 { 392 struct xe_device *xe = gt_to_xe(gt); 393 int i; 394 395 CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); 396 if (!fw_ref.domains) 397 return -ETIMEDOUT; 398 399 drm_printf(p, "PAT table:\n"); 400 401 for (i = 0; i < xe->pat.n_entries; i++) { 402 u32 pat = xe_mmio_read32(>->mmio, XE_REG(_PAT_INDEX(i))); 403 404 xelp_pat_entry_dump(p, i, pat); 405 } 406 407 return 0; 408 } 409 410 static const struct xe_pat_ops xelp_pat_ops = { 411 .program_graphics = program_pat, 412 .dump = xelp_dump, 413 }; 414 415 static int xehp_dump(struct xe_gt *gt, struct drm_printer *p) 416 { 417 struct xe_device *xe = gt_to_xe(gt); 418 int i; 419 420 CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); 421 if (!fw_ref.domains) 422 return -ETIMEDOUT; 423 424 drm_printf(p, "PAT table:\n"); 425 426 for (i = 0; i < xe->pat.n_entries; i++) { 427 u32 pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i))); 428 429 xelp_pat_entry_dump(p, i, pat); 430 } 431 432 return 0; 433 } 434 435 static const struct xe_pat_ops xehp_pat_ops = { 436 .program_graphics = program_pat_mcr, 437 .dump = xehp_dump, 438 }; 439 440 static int xehpc_dump(struct xe_gt *gt, struct drm_printer *p) 441 { 442 struct xe_device *xe = gt_to_xe(gt); 443 int i; 444 445 CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); 446 if (!fw_ref.domains) 447 return -ETIMEDOUT; 448 449 drm_printf(p, "PAT table:\n"); 450 451 for (i = 0; i < xe->pat.n_entries; i++) { 452 u32 pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i))); 453 454 xehpc_pat_entry_dump(p, i, pat); 455 } 456 457 return 0; 458 } 459 460 static const struct xe_pat_ops xehpc_pat_ops = { 461 .program_graphics = program_pat_mcr, 462 .dump = xehpc_dump, 463 }; 464 465 static int xelpg_dump(struct xe_gt *gt, struct drm_printer *p) 466 { 467 struct xe_device *xe = gt_to_xe(gt); 468 int i; 469 470 CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); 471 if (!fw_ref.domains) 472 return -ETIMEDOUT; 473 474 drm_printf(p, "PAT table:\n"); 475 476 for (i = 0; i < xe->pat.n_entries; i++) { 477 u32 pat; 478 479 if (xe_gt_is_media_type(gt)) 480 pat = xe_mmio_read32(>->mmio, XE_REG(_PAT_INDEX(i))); 481 else 482 pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i))); 483 484 xelpg_pat_entry_dump(p, i, pat); 485 } 486 487 return 0; 488 } 489 490 /* 491 * SAMedia register offsets are adjusted by the write methods and they target 492 * registers that are not MCR, while for normal GT they are MCR 493 */ 494 static const struct xe_pat_ops xelpg_pat_ops = { 495 .program_graphics = program_pat, 496 .program_media = program_pat_mcr, 497 .dump = xelpg_dump, 498 }; 499 500 static void xe2_pat_entry_dump(struct drm_printer *p, const char *label, u32 pat, bool rsvd) 501 { 502 drm_printf(p, "%s= [ %u, %u, %u, %u, %u, %u ] (%#8x)%s\n", label, 503 !!(pat & XE2_NO_PROMOTE), 504 !!(pat & XE2_COMP_EN), 505 REG_FIELD_GET(XE2_L3_CLOS, pat), 506 REG_FIELD_GET(XE2_L3_POLICY, pat), 507 REG_FIELD_GET(XE2_L4_POLICY, pat), 508 REG_FIELD_GET(XE2_COH_MODE, pat), 509 pat, rsvd ? " *" : ""); 510 } 511 512 static void xe3p_xpc_pat_entry_dump(struct drm_printer *p, const char *label, u32 pat, bool rsvd) 513 { 514 drm_printf(p, "%s= [ %u, %u, %u, %u, %u ] (%#8x)%s\n", label, 515 !!(pat & XE2_NO_PROMOTE), 516 REG_FIELD_GET(XE2_L3_CLOS, pat), 517 REG_FIELD_GET(XE2_L3_POLICY, pat), 518 REG_FIELD_GET(XE2_L4_POLICY, pat), 519 REG_FIELD_GET(XE2_COH_MODE, pat), 520 pat, rsvd ? " *" : ""); 521 } 522 523 static int xe2_dump(struct xe_gt *gt, struct drm_printer *p) 524 { 525 struct xe_device *xe = gt_to_xe(gt); 526 u32 pat; 527 int i; 528 char label[PAT_LABEL_LEN]; 529 530 CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); 531 if (!fw_ref.domains) 532 return -ETIMEDOUT; 533 534 drm_printf(p, "PAT table: (* = reserved entry)\n"); 535 536 for (i = 0; i < xe->pat.n_entries; i++) { 537 if (xe_gt_is_media_type(gt)) 538 pat = xe_mmio_read32(>->mmio, XE_REG(_PAT_INDEX(i))); 539 else 540 pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i))); 541 542 xe_pat_index_label(label, sizeof(label), i); 543 xe->pat.ops->entry_dump(p, label, pat, !xe->pat.table[i].valid); 544 } 545 546 /* 547 * Also print PTA_MODE, which describes how the hardware accesses 548 * PPGTT entries. 549 */ 550 if (xe_gt_is_media_type(gt)) 551 pat = xe_mmio_read32(>->mmio, XE_REG(_PAT_PTA)); 552 else 553 pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_PTA)); 554 555 drm_printf(p, "Page Table Access:\n"); 556 xe->pat.ops->entry_dump(p, "PTA_MODE", pat, false); 557 558 if (gt_tr_pta_entry(gt)) { 559 if (xe_gt_is_media_type(gt)) 560 pat = xe_mmio_read32(>->mmio, XE_REG(_PAT_TR_PTA)); 561 else 562 pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_TR_PTA)); 563 564 drm_printf(p, "TRTT Page Table Access:\n"); 565 xe->pat.ops->entry_dump(p, "TR_PTA_MODE", pat, false); 566 } 567 568 if (xe_gt_is_media_type(gt)) 569 pat = xe_mmio_read32(>->mmio, XE_REG(_PAT_ATS)); 570 else 571 pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_ATS)); 572 573 drm_printf(p, "PCIe ATS/PASID:\n"); 574 xe->pat.ops->entry_dump(p, "PAT_ATS ", pat, false); 575 576 return 0; 577 } 578 579 static const struct xe_pat_ops xe2_pat_ops = { 580 .program_graphics = program_pat_mcr, 581 .program_media = program_pat, 582 .dump = xe2_dump, 583 .entry_dump = xe2_pat_entry_dump, 584 }; 585 586 static const struct xe_pat_ops xe3p_xpc_pat_ops = { 587 .program_graphics = program_pat_mcr, 588 .program_media = program_pat, 589 .dump = xe2_dump, 590 .entry_dump = xe3p_xpc_pat_entry_dump, 591 }; 592 593 void xe_pat_init_early(struct xe_device *xe) 594 { 595 xe->pat.idx[XE_CACHE_WB_COMPRESSION] = XE_PAT_INVALID_IDX; 596 xe->pat.idx[XE_CACHE_NONE_COMPRESSION] = XE_PAT_INVALID_IDX; 597 if (GRAPHICS_VERx100(xe) == 3511) { 598 xe->pat.ops = &xe3p_xpc_pat_ops; 599 xe->pat.table = xe3p_xpc_pat_table; 600 xe->pat.pat_ats = &xe3p_xpc_pat_ats; 601 xe->pat.pat_primary_pta = &xe3p_xpc_pat_pta; 602 xe->pat.pat_media_pta = &xe3p_xpc_pat_pta; 603 xe->pat.n_entries = ARRAY_SIZE(xe3p_xpc_pat_table); 604 xe->pat.idx[XE_CACHE_NONE] = 3; 605 xe->pat.idx[XE_CACHE_WT] = 3; /* N/A (no display); use UC */ 606 xe->pat.idx[XE_CACHE_WB] = 2; 607 } else if (GRAPHICS_VER(xe) == 35) { 608 xe->pat.ops = &xe2_pat_ops; 609 xe->pat.table = xe3p_lpg_pat_table; 610 xe->pat.pat_ats = &xe2_pat_ats; 611 if (!IS_DGFX(xe)) { 612 xe->pat.pat_primary_pta = &xe3p_primary_pat_pta; 613 xe->pat.pat_media_pta = &xe3p_media_pat_pta; 614 xe->pat.pat_primary_tr_pta = &xe3p_pat_tr_pta; 615 xe->pat.pat_media_tr_pta = &xe3p_pat_tr_pta; 616 } 617 xe->pat.n_entries = ARRAY_SIZE(xe3p_lpg_pat_table); 618 xe->pat.idx[XE_CACHE_NONE] = 3; 619 xe->pat.idx[XE_CACHE_WT] = 15; 620 xe->pat.idx[XE_CACHE_WB] = 2; 621 xe->pat.idx[XE_CACHE_NONE_COMPRESSION] = 12; 622 xe->pat.idx[XE_CACHE_WB_COMPRESSION] = 16; 623 } else if (GRAPHICS_VER(xe) == 30 || GRAPHICS_VER(xe) == 20) { 624 xe->pat.ops = &xe2_pat_ops; 625 if (GRAPHICS_VER(xe) == 30) { 626 xe->pat.table = xe3_lpg_pat_table; 627 xe->pat.idx[XE_CACHE_WB_COMPRESSION] = 16; 628 } else { 629 xe->pat.table = xe2_pat_table; 630 } 631 xe->pat.pat_ats = &xe2_pat_ats; 632 if (IS_DGFX(xe)) { 633 xe->pat.pat_primary_pta = &xe2_pat_pta; 634 xe->pat.pat_media_pta = &xe2_pat_pta; 635 } 636 637 /* Wa_16023588340. XXX: Should use XE_WA */ 638 if (GRAPHICS_VERx100(xe) == 2001) 639 xe->pat.n_entries = 28; /* Disable CLOS3 */ 640 else 641 xe->pat.n_entries = ARRAY_SIZE(xe2_pat_table); 642 643 xe->pat.idx[XE_CACHE_NONE] = 3; 644 xe->pat.idx[XE_CACHE_WT] = 15; 645 xe->pat.idx[XE_CACHE_WB] = 2; 646 xe->pat.idx[XE_CACHE_NONE_COMPRESSION] = 12; /*Applicable on xe2 and beyond */ 647 } else if (xe->info.platform == XE_METEORLAKE) { 648 xe->pat.ops = &xelpg_pat_ops; 649 xe->pat.table = xelpg_pat_table; 650 xe->pat.n_entries = ARRAY_SIZE(xelpg_pat_table); 651 xe->pat.idx[XE_CACHE_NONE] = 2; 652 xe->pat.idx[XE_CACHE_WT] = 1; 653 xe->pat.idx[XE_CACHE_WB] = 3; 654 } else if (xe->info.platform == XE_PVC) { 655 xe->pat.ops = &xehpc_pat_ops; 656 xe->pat.table = xehpc_pat_table; 657 xe->pat.n_entries = ARRAY_SIZE(xehpc_pat_table); 658 xe->pat.idx[XE_CACHE_NONE] = 0; 659 xe->pat.idx[XE_CACHE_WT] = 2; 660 xe->pat.idx[XE_CACHE_WB] = 3; 661 } else if (xe->info.platform == XE_DG2) { 662 /* 663 * Table is the same as previous platforms, but programming 664 * method has changed. 665 */ 666 xe->pat.ops = &xehp_pat_ops; 667 xe->pat.table = xelp_pat_table; 668 xe->pat.n_entries = ARRAY_SIZE(xelp_pat_table); 669 xe->pat.idx[XE_CACHE_NONE] = 3; 670 xe->pat.idx[XE_CACHE_WT] = 2; 671 xe->pat.idx[XE_CACHE_WB] = 0; 672 } else if (GRAPHICS_VERx100(xe) <= 1210) { 673 WARN_ON_ONCE(!IS_DGFX(xe) && !xe->info.has_llc); 674 xe->pat.ops = &xelp_pat_ops; 675 xe->pat.table = xelp_pat_table; 676 xe->pat.n_entries = ARRAY_SIZE(xelp_pat_table); 677 xe->pat.idx[XE_CACHE_NONE] = 3; 678 xe->pat.idx[XE_CACHE_WT] = 2; 679 xe->pat.idx[XE_CACHE_WB] = 0; 680 } else { 681 /* 682 * Going forward we expect to need new PAT settings for most 683 * new platforms; failure to provide a new table can easily 684 * lead to subtle, hard-to-debug problems. If none of the 685 * conditions above match the platform we're running on we'll 686 * raise an error rather than trying to silently inherit the 687 * most recent platform's behavior. 688 */ 689 drm_err(&xe->drm, "Missing PAT table for platform with graphics version %d.%02d!\n", 690 GRAPHICS_VER(xe), GRAPHICS_VERx100(xe) % 100); 691 } 692 693 xe_assert(xe, xe->pat.ops->dump); 694 xe_assert(xe, xe->pat.ops->program_graphics); 695 xe_assert(xe, MEDIA_VER(xe) < 13 || xe->pat.ops->program_media); 696 xe_assert(xe, GRAPHICS_VER(xe) < 20 || xe->pat.ops->entry_dump); 697 } 698 699 void xe_pat_init(struct xe_gt *gt) 700 { 701 struct xe_device *xe = gt_to_xe(gt); 702 703 if (IS_SRIOV_VF(xe)) 704 return; 705 706 if (xe_gt_is_media_type(gt)) 707 xe->pat.ops->program_media(gt, xe->pat.table, xe->pat.n_entries); 708 else 709 xe->pat.ops->program_graphics(gt, xe->pat.table, xe->pat.n_entries); 710 } 711 712 /** 713 * xe_pat_dump() - Dump GT PAT table into a drm printer. 714 * @gt: the &xe_gt 715 * @p: the &drm_printer 716 * 717 * Return: 0 on success or a negative error code on failure. 718 */ 719 int xe_pat_dump(struct xe_gt *gt, struct drm_printer *p) 720 { 721 struct xe_device *xe = gt_to_xe(gt); 722 723 if (IS_SRIOV_VF(xe)) 724 return -EOPNOTSUPP; 725 726 return xe->pat.ops->dump(gt, p); 727 } 728 729 /** 730 * xe_pat_dump_sw_config() - Dump the software-configured GT PAT table into a drm printer. 731 * @gt: the &xe_gt 732 * @p: the &drm_printer 733 * 734 * Return: 0 on success or a negative error code on failure. 735 */ 736 int xe_pat_dump_sw_config(struct xe_gt *gt, struct drm_printer *p) 737 { 738 struct xe_device *xe = gt_to_xe(gt); 739 const struct xe_pat_table_entry *pta_entry = gt_pta_entry(gt); 740 const struct xe_pat_table_entry *tr_pta_entry = gt_tr_pta_entry(gt); 741 char label[PAT_LABEL_LEN]; 742 743 if (!xe->pat.table || !xe->pat.n_entries) 744 return -EOPNOTSUPP; 745 746 drm_printf(p, "PAT table:%s\n", GRAPHICS_VER(xe) >= 20 ? " (* = reserved entry)" : ""); 747 for (u32 i = 0; i < xe->pat.n_entries; i++) { 748 u32 pat = xe->pat.table[i].value; 749 750 if (GRAPHICS_VER(xe) >= 20) { 751 xe_pat_index_label(label, sizeof(label), i); 752 xe->pat.ops->entry_dump(p, label, pat, !xe->pat.table[i].valid); 753 } else if (xe->info.platform == XE_METEORLAKE) { 754 xelpg_pat_entry_dump(p, i, pat); 755 } else if (xe->info.platform == XE_PVC) { 756 xehpc_pat_entry_dump(p, i, pat); 757 } else if (xe->info.platform == XE_DG2 || GRAPHICS_VERx100(xe) <= 1210) { 758 xelp_pat_entry_dump(p, i, pat); 759 } else { 760 return -EOPNOTSUPP; 761 } 762 } 763 764 if (pta_entry) { 765 u32 pat = pta_entry->value; 766 767 drm_printf(p, "Page Table Access:\n"); 768 xe->pat.ops->entry_dump(p, "PTA_MODE", pat, false); 769 } 770 771 if (tr_pta_entry) { 772 u32 pat = tr_pta_entry->value; 773 774 drm_printf(p, "TRTT Page Table Access:\n"); 775 xe->pat.ops->entry_dump(p, "TR_PTA_MODE", pat, false); 776 } 777 778 if (xe->pat.pat_ats) { 779 u32 pat = xe->pat.pat_ats->value; 780 781 drm_printf(p, "PCIe ATS/PASID:\n"); 782 xe->pat.ops->entry_dump(p, "PAT_ATS ", pat, false); 783 } 784 785 drm_printf(p, "Cache Level:\n"); 786 drm_printf(p, "IDX[XE_CACHE_NONE] = %d\n", xe->pat.idx[XE_CACHE_NONE]); 787 drm_printf(p, "IDX[XE_CACHE_WT] = %d\n", xe->pat.idx[XE_CACHE_WT]); 788 drm_printf(p, "IDX[XE_CACHE_WB] = %d\n", xe->pat.idx[XE_CACHE_WB]); 789 if (GRAPHICS_VER(xe) >= 20) { 790 drm_printf(p, "IDX[XE_CACHE_NONE_COMPRESSION] = %d\n", 791 xe->pat.idx[XE_CACHE_NONE_COMPRESSION]); 792 drm_printf(p, "IDX[XE_CACHE_WB_COMPRESSION] = %d\n", 793 xe->pat.idx[XE_CACHE_WB_COMPRESSION]); 794 } 795 796 return 0; 797 } 798