1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2023 Intel Corporation 4 */ 5 6 #include "xe_pat.h" 7 8 #include <uapi/drm/xe_drm.h> 9 10 #include <generated/xe_wa_oob.h> 11 12 #include "regs/xe_gt_regs.h" 13 #include "regs/xe_reg_defs.h" 14 #include "xe_assert.h" 15 #include "xe_device.h" 16 #include "xe_force_wake.h" 17 #include "xe_gt.h" 18 #include "xe_gt_mcr.h" 19 #include "xe_mmio.h" 20 #include "xe_sriov.h" 21 #include "xe_wa.h" 22 23 #define _PAT_ATS 0x47fc 24 #define _PAT_INDEX(index) _PICK_EVEN_2RANGES(index, 8, \ 25 0x4800, 0x4804, \ 26 0x4848, 0x484c) 27 #define _PAT_PTA 0x4820 28 29 #define XE2_NO_PROMOTE REG_BIT(10) 30 #define XE2_COMP_EN REG_BIT(9) 31 #define XE2_L3_CLOS REG_GENMASK(7, 6) 32 #define XE2_L3_POLICY REG_GENMASK(5, 4) 33 #define XE2_L4_POLICY REG_GENMASK(3, 2) 34 #define XE2_COH_MODE REG_GENMASK(1, 0) 35 36 #define XELPG_L4_POLICY_MASK REG_GENMASK(3, 2) 37 #define XELPG_PAT_3_UC REG_FIELD_PREP(XELPG_L4_POLICY_MASK, 3) 38 #define XELPG_PAT_1_WT REG_FIELD_PREP(XELPG_L4_POLICY_MASK, 1) 39 #define XELPG_PAT_0_WB REG_FIELD_PREP(XELPG_L4_POLICY_MASK, 0) 40 #define XELPG_INDEX_COH_MODE_MASK REG_GENMASK(1, 0) 41 #define XELPG_3_COH_2W REG_FIELD_PREP(XELPG_INDEX_COH_MODE_MASK, 3) 42 #define XELPG_2_COH_1W REG_FIELD_PREP(XELPG_INDEX_COH_MODE_MASK, 2) 43 #define XELPG_0_COH_NON REG_FIELD_PREP(XELPG_INDEX_COH_MODE_MASK, 0) 44 45 #define XEHPC_CLOS_LEVEL_MASK REG_GENMASK(3, 2) 46 #define XEHPC_PAT_CLOS(x) REG_FIELD_PREP(XEHPC_CLOS_LEVEL_MASK, x) 47 48 #define XELP_MEM_TYPE_MASK REG_GENMASK(1, 0) 49 #define XELP_PAT_WB REG_FIELD_PREP(XELP_MEM_TYPE_MASK, 3) 50 #define XELP_PAT_WT REG_FIELD_PREP(XELP_MEM_TYPE_MASK, 2) 51 #define XELP_PAT_WC REG_FIELD_PREP(XELP_MEM_TYPE_MASK, 1) 52 #define XELP_PAT_UC REG_FIELD_PREP(XELP_MEM_TYPE_MASK, 0) 53 54 #define PAT_LABEL_LEN 20 55 56 static const char *XELP_MEM_TYPE_STR_MAP[] = { "UC", "WC", "WT", "WB" }; 57 58 static void xe_pat_index_label(char *label, size_t len, int index) 59 { 60 snprintf(label, len, "PAT[%2d] ", index); 61 } 62 63 static void xelp_pat_entry_dump(struct drm_printer *p, int index, u32 pat) 64 { 65 u8 mem_type = REG_FIELD_GET(XELP_MEM_TYPE_MASK, pat); 66 67 drm_printf(p, "PAT[%2d] = %s (%#8x)\n", index, 68 XELP_MEM_TYPE_STR_MAP[mem_type], pat); 69 } 70 71 static void xehpc_pat_entry_dump(struct drm_printer *p, int index, u32 pat) 72 { 73 drm_printf(p, "PAT[%2d] = [ %u, %u ] (%#8x)\n", index, 74 REG_FIELD_GET(XELP_MEM_TYPE_MASK, pat), 75 REG_FIELD_GET(XEHPC_CLOS_LEVEL_MASK, pat), pat); 76 } 77 78 static void xelpg_pat_entry_dump(struct drm_printer *p, int index, u32 pat) 79 { 80 drm_printf(p, "PAT[%2d] = [ %u, %u ] (%#8x)\n", index, 81 REG_FIELD_GET(XELPG_L4_POLICY_MASK, pat), 82 REG_FIELD_GET(XELPG_INDEX_COH_MODE_MASK, pat), pat); 83 } 84 85 struct xe_pat_ops { 86 void (*program_graphics)(struct xe_gt *gt, const struct xe_pat_table_entry table[], 87 int n_entries); 88 void (*program_media)(struct xe_gt *gt, const struct xe_pat_table_entry table[], 89 int n_entries); 90 int (*dump)(struct xe_gt *gt, struct drm_printer *p); 91 void (*entry_dump)(struct drm_printer *p, const char *label, u32 pat, bool rsvd); 92 }; 93 94 static const struct xe_pat_table_entry xelp_pat_table[] = { 95 [0] = { XELP_PAT_WB, XE_COH_AT_LEAST_1WAY }, 96 [1] = { XELP_PAT_WC, XE_COH_NONE }, 97 [2] = { XELP_PAT_WT, XE_COH_NONE }, 98 [3] = { XELP_PAT_UC, XE_COH_NONE }, 99 }; 100 101 static const struct xe_pat_table_entry xehpc_pat_table[] = { 102 [0] = { XELP_PAT_UC, XE_COH_NONE }, 103 [1] = { XELP_PAT_WC, XE_COH_NONE }, 104 [2] = { XELP_PAT_WT, XE_COH_NONE }, 105 [3] = { XELP_PAT_WB, XE_COH_AT_LEAST_1WAY }, 106 [4] = { XEHPC_PAT_CLOS(1) | XELP_PAT_WT, XE_COH_NONE }, 107 [5] = { XEHPC_PAT_CLOS(1) | XELP_PAT_WB, XE_COH_AT_LEAST_1WAY }, 108 [6] = { XEHPC_PAT_CLOS(2) | XELP_PAT_WT, XE_COH_NONE }, 109 [7] = { XEHPC_PAT_CLOS(2) | XELP_PAT_WB, XE_COH_AT_LEAST_1WAY }, 110 }; 111 112 static const struct xe_pat_table_entry xelpg_pat_table[] = { 113 [0] = { XELPG_PAT_0_WB, XE_COH_NONE }, 114 [1] = { XELPG_PAT_1_WT, XE_COH_NONE }, 115 [2] = { XELPG_PAT_3_UC, XE_COH_NONE }, 116 [3] = { XELPG_PAT_0_WB | XELPG_2_COH_1W, XE_COH_AT_LEAST_1WAY }, 117 [4] = { XELPG_PAT_0_WB | XELPG_3_COH_2W, XE_COH_AT_LEAST_1WAY }, 118 }; 119 120 /* 121 * The Xe2 table is getting large/complicated so it's easier to review if 122 * provided in a form that exactly matches the bspec's formatting. The meaning 123 * of the fields here are: 124 * - no_promote: 0=promotable, 1=no promote 125 * - comp_en: 0=disable, 1=enable 126 * - l3clos: L3 class of service (0-3) 127 * - l3_policy: 0=WB, 1=XD ("WB - Transient Display"), 128 * 2=XA ("WB - Transient App" for Xe3p), 3=UC 129 * - l4_policy: 0=WB, 1=WT, 3=UC 130 * - coh_mode: 0=no snoop, 2=1-way coherent, 3=2-way coherent 131 * 132 * Reserved entries should be programmed with the maximum caching, minimum 133 * coherency (which matches an all-0's encoding), so we can just omit them 134 * in the table. 135 * 136 * Note: There is an implicit assumption in the driver that compression and 137 * coh_1way+ are mutually exclusive for platforms prior to Xe3. Starting 138 * with Xe3, compression can be combined with coherency. If using compression 139 * with coherency, userptr and imported dma-buf from external device will 140 * have uncleared ccs state. See also xe_bo_needs_ccs_pages(). 141 */ 142 #define XE2_PAT(no_promote, comp_en, l3clos, l3_policy, l4_policy, __coh_mode) \ 143 { \ 144 .value = (no_promote ? XE2_NO_PROMOTE : 0) | \ 145 (comp_en ? XE2_COMP_EN : 0) | \ 146 REG_FIELD_PREP(XE2_L3_CLOS, l3clos) | \ 147 REG_FIELD_PREP(XE2_L3_POLICY, l3_policy) | \ 148 REG_FIELD_PREP(XE2_L4_POLICY, l4_policy) | \ 149 REG_FIELD_PREP(XE2_COH_MODE, __coh_mode), \ 150 .coh_mode = __coh_mode ? XE_COH_AT_LEAST_1WAY : XE_COH_NONE, \ 151 .valid = 1 \ 152 } 153 154 static const struct xe_pat_table_entry xe2_pat_table[] = { 155 [ 0] = XE2_PAT( 0, 0, 0, 0, 3, 0 ), 156 [ 1] = XE2_PAT( 0, 0, 0, 0, 3, 2 ), 157 [ 2] = XE2_PAT( 0, 0, 0, 0, 3, 3 ), 158 [ 3] = XE2_PAT( 0, 0, 0, 3, 3, 0 ), 159 [ 4] = XE2_PAT( 0, 0, 0, 3, 0, 2 ), 160 [ 5] = XE2_PAT( 0, 0, 0, 3, 3, 2 ), 161 [ 6] = XE2_PAT( 1, 0, 0, 1, 3, 0 ), 162 [ 7] = XE2_PAT( 0, 0, 0, 3, 0, 3 ), 163 [ 8] = XE2_PAT( 0, 0, 0, 3, 0, 0 ), 164 [ 9] = XE2_PAT( 0, 1, 0, 0, 3, 0 ), 165 [10] = XE2_PAT( 0, 1, 0, 3, 0, 0 ), 166 [11] = XE2_PAT( 1, 1, 0, 1, 3, 0 ), 167 [12] = XE2_PAT( 0, 1, 0, 3, 3, 0 ), 168 [13] = XE2_PAT( 0, 0, 0, 0, 0, 0 ), 169 [14] = XE2_PAT( 0, 1, 0, 0, 0, 0 ), 170 [15] = XE2_PAT( 1, 1, 0, 1, 1, 0 ), 171 /* 16..19 are reserved; leave set to all 0's */ 172 [20] = XE2_PAT( 0, 0, 1, 0, 3, 0 ), 173 [21] = XE2_PAT( 0, 1, 1, 0, 3, 0 ), 174 [22] = XE2_PAT( 0, 0, 1, 0, 3, 2 ), 175 [23] = XE2_PAT( 0, 0, 1, 0, 3, 3 ), 176 [24] = XE2_PAT( 0, 0, 2, 0, 3, 0 ), 177 [25] = XE2_PAT( 0, 1, 2, 0, 3, 0 ), 178 [26] = XE2_PAT( 0, 0, 2, 0, 3, 2 ), 179 [27] = XE2_PAT( 0, 0, 2, 0, 3, 3 ), 180 [28] = XE2_PAT( 0, 0, 3, 0, 3, 0 ), 181 [29] = XE2_PAT( 0, 1, 3, 0, 3, 0 ), 182 [30] = XE2_PAT( 0, 0, 3, 0, 3, 2 ), 183 [31] = XE2_PAT( 0, 0, 3, 0, 3, 3 ), 184 }; 185 186 static const struct xe_pat_table_entry xe3_lpg_pat_table[] = { 187 [ 0] = XE2_PAT( 0, 0, 0, 0, 3, 0 ), 188 [ 1] = XE2_PAT( 0, 0, 0, 0, 3, 2 ), 189 [ 2] = XE2_PAT( 0, 0, 0, 0, 3, 3 ), 190 [ 3] = XE2_PAT( 0, 0, 0, 3, 3, 0 ), 191 [ 4] = XE2_PAT( 0, 0, 0, 3, 0, 2 ), 192 [ 5] = XE2_PAT( 0, 0, 0, 3, 3, 2 ), 193 [ 6] = XE2_PAT( 1, 0, 0, 1, 3, 0 ), 194 [ 7] = XE2_PAT( 0, 0, 0, 3, 0, 3 ), 195 [ 8] = XE2_PAT( 0, 0, 0, 3, 0, 0 ), 196 [ 9] = XE2_PAT( 0, 1, 0, 0, 3, 0 ), 197 [10] = XE2_PAT( 0, 1, 0, 3, 0, 0 ), 198 [11] = XE2_PAT( 1, 1, 0, 1, 3, 0 ), 199 [12] = XE2_PAT( 0, 1, 0, 3, 3, 0 ), 200 [13] = XE2_PAT( 0, 0, 0, 0, 0, 0 ), 201 [14] = XE2_PAT( 0, 1, 0, 0, 0, 0 ), 202 [15] = XE2_PAT( 1, 1, 0, 1, 1, 0 ), 203 [16] = XE2_PAT( 0, 1, 0, 0, 3, 2 ), 204 /* 17..19 are reserved; leave set to all 0's */ 205 [20] = XE2_PAT( 0, 0, 1, 0, 3, 0 ), 206 [21] = XE2_PAT( 0, 1, 1, 0, 3, 0 ), 207 [22] = XE2_PAT( 0, 0, 1, 0, 3, 2 ), 208 [23] = XE2_PAT( 0, 0, 1, 0, 3, 3 ), 209 [24] = XE2_PAT( 0, 0, 2, 0, 3, 0 ), 210 [25] = XE2_PAT( 0, 1, 2, 0, 3, 0 ), 211 [26] = XE2_PAT( 0, 0, 2, 0, 3, 2 ), 212 [27] = XE2_PAT( 0, 0, 2, 0, 3, 3 ), 213 [28] = XE2_PAT( 0, 0, 3, 0, 3, 0 ), 214 [29] = XE2_PAT( 0, 1, 3, 0, 3, 0 ), 215 [30] = XE2_PAT( 0, 0, 3, 0, 3, 2 ), 216 [31] = XE2_PAT( 0, 0, 3, 0, 3, 3 ), 217 }; 218 /* Special PAT values programmed outside the main table */ 219 static const struct xe_pat_table_entry xe2_pat_ats = XE2_PAT( 0, 0, 0, 0, 3, 3 ); 220 static const struct xe_pat_table_entry xe2_pat_pta = XE2_PAT( 0, 0, 0, 0, 3, 0 ); 221 222 /* 223 * Xe3p_XPC PAT table uses the same layout as Xe2/Xe3, except that there's no 224 * option for compression. Also note that the "L3" and "L4" register fields 225 * actually control L2 and L3 cache respectively on this platform. 226 */ 227 #define XE3P_XPC_PAT(no_promote, l3clos, l3_policy, l4_policy, __coh_mode) \ 228 XE2_PAT(no_promote, 0, l3clos, l3_policy, l4_policy, __coh_mode) 229 230 static const struct xe_pat_table_entry xe3p_xpc_pat_ats = XE3P_XPC_PAT( 0, 0, 0, 0, 3 ); 231 static const struct xe_pat_table_entry xe3p_xpc_pat_pta = XE3P_XPC_PAT( 0, 0, 0, 0, 0 ); 232 233 static const struct xe_pat_table_entry xe3p_xpc_pat_table[] = { 234 [ 0] = XE3P_XPC_PAT( 0, 0, 0, 0, 0 ), 235 [ 1] = XE3P_XPC_PAT( 0, 0, 0, 0, 2 ), 236 [ 2] = XE3P_XPC_PAT( 0, 0, 0, 0, 3 ), 237 [ 3] = XE3P_XPC_PAT( 0, 0, 3, 3, 0 ), 238 [ 4] = XE3P_XPC_PAT( 0, 0, 3, 3, 2 ), 239 [ 5] = XE3P_XPC_PAT( 0, 0, 3, 0, 0 ), 240 [ 6] = XE3P_XPC_PAT( 0, 0, 3, 0, 2 ), 241 [ 7] = XE3P_XPC_PAT( 0, 0, 3, 0, 3 ), 242 [ 8] = XE3P_XPC_PAT( 0, 0, 0, 3, 0 ), 243 [ 9] = XE3P_XPC_PAT( 0, 0, 0, 3, 2 ), 244 [10] = XE3P_XPC_PAT( 0, 0, 0, 3, 3 ), 245 /* 11..22 are reserved; leave set to all 0's */ 246 [23] = XE3P_XPC_PAT( 0, 1, 0, 0, 0 ), 247 [24] = XE3P_XPC_PAT( 0, 1, 0, 0, 2 ), 248 [25] = XE3P_XPC_PAT( 0, 1, 0, 0, 3 ), 249 [26] = XE3P_XPC_PAT( 0, 2, 0, 0, 0 ), 250 [27] = XE3P_XPC_PAT( 0, 2, 0, 0, 2 ), 251 [28] = XE3P_XPC_PAT( 0, 2, 0, 0, 3 ), 252 [29] = XE3P_XPC_PAT( 0, 3, 0, 0, 0 ), 253 [30] = XE3P_XPC_PAT( 0, 3, 0, 0, 2 ), 254 [31] = XE3P_XPC_PAT( 0, 3, 0, 0, 3 ), 255 }; 256 257 static const struct xe_pat_table_entry xe3p_primary_pat_pta = XE2_PAT(0, 0, 0, 0, 0, 3); 258 static const struct xe_pat_table_entry xe3p_media_pat_pta = XE2_PAT(0, 0, 0, 0, 0, 2); 259 260 static const struct xe_pat_table_entry xe3p_lpg_pat_table[] = { 261 [ 0] = XE2_PAT( 0, 0, 0, 0, 3, 0 ), 262 [ 1] = XE2_PAT( 0, 0, 0, 0, 3, 2 ), 263 [ 2] = XE2_PAT( 0, 0, 0, 0, 3, 3 ), 264 [ 3] = XE2_PAT( 0, 0, 0, 3, 3, 0 ), 265 [ 4] = XE2_PAT( 0, 0, 0, 3, 0, 2 ), 266 [ 5] = XE2_PAT( 0, 0, 0, 3, 3, 2 ), 267 [ 6] = XE2_PAT( 1, 0, 0, 1, 3, 0 ), 268 [ 7] = XE2_PAT( 0, 0, 0, 3, 0, 3 ), 269 [ 8] = XE2_PAT( 0, 0, 0, 3, 0, 0 ), 270 [ 9] = XE2_PAT( 0, 1, 0, 0, 3, 0 ), 271 [10] = XE2_PAT( 0, 1, 0, 3, 0, 0 ), 272 [11] = XE2_PAT( 1, 1, 0, 1, 3, 0 ), 273 [12] = XE2_PAT( 0, 1, 0, 3, 3, 0 ), 274 [13] = XE2_PAT( 0, 0, 0, 0, 0, 0 ), 275 [14] = XE2_PAT( 0, 1, 0, 0, 0, 0 ), 276 [15] = XE2_PAT( 1, 1, 0, 1, 1, 0 ), 277 [16] = XE2_PAT( 0, 1, 0, 0, 3, 2 ), 278 /* 17 is reserved; leave set to all 0's */ 279 [18] = XE2_PAT( 1, 0, 0, 2, 3, 0 ), 280 [19] = XE2_PAT( 1, 0, 0, 2, 3, 2 ), 281 [20] = XE2_PAT( 0, 0, 1, 0, 3, 0 ), 282 [21] = XE2_PAT( 0, 1, 1, 0, 3, 0 ), 283 [22] = XE2_PAT( 0, 0, 1, 0, 3, 2 ), 284 [23] = XE2_PAT( 0, 0, 1, 0, 3, 3 ), 285 [24] = XE2_PAT( 0, 0, 2, 0, 3, 0 ), 286 [25] = XE2_PAT( 0, 1, 2, 0, 3, 0 ), 287 [26] = XE2_PAT( 0, 0, 2, 0, 3, 2 ), 288 [27] = XE2_PAT( 0, 0, 2, 0, 3, 3 ), 289 [28] = XE2_PAT( 0, 0, 3, 0, 3, 0 ), 290 [29] = XE2_PAT( 0, 1, 3, 0, 3, 0 ), 291 [30] = XE2_PAT( 0, 0, 3, 0, 3, 2 ), 292 [31] = XE2_PAT( 0, 0, 3, 0, 3, 3 ), 293 }; 294 295 u16 xe_pat_index_get_coh_mode(struct xe_device *xe, u16 pat_index) 296 { 297 WARN_ON(pat_index >= xe->pat.n_entries); 298 return xe->pat.table[pat_index].coh_mode; 299 } 300 301 bool xe_pat_index_get_comp_en(struct xe_device *xe, u16 pat_index) 302 { 303 WARN_ON(pat_index >= xe->pat.n_entries); 304 return !!(xe->pat.table[pat_index].value & XE2_COMP_EN); 305 } 306 307 u16 xe_pat_index_get_l3_policy(struct xe_device *xe, u16 pat_index) 308 { 309 WARN_ON(pat_index >= xe->pat.n_entries); 310 311 return REG_FIELD_GET(XE2_L3_POLICY, xe->pat.table[pat_index].value); 312 } 313 314 static const struct xe_pat_table_entry *gt_pta_entry(struct xe_gt *gt) 315 { 316 struct xe_device *xe = gt_to_xe(gt); 317 318 if (xe_gt_is_main_type(gt)) 319 return xe->pat.pat_primary_pta; 320 321 if (xe_gt_is_media_type(gt)) 322 return xe->pat.pat_media_pta; 323 324 xe_assert(xe, false); 325 return NULL; 326 } 327 328 static void program_pat(struct xe_gt *gt, const struct xe_pat_table_entry table[], 329 int n_entries) 330 { 331 struct xe_device *xe = gt_to_xe(gt); 332 const struct xe_pat_table_entry *pta_entry = gt_pta_entry(gt); 333 334 for (int i = 0; i < n_entries; i++) { 335 struct xe_reg reg = XE_REG(_PAT_INDEX(i)); 336 337 xe_mmio_write32(>->mmio, reg, table[i].value); 338 } 339 340 if (xe->pat.pat_ats) 341 xe_mmio_write32(>->mmio, XE_REG(_PAT_ATS), xe->pat.pat_ats->value); 342 343 if (pta_entry) 344 xe_mmio_write32(>->mmio, XE_REG(_PAT_PTA), pta_entry->value); 345 } 346 347 static void program_pat_mcr(struct xe_gt *gt, const struct xe_pat_table_entry table[], 348 int n_entries) 349 { 350 struct xe_device *xe = gt_to_xe(gt); 351 const struct xe_pat_table_entry *pta_entry = gt_pta_entry(gt); 352 353 for (int i = 0; i < n_entries; i++) { 354 struct xe_reg_mcr reg_mcr = XE_REG_MCR(_PAT_INDEX(i)); 355 356 xe_gt_mcr_multicast_write(gt, reg_mcr, table[i].value); 357 } 358 359 if (xe->pat.pat_ats) 360 xe_gt_mcr_multicast_write(gt, XE_REG_MCR(_PAT_ATS), xe->pat.pat_ats->value); 361 362 if (pta_entry) 363 xe_gt_mcr_multicast_write(gt, XE_REG_MCR(_PAT_PTA), pta_entry->value); 364 } 365 366 static int xelp_dump(struct xe_gt *gt, struct drm_printer *p) 367 { 368 struct xe_device *xe = gt_to_xe(gt); 369 int i; 370 371 CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); 372 if (!fw_ref.domains) 373 return -ETIMEDOUT; 374 375 drm_printf(p, "PAT table:\n"); 376 377 for (i = 0; i < xe->pat.n_entries; i++) { 378 u32 pat = xe_mmio_read32(>->mmio, XE_REG(_PAT_INDEX(i))); 379 380 xelp_pat_entry_dump(p, i, pat); 381 } 382 383 return 0; 384 } 385 386 static const struct xe_pat_ops xelp_pat_ops = { 387 .program_graphics = program_pat, 388 .dump = xelp_dump, 389 }; 390 391 static int xehp_dump(struct xe_gt *gt, struct drm_printer *p) 392 { 393 struct xe_device *xe = gt_to_xe(gt); 394 int i; 395 396 CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); 397 if (!fw_ref.domains) 398 return -ETIMEDOUT; 399 400 drm_printf(p, "PAT table:\n"); 401 402 for (i = 0; i < xe->pat.n_entries; i++) { 403 u32 pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i))); 404 405 xelp_pat_entry_dump(p, i, pat); 406 } 407 408 return 0; 409 } 410 411 static const struct xe_pat_ops xehp_pat_ops = { 412 .program_graphics = program_pat_mcr, 413 .dump = xehp_dump, 414 }; 415 416 static int xehpc_dump(struct xe_gt *gt, struct drm_printer *p) 417 { 418 struct xe_device *xe = gt_to_xe(gt); 419 int i; 420 421 CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); 422 if (!fw_ref.domains) 423 return -ETIMEDOUT; 424 425 drm_printf(p, "PAT table:\n"); 426 427 for (i = 0; i < xe->pat.n_entries; i++) { 428 u32 pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i))); 429 430 xehpc_pat_entry_dump(p, i, pat); 431 } 432 433 return 0; 434 } 435 436 static const struct xe_pat_ops xehpc_pat_ops = { 437 .program_graphics = program_pat_mcr, 438 .dump = xehpc_dump, 439 }; 440 441 static int xelpg_dump(struct xe_gt *gt, struct drm_printer *p) 442 { 443 struct xe_device *xe = gt_to_xe(gt); 444 int i; 445 446 CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); 447 if (!fw_ref.domains) 448 return -ETIMEDOUT; 449 450 drm_printf(p, "PAT table:\n"); 451 452 for (i = 0; i < xe->pat.n_entries; i++) { 453 u32 pat; 454 455 if (xe_gt_is_media_type(gt)) 456 pat = xe_mmio_read32(>->mmio, XE_REG(_PAT_INDEX(i))); 457 else 458 pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i))); 459 460 xelpg_pat_entry_dump(p, i, pat); 461 } 462 463 return 0; 464 } 465 466 /* 467 * SAMedia register offsets are adjusted by the write methods and they target 468 * registers that are not MCR, while for normal GT they are MCR 469 */ 470 static const struct xe_pat_ops xelpg_pat_ops = { 471 .program_graphics = program_pat, 472 .program_media = program_pat_mcr, 473 .dump = xelpg_dump, 474 }; 475 476 static void xe2_pat_entry_dump(struct drm_printer *p, const char *label, u32 pat, bool rsvd) 477 { 478 drm_printf(p, "%s= [ %u, %u, %u, %u, %u, %u ] (%#8x)%s\n", label, 479 !!(pat & XE2_NO_PROMOTE), 480 !!(pat & XE2_COMP_EN), 481 REG_FIELD_GET(XE2_L3_CLOS, pat), 482 REG_FIELD_GET(XE2_L3_POLICY, pat), 483 REG_FIELD_GET(XE2_L4_POLICY, pat), 484 REG_FIELD_GET(XE2_COH_MODE, pat), 485 pat, rsvd ? " *" : ""); 486 } 487 488 static void xe3p_xpc_pat_entry_dump(struct drm_printer *p, const char *label, u32 pat, bool rsvd) 489 { 490 drm_printf(p, "%s= [ %u, %u, %u, %u, %u ] (%#8x)%s\n", label, 491 !!(pat & XE2_NO_PROMOTE), 492 REG_FIELD_GET(XE2_L3_CLOS, pat), 493 REG_FIELD_GET(XE2_L3_POLICY, pat), 494 REG_FIELD_GET(XE2_L4_POLICY, pat), 495 REG_FIELD_GET(XE2_COH_MODE, pat), 496 pat, rsvd ? " *" : ""); 497 } 498 499 static int xe2_dump(struct xe_gt *gt, struct drm_printer *p) 500 { 501 struct xe_device *xe = gt_to_xe(gt); 502 u32 pat; 503 int i; 504 char label[PAT_LABEL_LEN]; 505 506 CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT); 507 if (!fw_ref.domains) 508 return -ETIMEDOUT; 509 510 drm_printf(p, "PAT table: (* = reserved entry)\n"); 511 512 for (i = 0; i < xe->pat.n_entries; i++) { 513 if (xe_gt_is_media_type(gt)) 514 pat = xe_mmio_read32(>->mmio, XE_REG(_PAT_INDEX(i))); 515 else 516 pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i))); 517 518 xe_pat_index_label(label, sizeof(label), i); 519 xe->pat.ops->entry_dump(p, label, pat, !xe->pat.table[i].valid); 520 } 521 522 /* 523 * Also print PTA_MODE, which describes how the hardware accesses 524 * PPGTT entries. 525 */ 526 if (xe_gt_is_media_type(gt)) 527 pat = xe_mmio_read32(>->mmio, XE_REG(_PAT_PTA)); 528 else 529 pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_PTA)); 530 531 drm_printf(p, "Page Table Access:\n"); 532 xe->pat.ops->entry_dump(p, "PTA_MODE", pat, false); 533 534 return 0; 535 } 536 537 static const struct xe_pat_ops xe2_pat_ops = { 538 .program_graphics = program_pat_mcr, 539 .program_media = program_pat, 540 .dump = xe2_dump, 541 .entry_dump = xe2_pat_entry_dump, 542 }; 543 544 static const struct xe_pat_ops xe3p_xpc_pat_ops = { 545 .program_graphics = program_pat_mcr, 546 .program_media = program_pat, 547 .dump = xe2_dump, 548 .entry_dump = xe3p_xpc_pat_entry_dump, 549 }; 550 551 void xe_pat_init_early(struct xe_device *xe) 552 { 553 xe->pat.idx[XE_CACHE_WB_COMPRESSION] = XE_PAT_INVALID_IDX; 554 if (GRAPHICS_VERx100(xe) == 3511) { 555 xe->pat.ops = &xe3p_xpc_pat_ops; 556 xe->pat.table = xe3p_xpc_pat_table; 557 xe->pat.pat_ats = &xe3p_xpc_pat_ats; 558 xe->pat.pat_primary_pta = &xe3p_xpc_pat_pta; 559 xe->pat.pat_media_pta = &xe3p_xpc_pat_pta; 560 xe->pat.n_entries = ARRAY_SIZE(xe3p_xpc_pat_table); 561 xe->pat.idx[XE_CACHE_NONE] = 3; 562 xe->pat.idx[XE_CACHE_WT] = 3; /* N/A (no display); use UC */ 563 xe->pat.idx[XE_CACHE_WB] = 2; 564 } else if (GRAPHICS_VER(xe) == 35) { 565 xe->pat.ops = &xe2_pat_ops; 566 xe->pat.table = xe3p_lpg_pat_table; 567 xe->pat.pat_ats = &xe2_pat_ats; 568 if (!IS_DGFX(xe)) { 569 xe->pat.pat_primary_pta = &xe3p_primary_pat_pta; 570 xe->pat.pat_media_pta = &xe3p_media_pat_pta; 571 } 572 xe->pat.n_entries = ARRAY_SIZE(xe3p_lpg_pat_table); 573 xe->pat.idx[XE_CACHE_NONE] = 3; 574 xe->pat.idx[XE_CACHE_WT] = 15; 575 xe->pat.idx[XE_CACHE_WB] = 2; 576 xe->pat.idx[XE_CACHE_NONE_COMPRESSION] = 12; 577 xe->pat.idx[XE_CACHE_WB_COMPRESSION] = 16; 578 } else if (GRAPHICS_VER(xe) == 30 || GRAPHICS_VER(xe) == 20) { 579 xe->pat.ops = &xe2_pat_ops; 580 if (GRAPHICS_VER(xe) == 30) { 581 xe->pat.table = xe3_lpg_pat_table; 582 xe->pat.idx[XE_CACHE_WB_COMPRESSION] = 16; 583 } else { 584 xe->pat.table = xe2_pat_table; 585 } 586 xe->pat.pat_ats = &xe2_pat_ats; 587 if (IS_DGFX(xe)) { 588 xe->pat.pat_primary_pta = &xe2_pat_pta; 589 xe->pat.pat_media_pta = &xe2_pat_pta; 590 } 591 592 /* Wa_16023588340. XXX: Should use XE_WA */ 593 if (GRAPHICS_VERx100(xe) == 2001) 594 xe->pat.n_entries = 28; /* Disable CLOS3 */ 595 else 596 xe->pat.n_entries = ARRAY_SIZE(xe2_pat_table); 597 598 xe->pat.idx[XE_CACHE_NONE] = 3; 599 xe->pat.idx[XE_CACHE_WT] = 15; 600 xe->pat.idx[XE_CACHE_WB] = 2; 601 xe->pat.idx[XE_CACHE_NONE_COMPRESSION] = 12; /*Applicable on xe2 and beyond */ 602 } else if (xe->info.platform == XE_METEORLAKE) { 603 xe->pat.ops = &xelpg_pat_ops; 604 xe->pat.table = xelpg_pat_table; 605 xe->pat.n_entries = ARRAY_SIZE(xelpg_pat_table); 606 xe->pat.idx[XE_CACHE_NONE] = 2; 607 xe->pat.idx[XE_CACHE_WT] = 1; 608 xe->pat.idx[XE_CACHE_WB] = 3; 609 } else if (xe->info.platform == XE_PVC) { 610 xe->pat.ops = &xehpc_pat_ops; 611 xe->pat.table = xehpc_pat_table; 612 xe->pat.n_entries = ARRAY_SIZE(xehpc_pat_table); 613 xe->pat.idx[XE_CACHE_NONE] = 0; 614 xe->pat.idx[XE_CACHE_WT] = 2; 615 xe->pat.idx[XE_CACHE_WB] = 3; 616 } else if (xe->info.platform == XE_DG2) { 617 /* 618 * Table is the same as previous platforms, but programming 619 * method has changed. 620 */ 621 xe->pat.ops = &xehp_pat_ops; 622 xe->pat.table = xelp_pat_table; 623 xe->pat.n_entries = ARRAY_SIZE(xelp_pat_table); 624 xe->pat.idx[XE_CACHE_NONE] = 3; 625 xe->pat.idx[XE_CACHE_WT] = 2; 626 xe->pat.idx[XE_CACHE_WB] = 0; 627 } else if (GRAPHICS_VERx100(xe) <= 1210) { 628 WARN_ON_ONCE(!IS_DGFX(xe) && !xe->info.has_llc); 629 xe->pat.ops = &xelp_pat_ops; 630 xe->pat.table = xelp_pat_table; 631 xe->pat.n_entries = ARRAY_SIZE(xelp_pat_table); 632 xe->pat.idx[XE_CACHE_NONE] = 3; 633 xe->pat.idx[XE_CACHE_WT] = 2; 634 xe->pat.idx[XE_CACHE_WB] = 0; 635 } else { 636 /* 637 * Going forward we expect to need new PAT settings for most 638 * new platforms; failure to provide a new table can easily 639 * lead to subtle, hard-to-debug problems. If none of the 640 * conditions above match the platform we're running on we'll 641 * raise an error rather than trying to silently inherit the 642 * most recent platform's behavior. 643 */ 644 drm_err(&xe->drm, "Missing PAT table for platform with graphics version %d.%02d!\n", 645 GRAPHICS_VER(xe), GRAPHICS_VERx100(xe) % 100); 646 } 647 648 xe_assert(xe, xe->pat.ops->dump); 649 xe_assert(xe, xe->pat.ops->program_graphics); 650 xe_assert(xe, MEDIA_VER(xe) < 13 || xe->pat.ops->program_media); 651 xe_assert(xe, GRAPHICS_VER(xe) < 20 || xe->pat.ops->entry_dump); 652 } 653 654 void xe_pat_init(struct xe_gt *gt) 655 { 656 struct xe_device *xe = gt_to_xe(gt); 657 658 if (IS_SRIOV_VF(xe)) 659 return; 660 661 if (xe_gt_is_media_type(gt)) 662 xe->pat.ops->program_media(gt, xe->pat.table, xe->pat.n_entries); 663 else 664 xe->pat.ops->program_graphics(gt, xe->pat.table, xe->pat.n_entries); 665 } 666 667 /** 668 * xe_pat_dump() - Dump GT PAT table into a drm printer. 669 * @gt: the &xe_gt 670 * @p: the &drm_printer 671 * 672 * Return: 0 on success or a negative error code on failure. 673 */ 674 int xe_pat_dump(struct xe_gt *gt, struct drm_printer *p) 675 { 676 struct xe_device *xe = gt_to_xe(gt); 677 678 if (IS_SRIOV_VF(xe)) 679 return -EOPNOTSUPP; 680 681 return xe->pat.ops->dump(gt, p); 682 } 683 684 /** 685 * xe_pat_dump_sw_config() - Dump the software-configured GT PAT table into a drm printer. 686 * @gt: the &xe_gt 687 * @p: the &drm_printer 688 * 689 * Return: 0 on success or a negative error code on failure. 690 */ 691 int xe_pat_dump_sw_config(struct xe_gt *gt, struct drm_printer *p) 692 { 693 struct xe_device *xe = gt_to_xe(gt); 694 const struct xe_pat_table_entry *pta_entry = gt_pta_entry(gt); 695 char label[PAT_LABEL_LEN]; 696 697 if (!xe->pat.table || !xe->pat.n_entries) 698 return -EOPNOTSUPP; 699 700 drm_printf(p, "PAT table:%s\n", GRAPHICS_VER(xe) >= 20 ? " (* = reserved entry)" : ""); 701 for (u32 i = 0; i < xe->pat.n_entries; i++) { 702 u32 pat = xe->pat.table[i].value; 703 704 if (GRAPHICS_VER(xe) >= 20) { 705 xe_pat_index_label(label, sizeof(label), i); 706 xe->pat.ops->entry_dump(p, label, pat, !xe->pat.table[i].valid); 707 } else if (xe->info.platform == XE_METEORLAKE) { 708 xelpg_pat_entry_dump(p, i, pat); 709 } else if (xe->info.platform == XE_PVC) { 710 xehpc_pat_entry_dump(p, i, pat); 711 } else if (xe->info.platform == XE_DG2 || GRAPHICS_VERx100(xe) <= 1210) { 712 xelp_pat_entry_dump(p, i, pat); 713 } else { 714 return -EOPNOTSUPP; 715 } 716 } 717 718 if (pta_entry) { 719 u32 pat = pta_entry->value; 720 721 drm_printf(p, "Page Table Access:\n"); 722 xe->pat.ops->entry_dump(p, "PTA_MODE", pat, false); 723 } 724 725 if (xe->pat.pat_ats) { 726 u32 pat = xe->pat.pat_ats->value; 727 728 drm_printf(p, "PCIe ATS/PASID:\n"); 729 xe->pat.ops->entry_dump(p, "PAT_ATS ", pat, false); 730 } 731 732 drm_printf(p, "Cache Level:\n"); 733 drm_printf(p, "IDX[XE_CACHE_NONE] = %d\n", xe->pat.idx[XE_CACHE_NONE]); 734 drm_printf(p, "IDX[XE_CACHE_WT] = %d\n", xe->pat.idx[XE_CACHE_WT]); 735 drm_printf(p, "IDX[XE_CACHE_WB] = %d\n", xe->pat.idx[XE_CACHE_WB]); 736 if (GRAPHICS_VER(xe) >= 20) { 737 drm_printf(p, "IDX[XE_CACHE_NONE_COMPRESSION] = %d\n", 738 xe->pat.idx[XE_CACHE_NONE_COMPRESSION]); 739 drm_printf(p, "IDX[XE_CACHE_WB_COMPRESSION] = %d\n", 740 xe->pat.idx[XE_CACHE_WB_COMPRESSION]); 741 } 742 743 return 0; 744 } 745