1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2023 Intel Corporation 4 */ 5 6 #include "xe_pat.h" 7 8 #include <drm/xe_drm.h> 9 10 #include "regs/xe_reg_defs.h" 11 #include "xe_assert.h" 12 #include "xe_device.h" 13 #include "xe_force_wake.h" 14 #include "xe_gt.h" 15 #include "xe_gt_mcr.h" 16 #include "xe_mmio.h" 17 #include "xe_sriov.h" 18 19 #define _PAT_ATS 0x47fc 20 #define _PAT_INDEX(index) _PICK_EVEN_2RANGES(index, 8, \ 21 0x4800, 0x4804, \ 22 0x4848, 0x484c) 23 #define _PAT_PTA 0x4820 24 25 #define XE2_NO_PROMOTE REG_BIT(10) 26 #define XE2_COMP_EN REG_BIT(9) 27 #define XE2_L3_CLOS REG_GENMASK(7, 6) 28 #define XE2_L3_POLICY REG_GENMASK(5, 4) 29 #define XE2_L4_POLICY REG_GENMASK(3, 2) 30 #define XE2_COH_MODE REG_GENMASK(1, 0) 31 32 #define XELPG_L4_POLICY_MASK REG_GENMASK(3, 2) 33 #define XELPG_PAT_3_UC REG_FIELD_PREP(XELPG_L4_POLICY_MASK, 3) 34 #define XELPG_PAT_1_WT REG_FIELD_PREP(XELPG_L4_POLICY_MASK, 1) 35 #define XELPG_PAT_0_WB REG_FIELD_PREP(XELPG_L4_POLICY_MASK, 0) 36 #define XELPG_INDEX_COH_MODE_MASK REG_GENMASK(1, 0) 37 #define XELPG_3_COH_2W REG_FIELD_PREP(XELPG_INDEX_COH_MODE_MASK, 3) 38 #define XELPG_2_COH_1W REG_FIELD_PREP(XELPG_INDEX_COH_MODE_MASK, 2) 39 #define XELPG_0_COH_NON REG_FIELD_PREP(XELPG_INDEX_COH_MODE_MASK, 0) 40 41 #define XEHPC_CLOS_LEVEL_MASK REG_GENMASK(3, 2) 42 #define XEHPC_PAT_CLOS(x) REG_FIELD_PREP(XEHPC_CLOS_LEVEL_MASK, x) 43 44 #define XELP_MEM_TYPE_MASK REG_GENMASK(1, 0) 45 #define XELP_PAT_WB REG_FIELD_PREP(XELP_MEM_TYPE_MASK, 3) 46 #define XELP_PAT_WT REG_FIELD_PREP(XELP_MEM_TYPE_MASK, 2) 47 #define XELP_PAT_WC REG_FIELD_PREP(XELP_MEM_TYPE_MASK, 1) 48 #define XELP_PAT_UC REG_FIELD_PREP(XELP_MEM_TYPE_MASK, 0) 49 50 static const char *XELP_MEM_TYPE_STR_MAP[] = { "UC", "WC", "WT", "WB" }; 51 52 struct xe_pat_ops { 53 void (*program_graphics)(struct xe_gt *gt, const struct xe_pat_table_entry table[], 54 int n_entries); 55 void (*program_media)(struct xe_gt *gt, const struct xe_pat_table_entry table[], 56 int n_entries); 57 void (*dump)(struct xe_gt *gt, struct drm_printer *p); 58 }; 59 60 static const struct xe_pat_table_entry xelp_pat_table[] = { 61 [0] = { XELP_PAT_WB, XE_COH_AT_LEAST_1WAY }, 62 [1] = { XELP_PAT_WC, XE_COH_NONE }, 63 [2] = { XELP_PAT_WT, XE_COH_NONE }, 64 [3] = { XELP_PAT_UC, XE_COH_NONE }, 65 }; 66 67 static const struct xe_pat_table_entry xehpc_pat_table[] = { 68 [0] = { XELP_PAT_UC, XE_COH_NONE }, 69 [1] = { XELP_PAT_WC, XE_COH_NONE }, 70 [2] = { XELP_PAT_WT, XE_COH_NONE }, 71 [3] = { XELP_PAT_WB, XE_COH_AT_LEAST_1WAY }, 72 [4] = { XEHPC_PAT_CLOS(1) | XELP_PAT_WT, XE_COH_NONE }, 73 [5] = { XEHPC_PAT_CLOS(1) | XELP_PAT_WB, XE_COH_AT_LEAST_1WAY }, 74 [6] = { XEHPC_PAT_CLOS(2) | XELP_PAT_WT, XE_COH_NONE }, 75 [7] = { XEHPC_PAT_CLOS(2) | XELP_PAT_WB, XE_COH_AT_LEAST_1WAY }, 76 }; 77 78 static const struct xe_pat_table_entry xelpg_pat_table[] = { 79 [0] = { XELPG_PAT_0_WB, XE_COH_NONE }, 80 [1] = { XELPG_PAT_1_WT, XE_COH_NONE }, 81 [2] = { XELPG_PAT_3_UC, XE_COH_NONE }, 82 [3] = { XELPG_PAT_0_WB | XELPG_2_COH_1W, XE_COH_AT_LEAST_1WAY }, 83 [4] = { XELPG_PAT_0_WB | XELPG_3_COH_2W, XE_COH_AT_LEAST_1WAY }, 84 }; 85 86 /* 87 * The Xe2 table is getting large/complicated so it's easier to review if 88 * provided in a form that exactly matches the bspec's formatting. The meaning 89 * of the fields here are: 90 * - no_promote: 0=promotable, 1=no promote 91 * - comp_en: 0=disable, 1=enable 92 * - l3clos: L3 class of service (0-3) 93 * - l3_policy: 0=WB, 1=XD ("WB - Transient Display"), 3=UC 94 * - l4_policy: 0=WB, 1=WT, 3=UC 95 * - coh_mode: 0=no snoop, 2=1-way coherent, 3=2-way coherent 96 * 97 * Reserved entries should be programmed with the maximum caching, minimum 98 * coherency (which matches an all-0's encoding), so we can just omit them 99 * in the table. 100 */ 101 #define XE2_PAT(no_promote, comp_en, l3clos, l3_policy, l4_policy, __coh_mode) \ 102 { \ 103 .value = (no_promote ? XE2_NO_PROMOTE : 0) | \ 104 (comp_en ? XE2_COMP_EN : 0) | \ 105 REG_FIELD_PREP(XE2_L3_CLOS, l3clos) | \ 106 REG_FIELD_PREP(XE2_L3_POLICY, l3_policy) | \ 107 REG_FIELD_PREP(XE2_L4_POLICY, l4_policy) | \ 108 REG_FIELD_PREP(XE2_COH_MODE, __coh_mode), \ 109 .coh_mode = __coh_mode ? XE_COH_AT_LEAST_1WAY : XE_COH_NONE \ 110 } 111 112 static const struct xe_pat_table_entry xe2_pat_table[] = { 113 [ 0] = XE2_PAT( 0, 0, 0, 0, 3, 0 ), 114 [ 1] = XE2_PAT( 0, 0, 0, 0, 3, 2 ), 115 [ 2] = XE2_PAT( 0, 0, 0, 0, 3, 3 ), 116 [ 3] = XE2_PAT( 0, 0, 0, 3, 3, 0 ), 117 [ 4] = XE2_PAT( 0, 0, 0, 3, 0, 2 ), 118 [ 5] = XE2_PAT( 0, 0, 0, 3, 3, 2 ), 119 [ 6] = XE2_PAT( 1, 0, 0, 1, 3, 0 ), 120 [ 7] = XE2_PAT( 0, 0, 0, 3, 0, 3 ), 121 [ 8] = XE2_PAT( 0, 0, 0, 3, 0, 0 ), 122 [ 9] = XE2_PAT( 0, 1, 0, 0, 3, 0 ), 123 [10] = XE2_PAT( 0, 1, 0, 3, 0, 0 ), 124 [11] = XE2_PAT( 1, 1, 0, 1, 3, 0 ), 125 [12] = XE2_PAT( 0, 1, 0, 3, 3, 0 ), 126 [13] = XE2_PAT( 0, 0, 0, 0, 0, 0 ), 127 [14] = XE2_PAT( 0, 1, 0, 0, 0, 0 ), 128 [15] = XE2_PAT( 1, 1, 0, 1, 1, 0 ), 129 /* 16..19 are reserved; leave set to all 0's */ 130 [20] = XE2_PAT( 0, 0, 1, 0, 3, 0 ), 131 [21] = XE2_PAT( 0, 1, 1, 0, 3, 0 ), 132 [22] = XE2_PAT( 0, 0, 1, 0, 3, 2 ), 133 [23] = XE2_PAT( 0, 0, 1, 0, 3, 3 ), 134 [24] = XE2_PAT( 0, 0, 2, 0, 3, 0 ), 135 [25] = XE2_PAT( 0, 1, 2, 0, 3, 0 ), 136 [26] = XE2_PAT( 0, 0, 2, 0, 3, 2 ), 137 [27] = XE2_PAT( 0, 0, 2, 0, 3, 3 ), 138 [28] = XE2_PAT( 0, 0, 3, 0, 3, 0 ), 139 [29] = XE2_PAT( 0, 1, 3, 0, 3, 0 ), 140 [30] = XE2_PAT( 0, 0, 3, 0, 3, 2 ), 141 [31] = XE2_PAT( 0, 0, 3, 0, 3, 3 ), 142 }; 143 144 /* Special PAT values programmed outside the main table */ 145 static const struct xe_pat_table_entry xe2_pat_ats = XE2_PAT( 0, 0, 0, 0, 3, 3 ); 146 static const struct xe_pat_table_entry xe2_pat_pta = XE2_PAT( 0, 0, 0, 0, 3, 0 ); 147 148 u16 xe_pat_index_get_coh_mode(struct xe_device *xe, u16 pat_index) 149 { 150 WARN_ON(pat_index >= xe->pat.n_entries); 151 return xe->pat.table[pat_index].coh_mode; 152 } 153 154 static void program_pat(struct xe_gt *gt, const struct xe_pat_table_entry table[], 155 int n_entries) 156 { 157 for (int i = 0; i < n_entries; i++) { 158 struct xe_reg reg = XE_REG(_PAT_INDEX(i)); 159 160 xe_mmio_write32(gt, reg, table[i].value); 161 } 162 } 163 164 static void program_pat_mcr(struct xe_gt *gt, const struct xe_pat_table_entry table[], 165 int n_entries) 166 { 167 for (int i = 0; i < n_entries; i++) { 168 struct xe_reg_mcr reg_mcr = XE_REG_MCR(_PAT_INDEX(i)); 169 170 xe_gt_mcr_multicast_write(gt, reg_mcr, table[i].value); 171 } 172 } 173 174 static void xelp_dump(struct xe_gt *gt, struct drm_printer *p) 175 { 176 struct xe_device *xe = gt_to_xe(gt); 177 int i, err; 178 179 err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); 180 if (err) 181 goto err_fw; 182 183 drm_printf(p, "PAT table:\n"); 184 185 for (i = 0; i < xe->pat.n_entries; i++) { 186 u32 pat = xe_mmio_read32(gt, XE_REG(_PAT_INDEX(i))); 187 u8 mem_type = REG_FIELD_GET(XELP_MEM_TYPE_MASK, pat); 188 189 drm_printf(p, "PAT[%2d] = %s (%#8x)\n", i, 190 XELP_MEM_TYPE_STR_MAP[mem_type], pat); 191 } 192 193 err = xe_force_wake_put(gt_to_fw(gt), XE_FW_GT); 194 err_fw: 195 xe_assert(xe, !err); 196 } 197 198 static const struct xe_pat_ops xelp_pat_ops = { 199 .program_graphics = program_pat, 200 .dump = xelp_dump, 201 }; 202 203 static void xehp_dump(struct xe_gt *gt, struct drm_printer *p) 204 { 205 struct xe_device *xe = gt_to_xe(gt); 206 int i, err; 207 208 err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); 209 if (err) 210 goto err_fw; 211 212 drm_printf(p, "PAT table:\n"); 213 214 for (i = 0; i < xe->pat.n_entries; i++) { 215 u32 pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i))); 216 u8 mem_type; 217 218 mem_type = REG_FIELD_GET(XELP_MEM_TYPE_MASK, pat); 219 220 drm_printf(p, "PAT[%2d] = %s (%#8x)\n", i, 221 XELP_MEM_TYPE_STR_MAP[mem_type], pat); 222 } 223 224 err = xe_force_wake_put(gt_to_fw(gt), XE_FW_GT); 225 err_fw: 226 xe_assert(xe, !err); 227 } 228 229 static const struct xe_pat_ops xehp_pat_ops = { 230 .program_graphics = program_pat_mcr, 231 .dump = xehp_dump, 232 }; 233 234 static void xehpc_dump(struct xe_gt *gt, struct drm_printer *p) 235 { 236 struct xe_device *xe = gt_to_xe(gt); 237 int i, err; 238 239 err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); 240 if (err) 241 goto err_fw; 242 243 drm_printf(p, "PAT table:\n"); 244 245 for (i = 0; i < xe->pat.n_entries; i++) { 246 u32 pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i))); 247 248 drm_printf(p, "PAT[%2d] = [ %u, %u ] (%#8x)\n", i, 249 REG_FIELD_GET(XELP_MEM_TYPE_MASK, pat), 250 REG_FIELD_GET(XEHPC_CLOS_LEVEL_MASK, pat), pat); 251 } 252 253 err = xe_force_wake_put(gt_to_fw(gt), XE_FW_GT); 254 err_fw: 255 xe_assert(xe, !err); 256 } 257 258 static const struct xe_pat_ops xehpc_pat_ops = { 259 .program_graphics = program_pat_mcr, 260 .dump = xehpc_dump, 261 }; 262 263 static void xelpg_dump(struct xe_gt *gt, struct drm_printer *p) 264 { 265 struct xe_device *xe = gt_to_xe(gt); 266 int i, err; 267 268 err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); 269 if (err) 270 goto err_fw; 271 272 drm_printf(p, "PAT table:\n"); 273 274 for (i = 0; i < xe->pat.n_entries; i++) { 275 u32 pat; 276 277 if (xe_gt_is_media_type(gt)) 278 pat = xe_mmio_read32(gt, XE_REG(_PAT_INDEX(i))); 279 else 280 pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i))); 281 282 drm_printf(p, "PAT[%2d] = [ %u, %u ] (%#8x)\n", i, 283 REG_FIELD_GET(XELPG_L4_POLICY_MASK, pat), 284 REG_FIELD_GET(XELPG_INDEX_COH_MODE_MASK, pat), pat); 285 } 286 287 err = xe_force_wake_put(gt_to_fw(gt), XE_FW_GT); 288 err_fw: 289 xe_assert(xe, !err); 290 } 291 292 /* 293 * SAMedia register offsets are adjusted by the write methods and they target 294 * registers that are not MCR, while for normal GT they are MCR 295 */ 296 static const struct xe_pat_ops xelpg_pat_ops = { 297 .program_graphics = program_pat, 298 .program_media = program_pat_mcr, 299 .dump = xelpg_dump, 300 }; 301 302 static void xe2lpg_program_pat(struct xe_gt *gt, const struct xe_pat_table_entry table[], 303 int n_entries) 304 { 305 program_pat_mcr(gt, table, n_entries); 306 xe_gt_mcr_multicast_write(gt, XE_REG_MCR(_PAT_ATS), xe2_pat_ats.value); 307 308 if (IS_DGFX(gt_to_xe(gt))) 309 xe_gt_mcr_multicast_write(gt, XE_REG_MCR(_PAT_PTA), xe2_pat_pta.value); 310 } 311 312 static void xe2lpm_program_pat(struct xe_gt *gt, const struct xe_pat_table_entry table[], 313 int n_entries) 314 { 315 program_pat(gt, table, n_entries); 316 xe_mmio_write32(gt, XE_REG(_PAT_ATS), xe2_pat_ats.value); 317 318 if (IS_DGFX(gt_to_xe(gt))) 319 xe_mmio_write32(gt, XE_REG(_PAT_PTA), xe2_pat_pta.value); 320 } 321 322 static void xe2_dump(struct xe_gt *gt, struct drm_printer *p) 323 { 324 struct xe_device *xe = gt_to_xe(gt); 325 int i, err; 326 u32 pat; 327 328 err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); 329 if (err) 330 goto err_fw; 331 332 drm_printf(p, "PAT table:\n"); 333 334 for (i = 0; i < xe->pat.n_entries; i++) { 335 if (xe_gt_is_media_type(gt)) 336 pat = xe_mmio_read32(gt, XE_REG(_PAT_INDEX(i))); 337 else 338 pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i))); 339 340 drm_printf(p, "PAT[%2d] = [ %u, %u, %u, %u, %u, %u ] (%#8x)\n", i, 341 !!(pat & XE2_NO_PROMOTE), 342 !!(pat & XE2_COMP_EN), 343 REG_FIELD_GET(XE2_L3_CLOS, pat), 344 REG_FIELD_GET(XE2_L3_POLICY, pat), 345 REG_FIELD_GET(XE2_L4_POLICY, pat), 346 REG_FIELD_GET(XE2_COH_MODE, pat), 347 pat); 348 } 349 350 /* 351 * Also print PTA_MODE, which describes how the hardware accesses 352 * PPGTT entries. 353 */ 354 if (xe_gt_is_media_type(gt)) 355 pat = xe_mmio_read32(gt, XE_REG(_PAT_PTA)); 356 else 357 pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_PTA)); 358 359 drm_printf(p, "Page Table Access:\n"); 360 drm_printf(p, "PTA_MODE= [ %u, %u, %u, %u, %u, %u ] (%#8x)\n", 361 !!(pat & XE2_NO_PROMOTE), 362 !!(pat & XE2_COMP_EN), 363 REG_FIELD_GET(XE2_L3_CLOS, pat), 364 REG_FIELD_GET(XE2_L3_POLICY, pat), 365 REG_FIELD_GET(XE2_L4_POLICY, pat), 366 REG_FIELD_GET(XE2_COH_MODE, pat), 367 pat); 368 369 err = xe_force_wake_put(gt_to_fw(gt), XE_FW_GT); 370 err_fw: 371 xe_assert(xe, !err); 372 } 373 374 static const struct xe_pat_ops xe2_pat_ops = { 375 .program_graphics = xe2lpg_program_pat, 376 .program_media = xe2lpm_program_pat, 377 .dump = xe2_dump, 378 }; 379 380 void xe_pat_init_early(struct xe_device *xe) 381 { 382 if (GRAPHICS_VER(xe) == 20) { 383 xe->pat.ops = &xe2_pat_ops; 384 xe->pat.table = xe2_pat_table; 385 xe->pat.n_entries = ARRAY_SIZE(xe2_pat_table); 386 xe->pat.idx[XE_CACHE_NONE] = 3; 387 xe->pat.idx[XE_CACHE_WT] = 15; 388 xe->pat.idx[XE_CACHE_WB] = 2; 389 xe->pat.idx[XE_CACHE_NONE_COMPRESSION] = 12; /*Applicable on xe2 and beyond */ 390 } else if (xe->info.platform == XE_METEORLAKE) { 391 xe->pat.ops = &xelpg_pat_ops; 392 xe->pat.table = xelpg_pat_table; 393 xe->pat.n_entries = ARRAY_SIZE(xelpg_pat_table); 394 xe->pat.idx[XE_CACHE_NONE] = 2; 395 xe->pat.idx[XE_CACHE_WT] = 1; 396 xe->pat.idx[XE_CACHE_WB] = 3; 397 } else if (xe->info.platform == XE_PVC) { 398 xe->pat.ops = &xehpc_pat_ops; 399 xe->pat.table = xehpc_pat_table; 400 xe->pat.n_entries = ARRAY_SIZE(xehpc_pat_table); 401 xe->pat.idx[XE_CACHE_NONE] = 0; 402 xe->pat.idx[XE_CACHE_WT] = 2; 403 xe->pat.idx[XE_CACHE_WB] = 3; 404 } else if (xe->info.platform == XE_DG2) { 405 /* 406 * Table is the same as previous platforms, but programming 407 * method has changed. 408 */ 409 xe->pat.ops = &xehp_pat_ops; 410 xe->pat.table = xelp_pat_table; 411 xe->pat.n_entries = ARRAY_SIZE(xelp_pat_table); 412 xe->pat.idx[XE_CACHE_NONE] = 3; 413 xe->pat.idx[XE_CACHE_WT] = 2; 414 xe->pat.idx[XE_CACHE_WB] = 0; 415 } else if (GRAPHICS_VERx100(xe) <= 1210) { 416 WARN_ON_ONCE(!IS_DGFX(xe) && !xe->info.has_llc); 417 xe->pat.ops = &xelp_pat_ops; 418 xe->pat.table = xelp_pat_table; 419 xe->pat.n_entries = ARRAY_SIZE(xelp_pat_table); 420 xe->pat.idx[XE_CACHE_NONE] = 3; 421 xe->pat.idx[XE_CACHE_WT] = 2; 422 xe->pat.idx[XE_CACHE_WB] = 0; 423 } else { 424 /* 425 * Going forward we expect to need new PAT settings for most 426 * new platforms; failure to provide a new table can easily 427 * lead to subtle, hard-to-debug problems. If none of the 428 * conditions above match the platform we're running on we'll 429 * raise an error rather than trying to silently inherit the 430 * most recent platform's behavior. 431 */ 432 drm_err(&xe->drm, "Missing PAT table for platform with graphics version %d.%02d!\n", 433 GRAPHICS_VER(xe), GRAPHICS_VERx100(xe) % 100); 434 } 435 436 /* VFs can't program nor dump PAT settings */ 437 if (IS_SRIOV_VF(xe)) 438 xe->pat.ops = NULL; 439 440 xe_assert(xe, !xe->pat.ops || xe->pat.ops->dump); 441 xe_assert(xe, !xe->pat.ops || xe->pat.ops->program_graphics); 442 xe_assert(xe, !xe->pat.ops || MEDIA_VER(xe) < 13 || xe->pat.ops->program_media); 443 } 444 445 void xe_pat_init(struct xe_gt *gt) 446 { 447 struct xe_device *xe = gt_to_xe(gt); 448 449 if (!xe->pat.ops) 450 return; 451 452 if (xe_gt_is_media_type(gt)) 453 xe->pat.ops->program_media(gt, xe->pat.table, xe->pat.n_entries); 454 else 455 xe->pat.ops->program_graphics(gt, xe->pat.table, xe->pat.n_entries); 456 } 457 458 void xe_pat_dump(struct xe_gt *gt, struct drm_printer *p) 459 { 460 struct xe_device *xe = gt_to_xe(gt); 461 462 if (!xe->pat.ops) 463 return; 464 465 xe->pat.ops->dump(gt, p); 466 } 467