1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include "xe_mocs.h" 7 8 #include "regs/xe_gt_regs.h" 9 #include "xe_bo.h" 10 #include "xe_device.h" 11 #include "xe_exec_queue.h" 12 #include "xe_force_wake.h" 13 #include "xe_gt.h" 14 #include "xe_gt_mcr.h" 15 #include "xe_gt_printk.h" 16 #include "xe_mmio.h" 17 #include "xe_platform_types.h" 18 #include "xe_pm.h" 19 #include "xe_sriov.h" 20 #include "xe_step_types.h" 21 22 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG) 23 #define mocs_dbg xe_gt_dbg 24 #else 25 __printf(2, 3) 26 static inline void mocs_dbg(const struct xe_gt *gt, 27 const char *format, ...) 28 { /* noop */ } 29 #endif 30 31 enum { 32 HAS_GLOBAL_MOCS = BIT(0), 33 HAS_LNCF_MOCS = BIT(1), 34 }; 35 36 struct xe_mocs_entry { 37 u32 control_value; 38 u16 l3cc_value; 39 u16 used; 40 }; 41 42 struct xe_mocs_info; 43 44 struct xe_mocs_ops { 45 void (*dump)(struct xe_mocs_info *mocs, unsigned int flags, 46 struct xe_gt *gt, struct drm_printer *p); 47 }; 48 49 struct xe_mocs_info { 50 unsigned int size; 51 unsigned int n_entries; 52 const struct xe_mocs_entry *table; 53 const struct xe_mocs_ops *ops; 54 u8 uc_index; 55 u8 wb_index; 56 u8 unused_entries_index; 57 }; 58 59 /* Defines for the tables (GLOB_MOCS_0 - GLOB_MOCS_16) */ 60 #define IG_PAT REG_BIT(8) 61 #define L3_CACHE_POLICY_MASK REG_GENMASK(5, 4) 62 #define L4_CACHE_POLICY_MASK REG_GENMASK(3, 2) 63 64 /* Helper defines */ 65 #define XELP_NUM_MOCS_ENTRIES 64 /* 63-64 are reserved, but configured. */ 66 #define PVC_NUM_MOCS_ENTRIES 3 67 #define MTL_NUM_MOCS_ENTRIES 16 68 #define XE2_NUM_MOCS_ENTRIES 16 69 70 /* (e)LLC caching options */ 71 /* 72 * Note: LE_0_PAGETABLE works only up to Gen11; for newer gens it means 73 * the same as LE_UC 74 */ 75 #define LE_0_PAGETABLE LE_CACHEABILITY(0) 76 #define LE_1_UC LE_CACHEABILITY(1) 77 #define LE_2_WT LE_CACHEABILITY(2) 78 #define LE_3_WB LE_CACHEABILITY(3) 79 80 /* Target cache */ 81 #define LE_TC_0_PAGETABLE LE_TGT_CACHE(0) 82 #define LE_TC_1_LLC LE_TGT_CACHE(1) 83 #define LE_TC_2_LLC_ELLC LE_TGT_CACHE(2) 84 #define LE_TC_3_LLC_ELLC_ALT LE_TGT_CACHE(3) 85 86 /* L3 caching options */ 87 #define L3_0_DIRECT L3_CACHEABILITY(0) 88 #define L3_1_UC L3_CACHEABILITY(1) 89 #define L3_2_RESERVED L3_CACHEABILITY(2) 90 #define L3_3_WB L3_CACHEABILITY(3) 91 92 /* L4 caching options */ 93 #define L4_0_WB REG_FIELD_PREP(L4_CACHE_POLICY_MASK, 0) 94 #define L4_1_WT REG_FIELD_PREP(L4_CACHE_POLICY_MASK, 1) 95 #define L4_3_UC REG_FIELD_PREP(L4_CACHE_POLICY_MASK, 3) 96 97 #define XE2_L3_0_WB REG_FIELD_PREP(L3_CACHE_POLICY_MASK, 0) 98 /* XD: WB Transient Display */ 99 #define XE2_L3_1_XD REG_FIELD_PREP(L3_CACHE_POLICY_MASK, 1) 100 #define XE2_L3_3_UC REG_FIELD_PREP(L3_CACHE_POLICY_MASK, 3) 101 102 #define XE2_L3_CLOS_MASK REG_GENMASK(7, 6) 103 104 #define MOCS_ENTRY(__idx, __control_value, __l3cc_value) \ 105 [__idx] = { \ 106 .control_value = __control_value, \ 107 .l3cc_value = __l3cc_value, \ 108 .used = 1, \ 109 } 110 111 /* 112 * MOCS tables 113 * 114 * These are the MOCS tables that are programmed across all the rings. 115 * The control value is programmed to all the rings that support the 116 * MOCS registers. While the l3cc_values are only programmed to the 117 * LNCFCMOCS0 - LNCFCMOCS32 registers. 118 * 119 * These tables are intended to be kept reasonably consistent across 120 * HW platforms, and for ICL+, be identical across OSes. To achieve 121 * that, the list of entries is published as part of bspec. 122 * 123 * Entries not part of the following tables are undefined as far as userspace is 124 * concerned and shouldn't be relied upon. The last few entries are reserved by 125 * the hardware. They should be initialized according to bspec and never used. 126 * 127 * NOTE1: These tables are part of bspec and defined as part of the hardware 128 * interface. It is expected that, for specific hardware platform, existing 129 * entries will remain constant and the table will only be updated by adding new 130 * entries, filling unused positions. 131 * 132 * NOTE2: Reserved and unspecified MOCS indices have been set to L3 WB. These 133 * reserved entries should never be used. They may be changed to low performant 134 * variants with better coherency in the future if more entries are needed. 135 */ 136 137 static const struct xe_mocs_entry gen12_mocs_desc[] = { 138 /* Base - L3 + LLC */ 139 MOCS_ENTRY(2, 140 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), 141 L3_3_WB), 142 /* Base - Uncached */ 143 MOCS_ENTRY(3, 144 LE_1_UC | LE_TC_1_LLC, 145 L3_1_UC), 146 /* Base - L3 */ 147 MOCS_ENTRY(4, 148 LE_1_UC | LE_TC_1_LLC, 149 L3_3_WB), 150 /* Base - LLC */ 151 MOCS_ENTRY(5, 152 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), 153 L3_1_UC), 154 /* Age 0 - LLC */ 155 MOCS_ENTRY(6, 156 LE_3_WB | LE_TC_1_LLC | LE_LRUM(1), 157 L3_1_UC), 158 /* Age 0 - L3 + LLC */ 159 MOCS_ENTRY(7, 160 LE_3_WB | LE_TC_1_LLC | LE_LRUM(1), 161 L3_3_WB), 162 /* Age: Don't Chg. - LLC */ 163 MOCS_ENTRY(8, 164 LE_3_WB | LE_TC_1_LLC | LE_LRUM(2), 165 L3_1_UC), 166 /* Age: Don't Chg. - L3 + LLC */ 167 MOCS_ENTRY(9, 168 LE_3_WB | LE_TC_1_LLC | LE_LRUM(2), 169 L3_3_WB), 170 /* No AOM - LLC */ 171 MOCS_ENTRY(10, 172 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_AOM(1), 173 L3_1_UC), 174 /* No AOM - L3 + LLC */ 175 MOCS_ENTRY(11, 176 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_AOM(1), 177 L3_3_WB), 178 /* No AOM; Age 0 - LLC */ 179 MOCS_ENTRY(12, 180 LE_3_WB | LE_TC_1_LLC | LE_LRUM(1) | LE_AOM(1), 181 L3_1_UC), 182 /* No AOM; Age 0 - L3 + LLC */ 183 MOCS_ENTRY(13, 184 LE_3_WB | LE_TC_1_LLC | LE_LRUM(1) | LE_AOM(1), 185 L3_3_WB), 186 /* No AOM; Age:DC - LLC */ 187 MOCS_ENTRY(14, 188 LE_3_WB | LE_TC_1_LLC | LE_LRUM(2) | LE_AOM(1), 189 L3_1_UC), 190 /* No AOM; Age:DC - L3 + LLC */ 191 MOCS_ENTRY(15, 192 LE_3_WB | LE_TC_1_LLC | LE_LRUM(2) | LE_AOM(1), 193 L3_3_WB), 194 /* Self-Snoop - L3 + LLC */ 195 MOCS_ENTRY(18, 196 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SSE(3), 197 L3_3_WB), 198 /* Skip Caching - L3 + LLC(12.5%) */ 199 MOCS_ENTRY(19, 200 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SCC(7), 201 L3_3_WB), 202 /* Skip Caching - L3 + LLC(25%) */ 203 MOCS_ENTRY(20, 204 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SCC(3), 205 L3_3_WB), 206 /* Skip Caching - L3 + LLC(50%) */ 207 MOCS_ENTRY(21, 208 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SCC(1), 209 L3_3_WB), 210 /* Skip Caching - L3 + LLC(75%) */ 211 MOCS_ENTRY(22, 212 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_RSC(1) | LE_SCC(3), 213 L3_3_WB), 214 /* Skip Caching - L3 + LLC(87.5%) */ 215 MOCS_ENTRY(23, 216 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_RSC(1) | LE_SCC(7), 217 L3_3_WB), 218 /* Implicitly enable L1 - HDC:L1 + L3 + LLC */ 219 MOCS_ENTRY(48, 220 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), 221 L3_3_WB), 222 /* Implicitly enable L1 - HDC:L1 + L3 */ 223 MOCS_ENTRY(49, 224 LE_1_UC | LE_TC_1_LLC, 225 L3_3_WB), 226 /* Implicitly enable L1 - HDC:L1 + LLC */ 227 MOCS_ENTRY(50, 228 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), 229 L3_1_UC), 230 /* Implicitly enable L1 - HDC:L1 */ 231 MOCS_ENTRY(51, 232 LE_1_UC | LE_TC_1_LLC, 233 L3_1_UC), 234 /* HW Special Case (CCS) */ 235 MOCS_ENTRY(60, 236 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), 237 L3_1_UC), 238 /* HW Special Case (Displayable) */ 239 MOCS_ENTRY(61, 240 LE_1_UC | LE_TC_1_LLC, 241 L3_3_WB), 242 /* HW Reserved - SW program but never use */ 243 MOCS_ENTRY(62, 244 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), 245 L3_1_UC), 246 /* HW Reserved - SW program but never use */ 247 MOCS_ENTRY(63, 248 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), 249 L3_1_UC) 250 }; 251 252 static bool regs_are_mcr(struct xe_gt *gt) 253 { 254 struct xe_device *xe = gt_to_xe(gt); 255 256 if (xe_gt_is_media_type(gt)) 257 return MEDIA_VER(xe) >= 20; 258 else 259 return GRAPHICS_VERx100(xe) >= 1250; 260 } 261 262 static void xelp_lncf_dump(struct xe_mocs_info *info, struct xe_gt *gt, struct drm_printer *p) 263 { 264 unsigned int i, j; 265 u32 reg_val; 266 267 drm_printf(p, "LNCFCMOCS[idx] = [ESC, SCC, L3CC] (value)\n\n"); 268 269 for (i = 0, j = 0; i < (info->n_entries + 1) / 2; i++, j++) { 270 if (regs_are_mcr(gt)) 271 reg_val = xe_gt_mcr_unicast_read_any(gt, XEHP_LNCFCMOCS(i)); 272 else 273 reg_val = xe_mmio_read32(gt, XELP_LNCFCMOCS(i)); 274 275 drm_printf(p, "LNCFCMOCS[%2d] = [%u, %u, %u] (%#8x)\n", 276 j++, 277 !!(reg_val & L3_ESC_MASK), 278 REG_FIELD_GET(L3_SCC_MASK, reg_val), 279 REG_FIELD_GET(L3_CACHEABILITY_MASK, reg_val), 280 reg_val); 281 282 drm_printf(p, "LNCFCMOCS[%2d] = [%u, %u, %u] (%#8x)\n", 283 j, 284 !!(reg_val & L3_UPPER_IDX_ESC_MASK), 285 REG_FIELD_GET(L3_UPPER_IDX_SCC_MASK, reg_val), 286 REG_FIELD_GET(L3_UPPER_IDX_CACHEABILITY_MASK, reg_val), 287 reg_val); 288 } 289 } 290 291 static void xelp_mocs_dump(struct xe_mocs_info *info, unsigned int flags, 292 struct xe_gt *gt, struct drm_printer *p) 293 { 294 unsigned int i; 295 u32 reg_val; 296 297 if (flags & HAS_GLOBAL_MOCS) { 298 drm_printf(p, "Global mocs table configuration:\n"); 299 drm_printf(p, "GLOB_MOCS[idx] = [LeCC, TC, LRUM, AOM, RSC, SCC, PFM, SCF, CoS, SSE] (value)\n\n"); 300 301 for (i = 0; i < info->n_entries; i++) { 302 if (regs_are_mcr(gt)) 303 reg_val = xe_gt_mcr_unicast_read_any(gt, XEHP_GLOBAL_MOCS(i)); 304 else 305 reg_val = xe_mmio_read32(gt, XELP_GLOBAL_MOCS(i)); 306 307 drm_printf(p, "GLOB_MOCS[%2d] = [%u, %u, %u, %u, %u, %u, %u, %u, %u, %u ] (%#8x)\n", 308 i, 309 REG_FIELD_GET(LE_CACHEABILITY_MASK, reg_val), 310 REG_FIELD_GET(LE_TGT_CACHE_MASK, reg_val), 311 REG_FIELD_GET(LE_LRUM_MASK, reg_val), 312 !!(reg_val & LE_AOM_MASK), 313 !!(reg_val & LE_RSC_MASK), 314 REG_FIELD_GET(LE_SCC_MASK, reg_val), 315 REG_FIELD_GET(LE_PFM_MASK, reg_val), 316 !!(reg_val & LE_SCF_MASK), 317 REG_FIELD_GET(LE_COS_MASK, reg_val), 318 REG_FIELD_GET(LE_SSE_MASK, reg_val), 319 reg_val); 320 } 321 } 322 323 xelp_lncf_dump(info, gt, p); 324 } 325 326 static const struct xe_mocs_ops xelp_mocs_ops = { 327 .dump = xelp_mocs_dump, 328 }; 329 330 static const struct xe_mocs_entry dg1_mocs_desc[] = { 331 /* UC */ 332 MOCS_ENTRY(1, 0, L3_1_UC), 333 /* WB - L3 */ 334 MOCS_ENTRY(5, 0, L3_3_WB), 335 /* WB - L3 50% */ 336 MOCS_ENTRY(6, 0, L3_ESC(1) | L3_SCC(1) | L3_3_WB), 337 /* WB - L3 25% */ 338 MOCS_ENTRY(7, 0, L3_ESC(1) | L3_SCC(3) | L3_3_WB), 339 /* WB - L3 12.5% */ 340 MOCS_ENTRY(8, 0, L3_ESC(1) | L3_SCC(7) | L3_3_WB), 341 342 /* HDC:L1 + L3 */ 343 MOCS_ENTRY(48, 0, L3_3_WB), 344 /* HDC:L1 */ 345 MOCS_ENTRY(49, 0, L3_1_UC), 346 347 /* HW Reserved */ 348 MOCS_ENTRY(60, 0, L3_1_UC), 349 MOCS_ENTRY(61, 0, L3_1_UC), 350 MOCS_ENTRY(62, 0, L3_1_UC), 351 MOCS_ENTRY(63, 0, L3_1_UC), 352 }; 353 354 static const struct xe_mocs_entry dg2_mocs_desc[] = { 355 /* UC - Coherent; GO:L3 */ 356 MOCS_ENTRY(0, 0, L3_1_UC | L3_LKUP(1)), 357 /* UC - Coherent; GO:Memory */ 358 MOCS_ENTRY(1, 0, L3_1_UC | L3_GLBGO(1) | L3_LKUP(1)), 359 /* UC - Non-Coherent; GO:Memory */ 360 MOCS_ENTRY(2, 0, L3_1_UC | L3_GLBGO(1)), 361 362 /* WB - LC */ 363 MOCS_ENTRY(3, 0, L3_3_WB | L3_LKUP(1)), 364 }; 365 366 static void xehp_lncf_dump(struct xe_mocs_info *info, unsigned int flags, 367 struct xe_gt *gt, struct drm_printer *p) 368 { 369 unsigned int i, j; 370 u32 reg_val; 371 372 drm_printf(p, "LNCFCMOCS[idx] = [UCL3LOOKUP, GLBGO, L3CC] (value)\n\n"); 373 374 for (i = 0, j = 0; i < (info->n_entries + 1) / 2; i++, j++) { 375 if (regs_are_mcr(gt)) 376 reg_val = xe_gt_mcr_unicast_read_any(gt, XEHP_LNCFCMOCS(i)); 377 else 378 reg_val = xe_mmio_read32(gt, XELP_LNCFCMOCS(i)); 379 380 drm_printf(p, "LNCFCMOCS[%2d] = [%u, %u, %u] (%#8x)\n", 381 j++, 382 !!(reg_val & L3_LKUP_MASK), 383 !!(reg_val & L3_GLBGO_MASK), 384 REG_FIELD_GET(L3_CACHEABILITY_MASK, reg_val), 385 reg_val); 386 387 drm_printf(p, "LNCFCMOCS[%2d] = [%u, %u, %u] (%#8x)\n", 388 j, 389 !!(reg_val & L3_UPPER_LKUP_MASK), 390 !!(reg_val & L3_UPPER_GLBGO_MASK), 391 REG_FIELD_GET(L3_UPPER_IDX_CACHEABILITY_MASK, reg_val), 392 reg_val); 393 } 394 } 395 396 static const struct xe_mocs_ops xehp_mocs_ops = { 397 .dump = xehp_lncf_dump, 398 }; 399 400 static const struct xe_mocs_entry pvc_mocs_desc[] = { 401 /* Error */ 402 MOCS_ENTRY(0, 0, L3_3_WB), 403 404 /* UC */ 405 MOCS_ENTRY(1, 0, L3_1_UC), 406 407 /* WB */ 408 MOCS_ENTRY(2, 0, L3_3_WB), 409 }; 410 411 static void pvc_mocs_dump(struct xe_mocs_info *info, unsigned int flags, struct xe_gt *gt, 412 struct drm_printer *p) 413 { 414 unsigned int i, j; 415 u32 reg_val; 416 417 drm_printf(p, "LNCFCMOCS[idx] = [ L3CC ] (value)\n\n"); 418 419 for (i = 0, j = 0; i < (info->n_entries + 1) / 2; i++, j++) { 420 if (regs_are_mcr(gt)) 421 reg_val = xe_gt_mcr_unicast_read_any(gt, XEHP_LNCFCMOCS(i)); 422 else 423 reg_val = xe_mmio_read32(gt, XELP_LNCFCMOCS(i)); 424 425 drm_printf(p, "LNCFCMOCS[%2d] = [ %u ] (%#8x)\n", 426 j++, 427 REG_FIELD_GET(L3_CACHEABILITY_MASK, reg_val), 428 reg_val); 429 430 drm_printf(p, "LNCFCMOCS[%2d] = [ %u ] (%#8x)\n", 431 j, 432 REG_FIELD_GET(L3_UPPER_IDX_CACHEABILITY_MASK, reg_val), 433 reg_val); 434 } 435 } 436 437 static const struct xe_mocs_ops pvc_mocs_ops = { 438 .dump = pvc_mocs_dump, 439 }; 440 441 static const struct xe_mocs_entry mtl_mocs_desc[] = { 442 /* Error - Reserved for Non-Use */ 443 MOCS_ENTRY(0, 444 0, 445 L3_LKUP(1) | L3_3_WB), 446 /* Cached - L3 + L4 */ 447 MOCS_ENTRY(1, 448 IG_PAT, 449 L3_LKUP(1) | L3_3_WB), 450 /* L4 - GO:L3 */ 451 MOCS_ENTRY(2, 452 IG_PAT, 453 L3_LKUP(1) | L3_1_UC), 454 /* Uncached - GO:L3 */ 455 MOCS_ENTRY(3, 456 IG_PAT | L4_3_UC, 457 L3_LKUP(1) | L3_1_UC), 458 /* L4 - GO:Mem */ 459 MOCS_ENTRY(4, 460 IG_PAT, 461 L3_LKUP(1) | L3_GLBGO(1) | L3_1_UC), 462 /* Uncached - GO:Mem */ 463 MOCS_ENTRY(5, 464 IG_PAT | L4_3_UC, 465 L3_LKUP(1) | L3_GLBGO(1) | L3_1_UC), 466 /* L4 - L3:NoLKUP; GO:L3 */ 467 MOCS_ENTRY(6, 468 IG_PAT, 469 L3_1_UC), 470 /* Uncached - L3:NoLKUP; GO:L3 */ 471 MOCS_ENTRY(7, 472 IG_PAT | L4_3_UC, 473 L3_1_UC), 474 /* L4 - L3:NoLKUP; GO:Mem */ 475 MOCS_ENTRY(8, 476 IG_PAT, 477 L3_GLBGO(1) | L3_1_UC), 478 /* Uncached - L3:NoLKUP; GO:Mem */ 479 MOCS_ENTRY(9, 480 IG_PAT | L4_3_UC, 481 L3_GLBGO(1) | L3_1_UC), 482 /* Display - L3; L4:WT */ 483 MOCS_ENTRY(14, 484 IG_PAT | L4_1_WT, 485 L3_LKUP(1) | L3_3_WB), 486 /* CCS - Non-Displayable */ 487 MOCS_ENTRY(15, 488 IG_PAT, 489 L3_GLBGO(1) | L3_1_UC), 490 }; 491 492 static void mtl_mocs_dump(struct xe_mocs_info *info, unsigned int flags, 493 struct xe_gt *gt, struct drm_printer *p) 494 { 495 unsigned int i; 496 u32 reg_val; 497 498 drm_printf(p, "Global mocs table configuration:\n"); 499 drm_printf(p, "GLOB_MOCS[idx] = [IG_PAT, L4_CACHE_POLICY] (value)\n\n"); 500 501 for (i = 0; i < info->n_entries; i++) { 502 if (regs_are_mcr(gt)) 503 reg_val = xe_gt_mcr_unicast_read_any(gt, XEHP_GLOBAL_MOCS(i)); 504 else 505 reg_val = xe_mmio_read32(gt, XELP_GLOBAL_MOCS(i)); 506 507 drm_printf(p, "GLOB_MOCS[%2d] = [%u, %u] (%#8x)\n", 508 i, 509 !!(reg_val & IG_PAT), 510 REG_FIELD_GET(L4_CACHE_POLICY_MASK, reg_val), 511 reg_val); 512 } 513 514 /* MTL lncf mocs table pattern is similar to that of xehp */ 515 xehp_lncf_dump(info, flags, gt, p); 516 } 517 518 static const struct xe_mocs_ops mtl_mocs_ops = { 519 .dump = mtl_mocs_dump, 520 }; 521 522 static const struct xe_mocs_entry xe2_mocs_table[] = { 523 /* Defer to PAT */ 524 MOCS_ENTRY(0, XE2_L3_0_WB | L4_3_UC, 0), 525 /* Cached L3, Uncached L4 */ 526 MOCS_ENTRY(1, IG_PAT | XE2_L3_0_WB | L4_3_UC, 0), 527 /* Uncached L3, Cached L4 */ 528 MOCS_ENTRY(2, IG_PAT | XE2_L3_3_UC | L4_0_WB, 0), 529 /* Uncached L3 + L4 */ 530 MOCS_ENTRY(3, IG_PAT | XE2_L3_3_UC | L4_3_UC, 0), 531 /* Cached L3 + L4 */ 532 MOCS_ENTRY(4, IG_PAT | XE2_L3_0_WB | L4_0_WB, 0), 533 }; 534 535 static void xe2_mocs_dump(struct xe_mocs_info *info, unsigned int flags, 536 struct xe_gt *gt, struct drm_printer *p) 537 { 538 unsigned int i; 539 u32 reg_val; 540 541 drm_printf(p, "Global mocs table configuration:\n"); 542 drm_printf(p, "GLOB_MOCS[idx] = [IG_PAT, L3_CLOS, L3_CACHE_POLICY, L4_CACHE_POLICY] (value)\n\n"); 543 544 for (i = 0; i < info->n_entries; i++) { 545 if (regs_are_mcr(gt)) 546 reg_val = xe_gt_mcr_unicast_read_any(gt, XEHP_GLOBAL_MOCS(i)); 547 else 548 reg_val = xe_mmio_read32(gt, XELP_GLOBAL_MOCS(i)); 549 550 drm_printf(p, "GLOB_MOCS[%2d] = [%u, %u, %u] (%#8x)\n", 551 i, 552 !!(reg_val & IG_PAT), 553 REG_FIELD_GET(XE2_L3_CLOS_MASK, reg_val), 554 REG_FIELD_GET(L4_CACHE_POLICY_MASK, reg_val), 555 reg_val); 556 } 557 } 558 559 static const struct xe_mocs_ops xe2_mocs_ops = { 560 .dump = xe2_mocs_dump, 561 }; 562 563 static unsigned int get_mocs_settings(struct xe_device *xe, 564 struct xe_mocs_info *info) 565 { 566 unsigned int flags = 0; 567 568 memset(info, 0, sizeof(struct xe_mocs_info)); 569 570 switch (xe->info.platform) { 571 case XE_LUNARLAKE: 572 case XE_BATTLEMAGE: 573 info->ops = &xe2_mocs_ops; 574 info->size = ARRAY_SIZE(xe2_mocs_table); 575 info->table = xe2_mocs_table; 576 info->n_entries = XE2_NUM_MOCS_ENTRIES; 577 info->uc_index = 3; 578 info->wb_index = 4; 579 info->unused_entries_index = 4; 580 break; 581 case XE_PVC: 582 info->ops = &pvc_mocs_ops; 583 info->size = ARRAY_SIZE(pvc_mocs_desc); 584 info->table = pvc_mocs_desc; 585 info->n_entries = PVC_NUM_MOCS_ENTRIES; 586 info->uc_index = 1; 587 info->wb_index = 2; 588 info->unused_entries_index = 2; 589 break; 590 case XE_METEORLAKE: 591 info->ops = &mtl_mocs_ops; 592 info->size = ARRAY_SIZE(mtl_mocs_desc); 593 info->table = mtl_mocs_desc; 594 info->n_entries = MTL_NUM_MOCS_ENTRIES; 595 info->uc_index = 9; 596 info->unused_entries_index = 1; 597 break; 598 case XE_DG2: 599 info->ops = &xehp_mocs_ops; 600 info->size = ARRAY_SIZE(dg2_mocs_desc); 601 info->table = dg2_mocs_desc; 602 info->uc_index = 1; 603 /* 604 * Last entry is RO on hardware, don't bother with what was 605 * written when checking later 606 */ 607 info->n_entries = XELP_NUM_MOCS_ENTRIES - 1; 608 info->unused_entries_index = 3; 609 break; 610 case XE_DG1: 611 info->ops = &xelp_mocs_ops; 612 info->size = ARRAY_SIZE(dg1_mocs_desc); 613 info->table = dg1_mocs_desc; 614 info->uc_index = 1; 615 info->n_entries = XELP_NUM_MOCS_ENTRIES; 616 info->unused_entries_index = 5; 617 break; 618 case XE_TIGERLAKE: 619 case XE_ROCKETLAKE: 620 case XE_ALDERLAKE_S: 621 case XE_ALDERLAKE_P: 622 case XE_ALDERLAKE_N: 623 info->ops = &xelp_mocs_ops; 624 info->size = ARRAY_SIZE(gen12_mocs_desc); 625 info->table = gen12_mocs_desc; 626 info->n_entries = XELP_NUM_MOCS_ENTRIES; 627 info->uc_index = 3; 628 info->unused_entries_index = 2; 629 break; 630 default: 631 drm_err(&xe->drm, "Platform that should have a MOCS table does not.\n"); 632 return 0; 633 } 634 635 /* 636 * Index 0 is a reserved/unused table entry on most platforms, but 637 * even on those where it does represent a legitimate MOCS entry, it 638 * never represents the "most cached, least coherent" behavior we want 639 * to populate undefined table rows with. So if unused_entries_index 640 * is still 0 at this point, we'll assume that it was omitted by 641 * mistake in the switch statement above. 642 */ 643 xe_assert(xe, info->unused_entries_index != 0); 644 645 xe_assert(xe, !info->ops || info->ops->dump); 646 647 if (XE_WARN_ON(info->size > info->n_entries)) { 648 info->table = NULL; 649 return 0; 650 } 651 652 if (!IS_DGFX(xe) || GRAPHICS_VER(xe) >= 20) 653 flags |= HAS_GLOBAL_MOCS; 654 if (GRAPHICS_VER(xe) < 20) 655 flags |= HAS_LNCF_MOCS; 656 657 return flags; 658 } 659 660 /* 661 * Get control_value from MOCS entry. If the table entry is not defined, the 662 * settings from unused_entries_index will be returned. 663 */ 664 static u32 get_entry_control(const struct xe_mocs_info *info, 665 unsigned int index) 666 { 667 if (index < info->size && info->table[index].used) 668 return info->table[index].control_value; 669 return info->table[info->unused_entries_index].control_value; 670 } 671 672 static void __init_mocs_table(struct xe_gt *gt, 673 const struct xe_mocs_info *info) 674 { 675 unsigned int i; 676 u32 mocs; 677 678 xe_gt_WARN_ONCE(gt, !info->unused_entries_index, 679 "Unused entries index should have been defined\n"); 680 681 mocs_dbg(gt, "mocs entries: %d\n", info->n_entries); 682 683 for (i = 0; i < info->n_entries; i++) { 684 mocs = get_entry_control(info, i); 685 686 mocs_dbg(gt, "GLOB_MOCS[%d] 0x%x 0x%x\n", i, 687 XELP_GLOBAL_MOCS(i).addr, mocs); 688 689 if (regs_are_mcr(gt)) 690 xe_gt_mcr_multicast_write(gt, XEHP_GLOBAL_MOCS(i), mocs); 691 else 692 xe_mmio_write32(gt, XELP_GLOBAL_MOCS(i), mocs); 693 } 694 } 695 696 /* 697 * Get l3cc_value from MOCS entry taking into account when it's not used 698 * then if unused_entries_index is not zero then its value will be returned 699 * otherwise I915_MOCS_PTE's value is returned in this case. 700 */ 701 static u16 get_entry_l3cc(const struct xe_mocs_info *info, 702 unsigned int index) 703 { 704 if (index < info->size && info->table[index].used) 705 return info->table[index].l3cc_value; 706 return info->table[info->unused_entries_index].l3cc_value; 707 } 708 709 static u32 l3cc_combine(u16 low, u16 high) 710 { 711 return low | (u32)high << 16; 712 } 713 714 static void init_l3cc_table(struct xe_gt *gt, 715 const struct xe_mocs_info *info) 716 { 717 unsigned int i; 718 u32 l3cc; 719 720 mocs_dbg(gt, "l3cc entries: %d\n", info->n_entries); 721 722 for (i = 0; i < (info->n_entries + 1) / 2; i++) { 723 l3cc = l3cc_combine(get_entry_l3cc(info, 2 * i), 724 get_entry_l3cc(info, 2 * i + 1)); 725 726 mocs_dbg(gt, "LNCFCMOCS[%d] 0x%x 0x%x\n", i, 727 XELP_LNCFCMOCS(i).addr, l3cc); 728 729 if (regs_are_mcr(gt)) 730 xe_gt_mcr_multicast_write(gt, XEHP_LNCFCMOCS(i), l3cc); 731 else 732 xe_mmio_write32(gt, XELP_LNCFCMOCS(i), l3cc); 733 } 734 } 735 736 void xe_mocs_init_early(struct xe_gt *gt) 737 { 738 struct xe_mocs_info table; 739 740 get_mocs_settings(gt_to_xe(gt), &table); 741 gt->mocs.uc_index = table.uc_index; 742 gt->mocs.wb_index = table.wb_index; 743 } 744 745 void xe_mocs_init(struct xe_gt *gt) 746 { 747 struct xe_mocs_info table; 748 unsigned int flags; 749 750 if (IS_SRIOV_VF(gt_to_xe(gt))) 751 return; 752 753 /* 754 * MOCS settings are split between "GLOB_MOCS" and/or "LNCFCMOCS" 755 * registers depending on platform. 756 * 757 * These registers should be programmed before GuC initialization 758 * since their values will affect some of the memory transactions 759 * performed by the GuC. 760 */ 761 flags = get_mocs_settings(gt_to_xe(gt), &table); 762 mocs_dbg(gt, "flag:0x%x\n", flags); 763 764 if (IS_SRIOV_VF(gt_to_xe(gt))) 765 return; 766 767 if (flags & HAS_GLOBAL_MOCS) 768 __init_mocs_table(gt, &table); 769 if (flags & HAS_LNCF_MOCS) 770 init_l3cc_table(gt, &table); 771 } 772 773 void xe_mocs_dump(struct xe_gt *gt, struct drm_printer *p) 774 { 775 struct xe_mocs_info table; 776 unsigned int flags; 777 u32 ret; 778 struct xe_device *xe = gt_to_xe(gt); 779 780 flags = get_mocs_settings(xe, &table); 781 782 if (!table.ops->dump) 783 return; 784 785 xe_pm_runtime_get_noresume(xe); 786 ret = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); 787 788 if (ret) 789 goto err_fw; 790 791 table.ops->dump(&table, flags, gt, p); 792 793 xe_force_wake_put(gt_to_fw(gt), XE_FW_GT); 794 795 err_fw: 796 xe_assert(xe, !ret); 797 xe_pm_runtime_put(xe); 798 } 799 800 #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) 801 #include "tests/xe_mocs.c" 802 #endif 803