1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include "xe_mocs.h" 7 8 #include "regs/xe_gt_regs.h" 9 #include "xe_device.h" 10 #include "xe_exec_queue.h" 11 #include "xe_force_wake.h" 12 #include "xe_gt.h" 13 #include "xe_gt_mcr.h" 14 #include "xe_gt_printk.h" 15 #include "xe_mmio.h" 16 #include "xe_platform_types.h" 17 #include "xe_pm.h" 18 #include "xe_sriov.h" 19 20 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG) 21 #define mocs_dbg xe_gt_dbg 22 #else 23 __printf(2, 3) 24 static inline void mocs_dbg(const struct xe_gt *gt, 25 const char *format, ...) 26 { /* noop */ } 27 #endif 28 29 enum { 30 HAS_GLOBAL_MOCS = BIT(0), 31 HAS_LNCF_MOCS = BIT(1), 32 }; 33 34 struct xe_mocs_entry { 35 u32 control_value; 36 u16 l3cc_value; 37 u16 used; 38 }; 39 40 struct xe_mocs_info; 41 42 struct xe_mocs_ops { 43 void (*dump)(struct xe_mocs_info *mocs, unsigned int flags, 44 struct xe_gt *gt, struct drm_printer *p); 45 }; 46 47 struct xe_mocs_info { 48 /* 49 * Size of the spec's suggested MOCS programming table. The list of 50 * table entries from the spec can potentially be smaller than the 51 * number of hardware registers used to program the MOCS table; in such 52 * cases the registers for the remaining indices will be programmed to 53 * match unused_entries_index. 54 */ 55 unsigned int table_size; 56 /* Number of MOCS entries supported by the hardware */ 57 unsigned int num_mocs_regs; 58 const struct xe_mocs_entry *table; 59 const struct xe_mocs_ops *ops; 60 u8 uc_index; 61 u8 wb_index; 62 u8 unused_entries_index; 63 }; 64 65 /* Defines for the tables (GLOB_MOCS_0 - GLOB_MOCS_16) */ 66 #define IG_PAT REG_BIT(8) 67 #define L3_CACHE_POLICY_MASK REG_GENMASK(5, 4) 68 #define L4_CACHE_POLICY_MASK REG_GENMASK(3, 2) 69 70 /* Helper defines */ 71 #define XELP_NUM_MOCS_ENTRIES 64 /* 63-64 are reserved, but configured. */ 72 #define PVC_NUM_MOCS_ENTRIES 3 73 #define MTL_NUM_MOCS_ENTRIES 16 74 #define XE2_NUM_MOCS_ENTRIES 16 75 76 /* (e)LLC caching options */ 77 /* 78 * Note: LE_0_PAGETABLE works only up to Gen11; for newer gens it means 79 * the same as LE_UC 80 */ 81 #define LE_0_PAGETABLE LE_CACHEABILITY(0) 82 #define LE_1_UC LE_CACHEABILITY(1) 83 #define LE_2_WT LE_CACHEABILITY(2) 84 #define LE_3_WB LE_CACHEABILITY(3) 85 86 /* Target cache */ 87 #define LE_TC_0_PAGETABLE LE_TGT_CACHE(0) 88 #define LE_TC_1_LLC LE_TGT_CACHE(1) 89 #define LE_TC_2_LLC_ELLC LE_TGT_CACHE(2) 90 #define LE_TC_3_LLC_ELLC_ALT LE_TGT_CACHE(3) 91 92 /* L3 caching options */ 93 #define L3_0_DIRECT L3_CACHEABILITY(0) 94 #define L3_1_UC L3_CACHEABILITY(1) 95 #define L3_2_RESERVED L3_CACHEABILITY(2) 96 #define L3_3_WB L3_CACHEABILITY(3) 97 98 /* L4 caching options */ 99 #define L4_0_WB REG_FIELD_PREP(L4_CACHE_POLICY_MASK, 0) 100 #define L4_1_WT REG_FIELD_PREP(L4_CACHE_POLICY_MASK, 1) 101 #define L4_3_UC REG_FIELD_PREP(L4_CACHE_POLICY_MASK, 3) 102 103 #define XE2_L3_0_WB REG_FIELD_PREP(L3_CACHE_POLICY_MASK, 0) 104 /* XD: WB Transient Display */ 105 #define XE2_L3_1_XD REG_FIELD_PREP(L3_CACHE_POLICY_MASK, 1) 106 #define XE2_L3_3_UC REG_FIELD_PREP(L3_CACHE_POLICY_MASK, 3) 107 108 #define XE2_L3_CLOS_MASK REG_GENMASK(7, 6) 109 110 #define MOCS_ENTRY(__idx, __control_value, __l3cc_value) \ 111 [__idx] = { \ 112 .control_value = __control_value, \ 113 .l3cc_value = __l3cc_value, \ 114 .used = 1, \ 115 } 116 117 /* 118 * MOCS tables 119 * 120 * These are the MOCS tables that are programmed across all the rings. 121 * The control value is programmed to all the rings that support the 122 * MOCS registers. While the l3cc_values are only programmed to the 123 * LNCFCMOCS0 - LNCFCMOCS32 registers. 124 * 125 * These tables are intended to be kept reasonably consistent across 126 * HW platforms, and for ICL+, be identical across OSes. To achieve 127 * that, the list of entries is published as part of bspec. 128 * 129 * Entries not part of the following tables are undefined as far as userspace is 130 * concerned and shouldn't be relied upon. The last few entries are reserved by 131 * the hardware. They should be initialized according to bspec and never used. 132 * 133 * NOTE1: These tables are part of bspec and defined as part of the hardware 134 * interface. It is expected that, for specific hardware platform, existing 135 * entries will remain constant and the table will only be updated by adding new 136 * entries, filling unused positions. 137 * 138 * NOTE2: Reserved and unspecified MOCS indices have been set to L3 WB. These 139 * reserved entries should never be used. They may be changed to low performant 140 * variants with better coherency in the future if more entries are needed. 141 */ 142 143 static const struct xe_mocs_entry gen12_mocs_desc[] = { 144 /* Base - L3 + LLC */ 145 MOCS_ENTRY(2, 146 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), 147 L3_3_WB), 148 /* Base - Uncached */ 149 MOCS_ENTRY(3, 150 LE_1_UC | LE_TC_1_LLC, 151 L3_1_UC), 152 /* Base - L3 */ 153 MOCS_ENTRY(4, 154 LE_1_UC | LE_TC_1_LLC, 155 L3_3_WB), 156 /* Base - LLC */ 157 MOCS_ENTRY(5, 158 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), 159 L3_1_UC), 160 /* Age 0 - LLC */ 161 MOCS_ENTRY(6, 162 LE_3_WB | LE_TC_1_LLC | LE_LRUM(1), 163 L3_1_UC), 164 /* Age 0 - L3 + LLC */ 165 MOCS_ENTRY(7, 166 LE_3_WB | LE_TC_1_LLC | LE_LRUM(1), 167 L3_3_WB), 168 /* Age: Don't Chg. - LLC */ 169 MOCS_ENTRY(8, 170 LE_3_WB | LE_TC_1_LLC | LE_LRUM(2), 171 L3_1_UC), 172 /* Age: Don't Chg. - L3 + LLC */ 173 MOCS_ENTRY(9, 174 LE_3_WB | LE_TC_1_LLC | LE_LRUM(2), 175 L3_3_WB), 176 /* No AOM - LLC */ 177 MOCS_ENTRY(10, 178 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_AOM(1), 179 L3_1_UC), 180 /* No AOM - L3 + LLC */ 181 MOCS_ENTRY(11, 182 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_AOM(1), 183 L3_3_WB), 184 /* No AOM; Age 0 - LLC */ 185 MOCS_ENTRY(12, 186 LE_3_WB | LE_TC_1_LLC | LE_LRUM(1) | LE_AOM(1), 187 L3_1_UC), 188 /* No AOM; Age 0 - L3 + LLC */ 189 MOCS_ENTRY(13, 190 LE_3_WB | LE_TC_1_LLC | LE_LRUM(1) | LE_AOM(1), 191 L3_3_WB), 192 /* No AOM; Age:DC - LLC */ 193 MOCS_ENTRY(14, 194 LE_3_WB | LE_TC_1_LLC | LE_LRUM(2) | LE_AOM(1), 195 L3_1_UC), 196 /* No AOM; Age:DC - L3 + LLC */ 197 MOCS_ENTRY(15, 198 LE_3_WB | LE_TC_1_LLC | LE_LRUM(2) | LE_AOM(1), 199 L3_3_WB), 200 /* Self-Snoop - L3 + LLC */ 201 MOCS_ENTRY(18, 202 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SSE(3), 203 L3_3_WB), 204 /* Skip Caching - L3 + LLC(12.5%) */ 205 MOCS_ENTRY(19, 206 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SCC(7), 207 L3_3_WB), 208 /* Skip Caching - L3 + LLC(25%) */ 209 MOCS_ENTRY(20, 210 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SCC(3), 211 L3_3_WB), 212 /* Skip Caching - L3 + LLC(50%) */ 213 MOCS_ENTRY(21, 214 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SCC(1), 215 L3_3_WB), 216 /* Skip Caching - L3 + LLC(75%) */ 217 MOCS_ENTRY(22, 218 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_RSC(1) | LE_SCC(3), 219 L3_3_WB), 220 /* Skip Caching - L3 + LLC(87.5%) */ 221 MOCS_ENTRY(23, 222 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_RSC(1) | LE_SCC(7), 223 L3_3_WB), 224 /* Implicitly enable L1 - HDC:L1 + L3 + LLC */ 225 MOCS_ENTRY(48, 226 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), 227 L3_3_WB), 228 /* Implicitly enable L1 - HDC:L1 + L3 */ 229 MOCS_ENTRY(49, 230 LE_1_UC | LE_TC_1_LLC, 231 L3_3_WB), 232 /* Implicitly enable L1 - HDC:L1 + LLC */ 233 MOCS_ENTRY(50, 234 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), 235 L3_1_UC), 236 /* Implicitly enable L1 - HDC:L1 */ 237 MOCS_ENTRY(51, 238 LE_1_UC | LE_TC_1_LLC, 239 L3_1_UC), 240 /* HW Special Case (CCS) */ 241 MOCS_ENTRY(60, 242 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), 243 L3_1_UC), 244 /* HW Special Case (Displayable) */ 245 MOCS_ENTRY(61, 246 LE_1_UC | LE_TC_1_LLC, 247 L3_3_WB), 248 /* HW Reserved - SW program but never use */ 249 MOCS_ENTRY(62, 250 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), 251 L3_1_UC), 252 /* HW Reserved - SW program but never use */ 253 MOCS_ENTRY(63, 254 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), 255 L3_1_UC) 256 }; 257 258 static bool regs_are_mcr(struct xe_gt *gt) 259 { 260 struct xe_device *xe = gt_to_xe(gt); 261 262 if (xe_gt_is_media_type(gt)) 263 return MEDIA_VER(xe) >= 20; 264 else 265 return GRAPHICS_VERx100(xe) >= 1250; 266 } 267 268 static void xelp_lncf_dump(struct xe_mocs_info *info, struct xe_gt *gt, struct drm_printer *p) 269 { 270 unsigned int i, j; 271 u32 reg_val; 272 273 drm_printf(p, "LNCFCMOCS[idx] = [ESC, SCC, L3CC] (value)\n\n"); 274 275 for (i = 0, j = 0; i < (info->num_mocs_regs + 1) / 2; i++, j++) { 276 if (regs_are_mcr(gt)) 277 reg_val = xe_gt_mcr_unicast_read_any(gt, XEHP_LNCFCMOCS(i)); 278 else 279 reg_val = xe_mmio_read32(>->mmio, XELP_LNCFCMOCS(i)); 280 281 drm_printf(p, "LNCFCMOCS[%2d] = [%u, %u, %u] (%#8x)\n", 282 j++, 283 !!(reg_val & L3_ESC_MASK), 284 REG_FIELD_GET(L3_SCC_MASK, reg_val), 285 REG_FIELD_GET(L3_CACHEABILITY_MASK, reg_val), 286 reg_val); 287 288 drm_printf(p, "LNCFCMOCS[%2d] = [%u, %u, %u] (%#8x)\n", 289 j, 290 !!(reg_val & L3_UPPER_IDX_ESC_MASK), 291 REG_FIELD_GET(L3_UPPER_IDX_SCC_MASK, reg_val), 292 REG_FIELD_GET(L3_UPPER_IDX_CACHEABILITY_MASK, reg_val), 293 reg_val); 294 } 295 } 296 297 static void xelp_mocs_dump(struct xe_mocs_info *info, unsigned int flags, 298 struct xe_gt *gt, struct drm_printer *p) 299 { 300 unsigned int i; 301 u32 reg_val; 302 303 if (flags & HAS_GLOBAL_MOCS) { 304 drm_printf(p, "Global mocs table configuration:\n"); 305 drm_printf(p, "GLOB_MOCS[idx] = [LeCC, TC, LRUM, AOM, RSC, SCC, PFM, SCF, CoS, SSE] (value)\n\n"); 306 307 for (i = 0; i < info->num_mocs_regs; i++) { 308 if (regs_are_mcr(gt)) 309 reg_val = xe_gt_mcr_unicast_read_any(gt, XEHP_GLOBAL_MOCS(i)); 310 else 311 reg_val = xe_mmio_read32(>->mmio, XELP_GLOBAL_MOCS(i)); 312 313 drm_printf(p, "GLOB_MOCS[%2d] = [%u, %u, %u, %u, %u, %u, %u, %u, %u, %u ] (%#8x)\n", 314 i, 315 REG_FIELD_GET(LE_CACHEABILITY_MASK, reg_val), 316 REG_FIELD_GET(LE_TGT_CACHE_MASK, reg_val), 317 REG_FIELD_GET(LE_LRUM_MASK, reg_val), 318 !!(reg_val & LE_AOM_MASK), 319 !!(reg_val & LE_RSC_MASK), 320 REG_FIELD_GET(LE_SCC_MASK, reg_val), 321 REG_FIELD_GET(LE_PFM_MASK, reg_val), 322 !!(reg_val & LE_SCF_MASK), 323 REG_FIELD_GET(LE_COS_MASK, reg_val), 324 REG_FIELD_GET(LE_SSE_MASK, reg_val), 325 reg_val); 326 } 327 } 328 329 xelp_lncf_dump(info, gt, p); 330 } 331 332 static const struct xe_mocs_ops xelp_mocs_ops = { 333 .dump = xelp_mocs_dump, 334 }; 335 336 static const struct xe_mocs_entry dg1_mocs_desc[] = { 337 /* UC */ 338 MOCS_ENTRY(1, 0, L3_1_UC), 339 /* WB - L3 */ 340 MOCS_ENTRY(5, 0, L3_3_WB), 341 /* WB - L3 50% */ 342 MOCS_ENTRY(6, 0, L3_ESC(1) | L3_SCC(1) | L3_3_WB), 343 /* WB - L3 25% */ 344 MOCS_ENTRY(7, 0, L3_ESC(1) | L3_SCC(3) | L3_3_WB), 345 /* WB - L3 12.5% */ 346 MOCS_ENTRY(8, 0, L3_ESC(1) | L3_SCC(7) | L3_3_WB), 347 348 /* HDC:L1 + L3 */ 349 MOCS_ENTRY(48, 0, L3_3_WB), 350 /* HDC:L1 */ 351 MOCS_ENTRY(49, 0, L3_1_UC), 352 353 /* HW Reserved */ 354 MOCS_ENTRY(60, 0, L3_1_UC), 355 MOCS_ENTRY(61, 0, L3_1_UC), 356 MOCS_ENTRY(62, 0, L3_1_UC), 357 MOCS_ENTRY(63, 0, L3_1_UC), 358 }; 359 360 static const struct xe_mocs_entry dg2_mocs_desc[] = { 361 /* UC - Coherent; GO:L3 */ 362 MOCS_ENTRY(0, 0, L3_1_UC | L3_LKUP(1)), 363 /* UC - Coherent; GO:Memory */ 364 MOCS_ENTRY(1, 0, L3_1_UC | L3_GLBGO(1) | L3_LKUP(1)), 365 /* UC - Non-Coherent; GO:Memory */ 366 MOCS_ENTRY(2, 0, L3_1_UC | L3_GLBGO(1)), 367 368 /* WB - LC */ 369 MOCS_ENTRY(3, 0, L3_3_WB | L3_LKUP(1)), 370 }; 371 372 static void xehp_lncf_dump(struct xe_mocs_info *info, unsigned int flags, 373 struct xe_gt *gt, struct drm_printer *p) 374 { 375 unsigned int i, j; 376 u32 reg_val; 377 378 drm_printf(p, "LNCFCMOCS[idx] = [UCL3LOOKUP, GLBGO, L3CC] (value)\n\n"); 379 380 for (i = 0, j = 0; i < (info->num_mocs_regs + 1) / 2; i++, j++) { 381 if (regs_are_mcr(gt)) 382 reg_val = xe_gt_mcr_unicast_read_any(gt, XEHP_LNCFCMOCS(i)); 383 else 384 reg_val = xe_mmio_read32(>->mmio, XELP_LNCFCMOCS(i)); 385 386 drm_printf(p, "LNCFCMOCS[%2d] = [%u, %u, %u] (%#8x)\n", 387 j++, 388 !!(reg_val & L3_LKUP_MASK), 389 !!(reg_val & L3_GLBGO_MASK), 390 REG_FIELD_GET(L3_CACHEABILITY_MASK, reg_val), 391 reg_val); 392 393 drm_printf(p, "LNCFCMOCS[%2d] = [%u, %u, %u] (%#8x)\n", 394 j, 395 !!(reg_val & L3_UPPER_LKUP_MASK), 396 !!(reg_val & L3_UPPER_GLBGO_MASK), 397 REG_FIELD_GET(L3_UPPER_IDX_CACHEABILITY_MASK, reg_val), 398 reg_val); 399 } 400 } 401 402 static const struct xe_mocs_ops xehp_mocs_ops = { 403 .dump = xehp_lncf_dump, 404 }; 405 406 static const struct xe_mocs_entry pvc_mocs_desc[] = { 407 /* Error */ 408 MOCS_ENTRY(0, 0, L3_3_WB), 409 410 /* UC */ 411 MOCS_ENTRY(1, 0, L3_1_UC), 412 413 /* WB */ 414 MOCS_ENTRY(2, 0, L3_3_WB), 415 }; 416 417 static void pvc_mocs_dump(struct xe_mocs_info *info, unsigned int flags, struct xe_gt *gt, 418 struct drm_printer *p) 419 { 420 unsigned int i, j; 421 u32 reg_val; 422 423 drm_printf(p, "LNCFCMOCS[idx] = [ L3CC ] (value)\n\n"); 424 425 for (i = 0, j = 0; i < (info->num_mocs_regs + 1) / 2; i++, j++) { 426 if (regs_are_mcr(gt)) 427 reg_val = xe_gt_mcr_unicast_read_any(gt, XEHP_LNCFCMOCS(i)); 428 else 429 reg_val = xe_mmio_read32(>->mmio, XELP_LNCFCMOCS(i)); 430 431 drm_printf(p, "LNCFCMOCS[%2d] = [ %u ] (%#8x)\n", 432 j++, 433 REG_FIELD_GET(L3_CACHEABILITY_MASK, reg_val), 434 reg_val); 435 436 drm_printf(p, "LNCFCMOCS[%2d] = [ %u ] (%#8x)\n", 437 j, 438 REG_FIELD_GET(L3_UPPER_IDX_CACHEABILITY_MASK, reg_val), 439 reg_val); 440 } 441 } 442 443 static const struct xe_mocs_ops pvc_mocs_ops = { 444 .dump = pvc_mocs_dump, 445 }; 446 447 static const struct xe_mocs_entry mtl_mocs_desc[] = { 448 /* Error - Reserved for Non-Use */ 449 MOCS_ENTRY(0, 450 0, 451 L3_LKUP(1) | L3_3_WB), 452 /* Cached - L3 + L4 */ 453 MOCS_ENTRY(1, 454 IG_PAT, 455 L3_LKUP(1) | L3_3_WB), 456 /* L4 - GO:L3 */ 457 MOCS_ENTRY(2, 458 IG_PAT, 459 L3_LKUP(1) | L3_1_UC), 460 /* Uncached - GO:L3 */ 461 MOCS_ENTRY(3, 462 IG_PAT | L4_3_UC, 463 L3_LKUP(1) | L3_1_UC), 464 /* L4 - GO:Mem */ 465 MOCS_ENTRY(4, 466 IG_PAT, 467 L3_LKUP(1) | L3_GLBGO(1) | L3_1_UC), 468 /* Uncached - GO:Mem */ 469 MOCS_ENTRY(5, 470 IG_PAT | L4_3_UC, 471 L3_LKUP(1) | L3_GLBGO(1) | L3_1_UC), 472 /* L4 - L3:NoLKUP; GO:L3 */ 473 MOCS_ENTRY(6, 474 IG_PAT, 475 L3_1_UC), 476 /* Uncached - L3:NoLKUP; GO:L3 */ 477 MOCS_ENTRY(7, 478 IG_PAT | L4_3_UC, 479 L3_1_UC), 480 /* L4 - L3:NoLKUP; GO:Mem */ 481 MOCS_ENTRY(8, 482 IG_PAT, 483 L3_GLBGO(1) | L3_1_UC), 484 /* Uncached - L3:NoLKUP; GO:Mem */ 485 MOCS_ENTRY(9, 486 IG_PAT | L4_3_UC, 487 L3_GLBGO(1) | L3_1_UC), 488 /* Display - L3; L4:WT */ 489 MOCS_ENTRY(14, 490 IG_PAT | L4_1_WT, 491 L3_LKUP(1) | L3_3_WB), 492 /* CCS - Non-Displayable */ 493 MOCS_ENTRY(15, 494 IG_PAT, 495 L3_GLBGO(1) | L3_1_UC), 496 }; 497 498 static void mtl_mocs_dump(struct xe_mocs_info *info, unsigned int flags, 499 struct xe_gt *gt, struct drm_printer *p) 500 { 501 unsigned int i; 502 u32 reg_val; 503 504 drm_printf(p, "Global mocs table configuration:\n"); 505 drm_printf(p, "GLOB_MOCS[idx] = [IG_PAT, L4_CACHE_POLICY] (value)\n\n"); 506 507 for (i = 0; i < info->num_mocs_regs; i++) { 508 if (regs_are_mcr(gt)) 509 reg_val = xe_gt_mcr_unicast_read_any(gt, XEHP_GLOBAL_MOCS(i)); 510 else 511 reg_val = xe_mmio_read32(>->mmio, XELP_GLOBAL_MOCS(i)); 512 513 drm_printf(p, "GLOB_MOCS[%2d] = [%u, %u] (%#8x)\n", 514 i, 515 !!(reg_val & IG_PAT), 516 REG_FIELD_GET(L4_CACHE_POLICY_MASK, reg_val), 517 reg_val); 518 } 519 520 /* MTL lncf mocs table pattern is similar to that of xehp */ 521 xehp_lncf_dump(info, flags, gt, p); 522 } 523 524 static const struct xe_mocs_ops mtl_mocs_ops = { 525 .dump = mtl_mocs_dump, 526 }; 527 528 static const struct xe_mocs_entry xe2_mocs_table[] = { 529 /* Defer to PAT */ 530 MOCS_ENTRY(0, XE2_L3_0_WB | L4_3_UC, 0), 531 /* Cached L3, Uncached L4 */ 532 MOCS_ENTRY(1, IG_PAT | XE2_L3_0_WB | L4_3_UC, 0), 533 /* Uncached L3, Cached L4 */ 534 MOCS_ENTRY(2, IG_PAT | XE2_L3_3_UC | L4_0_WB, 0), 535 /* Uncached L3 + L4 */ 536 MOCS_ENTRY(3, IG_PAT | XE2_L3_3_UC | L4_3_UC, 0), 537 /* Cached L3 + L4 */ 538 MOCS_ENTRY(4, IG_PAT | XE2_L3_0_WB | L4_0_WB, 0), 539 }; 540 541 static void xe2_mocs_dump(struct xe_mocs_info *info, unsigned int flags, 542 struct xe_gt *gt, struct drm_printer *p) 543 { 544 unsigned int i; 545 u32 reg_val; 546 547 drm_printf(p, "Global mocs table configuration:\n"); 548 drm_printf(p, "GLOB_MOCS[idx] = [IG_PAT, L3_CLOS, L3_CACHE_POLICY, L4_CACHE_POLICY] (value)\n\n"); 549 550 for (i = 0; i < info->num_mocs_regs; i++) { 551 if (regs_are_mcr(gt)) 552 reg_val = xe_gt_mcr_unicast_read_any(gt, XEHP_GLOBAL_MOCS(i)); 553 else 554 reg_val = xe_mmio_read32(>->mmio, XELP_GLOBAL_MOCS(i)); 555 556 drm_printf(p, "GLOB_MOCS[%2d] = [%u, %u, %u] (%#8x)\n", 557 i, 558 !!(reg_val & IG_PAT), 559 REG_FIELD_GET(XE2_L3_CLOS_MASK, reg_val), 560 REG_FIELD_GET(L4_CACHE_POLICY_MASK, reg_val), 561 reg_val); 562 } 563 } 564 565 static const struct xe_mocs_ops xe2_mocs_ops = { 566 .dump = xe2_mocs_dump, 567 }; 568 569 /* 570 * Note that the "L3" and "L4" register fields actually control the L2 and L3 571 * caches respectively on this platform. 572 */ 573 static const struct xe_mocs_entry xe3p_xpc_mocs_table[] = { 574 /* Defer to PAT */ 575 MOCS_ENTRY(0, XE2_L3_0_WB | L4_3_UC, 0), 576 /* UC */ 577 MOCS_ENTRY(1, IG_PAT | XE2_L3_3_UC | L4_3_UC, 0), 578 /* L2 */ 579 MOCS_ENTRY(2, IG_PAT | XE2_L3_0_WB | L4_3_UC, 0), 580 /* L3 */ 581 MOCS_ENTRY(3, IG_PAT | XE2_L3_3_UC | L4_0_WB, 0), 582 /* L2 + L3 */ 583 MOCS_ENTRY(4, IG_PAT | XE2_L3_0_WB | L4_0_WB, 0), 584 }; 585 586 static unsigned int get_mocs_settings(struct xe_device *xe, 587 struct xe_mocs_info *info) 588 { 589 unsigned int flags = 0; 590 591 memset(info, 0, sizeof(struct xe_mocs_info)); 592 593 switch (xe->info.platform) { 594 case XE_CRESCENTISLAND: 595 info->ops = &xe2_mocs_ops; 596 info->table_size = ARRAY_SIZE(xe3p_xpc_mocs_table); 597 info->table = xe3p_xpc_mocs_table; 598 info->num_mocs_regs = XE2_NUM_MOCS_ENTRIES; 599 info->uc_index = 1; 600 info->wb_index = 4; 601 info->unused_entries_index = 4; 602 break; 603 case XE_NOVALAKE_S: 604 case XE_PANTHERLAKE: 605 case XE_LUNARLAKE: 606 case XE_BATTLEMAGE: 607 info->ops = &xe2_mocs_ops; 608 info->table_size = ARRAY_SIZE(xe2_mocs_table); 609 info->table = xe2_mocs_table; 610 info->num_mocs_regs = XE2_NUM_MOCS_ENTRIES; 611 info->uc_index = 3; 612 info->wb_index = 4; 613 info->unused_entries_index = 4; 614 break; 615 case XE_PVC: 616 info->ops = &pvc_mocs_ops; 617 info->table_size = ARRAY_SIZE(pvc_mocs_desc); 618 info->table = pvc_mocs_desc; 619 info->num_mocs_regs = PVC_NUM_MOCS_ENTRIES; 620 info->uc_index = 1; 621 info->wb_index = 2; 622 info->unused_entries_index = 2; 623 break; 624 case XE_METEORLAKE: 625 info->ops = &mtl_mocs_ops; 626 info->table_size = ARRAY_SIZE(mtl_mocs_desc); 627 info->table = mtl_mocs_desc; 628 info->num_mocs_regs = MTL_NUM_MOCS_ENTRIES; 629 info->uc_index = 9; 630 info->unused_entries_index = 1; 631 break; 632 case XE_DG2: 633 info->ops = &xehp_mocs_ops; 634 info->table_size = ARRAY_SIZE(dg2_mocs_desc); 635 info->table = dg2_mocs_desc; 636 info->uc_index = 1; 637 /* 638 * Last entry is RO on hardware, don't bother with what was 639 * written when checking later 640 */ 641 info->num_mocs_regs = XELP_NUM_MOCS_ENTRIES - 1; 642 info->unused_entries_index = 3; 643 break; 644 case XE_DG1: 645 info->ops = &xelp_mocs_ops; 646 info->table_size = ARRAY_SIZE(dg1_mocs_desc); 647 info->table = dg1_mocs_desc; 648 info->uc_index = 1; 649 info->num_mocs_regs = XELP_NUM_MOCS_ENTRIES; 650 info->unused_entries_index = 5; 651 break; 652 case XE_TIGERLAKE: 653 case XE_ROCKETLAKE: 654 case XE_ALDERLAKE_S: 655 case XE_ALDERLAKE_P: 656 case XE_ALDERLAKE_N: 657 info->ops = &xelp_mocs_ops; 658 info->table_size = ARRAY_SIZE(gen12_mocs_desc); 659 info->table = gen12_mocs_desc; 660 info->num_mocs_regs = XELP_NUM_MOCS_ENTRIES; 661 info->uc_index = 3; 662 info->unused_entries_index = 2; 663 break; 664 default: 665 drm_err(&xe->drm, "Platform that should have a MOCS table does not.\n"); 666 return 0; 667 } 668 669 /* 670 * Index 0 is a reserved/unused table entry on most platforms, but 671 * even on those where it does represent a legitimate MOCS entry, it 672 * never represents the "most cached, least coherent" behavior we want 673 * to populate undefined table rows with. So if unused_entries_index 674 * is still 0 at this point, we'll assume that it was omitted by 675 * mistake in the switch statement above. 676 */ 677 xe_assert(xe, info->unused_entries_index != 0); 678 679 xe_assert(xe, info->ops && info->ops->dump); 680 xe_assert(xe, info->table_size <= info->num_mocs_regs); 681 682 if (!IS_DGFX(xe) || GRAPHICS_VER(xe) >= 20) 683 flags |= HAS_GLOBAL_MOCS; 684 if (GRAPHICS_VER(xe) < 20) 685 flags |= HAS_LNCF_MOCS; 686 687 return flags; 688 } 689 690 /* 691 * Get control_value from MOCS entry. If the table entry is not defined, the 692 * settings from unused_entries_index will be returned. 693 */ 694 static u32 get_entry_control(const struct xe_mocs_info *info, 695 unsigned int index) 696 { 697 if (index < info->table_size && info->table[index].used) 698 return info->table[index].control_value; 699 return info->table[info->unused_entries_index].control_value; 700 } 701 702 static void __init_mocs_table(struct xe_gt *gt, 703 const struct xe_mocs_info *info) 704 { 705 unsigned int i; 706 u32 mocs; 707 708 mocs_dbg(gt, "mocs entries: %d\n", info->num_mocs_regs); 709 710 for (i = 0; i < info->num_mocs_regs; i++) { 711 mocs = get_entry_control(info, i); 712 713 mocs_dbg(gt, "GLOB_MOCS[%d] 0x%x 0x%x\n", i, 714 XELP_GLOBAL_MOCS(i).addr, mocs); 715 716 if (regs_are_mcr(gt)) 717 xe_gt_mcr_multicast_write(gt, XEHP_GLOBAL_MOCS(i), mocs); 718 else 719 xe_mmio_write32(>->mmio, XELP_GLOBAL_MOCS(i), mocs); 720 } 721 } 722 723 /* 724 * Get l3cc_value from MOCS entry taking into account when it's not used 725 * then if unused_entries_index is not zero then its value will be returned 726 * otherwise I915_MOCS_PTE's value is returned in this case. 727 */ 728 static u16 get_entry_l3cc(const struct xe_mocs_info *info, 729 unsigned int index) 730 { 731 if (index < info->table_size && info->table[index].used) 732 return info->table[index].l3cc_value; 733 return info->table[info->unused_entries_index].l3cc_value; 734 } 735 736 static u32 l3cc_combine(u16 low, u16 high) 737 { 738 return low | (u32)high << 16; 739 } 740 741 static void init_l3cc_table(struct xe_gt *gt, 742 const struct xe_mocs_info *info) 743 { 744 unsigned int i; 745 u32 l3cc; 746 747 mocs_dbg(gt, "l3cc entries: %d\n", info->num_mocs_regs); 748 749 for (i = 0; i < (info->num_mocs_regs + 1) / 2; i++) { 750 l3cc = l3cc_combine(get_entry_l3cc(info, 2 * i), 751 get_entry_l3cc(info, 2 * i + 1)); 752 753 mocs_dbg(gt, "LNCFCMOCS[%d] 0x%x 0x%x\n", i, 754 XELP_LNCFCMOCS(i).addr, l3cc); 755 756 if (regs_are_mcr(gt)) 757 xe_gt_mcr_multicast_write(gt, XEHP_LNCFCMOCS(i), l3cc); 758 else 759 xe_mmio_write32(>->mmio, XELP_LNCFCMOCS(i), l3cc); 760 } 761 } 762 763 void xe_mocs_init_early(struct xe_gt *gt) 764 { 765 struct xe_mocs_info table; 766 767 get_mocs_settings(gt_to_xe(gt), &table); 768 gt->mocs.uc_index = table.uc_index; 769 gt->mocs.wb_index = table.wb_index; 770 } 771 772 void xe_mocs_init(struct xe_gt *gt) 773 { 774 struct xe_mocs_info table; 775 unsigned int flags; 776 777 if (IS_SRIOV_VF(gt_to_xe(gt))) 778 return; 779 780 /* 781 * MOCS settings are split between "GLOB_MOCS" and/or "LNCFCMOCS" 782 * registers depending on platform. 783 * 784 * These registers should be programmed before GuC initialization 785 * since their values will affect some of the memory transactions 786 * performed by the GuC. 787 */ 788 flags = get_mocs_settings(gt_to_xe(gt), &table); 789 mocs_dbg(gt, "flag:0x%x\n", flags); 790 791 if (IS_SRIOV_VF(gt_to_xe(gt))) 792 return; 793 794 if (flags & HAS_GLOBAL_MOCS) 795 __init_mocs_table(gt, &table); 796 if (flags & HAS_LNCF_MOCS) 797 init_l3cc_table(gt, &table); 798 } 799 800 /** 801 * xe_mocs_dump() - Dump MOCS table. 802 * @gt: the &xe_gt with MOCS table 803 * @p: the &drm_printer to dump info to 804 * 805 * Return: 0 on success or a negative error code on failure. 806 */ 807 int xe_mocs_dump(struct xe_gt *gt, struct drm_printer *p) 808 { 809 struct xe_device *xe = gt_to_xe(gt); 810 enum xe_force_wake_domains domain; 811 struct xe_mocs_info table; 812 unsigned int flags; 813 814 flags = get_mocs_settings(xe, &table); 815 816 domain = flags & HAS_LNCF_MOCS ? XE_FORCEWAKE_ALL : XE_FW_GT; 817 818 guard(xe_pm_runtime_noresume)(xe); 819 CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), domain); 820 if (!xe_force_wake_ref_has_domain(fw_ref.domains, domain)) 821 return -ETIMEDOUT; 822 823 table.ops->dump(&table, flags, gt, p); 824 825 return 0; 826 } 827 828 #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) 829 #include "tests/xe_mocs.c" 830 #endif 831