1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include "xe_mocs.h" 7 8 #include "regs/xe_gt_regs.h" 9 #include "xe_bo.h" 10 #include "xe_device.h" 11 #include "xe_exec_queue.h" 12 #include "xe_force_wake.h" 13 #include "xe_gt.h" 14 #include "xe_gt_mcr.h" 15 #include "xe_gt_printk.h" 16 #include "xe_mmio.h" 17 #include "xe_platform_types.h" 18 #include "xe_pm.h" 19 #include "xe_sriov.h" 20 #include "xe_step_types.h" 21 22 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG) 23 #define mocs_dbg xe_gt_dbg 24 #else 25 __printf(2, 3) 26 static inline void mocs_dbg(const struct xe_gt *gt, 27 const char *format, ...) 28 { /* noop */ } 29 #endif 30 31 enum { 32 HAS_GLOBAL_MOCS = BIT(0), 33 HAS_LNCF_MOCS = BIT(1), 34 }; 35 36 struct xe_mocs_entry { 37 u32 control_value; 38 u16 l3cc_value; 39 u16 used; 40 }; 41 42 struct xe_mocs_info; 43 44 struct xe_mocs_ops { 45 void (*dump)(struct xe_mocs_info *mocs, unsigned int flags, 46 struct xe_gt *gt, struct drm_printer *p); 47 }; 48 49 struct xe_mocs_info { 50 /* 51 * Size of the spec's suggested MOCS programming table. The list of 52 * table entries from the spec can potentially be smaller than the 53 * number of hardware registers used to program the MOCS table; in such 54 * cases the registers for the remaining indices will be programmed to 55 * match unused_entries_index. 56 */ 57 unsigned int table_size; 58 /* Number of MOCS entries supported by the hardware */ 59 unsigned int num_mocs_regs; 60 const struct xe_mocs_entry *table; 61 const struct xe_mocs_ops *ops; 62 u8 uc_index; 63 u8 wb_index; 64 u8 unused_entries_index; 65 }; 66 67 /* Defines for the tables (GLOB_MOCS_0 - GLOB_MOCS_16) */ 68 #define IG_PAT REG_BIT(8) 69 #define L3_CACHE_POLICY_MASK REG_GENMASK(5, 4) 70 #define L4_CACHE_POLICY_MASK REG_GENMASK(3, 2) 71 72 /* Helper defines */ 73 #define XELP_NUM_MOCS_ENTRIES 64 /* 63-64 are reserved, but configured. */ 74 #define PVC_NUM_MOCS_ENTRIES 3 75 #define MTL_NUM_MOCS_ENTRIES 16 76 #define XE2_NUM_MOCS_ENTRIES 16 77 78 /* (e)LLC caching options */ 79 /* 80 * Note: LE_0_PAGETABLE works only up to Gen11; for newer gens it means 81 * the same as LE_UC 82 */ 83 #define LE_0_PAGETABLE LE_CACHEABILITY(0) 84 #define LE_1_UC LE_CACHEABILITY(1) 85 #define LE_2_WT LE_CACHEABILITY(2) 86 #define LE_3_WB LE_CACHEABILITY(3) 87 88 /* Target cache */ 89 #define LE_TC_0_PAGETABLE LE_TGT_CACHE(0) 90 #define LE_TC_1_LLC LE_TGT_CACHE(1) 91 #define LE_TC_2_LLC_ELLC LE_TGT_CACHE(2) 92 #define LE_TC_3_LLC_ELLC_ALT LE_TGT_CACHE(3) 93 94 /* L3 caching options */ 95 #define L3_0_DIRECT L3_CACHEABILITY(0) 96 #define L3_1_UC L3_CACHEABILITY(1) 97 #define L3_2_RESERVED L3_CACHEABILITY(2) 98 #define L3_3_WB L3_CACHEABILITY(3) 99 100 /* L4 caching options */ 101 #define L4_0_WB REG_FIELD_PREP(L4_CACHE_POLICY_MASK, 0) 102 #define L4_1_WT REG_FIELD_PREP(L4_CACHE_POLICY_MASK, 1) 103 #define L4_3_UC REG_FIELD_PREP(L4_CACHE_POLICY_MASK, 3) 104 105 #define XE2_L3_0_WB REG_FIELD_PREP(L3_CACHE_POLICY_MASK, 0) 106 /* XD: WB Transient Display */ 107 #define XE2_L3_1_XD REG_FIELD_PREP(L3_CACHE_POLICY_MASK, 1) 108 #define XE2_L3_3_UC REG_FIELD_PREP(L3_CACHE_POLICY_MASK, 3) 109 110 #define XE2_L3_CLOS_MASK REG_GENMASK(7, 6) 111 112 #define MOCS_ENTRY(__idx, __control_value, __l3cc_value) \ 113 [__idx] = { \ 114 .control_value = __control_value, \ 115 .l3cc_value = __l3cc_value, \ 116 .used = 1, \ 117 } 118 119 /* 120 * MOCS tables 121 * 122 * These are the MOCS tables that are programmed across all the rings. 123 * The control value is programmed to all the rings that support the 124 * MOCS registers. While the l3cc_values are only programmed to the 125 * LNCFCMOCS0 - LNCFCMOCS32 registers. 126 * 127 * These tables are intended to be kept reasonably consistent across 128 * HW platforms, and for ICL+, be identical across OSes. To achieve 129 * that, the list of entries is published as part of bspec. 130 * 131 * Entries not part of the following tables are undefined as far as userspace is 132 * concerned and shouldn't be relied upon. The last few entries are reserved by 133 * the hardware. They should be initialized according to bspec and never used. 134 * 135 * NOTE1: These tables are part of bspec and defined as part of the hardware 136 * interface. It is expected that, for specific hardware platform, existing 137 * entries will remain constant and the table will only be updated by adding new 138 * entries, filling unused positions. 139 * 140 * NOTE2: Reserved and unspecified MOCS indices have been set to L3 WB. These 141 * reserved entries should never be used. They may be changed to low performant 142 * variants with better coherency in the future if more entries are needed. 143 */ 144 145 static const struct xe_mocs_entry gen12_mocs_desc[] = { 146 /* Base - L3 + LLC */ 147 MOCS_ENTRY(2, 148 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), 149 L3_3_WB), 150 /* Base - Uncached */ 151 MOCS_ENTRY(3, 152 LE_1_UC | LE_TC_1_LLC, 153 L3_1_UC), 154 /* Base - L3 */ 155 MOCS_ENTRY(4, 156 LE_1_UC | LE_TC_1_LLC, 157 L3_3_WB), 158 /* Base - LLC */ 159 MOCS_ENTRY(5, 160 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), 161 L3_1_UC), 162 /* Age 0 - LLC */ 163 MOCS_ENTRY(6, 164 LE_3_WB | LE_TC_1_LLC | LE_LRUM(1), 165 L3_1_UC), 166 /* Age 0 - L3 + LLC */ 167 MOCS_ENTRY(7, 168 LE_3_WB | LE_TC_1_LLC | LE_LRUM(1), 169 L3_3_WB), 170 /* Age: Don't Chg. - LLC */ 171 MOCS_ENTRY(8, 172 LE_3_WB | LE_TC_1_LLC | LE_LRUM(2), 173 L3_1_UC), 174 /* Age: Don't Chg. - L3 + LLC */ 175 MOCS_ENTRY(9, 176 LE_3_WB | LE_TC_1_LLC | LE_LRUM(2), 177 L3_3_WB), 178 /* No AOM - LLC */ 179 MOCS_ENTRY(10, 180 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_AOM(1), 181 L3_1_UC), 182 /* No AOM - L3 + LLC */ 183 MOCS_ENTRY(11, 184 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_AOM(1), 185 L3_3_WB), 186 /* No AOM; Age 0 - LLC */ 187 MOCS_ENTRY(12, 188 LE_3_WB | LE_TC_1_LLC | LE_LRUM(1) | LE_AOM(1), 189 L3_1_UC), 190 /* No AOM; Age 0 - L3 + LLC */ 191 MOCS_ENTRY(13, 192 LE_3_WB | LE_TC_1_LLC | LE_LRUM(1) | LE_AOM(1), 193 L3_3_WB), 194 /* No AOM; Age:DC - LLC */ 195 MOCS_ENTRY(14, 196 LE_3_WB | LE_TC_1_LLC | LE_LRUM(2) | LE_AOM(1), 197 L3_1_UC), 198 /* No AOM; Age:DC - L3 + LLC */ 199 MOCS_ENTRY(15, 200 LE_3_WB | LE_TC_1_LLC | LE_LRUM(2) | LE_AOM(1), 201 L3_3_WB), 202 /* Self-Snoop - L3 + LLC */ 203 MOCS_ENTRY(18, 204 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SSE(3), 205 L3_3_WB), 206 /* Skip Caching - L3 + LLC(12.5%) */ 207 MOCS_ENTRY(19, 208 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SCC(7), 209 L3_3_WB), 210 /* Skip Caching - L3 + LLC(25%) */ 211 MOCS_ENTRY(20, 212 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SCC(3), 213 L3_3_WB), 214 /* Skip Caching - L3 + LLC(50%) */ 215 MOCS_ENTRY(21, 216 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SCC(1), 217 L3_3_WB), 218 /* Skip Caching - L3 + LLC(75%) */ 219 MOCS_ENTRY(22, 220 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_RSC(1) | LE_SCC(3), 221 L3_3_WB), 222 /* Skip Caching - L3 + LLC(87.5%) */ 223 MOCS_ENTRY(23, 224 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_RSC(1) | LE_SCC(7), 225 L3_3_WB), 226 /* Implicitly enable L1 - HDC:L1 + L3 + LLC */ 227 MOCS_ENTRY(48, 228 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), 229 L3_3_WB), 230 /* Implicitly enable L1 - HDC:L1 + L3 */ 231 MOCS_ENTRY(49, 232 LE_1_UC | LE_TC_1_LLC, 233 L3_3_WB), 234 /* Implicitly enable L1 - HDC:L1 + LLC */ 235 MOCS_ENTRY(50, 236 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), 237 L3_1_UC), 238 /* Implicitly enable L1 - HDC:L1 */ 239 MOCS_ENTRY(51, 240 LE_1_UC | LE_TC_1_LLC, 241 L3_1_UC), 242 /* HW Special Case (CCS) */ 243 MOCS_ENTRY(60, 244 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), 245 L3_1_UC), 246 /* HW Special Case (Displayable) */ 247 MOCS_ENTRY(61, 248 LE_1_UC | LE_TC_1_LLC, 249 L3_3_WB), 250 /* HW Reserved - SW program but never use */ 251 MOCS_ENTRY(62, 252 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), 253 L3_1_UC), 254 /* HW Reserved - SW program but never use */ 255 MOCS_ENTRY(63, 256 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), 257 L3_1_UC) 258 }; 259 260 static bool regs_are_mcr(struct xe_gt *gt) 261 { 262 struct xe_device *xe = gt_to_xe(gt); 263 264 if (xe_gt_is_media_type(gt)) 265 return MEDIA_VER(xe) >= 20; 266 else 267 return GRAPHICS_VERx100(xe) >= 1250; 268 } 269 270 static void xelp_lncf_dump(struct xe_mocs_info *info, struct xe_gt *gt, struct drm_printer *p) 271 { 272 unsigned int i, j; 273 u32 reg_val; 274 275 drm_printf(p, "LNCFCMOCS[idx] = [ESC, SCC, L3CC] (value)\n\n"); 276 277 for (i = 0, j = 0; i < (info->num_mocs_regs + 1) / 2; i++, j++) { 278 if (regs_are_mcr(gt)) 279 reg_val = xe_gt_mcr_unicast_read_any(gt, XEHP_LNCFCMOCS(i)); 280 else 281 reg_val = xe_mmio_read32(gt, XELP_LNCFCMOCS(i)); 282 283 drm_printf(p, "LNCFCMOCS[%2d] = [%u, %u, %u] (%#8x)\n", 284 j++, 285 !!(reg_val & L3_ESC_MASK), 286 REG_FIELD_GET(L3_SCC_MASK, reg_val), 287 REG_FIELD_GET(L3_CACHEABILITY_MASK, reg_val), 288 reg_val); 289 290 drm_printf(p, "LNCFCMOCS[%2d] = [%u, %u, %u] (%#8x)\n", 291 j, 292 !!(reg_val & L3_UPPER_IDX_ESC_MASK), 293 REG_FIELD_GET(L3_UPPER_IDX_SCC_MASK, reg_val), 294 REG_FIELD_GET(L3_UPPER_IDX_CACHEABILITY_MASK, reg_val), 295 reg_val); 296 } 297 } 298 299 static void xelp_mocs_dump(struct xe_mocs_info *info, unsigned int flags, 300 struct xe_gt *gt, struct drm_printer *p) 301 { 302 unsigned int i; 303 u32 reg_val; 304 305 if (flags & HAS_GLOBAL_MOCS) { 306 drm_printf(p, "Global mocs table configuration:\n"); 307 drm_printf(p, "GLOB_MOCS[idx] = [LeCC, TC, LRUM, AOM, RSC, SCC, PFM, SCF, CoS, SSE] (value)\n\n"); 308 309 for (i = 0; i < info->num_mocs_regs; i++) { 310 if (regs_are_mcr(gt)) 311 reg_val = xe_gt_mcr_unicast_read_any(gt, XEHP_GLOBAL_MOCS(i)); 312 else 313 reg_val = xe_mmio_read32(gt, XELP_GLOBAL_MOCS(i)); 314 315 drm_printf(p, "GLOB_MOCS[%2d] = [%u, %u, %u, %u, %u, %u, %u, %u, %u, %u ] (%#8x)\n", 316 i, 317 REG_FIELD_GET(LE_CACHEABILITY_MASK, reg_val), 318 REG_FIELD_GET(LE_TGT_CACHE_MASK, reg_val), 319 REG_FIELD_GET(LE_LRUM_MASK, reg_val), 320 !!(reg_val & LE_AOM_MASK), 321 !!(reg_val & LE_RSC_MASK), 322 REG_FIELD_GET(LE_SCC_MASK, reg_val), 323 REG_FIELD_GET(LE_PFM_MASK, reg_val), 324 !!(reg_val & LE_SCF_MASK), 325 REG_FIELD_GET(LE_COS_MASK, reg_val), 326 REG_FIELD_GET(LE_SSE_MASK, reg_val), 327 reg_val); 328 } 329 } 330 331 xelp_lncf_dump(info, gt, p); 332 } 333 334 static const struct xe_mocs_ops xelp_mocs_ops = { 335 .dump = xelp_mocs_dump, 336 }; 337 338 static const struct xe_mocs_entry dg1_mocs_desc[] = { 339 /* UC */ 340 MOCS_ENTRY(1, 0, L3_1_UC), 341 /* WB - L3 */ 342 MOCS_ENTRY(5, 0, L3_3_WB), 343 /* WB - L3 50% */ 344 MOCS_ENTRY(6, 0, L3_ESC(1) | L3_SCC(1) | L3_3_WB), 345 /* WB - L3 25% */ 346 MOCS_ENTRY(7, 0, L3_ESC(1) | L3_SCC(3) | L3_3_WB), 347 /* WB - L3 12.5% */ 348 MOCS_ENTRY(8, 0, L3_ESC(1) | L3_SCC(7) | L3_3_WB), 349 350 /* HDC:L1 + L3 */ 351 MOCS_ENTRY(48, 0, L3_3_WB), 352 /* HDC:L1 */ 353 MOCS_ENTRY(49, 0, L3_1_UC), 354 355 /* HW Reserved */ 356 MOCS_ENTRY(60, 0, L3_1_UC), 357 MOCS_ENTRY(61, 0, L3_1_UC), 358 MOCS_ENTRY(62, 0, L3_1_UC), 359 MOCS_ENTRY(63, 0, L3_1_UC), 360 }; 361 362 static const struct xe_mocs_entry dg2_mocs_desc[] = { 363 /* UC - Coherent; GO:L3 */ 364 MOCS_ENTRY(0, 0, L3_1_UC | L3_LKUP(1)), 365 /* UC - Coherent; GO:Memory */ 366 MOCS_ENTRY(1, 0, L3_1_UC | L3_GLBGO(1) | L3_LKUP(1)), 367 /* UC - Non-Coherent; GO:Memory */ 368 MOCS_ENTRY(2, 0, L3_1_UC | L3_GLBGO(1)), 369 370 /* WB - LC */ 371 MOCS_ENTRY(3, 0, L3_3_WB | L3_LKUP(1)), 372 }; 373 374 static void xehp_lncf_dump(struct xe_mocs_info *info, unsigned int flags, 375 struct xe_gt *gt, struct drm_printer *p) 376 { 377 unsigned int i, j; 378 u32 reg_val; 379 380 drm_printf(p, "LNCFCMOCS[idx] = [UCL3LOOKUP, GLBGO, L3CC] (value)\n\n"); 381 382 for (i = 0, j = 0; i < (info->num_mocs_regs + 1) / 2; i++, j++) { 383 if (regs_are_mcr(gt)) 384 reg_val = xe_gt_mcr_unicast_read_any(gt, XEHP_LNCFCMOCS(i)); 385 else 386 reg_val = xe_mmio_read32(gt, XELP_LNCFCMOCS(i)); 387 388 drm_printf(p, "LNCFCMOCS[%2d] = [%u, %u, %u] (%#8x)\n", 389 j++, 390 !!(reg_val & L3_LKUP_MASK), 391 !!(reg_val & L3_GLBGO_MASK), 392 REG_FIELD_GET(L3_CACHEABILITY_MASK, reg_val), 393 reg_val); 394 395 drm_printf(p, "LNCFCMOCS[%2d] = [%u, %u, %u] (%#8x)\n", 396 j, 397 !!(reg_val & L3_UPPER_LKUP_MASK), 398 !!(reg_val & L3_UPPER_GLBGO_MASK), 399 REG_FIELD_GET(L3_UPPER_IDX_CACHEABILITY_MASK, reg_val), 400 reg_val); 401 } 402 } 403 404 static const struct xe_mocs_ops xehp_mocs_ops = { 405 .dump = xehp_lncf_dump, 406 }; 407 408 static const struct xe_mocs_entry pvc_mocs_desc[] = { 409 /* Error */ 410 MOCS_ENTRY(0, 0, L3_3_WB), 411 412 /* UC */ 413 MOCS_ENTRY(1, 0, L3_1_UC), 414 415 /* WB */ 416 MOCS_ENTRY(2, 0, L3_3_WB), 417 }; 418 419 static void pvc_mocs_dump(struct xe_mocs_info *info, unsigned int flags, struct xe_gt *gt, 420 struct drm_printer *p) 421 { 422 unsigned int i, j; 423 u32 reg_val; 424 425 drm_printf(p, "LNCFCMOCS[idx] = [ L3CC ] (value)\n\n"); 426 427 for (i = 0, j = 0; i < (info->num_mocs_regs + 1) / 2; i++, j++) { 428 if (regs_are_mcr(gt)) 429 reg_val = xe_gt_mcr_unicast_read_any(gt, XEHP_LNCFCMOCS(i)); 430 else 431 reg_val = xe_mmio_read32(gt, XELP_LNCFCMOCS(i)); 432 433 drm_printf(p, "LNCFCMOCS[%2d] = [ %u ] (%#8x)\n", 434 j++, 435 REG_FIELD_GET(L3_CACHEABILITY_MASK, reg_val), 436 reg_val); 437 438 drm_printf(p, "LNCFCMOCS[%2d] = [ %u ] (%#8x)\n", 439 j, 440 REG_FIELD_GET(L3_UPPER_IDX_CACHEABILITY_MASK, reg_val), 441 reg_val); 442 } 443 } 444 445 static const struct xe_mocs_ops pvc_mocs_ops = { 446 .dump = pvc_mocs_dump, 447 }; 448 449 static const struct xe_mocs_entry mtl_mocs_desc[] = { 450 /* Error - Reserved for Non-Use */ 451 MOCS_ENTRY(0, 452 0, 453 L3_LKUP(1) | L3_3_WB), 454 /* Cached - L3 + L4 */ 455 MOCS_ENTRY(1, 456 IG_PAT, 457 L3_LKUP(1) | L3_3_WB), 458 /* L4 - GO:L3 */ 459 MOCS_ENTRY(2, 460 IG_PAT, 461 L3_LKUP(1) | L3_1_UC), 462 /* Uncached - GO:L3 */ 463 MOCS_ENTRY(3, 464 IG_PAT | L4_3_UC, 465 L3_LKUP(1) | L3_1_UC), 466 /* L4 - GO:Mem */ 467 MOCS_ENTRY(4, 468 IG_PAT, 469 L3_LKUP(1) | L3_GLBGO(1) | L3_1_UC), 470 /* Uncached - GO:Mem */ 471 MOCS_ENTRY(5, 472 IG_PAT | L4_3_UC, 473 L3_LKUP(1) | L3_GLBGO(1) | L3_1_UC), 474 /* L4 - L3:NoLKUP; GO:L3 */ 475 MOCS_ENTRY(6, 476 IG_PAT, 477 L3_1_UC), 478 /* Uncached - L3:NoLKUP; GO:L3 */ 479 MOCS_ENTRY(7, 480 IG_PAT | L4_3_UC, 481 L3_1_UC), 482 /* L4 - L3:NoLKUP; GO:Mem */ 483 MOCS_ENTRY(8, 484 IG_PAT, 485 L3_GLBGO(1) | L3_1_UC), 486 /* Uncached - L3:NoLKUP; GO:Mem */ 487 MOCS_ENTRY(9, 488 IG_PAT | L4_3_UC, 489 L3_GLBGO(1) | L3_1_UC), 490 /* Display - L3; L4:WT */ 491 MOCS_ENTRY(14, 492 IG_PAT | L4_1_WT, 493 L3_LKUP(1) | L3_3_WB), 494 /* CCS - Non-Displayable */ 495 MOCS_ENTRY(15, 496 IG_PAT, 497 L3_GLBGO(1) | L3_1_UC), 498 }; 499 500 static void mtl_mocs_dump(struct xe_mocs_info *info, unsigned int flags, 501 struct xe_gt *gt, struct drm_printer *p) 502 { 503 unsigned int i; 504 u32 reg_val; 505 506 drm_printf(p, "Global mocs table configuration:\n"); 507 drm_printf(p, "GLOB_MOCS[idx] = [IG_PAT, L4_CACHE_POLICY] (value)\n\n"); 508 509 for (i = 0; i < info->num_mocs_regs; i++) { 510 if (regs_are_mcr(gt)) 511 reg_val = xe_gt_mcr_unicast_read_any(gt, XEHP_GLOBAL_MOCS(i)); 512 else 513 reg_val = xe_mmio_read32(gt, XELP_GLOBAL_MOCS(i)); 514 515 drm_printf(p, "GLOB_MOCS[%2d] = [%u, %u] (%#8x)\n", 516 i, 517 !!(reg_val & IG_PAT), 518 REG_FIELD_GET(L4_CACHE_POLICY_MASK, reg_val), 519 reg_val); 520 } 521 522 /* MTL lncf mocs table pattern is similar to that of xehp */ 523 xehp_lncf_dump(info, flags, gt, p); 524 } 525 526 static const struct xe_mocs_ops mtl_mocs_ops = { 527 .dump = mtl_mocs_dump, 528 }; 529 530 static const struct xe_mocs_entry xe2_mocs_table[] = { 531 /* Defer to PAT */ 532 MOCS_ENTRY(0, XE2_L3_0_WB | L4_3_UC, 0), 533 /* Cached L3, Uncached L4 */ 534 MOCS_ENTRY(1, IG_PAT | XE2_L3_0_WB | L4_3_UC, 0), 535 /* Uncached L3, Cached L4 */ 536 MOCS_ENTRY(2, IG_PAT | XE2_L3_3_UC | L4_0_WB, 0), 537 /* Uncached L3 + L4 */ 538 MOCS_ENTRY(3, IG_PAT | XE2_L3_3_UC | L4_3_UC, 0), 539 /* Cached L3 + L4 */ 540 MOCS_ENTRY(4, IG_PAT | XE2_L3_0_WB | L4_0_WB, 0), 541 }; 542 543 static void xe2_mocs_dump(struct xe_mocs_info *info, unsigned int flags, 544 struct xe_gt *gt, struct drm_printer *p) 545 { 546 unsigned int i; 547 u32 reg_val; 548 549 drm_printf(p, "Global mocs table configuration:\n"); 550 drm_printf(p, "GLOB_MOCS[idx] = [IG_PAT, L3_CLOS, L3_CACHE_POLICY, L4_CACHE_POLICY] (value)\n\n"); 551 552 for (i = 0; i < info->num_mocs_regs; i++) { 553 if (regs_are_mcr(gt)) 554 reg_val = xe_gt_mcr_unicast_read_any(gt, XEHP_GLOBAL_MOCS(i)); 555 else 556 reg_val = xe_mmio_read32(gt, XELP_GLOBAL_MOCS(i)); 557 558 drm_printf(p, "GLOB_MOCS[%2d] = [%u, %u, %u] (%#8x)\n", 559 i, 560 !!(reg_val & IG_PAT), 561 REG_FIELD_GET(XE2_L3_CLOS_MASK, reg_val), 562 REG_FIELD_GET(L4_CACHE_POLICY_MASK, reg_val), 563 reg_val); 564 } 565 } 566 567 static const struct xe_mocs_ops xe2_mocs_ops = { 568 .dump = xe2_mocs_dump, 569 }; 570 571 static unsigned int get_mocs_settings(struct xe_device *xe, 572 struct xe_mocs_info *info) 573 { 574 unsigned int flags = 0; 575 576 memset(info, 0, sizeof(struct xe_mocs_info)); 577 578 switch (xe->info.platform) { 579 case XE_LUNARLAKE: 580 case XE_BATTLEMAGE: 581 info->ops = &xe2_mocs_ops; 582 info->table_size = ARRAY_SIZE(xe2_mocs_table); 583 info->table = xe2_mocs_table; 584 info->num_mocs_regs = XE2_NUM_MOCS_ENTRIES; 585 info->uc_index = 3; 586 info->wb_index = 4; 587 info->unused_entries_index = 4; 588 break; 589 case XE_PVC: 590 info->ops = &pvc_mocs_ops; 591 info->table_size = ARRAY_SIZE(pvc_mocs_desc); 592 info->table = pvc_mocs_desc; 593 info->num_mocs_regs = PVC_NUM_MOCS_ENTRIES; 594 info->uc_index = 1; 595 info->wb_index = 2; 596 info->unused_entries_index = 2; 597 break; 598 case XE_METEORLAKE: 599 info->ops = &mtl_mocs_ops; 600 info->table_size = ARRAY_SIZE(mtl_mocs_desc); 601 info->table = mtl_mocs_desc; 602 info->num_mocs_regs = MTL_NUM_MOCS_ENTRIES; 603 info->uc_index = 9; 604 info->unused_entries_index = 1; 605 break; 606 case XE_DG2: 607 info->ops = &xehp_mocs_ops; 608 info->table_size = ARRAY_SIZE(dg2_mocs_desc); 609 info->table = dg2_mocs_desc; 610 info->uc_index = 1; 611 /* 612 * Last entry is RO on hardware, don't bother with what was 613 * written when checking later 614 */ 615 info->num_mocs_regs = XELP_NUM_MOCS_ENTRIES - 1; 616 info->unused_entries_index = 3; 617 break; 618 case XE_DG1: 619 info->ops = &xelp_mocs_ops; 620 info->table_size = ARRAY_SIZE(dg1_mocs_desc); 621 info->table = dg1_mocs_desc; 622 info->uc_index = 1; 623 info->num_mocs_regs = XELP_NUM_MOCS_ENTRIES; 624 info->unused_entries_index = 5; 625 break; 626 case XE_TIGERLAKE: 627 case XE_ROCKETLAKE: 628 case XE_ALDERLAKE_S: 629 case XE_ALDERLAKE_P: 630 case XE_ALDERLAKE_N: 631 info->ops = &xelp_mocs_ops; 632 info->table_size = ARRAY_SIZE(gen12_mocs_desc); 633 info->table = gen12_mocs_desc; 634 info->num_mocs_regs = XELP_NUM_MOCS_ENTRIES; 635 info->uc_index = 3; 636 info->unused_entries_index = 2; 637 break; 638 default: 639 drm_err(&xe->drm, "Platform that should have a MOCS table does not.\n"); 640 return 0; 641 } 642 643 /* 644 * Index 0 is a reserved/unused table entry on most platforms, but 645 * even on those where it does represent a legitimate MOCS entry, it 646 * never represents the "most cached, least coherent" behavior we want 647 * to populate undefined table rows with. So if unused_entries_index 648 * is still 0 at this point, we'll assume that it was omitted by 649 * mistake in the switch statement above. 650 */ 651 xe_assert(xe, info->unused_entries_index != 0); 652 653 xe_assert(xe, info->ops && info->ops->dump); 654 xe_assert(xe, info->table_size <= info->num_mocs_regs); 655 656 if (!IS_DGFX(xe) || GRAPHICS_VER(xe) >= 20) 657 flags |= HAS_GLOBAL_MOCS; 658 if (GRAPHICS_VER(xe) < 20) 659 flags |= HAS_LNCF_MOCS; 660 661 return flags; 662 } 663 664 /* 665 * Get control_value from MOCS entry. If the table entry is not defined, the 666 * settings from unused_entries_index will be returned. 667 */ 668 static u32 get_entry_control(const struct xe_mocs_info *info, 669 unsigned int index) 670 { 671 if (index < info->table_size && info->table[index].used) 672 return info->table[index].control_value; 673 return info->table[info->unused_entries_index].control_value; 674 } 675 676 static void __init_mocs_table(struct xe_gt *gt, 677 const struct xe_mocs_info *info) 678 { 679 unsigned int i; 680 u32 mocs; 681 682 mocs_dbg(gt, "mocs entries: %d\n", info->num_mocs_regs); 683 684 for (i = 0; i < info->num_mocs_regs; i++) { 685 mocs = get_entry_control(info, i); 686 687 mocs_dbg(gt, "GLOB_MOCS[%d] 0x%x 0x%x\n", i, 688 XELP_GLOBAL_MOCS(i).addr, mocs); 689 690 if (regs_are_mcr(gt)) 691 xe_gt_mcr_multicast_write(gt, XEHP_GLOBAL_MOCS(i), mocs); 692 else 693 xe_mmio_write32(gt, XELP_GLOBAL_MOCS(i), mocs); 694 } 695 } 696 697 /* 698 * Get l3cc_value from MOCS entry taking into account when it's not used 699 * then if unused_entries_index is not zero then its value will be returned 700 * otherwise I915_MOCS_PTE's value is returned in this case. 701 */ 702 static u16 get_entry_l3cc(const struct xe_mocs_info *info, 703 unsigned int index) 704 { 705 if (index < info->table_size && info->table[index].used) 706 return info->table[index].l3cc_value; 707 return info->table[info->unused_entries_index].l3cc_value; 708 } 709 710 static u32 l3cc_combine(u16 low, u16 high) 711 { 712 return low | (u32)high << 16; 713 } 714 715 static void init_l3cc_table(struct xe_gt *gt, 716 const struct xe_mocs_info *info) 717 { 718 unsigned int i; 719 u32 l3cc; 720 721 mocs_dbg(gt, "l3cc entries: %d\n", info->num_mocs_regs); 722 723 for (i = 0; i < (info->num_mocs_regs + 1) / 2; i++) { 724 l3cc = l3cc_combine(get_entry_l3cc(info, 2 * i), 725 get_entry_l3cc(info, 2 * i + 1)); 726 727 mocs_dbg(gt, "LNCFCMOCS[%d] 0x%x 0x%x\n", i, 728 XELP_LNCFCMOCS(i).addr, l3cc); 729 730 if (regs_are_mcr(gt)) 731 xe_gt_mcr_multicast_write(gt, XEHP_LNCFCMOCS(i), l3cc); 732 else 733 xe_mmio_write32(gt, XELP_LNCFCMOCS(i), l3cc); 734 } 735 } 736 737 void xe_mocs_init_early(struct xe_gt *gt) 738 { 739 struct xe_mocs_info table; 740 741 get_mocs_settings(gt_to_xe(gt), &table); 742 gt->mocs.uc_index = table.uc_index; 743 gt->mocs.wb_index = table.wb_index; 744 } 745 746 void xe_mocs_init(struct xe_gt *gt) 747 { 748 struct xe_mocs_info table; 749 unsigned int flags; 750 751 if (IS_SRIOV_VF(gt_to_xe(gt))) 752 return; 753 754 /* 755 * MOCS settings are split between "GLOB_MOCS" and/or "LNCFCMOCS" 756 * registers depending on platform. 757 * 758 * These registers should be programmed before GuC initialization 759 * since their values will affect some of the memory transactions 760 * performed by the GuC. 761 */ 762 flags = get_mocs_settings(gt_to_xe(gt), &table); 763 mocs_dbg(gt, "flag:0x%x\n", flags); 764 765 if (IS_SRIOV_VF(gt_to_xe(gt))) 766 return; 767 768 if (flags & HAS_GLOBAL_MOCS) 769 __init_mocs_table(gt, &table); 770 if (flags & HAS_LNCF_MOCS) 771 init_l3cc_table(gt, &table); 772 } 773 774 void xe_mocs_dump(struct xe_gt *gt, struct drm_printer *p) 775 { 776 struct xe_mocs_info table; 777 unsigned int flags; 778 u32 ret; 779 struct xe_device *xe = gt_to_xe(gt); 780 781 flags = get_mocs_settings(xe, &table); 782 783 xe_pm_runtime_get_noresume(xe); 784 ret = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); 785 786 if (ret) 787 goto err_fw; 788 789 table.ops->dump(&table, flags, gt, p); 790 791 xe_force_wake_put(gt_to_fw(gt), XE_FW_GT); 792 793 err_fw: 794 xe_assert(xe, !ret); 795 xe_pm_runtime_put(xe); 796 } 797 798 #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) 799 #include "tests/xe_mocs.c" 800 #endif 801