1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2014-2018 Intel Corporation 5 */ 6 7 #include "i915_drv.h" 8 #include "intel_context.h" 9 #include "intel_engine_pm.h" 10 #include "intel_gt.h" 11 #include "intel_ring.h" 12 #include "intel_workarounds.h" 13 14 /** 15 * DOC: Hardware workarounds 16 * 17 * This file is intended as a central place to implement most [1]_ of the 18 * required workarounds for hardware to work as originally intended. They fall 19 * in five basic categories depending on how/when they are applied: 20 * 21 * - Workarounds that touch registers that are saved/restored to/from the HW 22 * context image. The list is emitted (via Load Register Immediate commands) 23 * everytime a new context is created. 24 * - GT workarounds. The list of these WAs is applied whenever these registers 25 * revert to default values (on GPU reset, suspend/resume [2]_, etc..). 26 * - Display workarounds. The list is applied during display clock-gating 27 * initialization. 28 * - Workarounds that whitelist a privileged register, so that UMDs can manage 29 * them directly. This is just a special case of a MMMIO workaround (as we 30 * write the list of these to/be-whitelisted registers to some special HW 31 * registers). 32 * - Workaround batchbuffers, that get executed automatically by the hardware 33 * on every HW context restore. 34 * 35 * .. [1] Please notice that there are other WAs that, due to their nature, 36 * cannot be applied from a central place. Those are peppered around the rest 37 * of the code, as needed. 38 * 39 * .. [2] Technically, some registers are powercontext saved & restored, so they 40 * survive a suspend/resume. In practice, writing them again is not too 41 * costly and simplifies things. We can revisit this in the future. 42 * 43 * Layout 44 * ~~~~~~ 45 * 46 * Keep things in this file ordered by WA type, as per the above (context, GT, 47 * display, register whitelist, batchbuffer). Then, inside each type, keep the 48 * following order: 49 * 50 * - Infrastructure functions and macros 51 * - WAs per platform in standard gen/chrono order 52 * - Public functions to init or apply the given workaround type. 53 */ 54 55 /* 56 * KBL revision ID ordering is bizarre; higher revision ID's map to lower 57 * steppings in some cases. So rather than test against the revision ID 58 * directly, let's map that into our own range of increasing ID's that we 59 * can test against in a regular manner. 60 */ 61 62 const struct i915_rev_steppings kbl_revids[] = { 63 [0] = { .gt_stepping = KBL_REVID_A0, .disp_stepping = KBL_REVID_A0 }, 64 [1] = { .gt_stepping = KBL_REVID_B0, .disp_stepping = KBL_REVID_B0 }, 65 [2] = { .gt_stepping = KBL_REVID_C0, .disp_stepping = KBL_REVID_B0 }, 66 [3] = { .gt_stepping = KBL_REVID_D0, .disp_stepping = KBL_REVID_B0 }, 67 [4] = { .gt_stepping = KBL_REVID_F0, .disp_stepping = KBL_REVID_C0 }, 68 [5] = { .gt_stepping = KBL_REVID_C0, .disp_stepping = KBL_REVID_B1 }, 69 [6] = { .gt_stepping = KBL_REVID_D1, .disp_stepping = KBL_REVID_B1 }, 70 [7] = { .gt_stepping = KBL_REVID_G0, .disp_stepping = KBL_REVID_C0 }, 71 }; 72 73 const struct i915_rev_steppings tgl_uy_revids[] = { 74 [0] = { .gt_stepping = TGL_REVID_A0, .disp_stepping = TGL_REVID_A0 }, 75 [1] = { .gt_stepping = TGL_REVID_B0, .disp_stepping = TGL_REVID_C0 }, 76 [2] = { .gt_stepping = TGL_REVID_B1, .disp_stepping = TGL_REVID_C0 }, 77 [3] = { .gt_stepping = TGL_REVID_C0, .disp_stepping = TGL_REVID_D0 }, 78 }; 79 80 /* Same GT stepping between tgl_uy_revids and tgl_revids don't mean the same HW */ 81 const struct i915_rev_steppings tgl_revids[] = { 82 [0] = { .gt_stepping = TGL_REVID_A0, .disp_stepping = TGL_REVID_B0 }, 83 [1] = { .gt_stepping = TGL_REVID_B0, .disp_stepping = TGL_REVID_D0 }, 84 }; 85 86 static void wa_init_start(struct i915_wa_list *wal, const char *name, const char *engine_name) 87 { 88 wal->name = name; 89 wal->engine_name = engine_name; 90 } 91 92 #define WA_LIST_CHUNK (1 << 4) 93 94 static void wa_init_finish(struct i915_wa_list *wal) 95 { 96 /* Trim unused entries. */ 97 if (!IS_ALIGNED(wal->count, WA_LIST_CHUNK)) { 98 struct i915_wa *list = kmemdup(wal->list, 99 wal->count * sizeof(*list), 100 GFP_KERNEL); 101 102 if (list) { 103 kfree(wal->list); 104 wal->list = list; 105 } 106 } 107 108 if (!wal->count) 109 return; 110 111 DRM_DEBUG_DRIVER("Initialized %u %s workarounds on %s\n", 112 wal->wa_count, wal->name, wal->engine_name); 113 } 114 115 static void _wa_add(struct i915_wa_list *wal, const struct i915_wa *wa) 116 { 117 unsigned int addr = i915_mmio_reg_offset(wa->reg); 118 unsigned int start = 0, end = wal->count; 119 const unsigned int grow = WA_LIST_CHUNK; 120 struct i915_wa *wa_; 121 122 GEM_BUG_ON(!is_power_of_2(grow)); 123 124 if (IS_ALIGNED(wal->count, grow)) { /* Either uninitialized or full. */ 125 struct i915_wa *list; 126 127 list = kmalloc_array(ALIGN(wal->count + 1, grow), sizeof(*wa), 128 GFP_KERNEL); 129 if (!list) { 130 DRM_ERROR("No space for workaround init!\n"); 131 return; 132 } 133 134 if (wal->list) { 135 memcpy(list, wal->list, sizeof(*wa) * wal->count); 136 kfree(wal->list); 137 } 138 139 wal->list = list; 140 } 141 142 while (start < end) { 143 unsigned int mid = start + (end - start) / 2; 144 145 if (i915_mmio_reg_offset(wal->list[mid].reg) < addr) { 146 start = mid + 1; 147 } else if (i915_mmio_reg_offset(wal->list[mid].reg) > addr) { 148 end = mid; 149 } else { 150 wa_ = &wal->list[mid]; 151 152 if ((wa->clr | wa_->clr) && !(wa->clr & ~wa_->clr)) { 153 DRM_ERROR("Discarding overwritten w/a for reg %04x (clear: %08x, set: %08x)\n", 154 i915_mmio_reg_offset(wa_->reg), 155 wa_->clr, wa_->set); 156 157 wa_->set &= ~wa->clr; 158 } 159 160 wal->wa_count++; 161 wa_->set |= wa->set; 162 wa_->clr |= wa->clr; 163 wa_->read |= wa->read; 164 return; 165 } 166 } 167 168 wal->wa_count++; 169 wa_ = &wal->list[wal->count++]; 170 *wa_ = *wa; 171 172 while (wa_-- > wal->list) { 173 GEM_BUG_ON(i915_mmio_reg_offset(wa_[0].reg) == 174 i915_mmio_reg_offset(wa_[1].reg)); 175 if (i915_mmio_reg_offset(wa_[1].reg) > 176 i915_mmio_reg_offset(wa_[0].reg)) 177 break; 178 179 swap(wa_[1], wa_[0]); 180 } 181 } 182 183 static void wa_add(struct i915_wa_list *wal, i915_reg_t reg, 184 u32 clear, u32 set, u32 read_mask) 185 { 186 struct i915_wa wa = { 187 .reg = reg, 188 .clr = clear, 189 .set = set, 190 .read = read_mask, 191 }; 192 193 _wa_add(wal, &wa); 194 } 195 196 static void 197 wa_write_masked_or(struct i915_wa_list *wal, i915_reg_t reg, u32 clear, u32 set) 198 { 199 wa_add(wal, reg, clear, set, clear); 200 } 201 202 static void 203 wa_write(struct i915_wa_list *wal, i915_reg_t reg, u32 set) 204 { 205 wa_write_masked_or(wal, reg, ~0, set); 206 } 207 208 static void 209 wa_write_or(struct i915_wa_list *wal, i915_reg_t reg, u32 set) 210 { 211 wa_write_masked_or(wal, reg, set, set); 212 } 213 214 static void 215 wa_write_clr(struct i915_wa_list *wal, i915_reg_t reg, u32 clr) 216 { 217 wa_write_masked_or(wal, reg, clr, 0); 218 } 219 220 static void 221 wa_masked_en(struct i915_wa_list *wal, i915_reg_t reg, u32 val) 222 { 223 wa_add(wal, reg, 0, _MASKED_BIT_ENABLE(val), val); 224 } 225 226 static void 227 wa_masked_dis(struct i915_wa_list *wal, i915_reg_t reg, u32 val) 228 { 229 wa_add(wal, reg, 0, _MASKED_BIT_DISABLE(val), val); 230 } 231 232 #define WA_SET_BIT_MASKED(addr, mask) \ 233 wa_masked_en(wal, (addr), (mask)) 234 235 #define WA_CLR_BIT_MASKED(addr, mask) \ 236 wa_masked_dis(wal, (addr), (mask)) 237 238 #define WA_SET_FIELD_MASKED(addr, mask, value) \ 239 wa_write_masked_or(wal, (addr), 0, _MASKED_FIELD((mask), (value))) 240 241 static void gen6_ctx_workarounds_init(struct intel_engine_cs *engine, 242 struct i915_wa_list *wal) 243 { 244 WA_SET_BIT_MASKED(INSTPM, INSTPM_FORCE_ORDERING); 245 } 246 247 static void gen7_ctx_workarounds_init(struct intel_engine_cs *engine, 248 struct i915_wa_list *wal) 249 { 250 WA_SET_BIT_MASKED(INSTPM, INSTPM_FORCE_ORDERING); 251 } 252 253 static void gen8_ctx_workarounds_init(struct intel_engine_cs *engine, 254 struct i915_wa_list *wal) 255 { 256 WA_SET_BIT_MASKED(INSTPM, INSTPM_FORCE_ORDERING); 257 258 /* WaDisableAsyncFlipPerfMode:bdw,chv */ 259 WA_SET_BIT_MASKED(MI_MODE, ASYNC_FLIP_PERF_DISABLE); 260 261 /* WaDisablePartialInstShootdown:bdw,chv */ 262 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, 263 PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE); 264 265 /* Use Force Non-Coherent whenever executing a 3D context. This is a 266 * workaround for for a possible hang in the unlikely event a TLB 267 * invalidation occurs during a PSD flush. 268 */ 269 /* WaForceEnableNonCoherent:bdw,chv */ 270 /* WaHdcDisableFetchWhenMasked:bdw,chv */ 271 WA_SET_BIT_MASKED(HDC_CHICKEN0, 272 HDC_DONOT_FETCH_MEM_WHEN_MASKED | 273 HDC_FORCE_NON_COHERENT); 274 275 /* From the Haswell PRM, Command Reference: Registers, CACHE_MODE_0: 276 * "The Hierarchical Z RAW Stall Optimization allows non-overlapping 277 * polygons in the same 8x4 pixel/sample area to be processed without 278 * stalling waiting for the earlier ones to write to Hierarchical Z 279 * buffer." 280 * 281 * This optimization is off by default for BDW and CHV; turn it on. 282 */ 283 WA_CLR_BIT_MASKED(CACHE_MODE_0_GEN7, HIZ_RAW_STALL_OPT_DISABLE); 284 285 /* Wa4x4STCOptimizationDisable:bdw,chv */ 286 WA_SET_BIT_MASKED(CACHE_MODE_1, GEN8_4x4_STC_OPTIMIZATION_DISABLE); 287 288 /* 289 * BSpec recommends 8x4 when MSAA is used, 290 * however in practice 16x4 seems fastest. 291 * 292 * Note that PS/WM thread counts depend on the WIZ hashing 293 * disable bit, which we don't touch here, but it's good 294 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM). 295 */ 296 WA_SET_FIELD_MASKED(GEN7_GT_MODE, 297 GEN6_WIZ_HASHING_MASK, 298 GEN6_WIZ_HASHING_16x4); 299 } 300 301 static void bdw_ctx_workarounds_init(struct intel_engine_cs *engine, 302 struct i915_wa_list *wal) 303 { 304 struct drm_i915_private *i915 = engine->i915; 305 306 gen8_ctx_workarounds_init(engine, wal); 307 308 /* WaDisableThreadStallDopClockGating:bdw (pre-production) */ 309 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE); 310 311 /* WaDisableDopClockGating:bdw 312 * 313 * Also see the related UCGTCL1 write in bdw_init_clock_gating() 314 * to disable EUTC clock gating. 315 */ 316 WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2, 317 DOP_CLOCK_GATING_DISABLE); 318 319 WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3, 320 GEN8_SAMPLER_POWER_BYPASS_DIS); 321 322 WA_SET_BIT_MASKED(HDC_CHICKEN0, 323 /* WaForceContextSaveRestoreNonCoherent:bdw */ 324 HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT | 325 /* WaDisableFenceDestinationToSLM:bdw (pre-prod) */ 326 (IS_BDW_GT3(i915) ? HDC_FENCE_DEST_SLM_DISABLE : 0)); 327 } 328 329 static void chv_ctx_workarounds_init(struct intel_engine_cs *engine, 330 struct i915_wa_list *wal) 331 { 332 gen8_ctx_workarounds_init(engine, wal); 333 334 /* WaDisableThreadStallDopClockGating:chv */ 335 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE); 336 337 /* Improve HiZ throughput on CHV. */ 338 WA_SET_BIT_MASKED(HIZ_CHICKEN, CHV_HZ_8X8_MODE_IN_1X); 339 } 340 341 static void gen9_ctx_workarounds_init(struct intel_engine_cs *engine, 342 struct i915_wa_list *wal) 343 { 344 struct drm_i915_private *i915 = engine->i915; 345 346 if (HAS_LLC(i915)) { 347 /* WaCompressedResourceSamplerPbeMediaNewHashMode:skl,kbl 348 * 349 * Must match Display Engine. See 350 * WaCompressedResourceDisplayNewHashMode. 351 */ 352 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2, 353 GEN9_PBE_COMPRESSED_HASH_SELECTION); 354 WA_SET_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN7, 355 GEN9_SAMPLER_HASH_COMPRESSED_READ_ADDR); 356 } 357 358 /* WaClearFlowControlGpgpuContextSave:skl,bxt,kbl,glk,cfl */ 359 /* WaDisablePartialInstShootdown:skl,bxt,kbl,glk,cfl */ 360 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, 361 FLOW_CONTROL_ENABLE | 362 PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE); 363 364 /* WaEnableYV12BugFixInHalfSliceChicken7:skl,bxt,kbl,glk,cfl */ 365 /* WaEnableSamplerGPGPUPreemptionSupport:skl,bxt,kbl,cfl */ 366 WA_SET_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN7, 367 GEN9_ENABLE_YV12_BUGFIX | 368 GEN9_ENABLE_GPGPU_PREEMPTION); 369 370 /* Wa4x4STCOptimizationDisable:skl,bxt,kbl,glk,cfl */ 371 /* WaDisablePartialResolveInVc:skl,bxt,kbl,cfl */ 372 WA_SET_BIT_MASKED(CACHE_MODE_1, 373 GEN8_4x4_STC_OPTIMIZATION_DISABLE | 374 GEN9_PARTIAL_RESOLVE_IN_VC_DISABLE); 375 376 /* WaCcsTlbPrefetchDisable:skl,bxt,kbl,glk,cfl */ 377 WA_CLR_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN5, 378 GEN9_CCS_TLB_PREFETCH_ENABLE); 379 380 /* WaForceContextSaveRestoreNonCoherent:skl,bxt,kbl,cfl */ 381 WA_SET_BIT_MASKED(HDC_CHICKEN0, 382 HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT | 383 HDC_FORCE_CSR_NON_COHERENT_OVR_DISABLE); 384 385 /* WaForceEnableNonCoherent and WaDisableHDCInvalidation are 386 * both tied to WaForceContextSaveRestoreNonCoherent 387 * in some hsds for skl. We keep the tie for all gen9. The 388 * documentation is a bit hazy and so we want to get common behaviour, 389 * even though there is no clear evidence we would need both on kbl/bxt. 390 * This area has been source of system hangs so we play it safe 391 * and mimic the skl regardless of what bspec says. 392 * 393 * Use Force Non-Coherent whenever executing a 3D context. This 394 * is a workaround for a possible hang in the unlikely event 395 * a TLB invalidation occurs during a PSD flush. 396 */ 397 398 /* WaForceEnableNonCoherent:skl,bxt,kbl,cfl */ 399 WA_SET_BIT_MASKED(HDC_CHICKEN0, 400 HDC_FORCE_NON_COHERENT); 401 402 /* WaDisableSamplerPowerBypassForSOPingPong:skl,bxt,kbl,cfl */ 403 if (IS_SKYLAKE(i915) || 404 IS_KABYLAKE(i915) || 405 IS_COFFEELAKE(i915) || 406 IS_COMETLAKE(i915)) 407 WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3, 408 GEN8_SAMPLER_POWER_BYPASS_DIS); 409 410 /* WaDisableSTUnitPowerOptimization:skl,bxt,kbl,glk,cfl */ 411 WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN2, GEN8_ST_PO_DISABLE); 412 413 /* 414 * Supporting preemption with fine-granularity requires changes in the 415 * batch buffer programming. Since we can't break old userspace, we 416 * need to set our default preemption level to safe value. Userspace is 417 * still able to use more fine-grained preemption levels, since in 418 * WaEnablePreemptionGranularityControlByUMD we're whitelisting the 419 * per-ctx register. As such, WaDisable{3D,GPGPU}MidCmdPreemption are 420 * not real HW workarounds, but merely a way to start using preemption 421 * while maintaining old contract with userspace. 422 */ 423 424 /* WaDisable3DMidCmdPreemption:skl,bxt,glk,cfl,[cnl] */ 425 WA_CLR_BIT_MASKED(GEN8_CS_CHICKEN1, GEN9_PREEMPT_3D_OBJECT_LEVEL); 426 427 /* WaDisableGPGPUMidCmdPreemption:skl,bxt,blk,cfl,[cnl] */ 428 WA_SET_FIELD_MASKED(GEN8_CS_CHICKEN1, 429 GEN9_PREEMPT_GPGPU_LEVEL_MASK, 430 GEN9_PREEMPT_GPGPU_COMMAND_LEVEL); 431 432 /* WaClearHIZ_WM_CHICKEN3:bxt,glk */ 433 if (IS_GEN9_LP(i915)) 434 WA_SET_BIT_MASKED(GEN9_WM_CHICKEN3, GEN9_FACTOR_IN_CLR_VAL_HIZ); 435 } 436 437 static void skl_tune_iz_hashing(struct intel_engine_cs *engine, 438 struct i915_wa_list *wal) 439 { 440 struct intel_gt *gt = engine->gt; 441 u8 vals[3] = { 0, 0, 0 }; 442 unsigned int i; 443 444 for (i = 0; i < 3; i++) { 445 u8 ss; 446 447 /* 448 * Only consider slices where one, and only one, subslice has 7 449 * EUs 450 */ 451 if (!is_power_of_2(gt->info.sseu.subslice_7eu[i])) 452 continue; 453 454 /* 455 * subslice_7eu[i] != 0 (because of the check above) and 456 * ss_max == 4 (maximum number of subslices possible per slice) 457 * 458 * -> 0 <= ss <= 3; 459 */ 460 ss = ffs(gt->info.sseu.subslice_7eu[i]) - 1; 461 vals[i] = 3 - ss; 462 } 463 464 if (vals[0] == 0 && vals[1] == 0 && vals[2] == 0) 465 return; 466 467 /* Tune IZ hashing. See intel_device_info_runtime_init() */ 468 WA_SET_FIELD_MASKED(GEN7_GT_MODE, 469 GEN9_IZ_HASHING_MASK(2) | 470 GEN9_IZ_HASHING_MASK(1) | 471 GEN9_IZ_HASHING_MASK(0), 472 GEN9_IZ_HASHING(2, vals[2]) | 473 GEN9_IZ_HASHING(1, vals[1]) | 474 GEN9_IZ_HASHING(0, vals[0])); 475 } 476 477 static void skl_ctx_workarounds_init(struct intel_engine_cs *engine, 478 struct i915_wa_list *wal) 479 { 480 gen9_ctx_workarounds_init(engine, wal); 481 skl_tune_iz_hashing(engine, wal); 482 } 483 484 static void bxt_ctx_workarounds_init(struct intel_engine_cs *engine, 485 struct i915_wa_list *wal) 486 { 487 gen9_ctx_workarounds_init(engine, wal); 488 489 /* WaDisableThreadStallDopClockGating:bxt */ 490 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, 491 STALL_DOP_GATING_DISABLE); 492 493 /* WaToEnableHwFixForPushConstHWBug:bxt */ 494 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2, 495 GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION); 496 } 497 498 static void kbl_ctx_workarounds_init(struct intel_engine_cs *engine, 499 struct i915_wa_list *wal) 500 { 501 struct drm_i915_private *i915 = engine->i915; 502 503 gen9_ctx_workarounds_init(engine, wal); 504 505 /* WaToEnableHwFixForPushConstHWBug:kbl */ 506 if (IS_KBL_GT_REVID(i915, KBL_REVID_C0, REVID_FOREVER)) 507 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2, 508 GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION); 509 510 /* WaDisableSbeCacheDispatchPortSharing:kbl */ 511 WA_SET_BIT_MASKED(GEN7_HALF_SLICE_CHICKEN1, 512 GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE); 513 } 514 515 static void glk_ctx_workarounds_init(struct intel_engine_cs *engine, 516 struct i915_wa_list *wal) 517 { 518 gen9_ctx_workarounds_init(engine, wal); 519 520 /* WaToEnableHwFixForPushConstHWBug:glk */ 521 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2, 522 GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION); 523 } 524 525 static void cfl_ctx_workarounds_init(struct intel_engine_cs *engine, 526 struct i915_wa_list *wal) 527 { 528 gen9_ctx_workarounds_init(engine, wal); 529 530 /* WaToEnableHwFixForPushConstHWBug:cfl */ 531 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2, 532 GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION); 533 534 /* WaDisableSbeCacheDispatchPortSharing:cfl */ 535 WA_SET_BIT_MASKED(GEN7_HALF_SLICE_CHICKEN1, 536 GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE); 537 } 538 539 static void cnl_ctx_workarounds_init(struct intel_engine_cs *engine, 540 struct i915_wa_list *wal) 541 { 542 /* WaForceContextSaveRestoreNonCoherent:cnl */ 543 WA_SET_BIT_MASKED(CNL_HDC_CHICKEN0, 544 HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT); 545 546 /* WaDisableReplayBufferBankArbitrationOptimization:cnl */ 547 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2, 548 GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION); 549 550 /* WaPushConstantDereferenceHoldDisable:cnl */ 551 WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2, PUSH_CONSTANT_DEREF_DISABLE); 552 553 /* FtrEnableFastAnisoL1BankingFix:cnl */ 554 WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3, CNL_FAST_ANISO_L1_BANKING_FIX); 555 556 /* WaDisable3DMidCmdPreemption:cnl */ 557 WA_CLR_BIT_MASKED(GEN8_CS_CHICKEN1, GEN9_PREEMPT_3D_OBJECT_LEVEL); 558 559 /* WaDisableGPGPUMidCmdPreemption:cnl */ 560 WA_SET_FIELD_MASKED(GEN8_CS_CHICKEN1, 561 GEN9_PREEMPT_GPGPU_LEVEL_MASK, 562 GEN9_PREEMPT_GPGPU_COMMAND_LEVEL); 563 564 /* WaDisableEarlyEOT:cnl */ 565 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, DISABLE_EARLY_EOT); 566 } 567 568 static void icl_ctx_workarounds_init(struct intel_engine_cs *engine, 569 struct i915_wa_list *wal) 570 { 571 struct drm_i915_private *i915 = engine->i915; 572 573 /* WaDisableBankHangMode:icl */ 574 wa_write(wal, 575 GEN8_L3CNTLREG, 576 intel_uncore_read(engine->uncore, GEN8_L3CNTLREG) | 577 GEN8_ERRDETBCTRL); 578 579 /* Wa_1604370585:icl (pre-prod) 580 * Formerly known as WaPushConstantDereferenceHoldDisable 581 */ 582 if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_B0)) 583 WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2, 584 PUSH_CONSTANT_DEREF_DISABLE); 585 586 /* WaForceEnableNonCoherent:icl 587 * This is not the same workaround as in early Gen9 platforms, where 588 * lacking this could cause system hangs, but coherency performance 589 * overhead is high and only a few compute workloads really need it 590 * (the register is whitelisted in hardware now, so UMDs can opt in 591 * for coherency if they have a good reason). 592 */ 593 WA_SET_BIT_MASKED(ICL_HDC_MODE, HDC_FORCE_NON_COHERENT); 594 595 /* Wa_2006611047:icl (pre-prod) 596 * Formerly known as WaDisableImprovedTdlClkGating 597 */ 598 if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_A0)) 599 WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2, 600 GEN11_TDL_CLOCK_GATING_FIX_DISABLE); 601 602 /* Wa_2006665173:icl (pre-prod) */ 603 if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_A0)) 604 WA_SET_BIT_MASKED(GEN11_COMMON_SLICE_CHICKEN3, 605 GEN11_BLEND_EMB_FIX_DISABLE_IN_RCC); 606 607 /* WaEnableFloatBlendOptimization:icl */ 608 wa_write_masked_or(wal, 609 GEN10_CACHE_MODE_SS, 610 0, /* write-only, so skip validation */ 611 _MASKED_BIT_ENABLE(FLOAT_BLEND_OPTIMIZATION_ENABLE)); 612 613 /* WaDisableGPGPUMidThreadPreemption:icl */ 614 WA_SET_FIELD_MASKED(GEN8_CS_CHICKEN1, 615 GEN9_PREEMPT_GPGPU_LEVEL_MASK, 616 GEN9_PREEMPT_GPGPU_THREAD_GROUP_LEVEL); 617 618 /* allow headerless messages for preemptible GPGPU context */ 619 WA_SET_BIT_MASKED(GEN10_SAMPLER_MODE, 620 GEN11_SAMPLER_ENABLE_HEADLESS_MSG); 621 622 /* Wa_1604278689:icl,ehl */ 623 wa_write(wal, IVB_FBC_RT_BASE, 0xFFFFFFFF & ~ILK_FBC_RT_VALID); 624 wa_write_masked_or(wal, IVB_FBC_RT_BASE_UPPER, 625 0, /* write-only register; skip validation */ 626 0xFFFFFFFF); 627 628 /* Wa_1406306137:icl,ehl */ 629 wa_masked_en(wal, GEN9_ROW_CHICKEN4, GEN11_DIS_PICK_2ND_EU); 630 } 631 632 static void gen12_ctx_workarounds_init(struct intel_engine_cs *engine, 633 struct i915_wa_list *wal) 634 { 635 /* 636 * Wa_1409142259:tgl 637 * Wa_1409347922:tgl 638 * Wa_1409252684:tgl 639 * Wa_1409217633:tgl 640 * Wa_1409207793:tgl 641 * Wa_1409178076:tgl 642 * Wa_1408979724:tgl 643 * Wa_14010443199:rkl 644 * Wa_14010698770:rkl 645 */ 646 WA_SET_BIT_MASKED(GEN11_COMMON_SLICE_CHICKEN3, 647 GEN12_DISABLE_CPS_AWARE_COLOR_PIPE); 648 649 /* WaDisableGPGPUMidThreadPreemption:gen12 */ 650 WA_SET_FIELD_MASKED(GEN8_CS_CHICKEN1, 651 GEN9_PREEMPT_GPGPU_LEVEL_MASK, 652 GEN9_PREEMPT_GPGPU_THREAD_GROUP_LEVEL); 653 } 654 655 static void tgl_ctx_workarounds_init(struct intel_engine_cs *engine, 656 struct i915_wa_list *wal) 657 { 658 gen12_ctx_workarounds_init(engine, wal); 659 660 /* 661 * Wa_1604555607:tgl,rkl 662 * 663 * Note that the implementation of this workaround is further modified 664 * according to the FF_MODE2 guidance given by Wa_1608008084:gen12. 665 * FF_MODE2 register will return the wrong value when read. The default 666 * value for this register is zero for all fields and there are no bit 667 * masks. So instead of doing a RMW we should just write the GS Timer 668 * and TDS timer values for Wa_1604555607 and Wa_16011163337. 669 */ 670 wa_add(wal, 671 FF_MODE2, 672 FF_MODE2_GS_TIMER_MASK | FF_MODE2_TDS_TIMER_MASK, 673 FF_MODE2_GS_TIMER_224 | FF_MODE2_TDS_TIMER_128, 674 0); 675 } 676 677 static void dg1_ctx_workarounds_init(struct intel_engine_cs *engine, 678 struct i915_wa_list *wal) 679 { 680 gen12_ctx_workarounds_init(engine, wal); 681 682 /* Wa_1409044764 */ 683 WA_CLR_BIT_MASKED(GEN11_COMMON_SLICE_CHICKEN3, 684 DG1_FLOAT_POINT_BLEND_OPT_STRICT_MODE_EN); 685 686 /* Wa_22010493298 */ 687 WA_SET_BIT_MASKED(HIZ_CHICKEN, 688 DG1_HZ_READ_SUPPRESSION_OPTIMIZATION_DISABLE); 689 } 690 691 static void 692 __intel_engine_init_ctx_wa(struct intel_engine_cs *engine, 693 struct i915_wa_list *wal, 694 const char *name) 695 { 696 struct drm_i915_private *i915 = engine->i915; 697 698 if (engine->class != RENDER_CLASS) 699 return; 700 701 wa_init_start(wal, name, engine->name); 702 703 if (IS_DG1(i915)) 704 dg1_ctx_workarounds_init(engine, wal); 705 else if (IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915)) 706 tgl_ctx_workarounds_init(engine, wal); 707 else if (IS_GEN(i915, 12)) 708 gen12_ctx_workarounds_init(engine, wal); 709 else if (IS_GEN(i915, 11)) 710 icl_ctx_workarounds_init(engine, wal); 711 else if (IS_CANNONLAKE(i915)) 712 cnl_ctx_workarounds_init(engine, wal); 713 else if (IS_COFFEELAKE(i915) || IS_COMETLAKE(i915)) 714 cfl_ctx_workarounds_init(engine, wal); 715 else if (IS_GEMINILAKE(i915)) 716 glk_ctx_workarounds_init(engine, wal); 717 else if (IS_KABYLAKE(i915)) 718 kbl_ctx_workarounds_init(engine, wal); 719 else if (IS_BROXTON(i915)) 720 bxt_ctx_workarounds_init(engine, wal); 721 else if (IS_SKYLAKE(i915)) 722 skl_ctx_workarounds_init(engine, wal); 723 else if (IS_CHERRYVIEW(i915)) 724 chv_ctx_workarounds_init(engine, wal); 725 else if (IS_BROADWELL(i915)) 726 bdw_ctx_workarounds_init(engine, wal); 727 else if (IS_GEN(i915, 7)) 728 gen7_ctx_workarounds_init(engine, wal); 729 else if (IS_GEN(i915, 6)) 730 gen6_ctx_workarounds_init(engine, wal); 731 else if (INTEL_GEN(i915) < 8) 732 return; 733 else 734 MISSING_CASE(INTEL_GEN(i915)); 735 736 wa_init_finish(wal); 737 } 738 739 void intel_engine_init_ctx_wa(struct intel_engine_cs *engine) 740 { 741 __intel_engine_init_ctx_wa(engine, &engine->ctx_wa_list, "context"); 742 } 743 744 int intel_engine_emit_ctx_wa(struct i915_request *rq) 745 { 746 struct i915_wa_list *wal = &rq->engine->ctx_wa_list; 747 struct i915_wa *wa; 748 unsigned int i; 749 u32 *cs; 750 int ret; 751 752 if (wal->count == 0) 753 return 0; 754 755 ret = rq->engine->emit_flush(rq, EMIT_BARRIER); 756 if (ret) 757 return ret; 758 759 cs = intel_ring_begin(rq, (wal->count * 2 + 2)); 760 if (IS_ERR(cs)) 761 return PTR_ERR(cs); 762 763 *cs++ = MI_LOAD_REGISTER_IMM(wal->count); 764 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) { 765 *cs++ = i915_mmio_reg_offset(wa->reg); 766 *cs++ = wa->set; 767 } 768 *cs++ = MI_NOOP; 769 770 intel_ring_advance(rq, cs); 771 772 ret = rq->engine->emit_flush(rq, EMIT_BARRIER); 773 if (ret) 774 return ret; 775 776 return 0; 777 } 778 779 static void 780 gen4_gt_workarounds_init(struct drm_i915_private *i915, 781 struct i915_wa_list *wal) 782 { 783 /* WaDisable_RenderCache_OperationalFlush:gen4,ilk */ 784 wa_masked_dis(wal, CACHE_MODE_0, RC_OP_FLUSH_ENABLE); 785 } 786 787 static void 788 g4x_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) 789 { 790 gen4_gt_workarounds_init(i915, wal); 791 792 /* WaDisableRenderCachePipelinedFlush:g4x,ilk */ 793 wa_masked_en(wal, CACHE_MODE_0, CM0_PIPELINED_RENDER_FLUSH_DISABLE); 794 } 795 796 static void 797 ilk_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) 798 { 799 g4x_gt_workarounds_init(i915, wal); 800 801 wa_masked_en(wal, _3D_CHICKEN2, _3D_CHICKEN2_WM_READ_PIPELINED); 802 } 803 804 static void 805 snb_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) 806 { 807 /* WaDisableHiZPlanesWhenMSAAEnabled:snb */ 808 wa_masked_en(wal, 809 _3D_CHICKEN, 810 _3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB); 811 812 /* WaDisable_RenderCache_OperationalFlush:snb */ 813 wa_masked_dis(wal, CACHE_MODE_0, RC_OP_FLUSH_ENABLE); 814 815 /* 816 * BSpec recommends 8x4 when MSAA is used, 817 * however in practice 16x4 seems fastest. 818 * 819 * Note that PS/WM thread counts depend on the WIZ hashing 820 * disable bit, which we don't touch here, but it's good 821 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM). 822 */ 823 wa_add(wal, 824 GEN6_GT_MODE, 0, 825 _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4), 826 GEN6_WIZ_HASHING_16x4); 827 828 wa_masked_dis(wal, CACHE_MODE_0, CM0_STC_EVICT_DISABLE_LRA_SNB); 829 830 wa_masked_en(wal, 831 _3D_CHICKEN3, 832 /* WaStripsFansDisableFastClipPerformanceFix:snb */ 833 _3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL | 834 /* 835 * Bspec says: 836 * "This bit must be set if 3DSTATE_CLIP clip mode is set 837 * to normal and 3DSTATE_SF number of SF output attributes 838 * is more than 16." 839 */ 840 _3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH); 841 } 842 843 static void 844 ivb_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) 845 { 846 /* WaDisableEarlyCull:ivb */ 847 wa_masked_en(wal, _3D_CHICKEN3, _3D_CHICKEN_SF_DISABLE_OBJEND_CULL); 848 849 /* WaDisablePSDDualDispatchEnable:ivb */ 850 if (IS_IVB_GT1(i915)) 851 wa_masked_en(wal, 852 GEN7_HALF_SLICE_CHICKEN1, 853 GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE); 854 855 /* WaDisable_RenderCache_OperationalFlush:ivb */ 856 wa_masked_dis(wal, CACHE_MODE_0_GEN7, RC_OP_FLUSH_ENABLE); 857 858 /* Apply the WaDisableRHWOOptimizationForRenderHang:ivb workaround. */ 859 wa_masked_dis(wal, 860 GEN7_COMMON_SLICE_CHICKEN1, 861 GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC); 862 863 /* WaApplyL3ControlAndL3ChickenMode:ivb */ 864 wa_write(wal, GEN7_L3CNTLREG1, GEN7_WA_FOR_GEN7_L3_CONTROL); 865 wa_write(wal, GEN7_L3_CHICKEN_MODE_REGISTER, GEN7_WA_L3_CHICKEN_MODE); 866 867 /* WaForceL3Serialization:ivb */ 868 wa_write_clr(wal, GEN7_L3SQCREG4, L3SQ_URB_READ_CAM_MATCH_DISABLE); 869 870 /* 871 * WaVSThreadDispatchOverride:ivb,vlv 872 * 873 * This actually overrides the dispatch 874 * mode for all thread types. 875 */ 876 wa_write_masked_or(wal, GEN7_FF_THREAD_MODE, 877 GEN7_FF_SCHED_MASK, 878 GEN7_FF_TS_SCHED_HW | 879 GEN7_FF_VS_SCHED_HW | 880 GEN7_FF_DS_SCHED_HW); 881 882 if (0) { /* causes HiZ corruption on ivb:gt1 */ 883 /* enable HiZ Raw Stall Optimization */ 884 wa_masked_dis(wal, CACHE_MODE_0_GEN7, HIZ_RAW_STALL_OPT_DISABLE); 885 } 886 887 /* WaDisable4x2SubspanOptimization:ivb */ 888 wa_masked_en(wal, CACHE_MODE_1, PIXEL_SUBSPAN_COLLECT_OPT_DISABLE); 889 890 /* 891 * BSpec recommends 8x4 when MSAA is used, 892 * however in practice 16x4 seems fastest. 893 * 894 * Note that PS/WM thread counts depend on the WIZ hashing 895 * disable bit, which we don't touch here, but it's good 896 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM). 897 */ 898 wa_add(wal, GEN7_GT_MODE, 0, 899 _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4), 900 GEN6_WIZ_HASHING_16x4); 901 } 902 903 static void 904 vlv_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) 905 { 906 /* WaDisableEarlyCull:vlv */ 907 wa_masked_en(wal, _3D_CHICKEN3, _3D_CHICKEN_SF_DISABLE_OBJEND_CULL); 908 909 /* WaPsdDispatchEnable:vlv */ 910 /* WaDisablePSDDualDispatchEnable:vlv */ 911 wa_masked_en(wal, 912 GEN7_HALF_SLICE_CHICKEN1, 913 GEN7_MAX_PS_THREAD_DEP | 914 GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE); 915 916 /* WaDisable_RenderCache_OperationalFlush:vlv */ 917 wa_masked_dis(wal, CACHE_MODE_0_GEN7, RC_OP_FLUSH_ENABLE); 918 919 /* WaForceL3Serialization:vlv */ 920 wa_write_clr(wal, GEN7_L3SQCREG4, L3SQ_URB_READ_CAM_MATCH_DISABLE); 921 922 /* 923 * WaVSThreadDispatchOverride:ivb,vlv 924 * 925 * This actually overrides the dispatch 926 * mode for all thread types. 927 */ 928 wa_write_masked_or(wal, 929 GEN7_FF_THREAD_MODE, 930 GEN7_FF_SCHED_MASK, 931 GEN7_FF_TS_SCHED_HW | 932 GEN7_FF_VS_SCHED_HW | 933 GEN7_FF_DS_SCHED_HW); 934 935 /* 936 * BSpec says this must be set, even though 937 * WaDisable4x2SubspanOptimization isn't listed for VLV. 938 */ 939 wa_masked_en(wal, CACHE_MODE_1, PIXEL_SUBSPAN_COLLECT_OPT_DISABLE); 940 941 /* 942 * BSpec recommends 8x4 when MSAA is used, 943 * however in practice 16x4 seems fastest. 944 * 945 * Note that PS/WM thread counts depend on the WIZ hashing 946 * disable bit, which we don't touch here, but it's good 947 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM). 948 */ 949 wa_add(wal, GEN7_GT_MODE, 0, 950 _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4), 951 GEN6_WIZ_HASHING_16x4); 952 953 /* 954 * WaIncreaseL3CreditsForVLVB0:vlv 955 * This is the hardware default actually. 956 */ 957 wa_write(wal, GEN7_L3SQCREG1, VLV_B0_WA_L3SQCREG1_VALUE); 958 } 959 960 static void 961 hsw_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) 962 { 963 /* L3 caching of data atomics doesn't work -- disable it. */ 964 wa_write(wal, HSW_SCRATCH1, HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE); 965 966 wa_add(wal, 967 HSW_ROW_CHICKEN3, 0, 968 _MASKED_BIT_ENABLE(HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE), 969 0 /* XXX does this reg exist? */); 970 971 /* WaVSRefCountFullforceMissDisable:hsw */ 972 wa_write_clr(wal, GEN7_FF_THREAD_MODE, GEN7_FF_VS_REF_CNT_FFME); 973 974 wa_masked_dis(wal, 975 CACHE_MODE_0_GEN7, 976 /* WaDisable_RenderCache_OperationalFlush:hsw */ 977 RC_OP_FLUSH_ENABLE | 978 /* enable HiZ Raw Stall Optimization */ 979 HIZ_RAW_STALL_OPT_DISABLE); 980 981 /* WaDisable4x2SubspanOptimization:hsw */ 982 wa_masked_en(wal, CACHE_MODE_1, PIXEL_SUBSPAN_COLLECT_OPT_DISABLE); 983 984 /* 985 * BSpec recommends 8x4 when MSAA is used, 986 * however in practice 16x4 seems fastest. 987 * 988 * Note that PS/WM thread counts depend on the WIZ hashing 989 * disable bit, which we don't touch here, but it's good 990 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM). 991 */ 992 wa_add(wal, GEN7_GT_MODE, 0, 993 _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4), 994 GEN6_WIZ_HASHING_16x4); 995 996 /* WaSampleCChickenBitEnable:hsw */ 997 wa_masked_en(wal, HALF_SLICE_CHICKEN3, HSW_SAMPLE_C_PERFORMANCE); 998 } 999 1000 static void 1001 gen9_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) 1002 { 1003 /* WaDisableKillLogic:bxt,skl,kbl */ 1004 if (!IS_COFFEELAKE(i915) && !IS_COMETLAKE(i915)) 1005 wa_write_or(wal, 1006 GAM_ECOCHK, 1007 ECOCHK_DIS_TLB); 1008 1009 if (HAS_LLC(i915)) { 1010 /* WaCompressedResourceSamplerPbeMediaNewHashMode:skl,kbl 1011 * 1012 * Must match Display Engine. See 1013 * WaCompressedResourceDisplayNewHashMode. 1014 */ 1015 wa_write_or(wal, 1016 MMCD_MISC_CTRL, 1017 MMCD_PCLA | MMCD_HOTSPOT_EN); 1018 } 1019 1020 /* WaDisableHDCInvalidation:skl,bxt,kbl,cfl */ 1021 wa_write_or(wal, 1022 GAM_ECOCHK, 1023 BDW_DISABLE_HDC_INVALIDATION); 1024 } 1025 1026 static void 1027 skl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) 1028 { 1029 gen9_gt_workarounds_init(i915, wal); 1030 1031 /* WaDisableGafsUnitClkGating:skl */ 1032 wa_write_or(wal, 1033 GEN7_UCGCTL4, 1034 GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE); 1035 1036 /* WaInPlaceDecompressionHang:skl */ 1037 if (IS_SKL_REVID(i915, SKL_REVID_H0, REVID_FOREVER)) 1038 wa_write_or(wal, 1039 GEN9_GAMT_ECO_REG_RW_IA, 1040 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS); 1041 } 1042 1043 static void 1044 bxt_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) 1045 { 1046 gen9_gt_workarounds_init(i915, wal); 1047 1048 /* WaInPlaceDecompressionHang:bxt */ 1049 wa_write_or(wal, 1050 GEN9_GAMT_ECO_REG_RW_IA, 1051 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS); 1052 } 1053 1054 static void 1055 kbl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) 1056 { 1057 gen9_gt_workarounds_init(i915, wal); 1058 1059 /* WaDisableDynamicCreditSharing:kbl */ 1060 if (IS_KBL_GT_REVID(i915, 0, KBL_REVID_B0)) 1061 wa_write_or(wal, 1062 GAMT_CHKN_BIT_REG, 1063 GAMT_CHKN_DISABLE_DYNAMIC_CREDIT_SHARING); 1064 1065 /* WaDisableGafsUnitClkGating:kbl */ 1066 wa_write_or(wal, 1067 GEN7_UCGCTL4, 1068 GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE); 1069 1070 /* WaInPlaceDecompressionHang:kbl */ 1071 wa_write_or(wal, 1072 GEN9_GAMT_ECO_REG_RW_IA, 1073 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS); 1074 } 1075 1076 static void 1077 glk_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) 1078 { 1079 gen9_gt_workarounds_init(i915, wal); 1080 } 1081 1082 static void 1083 cfl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) 1084 { 1085 gen9_gt_workarounds_init(i915, wal); 1086 1087 /* WaDisableGafsUnitClkGating:cfl */ 1088 wa_write_or(wal, 1089 GEN7_UCGCTL4, 1090 GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE); 1091 1092 /* WaInPlaceDecompressionHang:cfl */ 1093 wa_write_or(wal, 1094 GEN9_GAMT_ECO_REG_RW_IA, 1095 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS); 1096 } 1097 1098 static void 1099 wa_init_mcr(struct drm_i915_private *i915, struct i915_wa_list *wal) 1100 { 1101 const struct sseu_dev_info *sseu = &i915->gt.info.sseu; 1102 unsigned int slice, subslice; 1103 u32 l3_en, mcr, mcr_mask; 1104 1105 GEM_BUG_ON(INTEL_GEN(i915) < 10); 1106 1107 /* 1108 * WaProgramMgsrForL3BankSpecificMmioReads: cnl,icl 1109 * L3Banks could be fused off in single slice scenario. If that is 1110 * the case, we might need to program MCR select to a valid L3Bank 1111 * by default, to make sure we correctly read certain registers 1112 * later on (in the range 0xB100 - 0xB3FF). 1113 * 1114 * WaProgramMgsrForCorrectSliceSpecificMmioReads:cnl,icl 1115 * Before any MMIO read into slice/subslice specific registers, MCR 1116 * packet control register needs to be programmed to point to any 1117 * enabled s/ss pair. Otherwise, incorrect values will be returned. 1118 * This means each subsequent MMIO read will be forwarded to an 1119 * specific s/ss combination, but this is OK since these registers 1120 * are consistent across s/ss in almost all cases. In the rare 1121 * occasions, such as INSTDONE, where this value is dependent 1122 * on s/ss combo, the read should be done with read_subslice_reg. 1123 * 1124 * Since GEN8_MCR_SELECTOR contains dual-purpose bits which select both 1125 * to which subslice, or to which L3 bank, the respective mmio reads 1126 * will go, we have to find a common index which works for both 1127 * accesses. 1128 * 1129 * Case where we cannot find a common index fortunately should not 1130 * happen in production hardware, so we only emit a warning instead of 1131 * implementing something more complex that requires checking the range 1132 * of every MMIO read. 1133 */ 1134 1135 if (INTEL_GEN(i915) >= 10 && is_power_of_2(sseu->slice_mask)) { 1136 u32 l3_fuse = 1137 intel_uncore_read(&i915->uncore, GEN10_MIRROR_FUSE3) & 1138 GEN10_L3BANK_MASK; 1139 1140 drm_dbg(&i915->drm, "L3 fuse = %x\n", l3_fuse); 1141 l3_en = ~(l3_fuse << GEN10_L3BANK_PAIR_COUNT | l3_fuse); 1142 } else { 1143 l3_en = ~0; 1144 } 1145 1146 slice = fls(sseu->slice_mask) - 1; 1147 subslice = fls(l3_en & intel_sseu_get_subslices(sseu, slice)); 1148 if (!subslice) { 1149 drm_warn(&i915->drm, 1150 "No common index found between subslice mask %x and L3 bank mask %x!\n", 1151 intel_sseu_get_subslices(sseu, slice), l3_en); 1152 subslice = fls(l3_en); 1153 drm_WARN_ON(&i915->drm, !subslice); 1154 } 1155 subslice--; 1156 1157 if (INTEL_GEN(i915) >= 11) { 1158 mcr = GEN11_MCR_SLICE(slice) | GEN11_MCR_SUBSLICE(subslice); 1159 mcr_mask = GEN11_MCR_SLICE_MASK | GEN11_MCR_SUBSLICE_MASK; 1160 } else { 1161 mcr = GEN8_MCR_SLICE(slice) | GEN8_MCR_SUBSLICE(subslice); 1162 mcr_mask = GEN8_MCR_SLICE_MASK | GEN8_MCR_SUBSLICE_MASK; 1163 } 1164 1165 drm_dbg(&i915->drm, "MCR slice/subslice = %x\n", mcr); 1166 1167 wa_write_masked_or(wal, GEN8_MCR_SELECTOR, mcr_mask, mcr); 1168 } 1169 1170 static void 1171 cnl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) 1172 { 1173 wa_init_mcr(i915, wal); 1174 1175 /* WaInPlaceDecompressionHang:cnl */ 1176 wa_write_or(wal, 1177 GEN9_GAMT_ECO_REG_RW_IA, 1178 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS); 1179 } 1180 1181 static void 1182 icl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) 1183 { 1184 wa_init_mcr(i915, wal); 1185 1186 /* WaInPlaceDecompressionHang:icl */ 1187 wa_write_or(wal, 1188 GEN9_GAMT_ECO_REG_RW_IA, 1189 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS); 1190 1191 /* WaModifyGamTlbPartitioning:icl */ 1192 wa_write_masked_or(wal, 1193 GEN11_GACB_PERF_CTRL, 1194 GEN11_HASH_CTRL_MASK, 1195 GEN11_HASH_CTRL_BIT0 | GEN11_HASH_CTRL_BIT4); 1196 1197 /* Wa_1405766107:icl 1198 * Formerly known as WaCL2SFHalfMaxAlloc 1199 */ 1200 wa_write_or(wal, 1201 GEN11_LSN_UNSLCVC, 1202 GEN11_LSN_UNSLCVC_GAFS_HALF_SF_MAXALLOC | 1203 GEN11_LSN_UNSLCVC_GAFS_HALF_CL2_MAXALLOC); 1204 1205 /* Wa_220166154:icl 1206 * Formerly known as WaDisCtxReload 1207 */ 1208 wa_write_or(wal, 1209 GEN8_GAMW_ECO_DEV_RW_IA, 1210 GAMW_ECO_DEV_CTX_RELOAD_DISABLE); 1211 1212 /* Wa_1405779004:icl (pre-prod) */ 1213 if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_A0)) 1214 wa_write_or(wal, 1215 SLICE_UNIT_LEVEL_CLKGATE, 1216 MSCUNIT_CLKGATE_DIS); 1217 1218 /* Wa_1406838659:icl (pre-prod) */ 1219 if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_B0)) 1220 wa_write_or(wal, 1221 INF_UNIT_LEVEL_CLKGATE, 1222 CGPSF_CLKGATE_DIS); 1223 1224 /* Wa_1406463099:icl 1225 * Formerly known as WaGamTlbPendError 1226 */ 1227 wa_write_or(wal, 1228 GAMT_CHKN_BIT_REG, 1229 GAMT_CHKN_DISABLE_L3_COH_PIPE); 1230 1231 /* Wa_1607087056:icl,ehl,jsl */ 1232 if (IS_ICELAKE(i915) || 1233 IS_JSL_EHL_REVID(i915, EHL_REVID_A0, EHL_REVID_A0)) { 1234 wa_write_or(wal, 1235 SLICE_UNIT_LEVEL_CLKGATE, 1236 L3_CLKGATE_DIS | L3_CR2X_CLKGATE_DIS); 1237 } 1238 } 1239 1240 static void 1241 gen12_gt_workarounds_init(struct drm_i915_private *i915, 1242 struct i915_wa_list *wal) 1243 { 1244 wa_init_mcr(i915, wal); 1245 } 1246 1247 static void 1248 tgl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) 1249 { 1250 gen12_gt_workarounds_init(i915, wal); 1251 1252 /* Wa_1409420604:tgl */ 1253 if (IS_TGL_UY_GT_REVID(i915, TGL_REVID_A0, TGL_REVID_A0)) 1254 wa_write_or(wal, 1255 SUBSLICE_UNIT_LEVEL_CLKGATE2, 1256 CPSSUNIT_CLKGATE_DIS); 1257 1258 /* Wa_1607087056:tgl also know as BUG:1409180338 */ 1259 if (IS_TGL_UY_GT_REVID(i915, TGL_REVID_A0, TGL_REVID_A0)) 1260 wa_write_or(wal, 1261 SLICE_UNIT_LEVEL_CLKGATE, 1262 L3_CLKGATE_DIS | L3_CR2X_CLKGATE_DIS); 1263 } 1264 1265 static void 1266 dg1_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) 1267 { 1268 gen12_gt_workarounds_init(i915, wal); 1269 1270 /* Wa_1607087056:dg1 */ 1271 if (IS_DG1_REVID(i915, DG1_REVID_A0, DG1_REVID_A0)) 1272 wa_write_or(wal, 1273 SLICE_UNIT_LEVEL_CLKGATE, 1274 L3_CLKGATE_DIS | L3_CR2X_CLKGATE_DIS); 1275 1276 /* Wa_1409420604:dg1 */ 1277 if (IS_DG1(i915)) 1278 wa_write_or(wal, 1279 SUBSLICE_UNIT_LEVEL_CLKGATE2, 1280 CPSSUNIT_CLKGATE_DIS); 1281 1282 /* Wa_1408615072:dg1 */ 1283 /* Empirical testing shows this register is unaffected by engine reset. */ 1284 if (IS_DG1(i915)) 1285 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE2, 1286 VSUNIT_CLKGATE_DIS_TGL); 1287 } 1288 1289 static void 1290 gt_init_workarounds(struct drm_i915_private *i915, struct i915_wa_list *wal) 1291 { 1292 if (IS_DG1(i915)) 1293 dg1_gt_workarounds_init(i915, wal); 1294 else if (IS_TIGERLAKE(i915)) 1295 tgl_gt_workarounds_init(i915, wal); 1296 else if (IS_GEN(i915, 12)) 1297 gen12_gt_workarounds_init(i915, wal); 1298 else if (IS_GEN(i915, 11)) 1299 icl_gt_workarounds_init(i915, wal); 1300 else if (IS_CANNONLAKE(i915)) 1301 cnl_gt_workarounds_init(i915, wal); 1302 else if (IS_COFFEELAKE(i915) || IS_COMETLAKE(i915)) 1303 cfl_gt_workarounds_init(i915, wal); 1304 else if (IS_GEMINILAKE(i915)) 1305 glk_gt_workarounds_init(i915, wal); 1306 else if (IS_KABYLAKE(i915)) 1307 kbl_gt_workarounds_init(i915, wal); 1308 else if (IS_BROXTON(i915)) 1309 bxt_gt_workarounds_init(i915, wal); 1310 else if (IS_SKYLAKE(i915)) 1311 skl_gt_workarounds_init(i915, wal); 1312 else if (IS_HASWELL(i915)) 1313 hsw_gt_workarounds_init(i915, wal); 1314 else if (IS_VALLEYVIEW(i915)) 1315 vlv_gt_workarounds_init(i915, wal); 1316 else if (IS_IVYBRIDGE(i915)) 1317 ivb_gt_workarounds_init(i915, wal); 1318 else if (IS_GEN(i915, 6)) 1319 snb_gt_workarounds_init(i915, wal); 1320 else if (IS_GEN(i915, 5)) 1321 ilk_gt_workarounds_init(i915, wal); 1322 else if (IS_G4X(i915)) 1323 g4x_gt_workarounds_init(i915, wal); 1324 else if (IS_GEN(i915, 4)) 1325 gen4_gt_workarounds_init(i915, wal); 1326 else if (INTEL_GEN(i915) <= 8) 1327 return; 1328 else 1329 MISSING_CASE(INTEL_GEN(i915)); 1330 } 1331 1332 void intel_gt_init_workarounds(struct drm_i915_private *i915) 1333 { 1334 struct i915_wa_list *wal = &i915->gt_wa_list; 1335 1336 wa_init_start(wal, "GT", "global"); 1337 gt_init_workarounds(i915, wal); 1338 wa_init_finish(wal); 1339 } 1340 1341 static enum forcewake_domains 1342 wal_get_fw_for_rmw(struct intel_uncore *uncore, const struct i915_wa_list *wal) 1343 { 1344 enum forcewake_domains fw = 0; 1345 struct i915_wa *wa; 1346 unsigned int i; 1347 1348 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) 1349 fw |= intel_uncore_forcewake_for_reg(uncore, 1350 wa->reg, 1351 FW_REG_READ | 1352 FW_REG_WRITE); 1353 1354 return fw; 1355 } 1356 1357 static bool 1358 wa_verify(const struct i915_wa *wa, u32 cur, const char *name, const char *from) 1359 { 1360 if ((cur ^ wa->set) & wa->read) { 1361 DRM_ERROR("%s workaround lost on %s! (%x=%x/%x, expected %x)\n", 1362 name, from, i915_mmio_reg_offset(wa->reg), 1363 cur, cur & wa->read, wa->set); 1364 1365 return false; 1366 } 1367 1368 return true; 1369 } 1370 1371 static void 1372 wa_list_apply(struct intel_uncore *uncore, const struct i915_wa_list *wal) 1373 { 1374 enum forcewake_domains fw; 1375 unsigned long flags; 1376 struct i915_wa *wa; 1377 unsigned int i; 1378 1379 if (!wal->count) 1380 return; 1381 1382 fw = wal_get_fw_for_rmw(uncore, wal); 1383 1384 spin_lock_irqsave(&uncore->lock, flags); 1385 intel_uncore_forcewake_get__locked(uncore, fw); 1386 1387 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) { 1388 if (wa->clr) 1389 intel_uncore_rmw_fw(uncore, wa->reg, wa->clr, wa->set); 1390 else 1391 intel_uncore_write_fw(uncore, wa->reg, wa->set); 1392 if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)) 1393 wa_verify(wa, 1394 intel_uncore_read_fw(uncore, wa->reg), 1395 wal->name, "application"); 1396 } 1397 1398 intel_uncore_forcewake_put__locked(uncore, fw); 1399 spin_unlock_irqrestore(&uncore->lock, flags); 1400 } 1401 1402 void intel_gt_apply_workarounds(struct intel_gt *gt) 1403 { 1404 wa_list_apply(gt->uncore, >->i915->gt_wa_list); 1405 } 1406 1407 static bool wa_list_verify(struct intel_uncore *uncore, 1408 const struct i915_wa_list *wal, 1409 const char *from) 1410 { 1411 struct i915_wa *wa; 1412 unsigned int i; 1413 bool ok = true; 1414 1415 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) 1416 ok &= wa_verify(wa, 1417 intel_uncore_read(uncore, wa->reg), 1418 wal->name, from); 1419 1420 return ok; 1421 } 1422 1423 bool intel_gt_verify_workarounds(struct intel_gt *gt, const char *from) 1424 { 1425 return wa_list_verify(gt->uncore, >->i915->gt_wa_list, from); 1426 } 1427 1428 static inline bool is_nonpriv_flags_valid(u32 flags) 1429 { 1430 /* Check only valid flag bits are set */ 1431 if (flags & ~RING_FORCE_TO_NONPRIV_MASK_VALID) 1432 return false; 1433 1434 /* NB: Only 3 out of 4 enum values are valid for access field */ 1435 if ((flags & RING_FORCE_TO_NONPRIV_ACCESS_MASK) == 1436 RING_FORCE_TO_NONPRIV_ACCESS_INVALID) 1437 return false; 1438 1439 return true; 1440 } 1441 1442 static void 1443 whitelist_reg_ext(struct i915_wa_list *wal, i915_reg_t reg, u32 flags) 1444 { 1445 struct i915_wa wa = { 1446 .reg = reg 1447 }; 1448 1449 if (GEM_DEBUG_WARN_ON(wal->count >= RING_MAX_NONPRIV_SLOTS)) 1450 return; 1451 1452 if (GEM_DEBUG_WARN_ON(!is_nonpriv_flags_valid(flags))) 1453 return; 1454 1455 wa.reg.reg |= flags; 1456 _wa_add(wal, &wa); 1457 } 1458 1459 static void 1460 whitelist_reg(struct i915_wa_list *wal, i915_reg_t reg) 1461 { 1462 whitelist_reg_ext(wal, reg, RING_FORCE_TO_NONPRIV_ACCESS_RW); 1463 } 1464 1465 static void gen9_whitelist_build(struct i915_wa_list *w) 1466 { 1467 /* WaVFEStateAfterPipeControlwithMediaStateClear:skl,bxt,glk,cfl */ 1468 whitelist_reg(w, GEN9_CTX_PREEMPT_REG); 1469 1470 /* WaEnablePreemptionGranularityControlByUMD:skl,bxt,kbl,cfl,[cnl] */ 1471 whitelist_reg(w, GEN8_CS_CHICKEN1); 1472 1473 /* WaAllowUMDToModifyHDCChicken1:skl,bxt,kbl,glk,cfl */ 1474 whitelist_reg(w, GEN8_HDC_CHICKEN1); 1475 1476 /* WaSendPushConstantsFromMMIO:skl,bxt */ 1477 whitelist_reg(w, COMMON_SLICE_CHICKEN2); 1478 } 1479 1480 static void skl_whitelist_build(struct intel_engine_cs *engine) 1481 { 1482 struct i915_wa_list *w = &engine->whitelist; 1483 1484 if (engine->class != RENDER_CLASS) 1485 return; 1486 1487 gen9_whitelist_build(w); 1488 1489 /* WaDisableLSQCROPERFforOCL:skl */ 1490 whitelist_reg(w, GEN8_L3SQCREG4); 1491 } 1492 1493 static void bxt_whitelist_build(struct intel_engine_cs *engine) 1494 { 1495 if (engine->class != RENDER_CLASS) 1496 return; 1497 1498 gen9_whitelist_build(&engine->whitelist); 1499 } 1500 1501 static void kbl_whitelist_build(struct intel_engine_cs *engine) 1502 { 1503 struct i915_wa_list *w = &engine->whitelist; 1504 1505 if (engine->class != RENDER_CLASS) 1506 return; 1507 1508 gen9_whitelist_build(w); 1509 1510 /* WaDisableLSQCROPERFforOCL:kbl */ 1511 whitelist_reg(w, GEN8_L3SQCREG4); 1512 } 1513 1514 static void glk_whitelist_build(struct intel_engine_cs *engine) 1515 { 1516 struct i915_wa_list *w = &engine->whitelist; 1517 1518 if (engine->class != RENDER_CLASS) 1519 return; 1520 1521 gen9_whitelist_build(w); 1522 1523 /* WA #0862: Userspace has to set "Barrier Mode" to avoid hangs. */ 1524 whitelist_reg(w, GEN9_SLICE_COMMON_ECO_CHICKEN1); 1525 } 1526 1527 static void cfl_whitelist_build(struct intel_engine_cs *engine) 1528 { 1529 struct i915_wa_list *w = &engine->whitelist; 1530 1531 if (engine->class != RENDER_CLASS) 1532 return; 1533 1534 gen9_whitelist_build(w); 1535 1536 /* 1537 * WaAllowPMDepthAndInvocationCountAccessFromUMD:cfl,whl,cml,aml 1538 * 1539 * This covers 4 register which are next to one another : 1540 * - PS_INVOCATION_COUNT 1541 * - PS_INVOCATION_COUNT_UDW 1542 * - PS_DEPTH_COUNT 1543 * - PS_DEPTH_COUNT_UDW 1544 */ 1545 whitelist_reg_ext(w, PS_INVOCATION_COUNT, 1546 RING_FORCE_TO_NONPRIV_ACCESS_RD | 1547 RING_FORCE_TO_NONPRIV_RANGE_4); 1548 } 1549 1550 static void cml_whitelist_build(struct intel_engine_cs *engine) 1551 { 1552 struct i915_wa_list *w = &engine->whitelist; 1553 1554 if (engine->class != RENDER_CLASS) 1555 whitelist_reg_ext(w, 1556 RING_CTX_TIMESTAMP(engine->mmio_base), 1557 RING_FORCE_TO_NONPRIV_ACCESS_RD); 1558 1559 cfl_whitelist_build(engine); 1560 } 1561 1562 static void cnl_whitelist_build(struct intel_engine_cs *engine) 1563 { 1564 struct i915_wa_list *w = &engine->whitelist; 1565 1566 if (engine->class != RENDER_CLASS) 1567 return; 1568 1569 /* WaEnablePreemptionGranularityControlByUMD:cnl */ 1570 whitelist_reg(w, GEN8_CS_CHICKEN1); 1571 } 1572 1573 static void icl_whitelist_build(struct intel_engine_cs *engine) 1574 { 1575 struct i915_wa_list *w = &engine->whitelist; 1576 1577 switch (engine->class) { 1578 case RENDER_CLASS: 1579 /* WaAllowUMDToModifyHalfSliceChicken7:icl */ 1580 whitelist_reg(w, GEN9_HALF_SLICE_CHICKEN7); 1581 1582 /* WaAllowUMDToModifySamplerMode:icl */ 1583 whitelist_reg(w, GEN10_SAMPLER_MODE); 1584 1585 /* WaEnableStateCacheRedirectToCS:icl */ 1586 whitelist_reg(w, GEN9_SLICE_COMMON_ECO_CHICKEN1); 1587 1588 /* 1589 * WaAllowPMDepthAndInvocationCountAccessFromUMD:icl 1590 * 1591 * This covers 4 register which are next to one another : 1592 * - PS_INVOCATION_COUNT 1593 * - PS_INVOCATION_COUNT_UDW 1594 * - PS_DEPTH_COUNT 1595 * - PS_DEPTH_COUNT_UDW 1596 */ 1597 whitelist_reg_ext(w, PS_INVOCATION_COUNT, 1598 RING_FORCE_TO_NONPRIV_ACCESS_RD | 1599 RING_FORCE_TO_NONPRIV_RANGE_4); 1600 break; 1601 1602 case VIDEO_DECODE_CLASS: 1603 /* hucStatusRegOffset */ 1604 whitelist_reg_ext(w, _MMIO(0x2000 + engine->mmio_base), 1605 RING_FORCE_TO_NONPRIV_ACCESS_RD); 1606 /* hucUKernelHdrInfoRegOffset */ 1607 whitelist_reg_ext(w, _MMIO(0x2014 + engine->mmio_base), 1608 RING_FORCE_TO_NONPRIV_ACCESS_RD); 1609 /* hucStatus2RegOffset */ 1610 whitelist_reg_ext(w, _MMIO(0x23B0 + engine->mmio_base), 1611 RING_FORCE_TO_NONPRIV_ACCESS_RD); 1612 whitelist_reg_ext(w, 1613 RING_CTX_TIMESTAMP(engine->mmio_base), 1614 RING_FORCE_TO_NONPRIV_ACCESS_RD); 1615 break; 1616 1617 default: 1618 whitelist_reg_ext(w, 1619 RING_CTX_TIMESTAMP(engine->mmio_base), 1620 RING_FORCE_TO_NONPRIV_ACCESS_RD); 1621 break; 1622 } 1623 } 1624 1625 static void tgl_whitelist_build(struct intel_engine_cs *engine) 1626 { 1627 struct i915_wa_list *w = &engine->whitelist; 1628 1629 switch (engine->class) { 1630 case RENDER_CLASS: 1631 /* 1632 * WaAllowPMDepthAndInvocationCountAccessFromUMD:tgl 1633 * Wa_1408556865:tgl 1634 * 1635 * This covers 4 registers which are next to one another : 1636 * - PS_INVOCATION_COUNT 1637 * - PS_INVOCATION_COUNT_UDW 1638 * - PS_DEPTH_COUNT 1639 * - PS_DEPTH_COUNT_UDW 1640 */ 1641 whitelist_reg_ext(w, PS_INVOCATION_COUNT, 1642 RING_FORCE_TO_NONPRIV_ACCESS_RD | 1643 RING_FORCE_TO_NONPRIV_RANGE_4); 1644 1645 /* Wa_1808121037:tgl */ 1646 whitelist_reg(w, GEN7_COMMON_SLICE_CHICKEN1); 1647 1648 /* Wa_1806527549:tgl */ 1649 whitelist_reg(w, HIZ_CHICKEN); 1650 break; 1651 default: 1652 whitelist_reg_ext(w, 1653 RING_CTX_TIMESTAMP(engine->mmio_base), 1654 RING_FORCE_TO_NONPRIV_ACCESS_RD); 1655 break; 1656 } 1657 } 1658 1659 static void dg1_whitelist_build(struct intel_engine_cs *engine) 1660 { 1661 struct i915_wa_list *w = &engine->whitelist; 1662 1663 tgl_whitelist_build(engine); 1664 1665 /* GEN:BUG:1409280441:dg1 */ 1666 if (IS_DG1_REVID(engine->i915, DG1_REVID_A0, DG1_REVID_A0) && 1667 (engine->class == RENDER_CLASS || 1668 engine->class == COPY_ENGINE_CLASS)) 1669 whitelist_reg_ext(w, RING_ID(engine->mmio_base), 1670 RING_FORCE_TO_NONPRIV_ACCESS_RD); 1671 } 1672 1673 void intel_engine_init_whitelist(struct intel_engine_cs *engine) 1674 { 1675 struct drm_i915_private *i915 = engine->i915; 1676 struct i915_wa_list *w = &engine->whitelist; 1677 1678 wa_init_start(w, "whitelist", engine->name); 1679 1680 if (IS_DG1(i915)) 1681 dg1_whitelist_build(engine); 1682 else if (IS_GEN(i915, 12)) 1683 tgl_whitelist_build(engine); 1684 else if (IS_GEN(i915, 11)) 1685 icl_whitelist_build(engine); 1686 else if (IS_CANNONLAKE(i915)) 1687 cnl_whitelist_build(engine); 1688 else if (IS_COMETLAKE(i915)) 1689 cml_whitelist_build(engine); 1690 else if (IS_COFFEELAKE(i915)) 1691 cfl_whitelist_build(engine); 1692 else if (IS_GEMINILAKE(i915)) 1693 glk_whitelist_build(engine); 1694 else if (IS_KABYLAKE(i915)) 1695 kbl_whitelist_build(engine); 1696 else if (IS_BROXTON(i915)) 1697 bxt_whitelist_build(engine); 1698 else if (IS_SKYLAKE(i915)) 1699 skl_whitelist_build(engine); 1700 else if (INTEL_GEN(i915) <= 8) 1701 return; 1702 else 1703 MISSING_CASE(INTEL_GEN(i915)); 1704 1705 wa_init_finish(w); 1706 } 1707 1708 void intel_engine_apply_whitelist(struct intel_engine_cs *engine) 1709 { 1710 const struct i915_wa_list *wal = &engine->whitelist; 1711 struct intel_uncore *uncore = engine->uncore; 1712 const u32 base = engine->mmio_base; 1713 struct i915_wa *wa; 1714 unsigned int i; 1715 1716 if (!wal->count) 1717 return; 1718 1719 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) 1720 intel_uncore_write(uncore, 1721 RING_FORCE_TO_NONPRIV(base, i), 1722 i915_mmio_reg_offset(wa->reg)); 1723 1724 /* And clear the rest just in case of garbage */ 1725 for (; i < RING_MAX_NONPRIV_SLOTS; i++) 1726 intel_uncore_write(uncore, 1727 RING_FORCE_TO_NONPRIV(base, i), 1728 i915_mmio_reg_offset(RING_NOPID(base))); 1729 } 1730 1731 static void 1732 rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) 1733 { 1734 struct drm_i915_private *i915 = engine->i915; 1735 1736 if (IS_DG1_REVID(i915, DG1_REVID_A0, DG1_REVID_A0) || 1737 IS_TGL_UY_GT_REVID(i915, TGL_REVID_A0, TGL_REVID_A0)) { 1738 /* 1739 * Wa_1607138336:tgl[a0],dg1[a0] 1740 * Wa_1607063988:tgl[a0],dg1[a0] 1741 */ 1742 wa_write_or(wal, 1743 GEN9_CTX_PREEMPT_REG, 1744 GEN12_DISABLE_POSH_BUSY_FF_DOP_CG); 1745 } 1746 1747 if (IS_TGL_UY_GT_REVID(i915, TGL_REVID_A0, TGL_REVID_A0)) { 1748 /* 1749 * Wa_1606679103:tgl 1750 * (see also Wa_1606682166:icl) 1751 */ 1752 wa_write_or(wal, 1753 GEN7_SARCHKMD, 1754 GEN7_DISABLE_SAMPLER_PREFETCH); 1755 1756 /* Wa_1408615072:tgl */ 1757 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE2, 1758 VSUNIT_CLKGATE_DIS_TGL); 1759 } 1760 1761 if (IS_DG1(i915) || IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915)) { 1762 /* Wa_1606931601:tgl,rkl,dg1 */ 1763 wa_masked_en(wal, GEN7_ROW_CHICKEN2, GEN12_DISABLE_EARLY_READ); 1764 1765 /* 1766 * Wa_1407928979:tgl A* 1767 * Wa_18011464164:tgl[B0+],dg1[B0+] 1768 * Wa_22010931296:tgl[B0+],dg1[B0+] 1769 * Wa_14010919138:rkl, dg1 1770 */ 1771 wa_write_or(wal, GEN7_FF_THREAD_MODE, 1772 GEN12_FF_TESSELATION_DOP_GATE_DISABLE); 1773 } 1774 1775 if (IS_DG1_REVID(i915, DG1_REVID_A0, DG1_REVID_A0) || 1776 IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915)) { 1777 /* Wa_1409804808:tgl,rkl,dg1[a0] */ 1778 wa_masked_en(wal, GEN7_ROW_CHICKEN2, 1779 GEN12_PUSH_CONST_DEREF_HOLD_DIS); 1780 1781 /* 1782 * Wa_1409085225:tgl 1783 * Wa_14010229206:tgl,rkl,dg1[a0] 1784 */ 1785 wa_masked_en(wal, GEN9_ROW_CHICKEN4, GEN12_DISABLE_TDL_PUSH); 1786 1787 /* 1788 * Wa_1607030317:tgl 1789 * Wa_1607186500:tgl 1790 * Wa_1607297627:tgl,rkl,dg1[a0] 1791 * 1792 * On TGL and RKL there are multiple entries for this WA in the 1793 * BSpec; some indicate this is an A0-only WA, others indicate 1794 * it applies to all steppings so we trust the "all steppings." 1795 * For DG1 this only applies to A0. 1796 */ 1797 wa_masked_en(wal, 1798 GEN6_RC_SLEEP_PSMI_CONTROL, 1799 GEN12_WAIT_FOR_EVENT_POWER_DOWN_DISABLE | 1800 GEN8_RC_SEMA_IDLE_MSG_DISABLE); 1801 1802 /* 1803 * Wa_1606700617:tgl 1804 * Wa_22010271021:tgl,rkl 1805 */ 1806 wa_masked_en(wal, 1807 GEN9_CS_DEBUG_MODE1, 1808 FF_DOP_CLOCK_GATE_DISABLE); 1809 } 1810 1811 if (IS_GEN(i915, 12)) { 1812 /* Wa_1406941453:gen12 */ 1813 wa_masked_en(wal, 1814 GEN10_SAMPLER_MODE, 1815 ENABLE_SMALLPL); 1816 } 1817 1818 if (IS_GEN(i915, 11)) { 1819 /* This is not an Wa. Enable for better image quality */ 1820 wa_masked_en(wal, 1821 _3D_CHICKEN3, 1822 _3D_CHICKEN3_AA_LINE_QUALITY_FIX_ENABLE); 1823 1824 /* WaPipelineFlushCoherentLines:icl */ 1825 wa_write_or(wal, 1826 GEN8_L3SQCREG4, 1827 GEN8_LQSC_FLUSH_COHERENT_LINES); 1828 1829 /* 1830 * Wa_1405543622:icl 1831 * Formerly known as WaGAPZPriorityScheme 1832 */ 1833 wa_write_or(wal, 1834 GEN8_GARBCNTL, 1835 GEN11_ARBITRATION_PRIO_ORDER_MASK); 1836 1837 /* 1838 * Wa_1604223664:icl 1839 * Formerly known as WaL3BankAddressHashing 1840 */ 1841 wa_write_masked_or(wal, 1842 GEN8_GARBCNTL, 1843 GEN11_HASH_CTRL_EXCL_MASK, 1844 GEN11_HASH_CTRL_EXCL_BIT0); 1845 wa_write_masked_or(wal, 1846 GEN11_GLBLINVL, 1847 GEN11_BANK_HASH_ADDR_EXCL_MASK, 1848 GEN11_BANK_HASH_ADDR_EXCL_BIT0); 1849 1850 /* 1851 * Wa_1405733216:icl 1852 * Formerly known as WaDisableCleanEvicts 1853 */ 1854 wa_write_or(wal, 1855 GEN8_L3SQCREG4, 1856 GEN11_LQSC_CLEAN_EVICT_DISABLE); 1857 1858 /* WaForwardProgressSoftReset:icl */ 1859 wa_write_or(wal, 1860 GEN10_SCRATCH_LNCF2, 1861 PMFLUSHDONE_LNICRSDROP | 1862 PMFLUSH_GAPL3UNBLOCK | 1863 PMFLUSHDONE_LNEBLK); 1864 1865 /* Wa_1406609255:icl (pre-prod) */ 1866 if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_B0)) 1867 wa_write_or(wal, 1868 GEN7_SARCHKMD, 1869 GEN7_DISABLE_DEMAND_PREFETCH); 1870 1871 /* Wa_1606682166:icl */ 1872 wa_write_or(wal, 1873 GEN7_SARCHKMD, 1874 GEN7_DISABLE_SAMPLER_PREFETCH); 1875 1876 /* Wa_1409178092:icl */ 1877 wa_write_masked_or(wal, 1878 GEN11_SCRATCH2, 1879 GEN11_COHERENT_PARTIAL_WRITE_MERGE_ENABLE, 1880 0); 1881 1882 /* WaEnable32PlaneMode:icl */ 1883 wa_masked_en(wal, GEN9_CSFE_CHICKEN1_RCS, 1884 GEN11_ENABLE_32_PLANE_MODE); 1885 1886 /* 1887 * Wa_1408615072:icl,ehl (vsunit) 1888 * Wa_1407596294:icl,ehl (hsunit) 1889 */ 1890 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE, 1891 VSUNIT_CLKGATE_DIS | HSUNIT_CLKGATE_DIS); 1892 1893 /* Wa_1407352427:icl,ehl */ 1894 wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE2, 1895 PSDUNIT_CLKGATE_DIS); 1896 1897 /* Wa_1406680159:icl,ehl */ 1898 wa_write_or(wal, 1899 SUBSLICE_UNIT_LEVEL_CLKGATE, 1900 GWUNIT_CLKGATE_DIS); 1901 1902 /* 1903 * Wa_1408767742:icl[a2..forever],ehl[all] 1904 * Wa_1605460711:icl[a0..c0] 1905 */ 1906 wa_write_or(wal, 1907 GEN7_FF_THREAD_MODE, 1908 GEN12_FF_TESSELATION_DOP_GATE_DISABLE); 1909 1910 /* Wa_22010271021:ehl */ 1911 if (IS_JSL_EHL(i915)) 1912 wa_masked_en(wal, 1913 GEN9_CS_DEBUG_MODE1, 1914 FF_DOP_CLOCK_GATE_DISABLE); 1915 } 1916 1917 if (IS_GEN_RANGE(i915, 9, 12)) { 1918 /* FtrPerCtxtPreemptionGranularityControl:skl,bxt,kbl,cfl,cnl,icl,tgl */ 1919 wa_masked_en(wal, 1920 GEN7_FF_SLICE_CS_CHICKEN1, 1921 GEN9_FFSC_PERCTX_PREEMPT_CTRL); 1922 } 1923 1924 if (IS_SKYLAKE(i915) || 1925 IS_KABYLAKE(i915) || 1926 IS_COFFEELAKE(i915) || 1927 IS_COMETLAKE(i915)) { 1928 /* WaEnableGapsTsvCreditFix:skl,kbl,cfl */ 1929 wa_write_or(wal, 1930 GEN8_GARBCNTL, 1931 GEN9_GAPS_TSV_CREDIT_DISABLE); 1932 } 1933 1934 if (IS_BROXTON(i915)) { 1935 /* WaDisablePooledEuLoadBalancingFix:bxt */ 1936 wa_masked_en(wal, 1937 FF_SLICE_CS_CHICKEN2, 1938 GEN9_POOLED_EU_LOAD_BALANCING_FIX_DISABLE); 1939 } 1940 1941 if (IS_GEN(i915, 9)) { 1942 /* WaContextSwitchWithConcurrentTLBInvalidate:skl,bxt,kbl,glk,cfl */ 1943 wa_masked_en(wal, 1944 GEN9_CSFE_CHICKEN1_RCS, 1945 GEN9_PREEMPT_GPGPU_SYNC_SWITCH_DISABLE); 1946 1947 /* WaEnableLbsSlaRetryTimerDecrement:skl,bxt,kbl,glk,cfl */ 1948 wa_write_or(wal, 1949 BDW_SCRATCH1, 1950 GEN9_LBS_SLA_RETRY_TIMER_DECREMENT_ENABLE); 1951 1952 /* WaProgramL3SqcReg1DefaultForPerf:bxt,glk */ 1953 if (IS_GEN9_LP(i915)) 1954 wa_write_masked_or(wal, 1955 GEN8_L3SQCREG1, 1956 L3_PRIO_CREDITS_MASK, 1957 L3_GENERAL_PRIO_CREDITS(62) | 1958 L3_HIGH_PRIO_CREDITS(2)); 1959 1960 /* WaOCLCoherentLineFlush:skl,bxt,kbl,cfl */ 1961 wa_write_or(wal, 1962 GEN8_L3SQCREG4, 1963 GEN8_LQSC_FLUSH_COHERENT_LINES); 1964 } 1965 1966 if (IS_GEN(i915, 7)) 1967 /* WaBCSVCSTlbInvalidationMode:ivb,vlv,hsw */ 1968 wa_masked_en(wal, 1969 GFX_MODE_GEN7, 1970 GFX_TLB_INVALIDATE_EXPLICIT | GFX_REPLAY_MODE); 1971 1972 if (IS_GEN_RANGE(i915, 6, 7)) 1973 /* 1974 * We need to disable the AsyncFlip performance optimisations in 1975 * order to use MI_WAIT_FOR_EVENT within the CS. It should 1976 * already be programmed to '1' on all products. 1977 * 1978 * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv 1979 */ 1980 wa_masked_en(wal, 1981 MI_MODE, 1982 ASYNC_FLIP_PERF_DISABLE); 1983 1984 if (IS_GEN(i915, 6)) { 1985 /* 1986 * Required for the hardware to program scanline values for 1987 * waiting 1988 * WaEnableFlushTlbInvalidationMode:snb 1989 */ 1990 wa_masked_en(wal, 1991 GFX_MODE, 1992 GFX_TLB_INVALIDATE_EXPLICIT); 1993 1994 /* 1995 * From the Sandybridge PRM, volume 1 part 3, page 24: 1996 * "If this bit is set, STCunit will have LRA as replacement 1997 * policy. [...] This bit must be reset. LRA replacement 1998 * policy is not supported." 1999 */ 2000 wa_masked_dis(wal, 2001 CACHE_MODE_0, 2002 CM0_STC_EVICT_DISABLE_LRA_SNB); 2003 } 2004 2005 if (IS_GEN_RANGE(i915, 4, 6)) 2006 /* WaTimedSingleVertexDispatch:cl,bw,ctg,elk,ilk,snb */ 2007 wa_add(wal, MI_MODE, 2008 0, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH), 2009 /* XXX bit doesn't stick on Broadwater */ 2010 IS_I965G(i915) ? 0 : VS_TIMER_DISPATCH); 2011 2012 if (IS_GEN(i915, 4)) 2013 /* 2014 * Disable CONSTANT_BUFFER before it is loaded from the context 2015 * image. For as it is loaded, it is executed and the stored 2016 * address may no longer be valid, leading to a GPU hang. 2017 * 2018 * This imposes the requirement that userspace reload their 2019 * CONSTANT_BUFFER on every batch, fortunately a requirement 2020 * they are already accustomed to from before contexts were 2021 * enabled. 2022 */ 2023 wa_add(wal, ECOSKPD, 2024 0, _MASKED_BIT_ENABLE(ECO_CONSTANT_BUFFER_SR_DISABLE), 2025 0 /* XXX bit doesn't stick on Broadwater */); 2026 } 2027 2028 static void 2029 xcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) 2030 { 2031 struct drm_i915_private *i915 = engine->i915; 2032 2033 /* WaKBLVECSSemaphoreWaitPoll:kbl */ 2034 if (IS_KBL_GT_REVID(i915, KBL_REVID_A0, KBL_REVID_E0)) { 2035 wa_write(wal, 2036 RING_SEMA_WAIT_POLL(engine->mmio_base), 2037 1); 2038 } 2039 } 2040 2041 static void 2042 engine_init_workarounds(struct intel_engine_cs *engine, struct i915_wa_list *wal) 2043 { 2044 if (I915_SELFTEST_ONLY(INTEL_GEN(engine->i915) < 4)) 2045 return; 2046 2047 if (engine->class == RENDER_CLASS) 2048 rcs_engine_wa_init(engine, wal); 2049 else 2050 xcs_engine_wa_init(engine, wal); 2051 } 2052 2053 void intel_engine_init_workarounds(struct intel_engine_cs *engine) 2054 { 2055 struct i915_wa_list *wal = &engine->wa_list; 2056 2057 if (INTEL_GEN(engine->i915) < 4) 2058 return; 2059 2060 wa_init_start(wal, "engine", engine->name); 2061 engine_init_workarounds(engine, wal); 2062 wa_init_finish(wal); 2063 } 2064 2065 void intel_engine_apply_workarounds(struct intel_engine_cs *engine) 2066 { 2067 wa_list_apply(engine->uncore, &engine->wa_list); 2068 } 2069 2070 static struct i915_vma * 2071 create_scratch(struct i915_address_space *vm, int count) 2072 { 2073 struct drm_i915_gem_object *obj; 2074 struct i915_vma *vma; 2075 unsigned int size; 2076 int err; 2077 2078 size = round_up(count * sizeof(u32), PAGE_SIZE); 2079 obj = i915_gem_object_create_internal(vm->i915, size); 2080 if (IS_ERR(obj)) 2081 return ERR_CAST(obj); 2082 2083 i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC); 2084 2085 vma = i915_vma_instance(obj, vm, NULL); 2086 if (IS_ERR(vma)) { 2087 err = PTR_ERR(vma); 2088 goto err_obj; 2089 } 2090 2091 err = i915_vma_pin(vma, 0, 0, 2092 i915_vma_is_ggtt(vma) ? PIN_GLOBAL : PIN_USER); 2093 if (err) 2094 goto err_obj; 2095 2096 return vma; 2097 2098 err_obj: 2099 i915_gem_object_put(obj); 2100 return ERR_PTR(err); 2101 } 2102 2103 struct mcr_range { 2104 u32 start; 2105 u32 end; 2106 }; 2107 2108 static const struct mcr_range mcr_ranges_gen8[] = { 2109 { .start = 0x5500, .end = 0x55ff }, 2110 { .start = 0x7000, .end = 0x7fff }, 2111 { .start = 0x9400, .end = 0x97ff }, 2112 { .start = 0xb000, .end = 0xb3ff }, 2113 { .start = 0xe000, .end = 0xe7ff }, 2114 {}, 2115 }; 2116 2117 static const struct mcr_range mcr_ranges_gen12[] = { 2118 { .start = 0x8150, .end = 0x815f }, 2119 { .start = 0x9520, .end = 0x955f }, 2120 { .start = 0xb100, .end = 0xb3ff }, 2121 { .start = 0xde80, .end = 0xe8ff }, 2122 { .start = 0x24a00, .end = 0x24a7f }, 2123 {}, 2124 }; 2125 2126 static bool mcr_range(struct drm_i915_private *i915, u32 offset) 2127 { 2128 const struct mcr_range *mcr_ranges; 2129 int i; 2130 2131 if (INTEL_GEN(i915) >= 12) 2132 mcr_ranges = mcr_ranges_gen12; 2133 else if (INTEL_GEN(i915) >= 8) 2134 mcr_ranges = mcr_ranges_gen8; 2135 else 2136 return false; 2137 2138 /* 2139 * Registers in these ranges are affected by the MCR selector 2140 * which only controls CPU initiated MMIO. Routing does not 2141 * work for CS access so we cannot verify them on this path. 2142 */ 2143 for (i = 0; mcr_ranges[i].start; i++) 2144 if (offset >= mcr_ranges[i].start && 2145 offset <= mcr_ranges[i].end) 2146 return true; 2147 2148 return false; 2149 } 2150 2151 static int 2152 wa_list_srm(struct i915_request *rq, 2153 const struct i915_wa_list *wal, 2154 struct i915_vma *vma) 2155 { 2156 struct drm_i915_private *i915 = rq->engine->i915; 2157 unsigned int i, count = 0; 2158 const struct i915_wa *wa; 2159 u32 srm, *cs; 2160 2161 srm = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT; 2162 if (INTEL_GEN(i915) >= 8) 2163 srm++; 2164 2165 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) { 2166 if (!mcr_range(i915, i915_mmio_reg_offset(wa->reg))) 2167 count++; 2168 } 2169 2170 cs = intel_ring_begin(rq, 4 * count); 2171 if (IS_ERR(cs)) 2172 return PTR_ERR(cs); 2173 2174 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) { 2175 u32 offset = i915_mmio_reg_offset(wa->reg); 2176 2177 if (mcr_range(i915, offset)) 2178 continue; 2179 2180 *cs++ = srm; 2181 *cs++ = offset; 2182 *cs++ = i915_ggtt_offset(vma) + sizeof(u32) * i; 2183 *cs++ = 0; 2184 } 2185 intel_ring_advance(rq, cs); 2186 2187 return 0; 2188 } 2189 2190 static int engine_wa_list_verify(struct intel_context *ce, 2191 const struct i915_wa_list * const wal, 2192 const char *from) 2193 { 2194 const struct i915_wa *wa; 2195 struct i915_request *rq; 2196 struct i915_vma *vma; 2197 struct i915_gem_ww_ctx ww; 2198 unsigned int i; 2199 u32 *results; 2200 int err; 2201 2202 if (!wal->count) 2203 return 0; 2204 2205 vma = create_scratch(&ce->engine->gt->ggtt->vm, wal->count); 2206 if (IS_ERR(vma)) 2207 return PTR_ERR(vma); 2208 2209 intel_engine_pm_get(ce->engine); 2210 i915_gem_ww_ctx_init(&ww, false); 2211 retry: 2212 err = i915_gem_object_lock(vma->obj, &ww); 2213 if (err == 0) 2214 err = intel_context_pin_ww(ce, &ww); 2215 if (err) 2216 goto err_pm; 2217 2218 rq = i915_request_create(ce); 2219 if (IS_ERR(rq)) { 2220 err = PTR_ERR(rq); 2221 goto err_unpin; 2222 } 2223 2224 err = i915_request_await_object(rq, vma->obj, true); 2225 if (err == 0) 2226 err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE); 2227 if (err == 0) 2228 err = wa_list_srm(rq, wal, vma); 2229 2230 i915_request_get(rq); 2231 if (err) 2232 i915_request_set_error_once(rq, err); 2233 i915_request_add(rq); 2234 2235 if (err) 2236 goto err_rq; 2237 2238 if (i915_request_wait(rq, 0, HZ / 5) < 0) { 2239 err = -ETIME; 2240 goto err_rq; 2241 } 2242 2243 results = i915_gem_object_pin_map(vma->obj, I915_MAP_WB); 2244 if (IS_ERR(results)) { 2245 err = PTR_ERR(results); 2246 goto err_rq; 2247 } 2248 2249 err = 0; 2250 for (i = 0, wa = wal->list; i < wal->count; i++, wa++) { 2251 if (mcr_range(rq->engine->i915, i915_mmio_reg_offset(wa->reg))) 2252 continue; 2253 2254 if (!wa_verify(wa, results[i], wal->name, from)) 2255 err = -ENXIO; 2256 } 2257 2258 i915_gem_object_unpin_map(vma->obj); 2259 2260 err_rq: 2261 i915_request_put(rq); 2262 err_unpin: 2263 intel_context_unpin(ce); 2264 err_pm: 2265 if (err == -EDEADLK) { 2266 err = i915_gem_ww_ctx_backoff(&ww); 2267 if (!err) 2268 goto retry; 2269 } 2270 i915_gem_ww_ctx_fini(&ww); 2271 intel_engine_pm_put(ce->engine); 2272 i915_vma_unpin(vma); 2273 i915_vma_put(vma); 2274 return err; 2275 } 2276 2277 int intel_engine_verify_workarounds(struct intel_engine_cs *engine, 2278 const char *from) 2279 { 2280 return engine_wa_list_verify(engine->kernel_context, 2281 &engine->wa_list, 2282 from); 2283 } 2284 2285 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 2286 #include "selftest_workarounds.c" 2287 #endif 2288