1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2019 Intel Corporation 4 */ 5 6 #include "i915_drv.h" 7 #include "intel_gt.h" 8 #include "intel_gt_pm.h" 9 #include "intel_gt_requests.h" 10 #include "intel_mocs.h" 11 #include "intel_rc6.h" 12 #include "intel_rps.h" 13 #include "intel_uncore.h" 14 #include "intel_pm.h" 15 16 void intel_gt_init_early(struct intel_gt *gt, struct drm_i915_private *i915) 17 { 18 gt->i915 = i915; 19 gt->uncore = &i915->uncore; 20 21 spin_lock_init(>->irq_lock); 22 23 INIT_LIST_HEAD(>->closed_vma); 24 spin_lock_init(>->closed_lock); 25 26 intel_gt_init_reset(gt); 27 intel_gt_init_requests(gt); 28 intel_gt_pm_init_early(gt); 29 30 intel_rps_init_early(>->rps); 31 intel_uc_init_early(>->uc); 32 } 33 34 void intel_gt_init_hw_early(struct intel_gt *gt, struct i915_ggtt *ggtt) 35 { 36 gt->ggtt = ggtt; 37 38 intel_gt_sanitize(gt, false); 39 } 40 41 static void init_unused_ring(struct intel_gt *gt, u32 base) 42 { 43 struct intel_uncore *uncore = gt->uncore; 44 45 intel_uncore_write(uncore, RING_CTL(base), 0); 46 intel_uncore_write(uncore, RING_HEAD(base), 0); 47 intel_uncore_write(uncore, RING_TAIL(base), 0); 48 intel_uncore_write(uncore, RING_START(base), 0); 49 } 50 51 static void init_unused_rings(struct intel_gt *gt) 52 { 53 struct drm_i915_private *i915 = gt->i915; 54 55 if (IS_I830(i915)) { 56 init_unused_ring(gt, PRB1_BASE); 57 init_unused_ring(gt, SRB0_BASE); 58 init_unused_ring(gt, SRB1_BASE); 59 init_unused_ring(gt, SRB2_BASE); 60 init_unused_ring(gt, SRB3_BASE); 61 } else if (IS_GEN(i915, 2)) { 62 init_unused_ring(gt, SRB0_BASE); 63 init_unused_ring(gt, SRB1_BASE); 64 } else if (IS_GEN(i915, 3)) { 65 init_unused_ring(gt, PRB1_BASE); 66 init_unused_ring(gt, PRB2_BASE); 67 } 68 } 69 70 int intel_gt_init_hw(struct intel_gt *gt) 71 { 72 struct drm_i915_private *i915 = gt->i915; 73 struct intel_uncore *uncore = gt->uncore; 74 int ret; 75 76 BUG_ON(!i915->kernel_context); 77 ret = intel_gt_terminally_wedged(gt); 78 if (ret) 79 return ret; 80 81 gt->last_init_time = ktime_get(); 82 83 /* Double layer security blanket, see i915_gem_init() */ 84 intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL); 85 86 if (HAS_EDRAM(i915) && INTEL_GEN(i915) < 9) 87 intel_uncore_rmw(uncore, HSW_IDICR, 0, IDIHASHMSK(0xf)); 88 89 if (IS_HASWELL(i915)) 90 intel_uncore_write(uncore, 91 MI_PREDICATE_RESULT_2, 92 IS_HSW_GT3(i915) ? 93 LOWER_SLICE_ENABLED : LOWER_SLICE_DISABLED); 94 95 /* Apply the GT workarounds... */ 96 intel_gt_apply_workarounds(gt); 97 /* ...and determine whether they are sticking. */ 98 intel_gt_verify_workarounds(gt, "init"); 99 100 intel_gt_init_swizzling(gt); 101 102 /* 103 * At least 830 can leave some of the unused rings 104 * "active" (ie. head != tail) after resume which 105 * will prevent c3 entry. Makes sure all unused rings 106 * are totally idle. 107 */ 108 init_unused_rings(gt); 109 110 ret = i915_ppgtt_init_hw(gt); 111 if (ret) { 112 DRM_ERROR("Enabling PPGTT failed (%d)\n", ret); 113 goto out; 114 } 115 116 /* We can't enable contexts until all firmware is loaded */ 117 ret = intel_uc_init_hw(>->uc); 118 if (ret) { 119 i915_probe_error(i915, "Enabling uc failed (%d)\n", ret); 120 goto out; 121 } 122 123 intel_mocs_init(gt); 124 125 out: 126 intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL); 127 return ret; 128 } 129 130 static void rmw_set(struct intel_uncore *uncore, i915_reg_t reg, u32 set) 131 { 132 intel_uncore_rmw(uncore, reg, 0, set); 133 } 134 135 static void rmw_clear(struct intel_uncore *uncore, i915_reg_t reg, u32 clr) 136 { 137 intel_uncore_rmw(uncore, reg, clr, 0); 138 } 139 140 static void clear_register(struct intel_uncore *uncore, i915_reg_t reg) 141 { 142 intel_uncore_rmw(uncore, reg, 0, 0); 143 } 144 145 static void gen8_clear_engine_error_register(struct intel_engine_cs *engine) 146 { 147 GEN6_RING_FAULT_REG_RMW(engine, RING_FAULT_VALID, 0); 148 GEN6_RING_FAULT_REG_POSTING_READ(engine); 149 } 150 151 void 152 intel_gt_clear_error_registers(struct intel_gt *gt, 153 intel_engine_mask_t engine_mask) 154 { 155 struct drm_i915_private *i915 = gt->i915; 156 struct intel_uncore *uncore = gt->uncore; 157 u32 eir; 158 159 if (!IS_GEN(i915, 2)) 160 clear_register(uncore, PGTBL_ER); 161 162 if (INTEL_GEN(i915) < 4) 163 clear_register(uncore, IPEIR(RENDER_RING_BASE)); 164 else 165 clear_register(uncore, IPEIR_I965); 166 167 clear_register(uncore, EIR); 168 eir = intel_uncore_read(uncore, EIR); 169 if (eir) { 170 /* 171 * some errors might have become stuck, 172 * mask them. 173 */ 174 DRM_DEBUG_DRIVER("EIR stuck: 0x%08x, masking\n", eir); 175 rmw_set(uncore, EMR, eir); 176 intel_uncore_write(uncore, GEN2_IIR, 177 I915_MASTER_ERROR_INTERRUPT); 178 } 179 180 if (INTEL_GEN(i915) >= 12) { 181 rmw_clear(uncore, GEN12_RING_FAULT_REG, RING_FAULT_VALID); 182 intel_uncore_posting_read(uncore, GEN12_RING_FAULT_REG); 183 } else if (INTEL_GEN(i915) >= 8) { 184 rmw_clear(uncore, GEN8_RING_FAULT_REG, RING_FAULT_VALID); 185 intel_uncore_posting_read(uncore, GEN8_RING_FAULT_REG); 186 } else if (INTEL_GEN(i915) >= 6) { 187 struct intel_engine_cs *engine; 188 enum intel_engine_id id; 189 190 for_each_engine_masked(engine, gt, engine_mask, id) 191 gen8_clear_engine_error_register(engine); 192 } 193 } 194 195 static void gen6_check_faults(struct intel_gt *gt) 196 { 197 struct intel_engine_cs *engine; 198 enum intel_engine_id id; 199 u32 fault; 200 201 for_each_engine(engine, gt, id) { 202 fault = GEN6_RING_FAULT_REG_READ(engine); 203 if (fault & RING_FAULT_VALID) { 204 DRM_DEBUG_DRIVER("Unexpected fault\n" 205 "\tAddr: 0x%08lx\n" 206 "\tAddress space: %s\n" 207 "\tSource ID: %d\n" 208 "\tType: %d\n", 209 fault & PAGE_MASK, 210 fault & RING_FAULT_GTTSEL_MASK ? 211 "GGTT" : "PPGTT", 212 RING_FAULT_SRCID(fault), 213 RING_FAULT_FAULT_TYPE(fault)); 214 } 215 } 216 } 217 218 static void gen8_check_faults(struct intel_gt *gt) 219 { 220 struct intel_uncore *uncore = gt->uncore; 221 i915_reg_t fault_reg, fault_data0_reg, fault_data1_reg; 222 u32 fault; 223 224 if (INTEL_GEN(gt->i915) >= 12) { 225 fault_reg = GEN12_RING_FAULT_REG; 226 fault_data0_reg = GEN12_FAULT_TLB_DATA0; 227 fault_data1_reg = GEN12_FAULT_TLB_DATA1; 228 } else { 229 fault_reg = GEN8_RING_FAULT_REG; 230 fault_data0_reg = GEN8_FAULT_TLB_DATA0; 231 fault_data1_reg = GEN8_FAULT_TLB_DATA1; 232 } 233 234 fault = intel_uncore_read(uncore, fault_reg); 235 if (fault & RING_FAULT_VALID) { 236 u32 fault_data0, fault_data1; 237 u64 fault_addr; 238 239 fault_data0 = intel_uncore_read(uncore, fault_data0_reg); 240 fault_data1 = intel_uncore_read(uncore, fault_data1_reg); 241 242 fault_addr = ((u64)(fault_data1 & FAULT_VA_HIGH_BITS) << 44) | 243 ((u64)fault_data0 << 12); 244 245 DRM_DEBUG_DRIVER("Unexpected fault\n" 246 "\tAddr: 0x%08x_%08x\n" 247 "\tAddress space: %s\n" 248 "\tEngine ID: %d\n" 249 "\tSource ID: %d\n" 250 "\tType: %d\n", 251 upper_32_bits(fault_addr), 252 lower_32_bits(fault_addr), 253 fault_data1 & FAULT_GTT_SEL ? "GGTT" : "PPGTT", 254 GEN8_RING_FAULT_ENGINE_ID(fault), 255 RING_FAULT_SRCID(fault), 256 RING_FAULT_FAULT_TYPE(fault)); 257 } 258 } 259 260 void intel_gt_check_and_clear_faults(struct intel_gt *gt) 261 { 262 struct drm_i915_private *i915 = gt->i915; 263 264 /* From GEN8 onwards we only have one 'All Engine Fault Register' */ 265 if (INTEL_GEN(i915) >= 8) 266 gen8_check_faults(gt); 267 else if (INTEL_GEN(i915) >= 6) 268 gen6_check_faults(gt); 269 else 270 return; 271 272 intel_gt_clear_error_registers(gt, ALL_ENGINES); 273 } 274 275 void intel_gt_flush_ggtt_writes(struct intel_gt *gt) 276 { 277 struct intel_uncore *uncore = gt->uncore; 278 intel_wakeref_t wakeref; 279 280 /* 281 * No actual flushing is required for the GTT write domain for reads 282 * from the GTT domain. Writes to it "immediately" go to main memory 283 * as far as we know, so there's no chipset flush. It also doesn't 284 * land in the GPU render cache. 285 * 286 * However, we do have to enforce the order so that all writes through 287 * the GTT land before any writes to the device, such as updates to 288 * the GATT itself. 289 * 290 * We also have to wait a bit for the writes to land from the GTT. 291 * An uncached read (i.e. mmio) seems to be ideal for the round-trip 292 * timing. This issue has only been observed when switching quickly 293 * between GTT writes and CPU reads from inside the kernel on recent hw, 294 * and it appears to only affect discrete GTT blocks (i.e. on LLC 295 * system agents we cannot reproduce this behaviour, until Cannonlake 296 * that was!). 297 */ 298 299 wmb(); 300 301 if (INTEL_INFO(gt->i915)->has_coherent_ggtt) 302 return; 303 304 intel_gt_chipset_flush(gt); 305 306 with_intel_runtime_pm(uncore->rpm, wakeref) { 307 unsigned long flags; 308 309 spin_lock_irqsave(&uncore->lock, flags); 310 intel_uncore_posting_read_fw(uncore, 311 RING_HEAD(RENDER_RING_BASE)); 312 spin_unlock_irqrestore(&uncore->lock, flags); 313 } 314 } 315 316 void intel_gt_chipset_flush(struct intel_gt *gt) 317 { 318 wmb(); 319 if (INTEL_GEN(gt->i915) < 6) 320 intel_gtt_chipset_flush(); 321 } 322 323 void intel_gt_driver_register(struct intel_gt *gt) 324 { 325 intel_rps_driver_register(>->rps); 326 } 327 328 static int intel_gt_init_scratch(struct intel_gt *gt, unsigned int size) 329 { 330 struct drm_i915_private *i915 = gt->i915; 331 struct drm_i915_gem_object *obj; 332 struct i915_vma *vma; 333 int ret; 334 335 obj = i915_gem_object_create_stolen(i915, size); 336 if (IS_ERR(obj)) 337 obj = i915_gem_object_create_internal(i915, size); 338 if (IS_ERR(obj)) { 339 DRM_ERROR("Failed to allocate scratch page\n"); 340 return PTR_ERR(obj); 341 } 342 343 vma = i915_vma_instance(obj, >->ggtt->vm, NULL); 344 if (IS_ERR(vma)) { 345 ret = PTR_ERR(vma); 346 goto err_unref; 347 } 348 349 ret = i915_vma_pin(vma, 0, 0, PIN_GLOBAL | PIN_HIGH); 350 if (ret) 351 goto err_unref; 352 353 gt->scratch = i915_vma_make_unshrinkable(vma); 354 355 return 0; 356 357 err_unref: 358 i915_gem_object_put(obj); 359 return ret; 360 } 361 362 static void intel_gt_fini_scratch(struct intel_gt *gt) 363 { 364 i915_vma_unpin_and_release(>->scratch, 0); 365 } 366 367 int intel_gt_init(struct intel_gt *gt) 368 { 369 int err; 370 371 err = intel_gt_init_scratch(gt, IS_GEN(gt->i915, 2) ? SZ_256K : SZ_4K); 372 if (err) 373 return err; 374 375 intel_gt_pm_init(gt); 376 377 return 0; 378 } 379 380 void intel_gt_driver_remove(struct intel_gt *gt) 381 { 382 GEM_BUG_ON(gt->awake); 383 } 384 385 void intel_gt_driver_unregister(struct intel_gt *gt) 386 { 387 intel_rps_driver_unregister(>->rps); 388 } 389 390 void intel_gt_driver_release(struct intel_gt *gt) 391 { 392 intel_gt_pm_fini(gt); 393 intel_gt_fini_scratch(gt); 394 } 395 396 void intel_gt_driver_late_release(struct intel_gt *gt) 397 { 398 intel_uc_driver_late_release(>->uc); 399 intel_gt_fini_reset(gt); 400 } 401