1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2020 Intel Corporation 4 */ 5 6 #include <asm/set_memory.h> 7 #include <asm/smp.h> 8 #include <linux/types.h> 9 #include <linux/stop_machine.h> 10 11 #include <drm/drm_managed.h> 12 #include <drm/i915_drm.h> 13 #include <drm/intel-gtt.h> 14 15 #include "display/intel_display.h" 16 #include "gem/i915_gem_lmem.h" 17 18 #include "intel_context.h" 19 #include "intel_ggtt_gmch.h" 20 #include "intel_gpu_commands.h" 21 #include "intel_gt.h" 22 #include "intel_gt_regs.h" 23 #include "intel_pci_config.h" 24 #include "intel_ring.h" 25 #include "i915_drv.h" 26 #include "i915_pci.h" 27 #include "i915_request.h" 28 #include "i915_scatterlist.h" 29 #include "i915_utils.h" 30 #include "i915_vgpu.h" 31 32 #include "intel_gtt.h" 33 #include "gen8_ppgtt.h" 34 #include "intel_engine_pm.h" 35 36 static void i915_ggtt_color_adjust(const struct drm_mm_node *node, 37 unsigned long color, 38 u64 *start, 39 u64 *end) 40 { 41 if (i915_node_color_differs(node, color)) 42 *start += I915_GTT_PAGE_SIZE; 43 44 /* 45 * Also leave a space between the unallocated reserved node after the 46 * GTT and any objects within the GTT, i.e. we use the color adjustment 47 * to insert a guard page to prevent prefetches crossing over the 48 * GTT boundary. 49 */ 50 node = list_next_entry(node, node_list); 51 if (node->color != color) 52 *end -= I915_GTT_PAGE_SIZE; 53 } 54 55 static int ggtt_init_hw(struct i915_ggtt *ggtt) 56 { 57 struct drm_i915_private *i915 = ggtt->vm.i915; 58 59 i915_address_space_init(&ggtt->vm, VM_CLASS_GGTT); 60 61 ggtt->vm.is_ggtt = true; 62 63 /* Only VLV supports read-only GGTT mappings */ 64 ggtt->vm.has_read_only = IS_VALLEYVIEW(i915); 65 66 if (!HAS_LLC(i915) && !HAS_PPGTT(i915)) 67 ggtt->vm.mm.color_adjust = i915_ggtt_color_adjust; 68 69 if (ggtt->mappable_end) { 70 if (!io_mapping_init_wc(&ggtt->iomap, 71 ggtt->gmadr.start, 72 ggtt->mappable_end)) { 73 ggtt->vm.cleanup(&ggtt->vm); 74 return -EIO; 75 } 76 77 ggtt->mtrr = arch_phys_wc_add(ggtt->gmadr.start, 78 ggtt->mappable_end); 79 } 80 81 intel_ggtt_init_fences(ggtt); 82 83 return 0; 84 } 85 86 /** 87 * i915_ggtt_init_hw - Initialize GGTT hardware 88 * @i915: i915 device 89 */ 90 int i915_ggtt_init_hw(struct drm_i915_private *i915) 91 { 92 int ret; 93 94 /* 95 * Note that we use page colouring to enforce a guard page at the 96 * end of the address space. This is required as the CS may prefetch 97 * beyond the end of the batch buffer, across the page boundary, 98 * and beyond the end of the GTT if we do not provide a guard. 99 */ 100 ret = ggtt_init_hw(to_gt(i915)->ggtt); 101 if (ret) 102 return ret; 103 104 return 0; 105 } 106 107 /** 108 * i915_ggtt_suspend_vm - Suspend the memory mappings for a GGTT or DPT VM 109 * @vm: The VM to suspend the mappings for 110 * 111 * Suspend the memory mappings for all objects mapped to HW via the GGTT or a 112 * DPT page table. 113 */ 114 void i915_ggtt_suspend_vm(struct i915_address_space *vm) 115 { 116 struct i915_vma *vma, *vn; 117 int save_skip_rewrite; 118 119 drm_WARN_ON(&vm->i915->drm, !vm->is_ggtt && !vm->is_dpt); 120 121 retry: 122 i915_gem_drain_freed_objects(vm->i915); 123 124 mutex_lock(&vm->mutex); 125 126 /* 127 * Skip rewriting PTE on VMA unbind. 128 * FIXME: Use an argument to i915_vma_unbind() instead? 129 */ 130 save_skip_rewrite = vm->skip_pte_rewrite; 131 vm->skip_pte_rewrite = true; 132 133 list_for_each_entry_safe(vma, vn, &vm->bound_list, vm_link) { 134 struct drm_i915_gem_object *obj = vma->obj; 135 136 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 137 138 if (i915_vma_is_pinned(vma) || !i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND)) 139 continue; 140 141 /* unlikely to race when GPU is idle, so no worry about slowpath.. */ 142 if (WARN_ON(!i915_gem_object_trylock(obj, NULL))) { 143 /* 144 * No dead objects should appear here, GPU should be 145 * completely idle, and userspace suspended 146 */ 147 i915_gem_object_get(obj); 148 149 mutex_unlock(&vm->mutex); 150 151 i915_gem_object_lock(obj, NULL); 152 GEM_WARN_ON(i915_vma_unbind(vma)); 153 i915_gem_object_unlock(obj); 154 i915_gem_object_put(obj); 155 156 vm->skip_pte_rewrite = save_skip_rewrite; 157 goto retry; 158 } 159 160 if (!i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND)) { 161 i915_vma_wait_for_bind(vma); 162 163 __i915_vma_evict(vma, false); 164 drm_mm_remove_node(&vma->node); 165 } 166 167 i915_gem_object_unlock(obj); 168 } 169 170 vm->clear_range(vm, 0, vm->total); 171 172 vm->skip_pte_rewrite = save_skip_rewrite; 173 174 mutex_unlock(&vm->mutex); 175 } 176 177 void i915_ggtt_suspend(struct i915_ggtt *ggtt) 178 { 179 struct intel_gt *gt; 180 181 i915_ggtt_suspend_vm(&ggtt->vm); 182 ggtt->invalidate(ggtt); 183 184 list_for_each_entry(gt, &ggtt->gt_list, ggtt_link) 185 intel_gt_check_and_clear_faults(gt); 186 } 187 188 void gen6_ggtt_invalidate(struct i915_ggtt *ggtt) 189 { 190 struct intel_uncore *uncore = ggtt->vm.gt->uncore; 191 192 spin_lock_irq(&uncore->lock); 193 intel_uncore_write_fw(uncore, GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN); 194 intel_uncore_read_fw(uncore, GFX_FLSH_CNTL_GEN6); 195 spin_unlock_irq(&uncore->lock); 196 } 197 198 static bool needs_wc_ggtt_mapping(struct drm_i915_private *i915) 199 { 200 /* 201 * On BXT+/ICL+ writes larger than 64 bit to the GTT pagetable range 202 * will be dropped. For WC mappings in general we have 64 byte burst 203 * writes when the WC buffer is flushed, so we can't use it, but have to 204 * resort to an uncached mapping. The WC issue is easily caught by the 205 * readback check when writing GTT PTE entries. 206 */ 207 if (!IS_GEN9_LP(i915) && GRAPHICS_VER(i915) < 11) 208 return true; 209 210 return false; 211 } 212 213 static void gen8_ggtt_invalidate(struct i915_ggtt *ggtt) 214 { 215 struct intel_uncore *uncore = ggtt->vm.gt->uncore; 216 217 /* 218 * Note that as an uncached mmio write, this will flush the 219 * WCB of the writes into the GGTT before it triggers the invalidate. 220 * 221 * Only perform this when GGTT is mapped as WC, see ggtt_probe_common(). 222 */ 223 if (needs_wc_ggtt_mapping(ggtt->vm.i915)) 224 intel_uncore_write_fw(uncore, GFX_FLSH_CNTL_GEN6, 225 GFX_FLSH_CNTL_EN); 226 } 227 228 static void guc_ggtt_ct_invalidate(struct intel_gt *gt) 229 { 230 struct intel_uncore *uncore = gt->uncore; 231 intel_wakeref_t wakeref; 232 233 with_intel_runtime_pm_if_active(uncore->rpm, wakeref) { 234 struct intel_guc *guc = >->uc.guc; 235 236 intel_guc_invalidate_tlb_guc(guc); 237 } 238 } 239 240 static void guc_ggtt_invalidate(struct i915_ggtt *ggtt) 241 { 242 struct drm_i915_private *i915 = ggtt->vm.i915; 243 struct intel_gt *gt; 244 245 gen8_ggtt_invalidate(ggtt); 246 247 list_for_each_entry(gt, &ggtt->gt_list, ggtt_link) { 248 if (intel_guc_tlb_invalidation_is_available(>->uc.guc)) { 249 guc_ggtt_ct_invalidate(gt); 250 } else if (GRAPHICS_VER(i915) >= 12) { 251 intel_uncore_write_fw(gt->uncore, 252 GEN12_GUC_TLB_INV_CR, 253 GEN12_GUC_TLB_INV_CR_INVALIDATE); 254 } else { 255 intel_uncore_write_fw(gt->uncore, 256 GEN8_GTCR, GEN8_GTCR_INVALIDATE); 257 } 258 } 259 } 260 261 static u64 mtl_ggtt_pte_encode(dma_addr_t addr, 262 unsigned int pat_index, 263 u32 flags) 264 { 265 gen8_pte_t pte = addr | GEN8_PAGE_PRESENT; 266 267 WARN_ON_ONCE(addr & ~GEN12_GGTT_PTE_ADDR_MASK); 268 269 if (flags & PTE_LM) 270 pte |= GEN12_GGTT_PTE_LM; 271 272 if (pat_index & BIT(0)) 273 pte |= MTL_GGTT_PTE_PAT0; 274 275 if (pat_index & BIT(1)) 276 pte |= MTL_GGTT_PTE_PAT1; 277 278 return pte; 279 } 280 281 u64 gen8_ggtt_pte_encode(dma_addr_t addr, 282 unsigned int pat_index, 283 u32 flags) 284 { 285 gen8_pte_t pte = addr | GEN8_PAGE_PRESENT; 286 287 if (flags & PTE_LM) 288 pte |= GEN12_GGTT_PTE_LM; 289 290 return pte; 291 } 292 293 static bool should_update_ggtt_with_bind(struct i915_ggtt *ggtt) 294 { 295 struct intel_gt *gt = ggtt->vm.gt; 296 297 return intel_gt_is_bind_context_ready(gt); 298 } 299 300 static struct intel_context *gen8_ggtt_bind_get_ce(struct i915_ggtt *ggtt) 301 { 302 struct intel_context *ce; 303 struct intel_gt *gt = ggtt->vm.gt; 304 305 if (intel_gt_is_wedged(gt)) 306 return NULL; 307 308 ce = gt->engine[BCS0]->bind_context; 309 GEM_BUG_ON(!ce); 310 311 /* 312 * If the GT is not awake already at this stage then fallback 313 * to pci based GGTT update otherwise __intel_wakeref_get_first() 314 * would conflict with fs_reclaim trying to allocate memory while 315 * doing rpm_resume(). 316 */ 317 if (!intel_gt_pm_get_if_awake(gt)) 318 return NULL; 319 320 intel_engine_pm_get(ce->engine); 321 322 return ce; 323 } 324 325 static void gen8_ggtt_bind_put_ce(struct intel_context *ce) 326 { 327 intel_engine_pm_put(ce->engine); 328 intel_gt_pm_put(ce->engine->gt); 329 } 330 331 static bool gen8_ggtt_bind_ptes(struct i915_ggtt *ggtt, u32 offset, 332 struct sg_table *pages, u32 num_entries, 333 const gen8_pte_t pte) 334 { 335 struct i915_sched_attr attr = {}; 336 struct intel_gt *gt = ggtt->vm.gt; 337 const gen8_pte_t scratch_pte = ggtt->vm.scratch[0]->encode; 338 struct sgt_iter iter; 339 struct i915_request *rq; 340 struct intel_context *ce; 341 u32 *cs; 342 343 if (!num_entries) 344 return true; 345 346 ce = gen8_ggtt_bind_get_ce(ggtt); 347 if (!ce) 348 return false; 349 350 if (pages) 351 iter = __sgt_iter(pages->sgl, true); 352 353 while (num_entries) { 354 int count = 0; 355 dma_addr_t addr; 356 /* 357 * MI_UPDATE_GTT can update 512 entries in a single command but 358 * that end up with engine reset, 511 works. 359 */ 360 u32 n_ptes = min_t(u32, 511, num_entries); 361 362 if (mutex_lock_interruptible(&ce->timeline->mutex)) 363 goto put_ce; 364 365 intel_context_enter(ce); 366 rq = __i915_request_create(ce, GFP_NOWAIT | GFP_ATOMIC); 367 intel_context_exit(ce); 368 if (IS_ERR(rq)) { 369 GT_TRACE(gt, "Failed to get bind request\n"); 370 mutex_unlock(&ce->timeline->mutex); 371 goto put_ce; 372 } 373 374 cs = intel_ring_begin(rq, 2 * n_ptes + 2); 375 if (IS_ERR(cs)) { 376 GT_TRACE(gt, "Failed to ring space for GGTT bind\n"); 377 i915_request_set_error_once(rq, PTR_ERR(cs)); 378 /* once a request is created, it must be queued */ 379 goto queue_err_rq; 380 } 381 382 *cs++ = MI_UPDATE_GTT | (2 * n_ptes); 383 *cs++ = offset << 12; 384 385 if (pages) { 386 for_each_sgt_daddr_next(addr, iter) { 387 if (count == n_ptes) 388 break; 389 *cs++ = lower_32_bits(pte | addr); 390 *cs++ = upper_32_bits(pte | addr); 391 count++; 392 } 393 /* fill remaining with scratch pte, if any */ 394 if (count < n_ptes) { 395 memset64((u64 *)cs, scratch_pte, 396 n_ptes - count); 397 cs += (n_ptes - count) * 2; 398 } 399 } else { 400 memset64((u64 *)cs, pte, n_ptes); 401 cs += n_ptes * 2; 402 } 403 404 intel_ring_advance(rq, cs); 405 queue_err_rq: 406 i915_request_get(rq); 407 __i915_request_commit(rq); 408 __i915_request_queue(rq, &attr); 409 410 mutex_unlock(&ce->timeline->mutex); 411 /* This will break if the request is complete or after engine reset */ 412 i915_request_wait(rq, 0, MAX_SCHEDULE_TIMEOUT); 413 if (rq->fence.error) 414 goto err_rq; 415 416 i915_request_put(rq); 417 418 num_entries -= n_ptes; 419 offset += n_ptes; 420 } 421 422 gen8_ggtt_bind_put_ce(ce); 423 return true; 424 425 err_rq: 426 i915_request_put(rq); 427 put_ce: 428 gen8_ggtt_bind_put_ce(ce); 429 return false; 430 } 431 432 static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte) 433 { 434 writeq(pte, addr); 435 } 436 437 static void gen8_ggtt_insert_page(struct i915_address_space *vm, 438 dma_addr_t addr, 439 u64 offset, 440 unsigned int pat_index, 441 u32 flags) 442 { 443 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 444 gen8_pte_t __iomem *pte = 445 (gen8_pte_t __iomem *)ggtt->gsm + offset / I915_GTT_PAGE_SIZE; 446 447 gen8_set_pte(pte, ggtt->vm.pte_encode(addr, pat_index, flags)); 448 449 ggtt->invalidate(ggtt); 450 } 451 452 static void gen8_ggtt_insert_page_bind(struct i915_address_space *vm, 453 dma_addr_t addr, u64 offset, 454 unsigned int pat_index, u32 flags) 455 { 456 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 457 gen8_pte_t pte; 458 459 pte = ggtt->vm.pte_encode(addr, pat_index, flags); 460 if (should_update_ggtt_with_bind(i915_vm_to_ggtt(vm)) && 461 gen8_ggtt_bind_ptes(ggtt, offset, NULL, 1, pte)) 462 return ggtt->invalidate(ggtt); 463 464 gen8_ggtt_insert_page(vm, addr, offset, pat_index, flags); 465 } 466 467 static void gen8_ggtt_insert_entries(struct i915_address_space *vm, 468 struct i915_vma_resource *vma_res, 469 unsigned int pat_index, 470 u32 flags) 471 { 472 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 473 const gen8_pte_t pte_encode = ggtt->vm.pte_encode(0, pat_index, flags); 474 gen8_pte_t __iomem *gte; 475 gen8_pte_t __iomem *end; 476 struct sgt_iter iter; 477 dma_addr_t addr; 478 479 /* 480 * Note that we ignore PTE_READ_ONLY here. The caller must be careful 481 * not to allow the user to override access to a read only page. 482 */ 483 484 gte = (gen8_pte_t __iomem *)ggtt->gsm; 485 gte += (vma_res->start - vma_res->guard) / I915_GTT_PAGE_SIZE; 486 end = gte + vma_res->guard / I915_GTT_PAGE_SIZE; 487 while (gte < end) 488 gen8_set_pte(gte++, vm->scratch[0]->encode); 489 end += (vma_res->node_size + vma_res->guard) / I915_GTT_PAGE_SIZE; 490 491 for_each_sgt_daddr(addr, iter, vma_res->bi.pages) 492 gen8_set_pte(gte++, pte_encode | addr); 493 GEM_BUG_ON(gte > end); 494 495 /* Fill the allocated but "unused" space beyond the end of the buffer */ 496 while (gte < end) 497 gen8_set_pte(gte++, vm->scratch[0]->encode); 498 499 /* 500 * We want to flush the TLBs only after we're certain all the PTE 501 * updates have finished. 502 */ 503 ggtt->invalidate(ggtt); 504 } 505 506 static bool __gen8_ggtt_insert_entries_bind(struct i915_address_space *vm, 507 struct i915_vma_resource *vma_res, 508 unsigned int pat_index, u32 flags) 509 { 510 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 511 gen8_pte_t scratch_pte = vm->scratch[0]->encode; 512 gen8_pte_t pte_encode; 513 u64 start, end; 514 515 pte_encode = ggtt->vm.pte_encode(0, pat_index, flags); 516 start = (vma_res->start - vma_res->guard) / I915_GTT_PAGE_SIZE; 517 end = start + vma_res->guard / I915_GTT_PAGE_SIZE; 518 if (!gen8_ggtt_bind_ptes(ggtt, start, NULL, end - start, scratch_pte)) 519 goto err; 520 521 start = end; 522 end += (vma_res->node_size + vma_res->guard) / I915_GTT_PAGE_SIZE; 523 if (!gen8_ggtt_bind_ptes(ggtt, start, vma_res->bi.pages, 524 vma_res->node_size / I915_GTT_PAGE_SIZE, pte_encode)) 525 goto err; 526 527 start += vma_res->node_size / I915_GTT_PAGE_SIZE; 528 if (!gen8_ggtt_bind_ptes(ggtt, start, NULL, end - start, scratch_pte)) 529 goto err; 530 531 return true; 532 533 err: 534 return false; 535 } 536 537 static void gen8_ggtt_insert_entries_bind(struct i915_address_space *vm, 538 struct i915_vma_resource *vma_res, 539 unsigned int pat_index, u32 flags) 540 { 541 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 542 543 if (should_update_ggtt_with_bind(i915_vm_to_ggtt(vm)) && 544 __gen8_ggtt_insert_entries_bind(vm, vma_res, pat_index, flags)) 545 return ggtt->invalidate(ggtt); 546 547 gen8_ggtt_insert_entries(vm, vma_res, pat_index, flags); 548 } 549 550 static void gen8_ggtt_clear_range(struct i915_address_space *vm, 551 u64 start, u64 length) 552 { 553 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 554 unsigned int first_entry = start / I915_GTT_PAGE_SIZE; 555 unsigned int num_entries = length / I915_GTT_PAGE_SIZE; 556 const gen8_pte_t scratch_pte = vm->scratch[0]->encode; 557 gen8_pte_t __iomem *gtt_base = 558 (gen8_pte_t __iomem *)ggtt->gsm + first_entry; 559 const int max_entries = ggtt_total_entries(ggtt) - first_entry; 560 int i; 561 562 if (WARN(num_entries > max_entries, 563 "First entry = %d; Num entries = %d (max=%d)\n", 564 first_entry, num_entries, max_entries)) 565 num_entries = max_entries; 566 567 for (i = 0; i < num_entries; i++) 568 gen8_set_pte(>t_base[i], scratch_pte); 569 } 570 571 static void gen8_ggtt_scratch_range_bind(struct i915_address_space *vm, 572 u64 start, u64 length) 573 { 574 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 575 unsigned int first_entry = start / I915_GTT_PAGE_SIZE; 576 unsigned int num_entries = length / I915_GTT_PAGE_SIZE; 577 const gen8_pte_t scratch_pte = vm->scratch[0]->encode; 578 const int max_entries = ggtt_total_entries(ggtt) - first_entry; 579 580 if (WARN(num_entries > max_entries, 581 "First entry = %d; Num entries = %d (max=%d)\n", 582 first_entry, num_entries, max_entries)) 583 num_entries = max_entries; 584 585 if (should_update_ggtt_with_bind(ggtt) && gen8_ggtt_bind_ptes(ggtt, first_entry, 586 NULL, num_entries, scratch_pte)) 587 return ggtt->invalidate(ggtt); 588 589 gen8_ggtt_clear_range(vm, start, length); 590 } 591 592 static void gen6_ggtt_insert_page(struct i915_address_space *vm, 593 dma_addr_t addr, 594 u64 offset, 595 unsigned int pat_index, 596 u32 flags) 597 { 598 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 599 gen6_pte_t __iomem *pte = 600 (gen6_pte_t __iomem *)ggtt->gsm + offset / I915_GTT_PAGE_SIZE; 601 602 iowrite32(vm->pte_encode(addr, pat_index, flags), pte); 603 604 ggtt->invalidate(ggtt); 605 } 606 607 /* 608 * Binds an object into the global gtt with the specified cache level. 609 * The object will be accessible to the GPU via commands whose operands 610 * reference offsets within the global GTT as well as accessible by the GPU 611 * through the GMADR mapped BAR (i915->mm.gtt->gtt). 612 */ 613 static void gen6_ggtt_insert_entries(struct i915_address_space *vm, 614 struct i915_vma_resource *vma_res, 615 unsigned int pat_index, 616 u32 flags) 617 { 618 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 619 gen6_pte_t __iomem *gte; 620 gen6_pte_t __iomem *end; 621 struct sgt_iter iter; 622 dma_addr_t addr; 623 624 gte = (gen6_pte_t __iomem *)ggtt->gsm; 625 gte += (vma_res->start - vma_res->guard) / I915_GTT_PAGE_SIZE; 626 627 end = gte + vma_res->guard / I915_GTT_PAGE_SIZE; 628 while (gte < end) 629 iowrite32(vm->scratch[0]->encode, gte++); 630 end += (vma_res->node_size + vma_res->guard) / I915_GTT_PAGE_SIZE; 631 for_each_sgt_daddr(addr, iter, vma_res->bi.pages) 632 iowrite32(vm->pte_encode(addr, pat_index, flags), gte++); 633 GEM_BUG_ON(gte > end); 634 635 /* Fill the allocated but "unused" space beyond the end of the buffer */ 636 while (gte < end) 637 iowrite32(vm->scratch[0]->encode, gte++); 638 639 /* 640 * We want to flush the TLBs only after we're certain all the PTE 641 * updates have finished. 642 */ 643 ggtt->invalidate(ggtt); 644 } 645 646 static void nop_clear_range(struct i915_address_space *vm, 647 u64 start, u64 length) 648 { 649 } 650 651 static void bxt_vtd_ggtt_wa(struct i915_address_space *vm) 652 { 653 /* 654 * Make sure the internal GAM fifo has been cleared of all GTT 655 * writes before exiting stop_machine(). This guarantees that 656 * any aperture accesses waiting to start in another process 657 * cannot back up behind the GTT writes causing a hang. 658 * The register can be any arbitrary GAM register. 659 */ 660 intel_uncore_posting_read_fw(vm->gt->uncore, GFX_FLSH_CNTL_GEN6); 661 } 662 663 struct insert_page { 664 struct i915_address_space *vm; 665 dma_addr_t addr; 666 u64 offset; 667 unsigned int pat_index; 668 }; 669 670 static int bxt_vtd_ggtt_insert_page__cb(void *_arg) 671 { 672 struct insert_page *arg = _arg; 673 674 gen8_ggtt_insert_page(arg->vm, arg->addr, arg->offset, 675 arg->pat_index, 0); 676 bxt_vtd_ggtt_wa(arg->vm); 677 678 return 0; 679 } 680 681 static void bxt_vtd_ggtt_insert_page__BKL(struct i915_address_space *vm, 682 dma_addr_t addr, 683 u64 offset, 684 unsigned int pat_index, 685 u32 unused) 686 { 687 struct insert_page arg = { vm, addr, offset, pat_index }; 688 689 stop_machine(bxt_vtd_ggtt_insert_page__cb, &arg, NULL); 690 } 691 692 struct insert_entries { 693 struct i915_address_space *vm; 694 struct i915_vma_resource *vma_res; 695 unsigned int pat_index; 696 u32 flags; 697 }; 698 699 static int bxt_vtd_ggtt_insert_entries__cb(void *_arg) 700 { 701 struct insert_entries *arg = _arg; 702 703 gen8_ggtt_insert_entries(arg->vm, arg->vma_res, 704 arg->pat_index, arg->flags); 705 bxt_vtd_ggtt_wa(arg->vm); 706 707 return 0; 708 } 709 710 static void bxt_vtd_ggtt_insert_entries__BKL(struct i915_address_space *vm, 711 struct i915_vma_resource *vma_res, 712 unsigned int pat_index, 713 u32 flags) 714 { 715 struct insert_entries arg = { vm, vma_res, pat_index, flags }; 716 717 stop_machine(bxt_vtd_ggtt_insert_entries__cb, &arg, NULL); 718 } 719 720 static void gen6_ggtt_clear_range(struct i915_address_space *vm, 721 u64 start, u64 length) 722 { 723 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 724 unsigned int first_entry = start / I915_GTT_PAGE_SIZE; 725 unsigned int num_entries = length / I915_GTT_PAGE_SIZE; 726 gen6_pte_t scratch_pte, __iomem *gtt_base = 727 (gen6_pte_t __iomem *)ggtt->gsm + first_entry; 728 const int max_entries = ggtt_total_entries(ggtt) - first_entry; 729 int i; 730 731 if (WARN(num_entries > max_entries, 732 "First entry = %d; Num entries = %d (max=%d)\n", 733 first_entry, num_entries, max_entries)) 734 num_entries = max_entries; 735 736 scratch_pte = vm->scratch[0]->encode; 737 for (i = 0; i < num_entries; i++) 738 iowrite32(scratch_pte, >t_base[i]); 739 } 740 741 void intel_ggtt_bind_vma(struct i915_address_space *vm, 742 struct i915_vm_pt_stash *stash, 743 struct i915_vma_resource *vma_res, 744 unsigned int pat_index, 745 u32 flags) 746 { 747 u32 pte_flags; 748 749 if (vma_res->bound_flags & (~flags & I915_VMA_BIND_MASK)) 750 return; 751 752 vma_res->bound_flags |= flags; 753 754 /* Applicable to VLV (gen8+ do not support RO in the GGTT) */ 755 pte_flags = 0; 756 if (vma_res->bi.readonly) 757 pte_flags |= PTE_READ_ONLY; 758 if (vma_res->bi.lmem) 759 pte_flags |= PTE_LM; 760 761 vm->insert_entries(vm, vma_res, pat_index, pte_flags); 762 vma_res->page_sizes_gtt = I915_GTT_PAGE_SIZE; 763 } 764 765 void intel_ggtt_unbind_vma(struct i915_address_space *vm, 766 struct i915_vma_resource *vma_res) 767 { 768 vm->clear_range(vm, vma_res->start, vma_res->vma_size); 769 } 770 771 /* 772 * Reserve the top of the GuC address space for firmware images. Addresses 773 * beyond GUC_GGTT_TOP in the GuC address space are inaccessible by GuC, 774 * which makes for a suitable range to hold GuC/HuC firmware images if the 775 * size of the GGTT is 4G. However, on a 32-bit platform the size of the GGTT 776 * is limited to 2G, which is less than GUC_GGTT_TOP, but we reserve a chunk 777 * of the same size anyway, which is far more than needed, to keep the logic 778 * in uc_fw_ggtt_offset() simple. 779 */ 780 #define GUC_TOP_RESERVE_SIZE (SZ_4G - GUC_GGTT_TOP) 781 782 static int ggtt_reserve_guc_top(struct i915_ggtt *ggtt) 783 { 784 u64 offset; 785 int ret; 786 787 if (!intel_uc_uses_guc(&ggtt->vm.gt->uc)) 788 return 0; 789 790 GEM_BUG_ON(ggtt->vm.total <= GUC_TOP_RESERVE_SIZE); 791 offset = ggtt->vm.total - GUC_TOP_RESERVE_SIZE; 792 793 ret = i915_gem_gtt_reserve(&ggtt->vm, NULL, &ggtt->uc_fw, 794 GUC_TOP_RESERVE_SIZE, offset, 795 I915_COLOR_UNEVICTABLE, PIN_NOEVICT); 796 if (ret) 797 drm_dbg(&ggtt->vm.i915->drm, 798 "Failed to reserve top of GGTT for GuC\n"); 799 800 return ret; 801 } 802 803 static void ggtt_release_guc_top(struct i915_ggtt *ggtt) 804 { 805 if (drm_mm_node_allocated(&ggtt->uc_fw)) 806 drm_mm_remove_node(&ggtt->uc_fw); 807 } 808 809 static void cleanup_init_ggtt(struct i915_ggtt *ggtt) 810 { 811 ggtt_release_guc_top(ggtt); 812 if (drm_mm_node_allocated(&ggtt->error_capture)) 813 drm_mm_remove_node(&ggtt->error_capture); 814 mutex_destroy(&ggtt->error_mutex); 815 } 816 817 static int init_ggtt(struct i915_ggtt *ggtt) 818 { 819 /* 820 * Let GEM Manage all of the aperture. 821 * 822 * However, leave one page at the end still bound to the scratch page. 823 * There are a number of places where the hardware apparently prefetches 824 * past the end of the object, and we've seen multiple hangs with the 825 * GPU head pointer stuck in a batchbuffer bound at the last page of the 826 * aperture. One page should be enough to keep any prefetching inside 827 * of the aperture. 828 */ 829 unsigned long hole_start, hole_end; 830 struct drm_mm_node *entry; 831 int ret; 832 833 /* 834 * GuC requires all resources that we're sharing with it to be placed in 835 * non-WOPCM memory. If GuC is not present or not in use we still need a 836 * small bias as ring wraparound at offset 0 sometimes hangs. No idea 837 * why. 838 */ 839 ggtt->pin_bias = max_t(u32, I915_GTT_PAGE_SIZE, 840 intel_wopcm_guc_size(&ggtt->vm.gt->wopcm)); 841 842 ret = intel_vgt_balloon(ggtt); 843 if (ret) 844 return ret; 845 846 mutex_init(&ggtt->error_mutex); 847 if (ggtt->mappable_end) { 848 /* 849 * Reserve a mappable slot for our lockless error capture. 850 * 851 * We strongly prefer taking address 0x0 in order to protect 852 * other critical buffers against accidental overwrites, 853 * as writing to address 0 is a very common mistake. 854 * 855 * Since 0 may already be in use by the system (e.g. the BIOS 856 * framebuffer), we let the reservation fail quietly and hope 857 * 0 remains reserved always. 858 * 859 * If we fail to reserve 0, and then fail to find any space 860 * for an error-capture, remain silent. We can afford not 861 * to reserve an error_capture node as we have fallback 862 * paths, and we trust that 0 will remain reserved. However, 863 * the only likely reason for failure to insert is a driver 864 * bug, which we expect to cause other failures... 865 * 866 * Since CPU can perform speculative reads on error capture 867 * (write-combining allows it) add scratch page after error 868 * capture to avoid DMAR errors. 869 */ 870 ggtt->error_capture.size = 2 * I915_GTT_PAGE_SIZE; 871 ggtt->error_capture.color = I915_COLOR_UNEVICTABLE; 872 if (drm_mm_reserve_node(&ggtt->vm.mm, &ggtt->error_capture)) 873 drm_mm_insert_node_in_range(&ggtt->vm.mm, 874 &ggtt->error_capture, 875 ggtt->error_capture.size, 0, 876 ggtt->error_capture.color, 877 0, ggtt->mappable_end, 878 DRM_MM_INSERT_LOW); 879 } 880 if (drm_mm_node_allocated(&ggtt->error_capture)) { 881 u64 start = ggtt->error_capture.start; 882 u64 size = ggtt->error_capture.size; 883 884 ggtt->vm.scratch_range(&ggtt->vm, start, size); 885 drm_dbg(&ggtt->vm.i915->drm, 886 "Reserved GGTT:[%llx, %llx] for use by error capture\n", 887 start, start + size); 888 } 889 890 /* 891 * The upper portion of the GuC address space has a sizeable hole 892 * (several MB) that is inaccessible by GuC. Reserve this range within 893 * GGTT as it can comfortably hold GuC/HuC firmware images. 894 */ 895 ret = ggtt_reserve_guc_top(ggtt); 896 if (ret) 897 goto err; 898 899 /* Clear any non-preallocated blocks */ 900 drm_mm_for_each_hole(entry, &ggtt->vm.mm, hole_start, hole_end) { 901 drm_dbg(&ggtt->vm.i915->drm, 902 "clearing unused GTT space: [%lx, %lx]\n", 903 hole_start, hole_end); 904 ggtt->vm.clear_range(&ggtt->vm, hole_start, 905 hole_end - hole_start); 906 } 907 908 /* And finally clear the reserved guard page */ 909 ggtt->vm.clear_range(&ggtt->vm, ggtt->vm.total - PAGE_SIZE, PAGE_SIZE); 910 911 return 0; 912 913 err: 914 cleanup_init_ggtt(ggtt); 915 return ret; 916 } 917 918 static void aliasing_gtt_bind_vma(struct i915_address_space *vm, 919 struct i915_vm_pt_stash *stash, 920 struct i915_vma_resource *vma_res, 921 unsigned int pat_index, 922 u32 flags) 923 { 924 u32 pte_flags; 925 926 /* Currently applicable only to VLV */ 927 pte_flags = 0; 928 if (vma_res->bi.readonly) 929 pte_flags |= PTE_READ_ONLY; 930 931 if (flags & I915_VMA_LOCAL_BIND) 932 ppgtt_bind_vma(&i915_vm_to_ggtt(vm)->alias->vm, 933 stash, vma_res, pat_index, flags); 934 935 if (flags & I915_VMA_GLOBAL_BIND) 936 vm->insert_entries(vm, vma_res, pat_index, pte_flags); 937 938 vma_res->bound_flags |= flags; 939 } 940 941 static void aliasing_gtt_unbind_vma(struct i915_address_space *vm, 942 struct i915_vma_resource *vma_res) 943 { 944 if (vma_res->bound_flags & I915_VMA_GLOBAL_BIND) 945 vm->clear_range(vm, vma_res->start, vma_res->vma_size); 946 947 if (vma_res->bound_flags & I915_VMA_LOCAL_BIND) 948 ppgtt_unbind_vma(&i915_vm_to_ggtt(vm)->alias->vm, vma_res); 949 } 950 951 static int init_aliasing_ppgtt(struct i915_ggtt *ggtt) 952 { 953 struct i915_vm_pt_stash stash = {}; 954 struct i915_ppgtt *ppgtt; 955 int err; 956 957 ppgtt = i915_ppgtt_create(ggtt->vm.gt, 0); 958 if (IS_ERR(ppgtt)) 959 return PTR_ERR(ppgtt); 960 961 if (GEM_WARN_ON(ppgtt->vm.total < ggtt->vm.total)) { 962 err = -ENODEV; 963 goto err_ppgtt; 964 } 965 966 err = i915_vm_alloc_pt_stash(&ppgtt->vm, &stash, ggtt->vm.total); 967 if (err) 968 goto err_ppgtt; 969 970 i915_gem_object_lock(ppgtt->vm.scratch[0], NULL); 971 err = i915_vm_map_pt_stash(&ppgtt->vm, &stash); 972 i915_gem_object_unlock(ppgtt->vm.scratch[0]); 973 if (err) 974 goto err_stash; 975 976 /* 977 * Note we only pre-allocate as far as the end of the global 978 * GTT. On 48b / 4-level page-tables, the difference is very, 979 * very significant! We have to preallocate as GVT/vgpu does 980 * not like the page directory disappearing. 981 */ 982 ppgtt->vm.allocate_va_range(&ppgtt->vm, &stash, 0, ggtt->vm.total); 983 984 ggtt->alias = ppgtt; 985 ggtt->vm.bind_async_flags |= ppgtt->vm.bind_async_flags; 986 987 GEM_BUG_ON(ggtt->vm.vma_ops.bind_vma != intel_ggtt_bind_vma); 988 ggtt->vm.vma_ops.bind_vma = aliasing_gtt_bind_vma; 989 990 GEM_BUG_ON(ggtt->vm.vma_ops.unbind_vma != intel_ggtt_unbind_vma); 991 ggtt->vm.vma_ops.unbind_vma = aliasing_gtt_unbind_vma; 992 993 i915_vm_free_pt_stash(&ppgtt->vm, &stash); 994 return 0; 995 996 err_stash: 997 i915_vm_free_pt_stash(&ppgtt->vm, &stash); 998 err_ppgtt: 999 i915_vm_put(&ppgtt->vm); 1000 return err; 1001 } 1002 1003 static void fini_aliasing_ppgtt(struct i915_ggtt *ggtt) 1004 { 1005 struct i915_ppgtt *ppgtt; 1006 1007 ppgtt = fetch_and_zero(&ggtt->alias); 1008 if (!ppgtt) 1009 return; 1010 1011 i915_vm_put(&ppgtt->vm); 1012 1013 ggtt->vm.vma_ops.bind_vma = intel_ggtt_bind_vma; 1014 ggtt->vm.vma_ops.unbind_vma = intel_ggtt_unbind_vma; 1015 } 1016 1017 int i915_init_ggtt(struct drm_i915_private *i915) 1018 { 1019 int ret; 1020 1021 ret = init_ggtt(to_gt(i915)->ggtt); 1022 if (ret) 1023 return ret; 1024 1025 if (INTEL_PPGTT(i915) == INTEL_PPGTT_ALIASING) { 1026 ret = init_aliasing_ppgtt(to_gt(i915)->ggtt); 1027 if (ret) 1028 cleanup_init_ggtt(to_gt(i915)->ggtt); 1029 } 1030 1031 return 0; 1032 } 1033 1034 static void ggtt_cleanup_hw(struct i915_ggtt *ggtt) 1035 { 1036 struct i915_vma *vma, *vn; 1037 1038 flush_workqueue(ggtt->vm.i915->wq); 1039 i915_gem_drain_freed_objects(ggtt->vm.i915); 1040 1041 mutex_lock(&ggtt->vm.mutex); 1042 1043 ggtt->vm.skip_pte_rewrite = true; 1044 1045 list_for_each_entry_safe(vma, vn, &ggtt->vm.bound_list, vm_link) { 1046 struct drm_i915_gem_object *obj = vma->obj; 1047 bool trylock; 1048 1049 trylock = i915_gem_object_trylock(obj, NULL); 1050 WARN_ON(!trylock); 1051 1052 WARN_ON(__i915_vma_unbind(vma)); 1053 if (trylock) 1054 i915_gem_object_unlock(obj); 1055 } 1056 1057 if (drm_mm_node_allocated(&ggtt->error_capture)) 1058 drm_mm_remove_node(&ggtt->error_capture); 1059 mutex_destroy(&ggtt->error_mutex); 1060 1061 ggtt_release_guc_top(ggtt); 1062 intel_vgt_deballoon(ggtt); 1063 1064 ggtt->vm.cleanup(&ggtt->vm); 1065 1066 mutex_unlock(&ggtt->vm.mutex); 1067 i915_address_space_fini(&ggtt->vm); 1068 1069 arch_phys_wc_del(ggtt->mtrr); 1070 1071 if (ggtt->iomap.size) 1072 io_mapping_fini(&ggtt->iomap); 1073 } 1074 1075 /** 1076 * i915_ggtt_driver_release - Clean up GGTT hardware initialization 1077 * @i915: i915 device 1078 */ 1079 void i915_ggtt_driver_release(struct drm_i915_private *i915) 1080 { 1081 struct i915_ggtt *ggtt = to_gt(i915)->ggtt; 1082 1083 fini_aliasing_ppgtt(ggtt); 1084 1085 intel_ggtt_fini_fences(ggtt); 1086 ggtt_cleanup_hw(ggtt); 1087 } 1088 1089 /** 1090 * i915_ggtt_driver_late_release - Cleanup of GGTT that needs to be done after 1091 * all free objects have been drained. 1092 * @i915: i915 device 1093 */ 1094 void i915_ggtt_driver_late_release(struct drm_i915_private *i915) 1095 { 1096 struct i915_ggtt *ggtt = to_gt(i915)->ggtt; 1097 1098 GEM_WARN_ON(kref_read(&ggtt->vm.resv_ref) != 1); 1099 dma_resv_fini(&ggtt->vm._resv); 1100 } 1101 1102 static unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl) 1103 { 1104 snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT; 1105 snb_gmch_ctl &= SNB_GMCH_GGMS_MASK; 1106 return snb_gmch_ctl << 20; 1107 } 1108 1109 static unsigned int gen8_get_total_gtt_size(u16 bdw_gmch_ctl) 1110 { 1111 bdw_gmch_ctl >>= BDW_GMCH_GGMS_SHIFT; 1112 bdw_gmch_ctl &= BDW_GMCH_GGMS_MASK; 1113 if (bdw_gmch_ctl) 1114 bdw_gmch_ctl = 1 << bdw_gmch_ctl; 1115 1116 #ifdef CONFIG_X86_32 1117 /* Limit 32b platforms to a 2GB GGTT: 4 << 20 / pte size * I915_GTT_PAGE_SIZE */ 1118 if (bdw_gmch_ctl > 4) 1119 bdw_gmch_ctl = 4; 1120 #endif 1121 1122 return bdw_gmch_ctl << 20; 1123 } 1124 1125 static unsigned int chv_get_total_gtt_size(u16 gmch_ctrl) 1126 { 1127 gmch_ctrl >>= SNB_GMCH_GGMS_SHIFT; 1128 gmch_ctrl &= SNB_GMCH_GGMS_MASK; 1129 1130 if (gmch_ctrl) 1131 return 1 << (20 + gmch_ctrl); 1132 1133 return 0; 1134 } 1135 1136 static unsigned int gen6_gttmmadr_size(struct drm_i915_private *i915) 1137 { 1138 /* 1139 * GEN6: GTTMMADR size is 4MB and GTTADR starts at 2MB offset 1140 * GEN8: GTTMMADR size is 16MB and GTTADR starts at 8MB offset 1141 */ 1142 GEM_BUG_ON(GRAPHICS_VER(i915) < 6); 1143 return (GRAPHICS_VER(i915) < 8) ? SZ_4M : SZ_16M; 1144 } 1145 1146 static unsigned int gen6_gttadr_offset(struct drm_i915_private *i915) 1147 { 1148 return gen6_gttmmadr_size(i915) / 2; 1149 } 1150 1151 static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size) 1152 { 1153 struct drm_i915_private *i915 = ggtt->vm.i915; 1154 struct pci_dev *pdev = to_pci_dev(i915->drm.dev); 1155 phys_addr_t phys_addr; 1156 u32 pte_flags; 1157 int ret; 1158 1159 GEM_WARN_ON(pci_resource_len(pdev, GEN4_GTTMMADR_BAR) != gen6_gttmmadr_size(i915)); 1160 phys_addr = pci_resource_start(pdev, GEN4_GTTMMADR_BAR) + gen6_gttadr_offset(i915); 1161 1162 if (needs_wc_ggtt_mapping(i915)) 1163 ggtt->gsm = ioremap_wc(phys_addr, size); 1164 else 1165 ggtt->gsm = ioremap(phys_addr, size); 1166 1167 if (!ggtt->gsm) { 1168 drm_err(&i915->drm, "Failed to map the ggtt page table\n"); 1169 return -ENOMEM; 1170 } 1171 1172 kref_init(&ggtt->vm.resv_ref); 1173 ret = setup_scratch_page(&ggtt->vm); 1174 if (ret) { 1175 drm_err(&i915->drm, "Scratch setup failed\n"); 1176 /* iounmap will also get called at remove, but meh */ 1177 iounmap(ggtt->gsm); 1178 return ret; 1179 } 1180 1181 pte_flags = 0; 1182 if (i915_gem_object_is_lmem(ggtt->vm.scratch[0])) 1183 pte_flags |= PTE_LM; 1184 1185 ggtt->vm.scratch[0]->encode = 1186 ggtt->vm.pte_encode(px_dma(ggtt->vm.scratch[0]), 1187 i915_gem_get_pat_index(i915, 1188 I915_CACHE_NONE), 1189 pte_flags); 1190 1191 return 0; 1192 } 1193 1194 static void gen6_gmch_remove(struct i915_address_space *vm) 1195 { 1196 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 1197 1198 iounmap(ggtt->gsm); 1199 free_scratch(vm); 1200 } 1201 1202 static struct resource pci_resource(struct pci_dev *pdev, int bar) 1203 { 1204 return DEFINE_RES_MEM(pci_resource_start(pdev, bar), 1205 pci_resource_len(pdev, bar)); 1206 } 1207 1208 static int gen8_gmch_probe(struct i915_ggtt *ggtt) 1209 { 1210 struct drm_i915_private *i915 = ggtt->vm.i915; 1211 struct pci_dev *pdev = to_pci_dev(i915->drm.dev); 1212 unsigned int size; 1213 u16 snb_gmch_ctl; 1214 1215 if (!HAS_LMEM(i915) && !HAS_LMEMBAR_SMEM_STOLEN(i915)) { 1216 if (!i915_pci_resource_valid(pdev, GEN4_GMADR_BAR)) 1217 return -ENXIO; 1218 1219 ggtt->gmadr = pci_resource(pdev, GEN4_GMADR_BAR); 1220 ggtt->mappable_end = resource_size(&ggtt->gmadr); 1221 } 1222 1223 pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl); 1224 if (IS_CHERRYVIEW(i915)) 1225 size = chv_get_total_gtt_size(snb_gmch_ctl); 1226 else 1227 size = gen8_get_total_gtt_size(snb_gmch_ctl); 1228 1229 ggtt->vm.alloc_pt_dma = alloc_pt_dma; 1230 ggtt->vm.alloc_scratch_dma = alloc_pt_dma; 1231 ggtt->vm.lmem_pt_obj_flags = I915_BO_ALLOC_PM_EARLY; 1232 1233 ggtt->vm.total = (size / sizeof(gen8_pte_t)) * I915_GTT_PAGE_SIZE; 1234 ggtt->vm.cleanup = gen6_gmch_remove; 1235 ggtt->vm.insert_page = gen8_ggtt_insert_page; 1236 ggtt->vm.clear_range = nop_clear_range; 1237 ggtt->vm.scratch_range = gen8_ggtt_clear_range; 1238 1239 ggtt->vm.insert_entries = gen8_ggtt_insert_entries; 1240 1241 /* 1242 * Serialize GTT updates with aperture access on BXT if VT-d is on, 1243 * and always on CHV. 1244 */ 1245 if (intel_vm_no_concurrent_access_wa(i915)) { 1246 ggtt->vm.insert_entries = bxt_vtd_ggtt_insert_entries__BKL; 1247 ggtt->vm.insert_page = bxt_vtd_ggtt_insert_page__BKL; 1248 1249 /* 1250 * Calling stop_machine() version of GGTT update function 1251 * at error capture/reset path will raise lockdep warning. 1252 * Allow calling gen8_ggtt_insert_* directly at reset path 1253 * which is safe from parallel GGTT updates. 1254 */ 1255 ggtt->vm.raw_insert_page = gen8_ggtt_insert_page; 1256 ggtt->vm.raw_insert_entries = gen8_ggtt_insert_entries; 1257 1258 ggtt->vm.bind_async_flags = 1259 I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND; 1260 } 1261 1262 if (i915_ggtt_require_binder(i915)) { 1263 ggtt->vm.scratch_range = gen8_ggtt_scratch_range_bind; 1264 ggtt->vm.insert_page = gen8_ggtt_insert_page_bind; 1265 ggtt->vm.insert_entries = gen8_ggtt_insert_entries_bind; 1266 /* 1267 * On GPU is hung, we might bind VMAs for error capture. 1268 * Fallback to CPU GGTT updates in that case. 1269 */ 1270 ggtt->vm.raw_insert_page = gen8_ggtt_insert_page; 1271 } 1272 1273 if (intel_uc_wants_guc_submission(&ggtt->vm.gt->uc)) 1274 ggtt->invalidate = guc_ggtt_invalidate; 1275 else 1276 ggtt->invalidate = gen8_ggtt_invalidate; 1277 1278 ggtt->vm.vma_ops.bind_vma = intel_ggtt_bind_vma; 1279 ggtt->vm.vma_ops.unbind_vma = intel_ggtt_unbind_vma; 1280 1281 if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70)) 1282 ggtt->vm.pte_encode = mtl_ggtt_pte_encode; 1283 else 1284 ggtt->vm.pte_encode = gen8_ggtt_pte_encode; 1285 1286 return ggtt_probe_common(ggtt, size); 1287 } 1288 1289 /* 1290 * For pre-gen8 platforms pat_index is the same as enum i915_cache_level, 1291 * so the switch-case statements in these PTE encode functions are still valid. 1292 * See translation table LEGACY_CACHELEVEL. 1293 */ 1294 static u64 snb_pte_encode(dma_addr_t addr, 1295 unsigned int pat_index, 1296 u32 flags) 1297 { 1298 gen6_pte_t pte = GEN6_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID; 1299 1300 switch (pat_index) { 1301 case I915_CACHE_L3_LLC: 1302 case I915_CACHE_LLC: 1303 pte |= GEN6_PTE_CACHE_LLC; 1304 break; 1305 case I915_CACHE_NONE: 1306 pte |= GEN6_PTE_UNCACHED; 1307 break; 1308 default: 1309 MISSING_CASE(pat_index); 1310 } 1311 1312 return pte; 1313 } 1314 1315 static u64 ivb_pte_encode(dma_addr_t addr, 1316 unsigned int pat_index, 1317 u32 flags) 1318 { 1319 gen6_pte_t pte = GEN6_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID; 1320 1321 switch (pat_index) { 1322 case I915_CACHE_L3_LLC: 1323 pte |= GEN7_PTE_CACHE_L3_LLC; 1324 break; 1325 case I915_CACHE_LLC: 1326 pte |= GEN6_PTE_CACHE_LLC; 1327 break; 1328 case I915_CACHE_NONE: 1329 pte |= GEN6_PTE_UNCACHED; 1330 break; 1331 default: 1332 MISSING_CASE(pat_index); 1333 } 1334 1335 return pte; 1336 } 1337 1338 static u64 byt_pte_encode(dma_addr_t addr, 1339 unsigned int pat_index, 1340 u32 flags) 1341 { 1342 gen6_pte_t pte = GEN6_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID; 1343 1344 if (!(flags & PTE_READ_ONLY)) 1345 pte |= BYT_PTE_WRITEABLE; 1346 1347 if (pat_index != I915_CACHE_NONE) 1348 pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES; 1349 1350 return pte; 1351 } 1352 1353 static u64 hsw_pte_encode(dma_addr_t addr, 1354 unsigned int pat_index, 1355 u32 flags) 1356 { 1357 gen6_pte_t pte = HSW_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID; 1358 1359 if (pat_index != I915_CACHE_NONE) 1360 pte |= HSW_WB_LLC_AGE3; 1361 1362 return pte; 1363 } 1364 1365 static u64 iris_pte_encode(dma_addr_t addr, 1366 unsigned int pat_index, 1367 u32 flags) 1368 { 1369 gen6_pte_t pte = HSW_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID; 1370 1371 switch (pat_index) { 1372 case I915_CACHE_NONE: 1373 break; 1374 case I915_CACHE_WT: 1375 pte |= HSW_WT_ELLC_LLC_AGE3; 1376 break; 1377 default: 1378 pte |= HSW_WB_ELLC_LLC_AGE3; 1379 break; 1380 } 1381 1382 return pte; 1383 } 1384 1385 static int gen6_gmch_probe(struct i915_ggtt *ggtt) 1386 { 1387 struct drm_i915_private *i915 = ggtt->vm.i915; 1388 struct pci_dev *pdev = to_pci_dev(i915->drm.dev); 1389 unsigned int size; 1390 u16 snb_gmch_ctl; 1391 1392 if (!i915_pci_resource_valid(pdev, GEN4_GMADR_BAR)) 1393 return -ENXIO; 1394 1395 ggtt->gmadr = pci_resource(pdev, GEN4_GMADR_BAR); 1396 ggtt->mappable_end = resource_size(&ggtt->gmadr); 1397 1398 /* 1399 * 64/512MB is the current min/max we actually know of, but this is 1400 * just a coarse sanity check. 1401 */ 1402 if (ggtt->mappable_end < (64 << 20) || 1403 ggtt->mappable_end > (512 << 20)) { 1404 drm_err(&i915->drm, "Unknown GMADR size (%pa)\n", 1405 &ggtt->mappable_end); 1406 return -ENXIO; 1407 } 1408 1409 pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl); 1410 1411 size = gen6_get_total_gtt_size(snb_gmch_ctl); 1412 ggtt->vm.total = (size / sizeof(gen6_pte_t)) * I915_GTT_PAGE_SIZE; 1413 1414 ggtt->vm.alloc_pt_dma = alloc_pt_dma; 1415 ggtt->vm.alloc_scratch_dma = alloc_pt_dma; 1416 1417 ggtt->vm.clear_range = nop_clear_range; 1418 if (!HAS_FULL_PPGTT(i915)) 1419 ggtt->vm.clear_range = gen6_ggtt_clear_range; 1420 ggtt->vm.scratch_range = gen6_ggtt_clear_range; 1421 ggtt->vm.insert_page = gen6_ggtt_insert_page; 1422 ggtt->vm.insert_entries = gen6_ggtt_insert_entries; 1423 ggtt->vm.cleanup = gen6_gmch_remove; 1424 1425 ggtt->invalidate = gen6_ggtt_invalidate; 1426 1427 if (HAS_EDRAM(i915)) 1428 ggtt->vm.pte_encode = iris_pte_encode; 1429 else if (IS_HASWELL(i915)) 1430 ggtt->vm.pte_encode = hsw_pte_encode; 1431 else if (IS_VALLEYVIEW(i915)) 1432 ggtt->vm.pte_encode = byt_pte_encode; 1433 else if (GRAPHICS_VER(i915) >= 7) 1434 ggtt->vm.pte_encode = ivb_pte_encode; 1435 else 1436 ggtt->vm.pte_encode = snb_pte_encode; 1437 1438 ggtt->vm.vma_ops.bind_vma = intel_ggtt_bind_vma; 1439 ggtt->vm.vma_ops.unbind_vma = intel_ggtt_unbind_vma; 1440 1441 return ggtt_probe_common(ggtt, size); 1442 } 1443 1444 static int ggtt_probe_hw(struct i915_ggtt *ggtt, struct intel_gt *gt) 1445 { 1446 struct drm_i915_private *i915 = gt->i915; 1447 int ret; 1448 1449 ggtt->vm.gt = gt; 1450 ggtt->vm.i915 = i915; 1451 ggtt->vm.dma = i915->drm.dev; 1452 dma_resv_init(&ggtt->vm._resv); 1453 1454 if (GRAPHICS_VER(i915) >= 8) 1455 ret = gen8_gmch_probe(ggtt); 1456 else if (GRAPHICS_VER(i915) >= 6) 1457 ret = gen6_gmch_probe(ggtt); 1458 else 1459 ret = intel_ggtt_gmch_probe(ggtt); 1460 1461 if (ret) { 1462 dma_resv_fini(&ggtt->vm._resv); 1463 return ret; 1464 } 1465 1466 if ((ggtt->vm.total - 1) >> 32) { 1467 drm_err(&i915->drm, 1468 "We never expected a Global GTT with more than 32bits" 1469 " of address space! Found %lldM!\n", 1470 ggtt->vm.total >> 20); 1471 ggtt->vm.total = 1ULL << 32; 1472 ggtt->mappable_end = 1473 min_t(u64, ggtt->mappable_end, ggtt->vm.total); 1474 } 1475 1476 if (ggtt->mappable_end > ggtt->vm.total) { 1477 drm_err(&i915->drm, 1478 "mappable aperture extends past end of GGTT," 1479 " aperture=%pa, total=%llx\n", 1480 &ggtt->mappable_end, ggtt->vm.total); 1481 ggtt->mappable_end = ggtt->vm.total; 1482 } 1483 1484 /* GMADR is the PCI mmio aperture into the global GTT. */ 1485 drm_dbg(&i915->drm, "GGTT size = %lluM\n", ggtt->vm.total >> 20); 1486 drm_dbg(&i915->drm, "GMADR size = %lluM\n", 1487 (u64)ggtt->mappable_end >> 20); 1488 drm_dbg(&i915->drm, "DSM size = %lluM\n", 1489 (u64)resource_size(&intel_graphics_stolen_res) >> 20); 1490 1491 return 0; 1492 } 1493 1494 /** 1495 * i915_ggtt_probe_hw - Probe GGTT hardware location 1496 * @i915: i915 device 1497 */ 1498 int i915_ggtt_probe_hw(struct drm_i915_private *i915) 1499 { 1500 struct intel_gt *gt; 1501 int ret, i; 1502 1503 for_each_gt(gt, i915, i) { 1504 ret = intel_gt_assign_ggtt(gt); 1505 if (ret) 1506 return ret; 1507 } 1508 1509 ret = ggtt_probe_hw(to_gt(i915)->ggtt, to_gt(i915)); 1510 if (ret) 1511 return ret; 1512 1513 if (i915_vtd_active(i915)) 1514 drm_info(&i915->drm, "VT-d active for gfx access\n"); 1515 1516 return 0; 1517 } 1518 1519 struct i915_ggtt *i915_ggtt_create(struct drm_i915_private *i915) 1520 { 1521 struct i915_ggtt *ggtt; 1522 1523 ggtt = drmm_kzalloc(&i915->drm, sizeof(*ggtt), GFP_KERNEL); 1524 if (!ggtt) 1525 return ERR_PTR(-ENOMEM); 1526 1527 INIT_LIST_HEAD(&ggtt->gt_list); 1528 1529 return ggtt; 1530 } 1531 1532 int i915_ggtt_enable_hw(struct drm_i915_private *i915) 1533 { 1534 if (GRAPHICS_VER(i915) < 6) 1535 return intel_ggtt_gmch_enable_hw(i915); 1536 1537 return 0; 1538 } 1539 1540 /** 1541 * i915_ggtt_resume_vm - Restore the memory mappings for a GGTT or DPT VM 1542 * @vm: The VM to restore the mappings for 1543 * 1544 * Restore the memory mappings for all objects mapped to HW via the GGTT or a 1545 * DPT page table. 1546 * 1547 * Returns %true if restoring the mapping for any object that was in a write 1548 * domain before suspend. 1549 */ 1550 bool i915_ggtt_resume_vm(struct i915_address_space *vm) 1551 { 1552 struct i915_vma *vma; 1553 bool write_domain_objs = false; 1554 1555 drm_WARN_ON(&vm->i915->drm, !vm->is_ggtt && !vm->is_dpt); 1556 1557 /* First fill our portion of the GTT with scratch pages */ 1558 vm->clear_range(vm, 0, vm->total); 1559 1560 /* clflush objects bound into the GGTT and rebind them. */ 1561 list_for_each_entry(vma, &vm->bound_list, vm_link) { 1562 struct drm_i915_gem_object *obj = vma->obj; 1563 unsigned int was_bound = 1564 atomic_read(&vma->flags) & I915_VMA_BIND_MASK; 1565 1566 GEM_BUG_ON(!was_bound); 1567 1568 /* 1569 * Clear the bound flags of the vma resource to allow 1570 * ptes to be repopulated. 1571 */ 1572 vma->resource->bound_flags = 0; 1573 vma->ops->bind_vma(vm, NULL, vma->resource, 1574 obj ? obj->pat_index : 1575 i915_gem_get_pat_index(vm->i915, 1576 I915_CACHE_NONE), 1577 was_bound); 1578 1579 if (obj) { /* only used during resume => exclusive access */ 1580 write_domain_objs |= fetch_and_zero(&obj->write_domain); 1581 obj->read_domains |= I915_GEM_DOMAIN_GTT; 1582 } 1583 } 1584 1585 return write_domain_objs; 1586 } 1587 1588 void i915_ggtt_resume(struct i915_ggtt *ggtt) 1589 { 1590 struct intel_gt *gt; 1591 bool flush; 1592 1593 list_for_each_entry(gt, &ggtt->gt_list, ggtt_link) 1594 intel_gt_check_and_clear_faults(gt); 1595 1596 flush = i915_ggtt_resume_vm(&ggtt->vm); 1597 1598 if (drm_mm_node_allocated(&ggtt->error_capture)) 1599 ggtt->vm.scratch_range(&ggtt->vm, ggtt->error_capture.start, 1600 ggtt->error_capture.size); 1601 1602 list_for_each_entry(gt, &ggtt->gt_list, ggtt_link) 1603 intel_uc_resume_mappings(>->uc); 1604 1605 ggtt->invalidate(ggtt); 1606 1607 if (flush) 1608 wbinvd_on_all_cpus(); 1609 1610 intel_ggtt_restore_fences(ggtt); 1611 } 1612