1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2020 Intel Corporation 4 */ 5 6 #include <linux/stop_machine.h> 7 8 #include <asm/set_memory.h> 9 #include <asm/smp.h> 10 11 #include <drm/i915_drm.h> 12 13 #include "gem/i915_gem_lmem.h" 14 15 #include "intel_gt.h" 16 #include "i915_drv.h" 17 #include "i915_scatterlist.h" 18 #include "i915_vgpu.h" 19 20 #include "intel_gtt.h" 21 #include "gen8_ppgtt.h" 22 23 static int 24 i915_get_ggtt_vma_pages(struct i915_vma *vma); 25 26 static void i915_ggtt_color_adjust(const struct drm_mm_node *node, 27 unsigned long color, 28 u64 *start, 29 u64 *end) 30 { 31 if (i915_node_color_differs(node, color)) 32 *start += I915_GTT_PAGE_SIZE; 33 34 /* 35 * Also leave a space between the unallocated reserved node after the 36 * GTT and any objects within the GTT, i.e. we use the color adjustment 37 * to insert a guard page to prevent prefetches crossing over the 38 * GTT boundary. 39 */ 40 node = list_next_entry(node, node_list); 41 if (node->color != color) 42 *end -= I915_GTT_PAGE_SIZE; 43 } 44 45 static int ggtt_init_hw(struct i915_ggtt *ggtt) 46 { 47 struct drm_i915_private *i915 = ggtt->vm.i915; 48 49 i915_address_space_init(&ggtt->vm, VM_CLASS_GGTT); 50 51 ggtt->vm.is_ggtt = true; 52 53 /* Only VLV supports read-only GGTT mappings */ 54 ggtt->vm.has_read_only = IS_VALLEYVIEW(i915); 55 56 if (!HAS_LLC(i915) && !HAS_PPGTT(i915)) 57 ggtt->vm.mm.color_adjust = i915_ggtt_color_adjust; 58 59 if (ggtt->mappable_end) { 60 if (!io_mapping_init_wc(&ggtt->iomap, 61 ggtt->gmadr.start, 62 ggtt->mappable_end)) { 63 ggtt->vm.cleanup(&ggtt->vm); 64 return -EIO; 65 } 66 67 ggtt->mtrr = arch_phys_wc_add(ggtt->gmadr.start, 68 ggtt->mappable_end); 69 } 70 71 intel_ggtt_init_fences(ggtt); 72 73 return 0; 74 } 75 76 /** 77 * i915_ggtt_init_hw - Initialize GGTT hardware 78 * @i915: i915 device 79 */ 80 int i915_ggtt_init_hw(struct drm_i915_private *i915) 81 { 82 int ret; 83 84 /* 85 * Note that we use page colouring to enforce a guard page at the 86 * end of the address space. This is required as the CS may prefetch 87 * beyond the end of the batch buffer, across the page boundary, 88 * and beyond the end of the GTT if we do not provide a guard. 89 */ 90 ret = ggtt_init_hw(&i915->ggtt); 91 if (ret) 92 return ret; 93 94 return 0; 95 } 96 97 /* 98 * Certain Gen5 chipsets require idling the GPU before 99 * unmapping anything from the GTT when VT-d is enabled. 100 */ 101 static bool needs_idle_maps(struct drm_i915_private *i915) 102 { 103 /* 104 * Query intel_iommu to see if we need the workaround. Presumably that 105 * was loaded first. 106 */ 107 if (!intel_vtd_active()) 108 return false; 109 110 if (IS_GEN(i915, 5) && IS_MOBILE(i915)) 111 return true; 112 113 if (IS_GEN(i915, 12)) 114 return true; /* XXX DMAR fault reason 7 */ 115 116 return false; 117 } 118 119 void i915_ggtt_suspend(struct i915_ggtt *ggtt) 120 { 121 struct i915_vma *vma, *vn; 122 int open; 123 124 mutex_lock(&ggtt->vm.mutex); 125 126 /* Skip rewriting PTE on VMA unbind. */ 127 open = atomic_xchg(&ggtt->vm.open, 0); 128 129 list_for_each_entry_safe(vma, vn, &ggtt->vm.bound_list, vm_link) { 130 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 131 i915_vma_wait_for_bind(vma); 132 133 if (i915_vma_is_pinned(vma)) 134 continue; 135 136 if (!i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND)) { 137 __i915_vma_evict(vma); 138 drm_mm_remove_node(&vma->node); 139 } 140 } 141 142 ggtt->vm.clear_range(&ggtt->vm, 0, ggtt->vm.total); 143 ggtt->invalidate(ggtt); 144 atomic_set(&ggtt->vm.open, open); 145 146 mutex_unlock(&ggtt->vm.mutex); 147 148 intel_gt_check_and_clear_faults(ggtt->vm.gt); 149 } 150 151 void gen6_ggtt_invalidate(struct i915_ggtt *ggtt) 152 { 153 struct intel_uncore *uncore = ggtt->vm.gt->uncore; 154 155 spin_lock_irq(&uncore->lock); 156 intel_uncore_write_fw(uncore, GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN); 157 intel_uncore_read_fw(uncore, GFX_FLSH_CNTL_GEN6); 158 spin_unlock_irq(&uncore->lock); 159 } 160 161 static void gen8_ggtt_invalidate(struct i915_ggtt *ggtt) 162 { 163 struct intel_uncore *uncore = ggtt->vm.gt->uncore; 164 165 /* 166 * Note that as an uncached mmio write, this will flush the 167 * WCB of the writes into the GGTT before it triggers the invalidate. 168 */ 169 intel_uncore_write_fw(uncore, GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN); 170 } 171 172 static void guc_ggtt_invalidate(struct i915_ggtt *ggtt) 173 { 174 struct intel_uncore *uncore = ggtt->vm.gt->uncore; 175 struct drm_i915_private *i915 = ggtt->vm.i915; 176 177 gen8_ggtt_invalidate(ggtt); 178 179 if (INTEL_GEN(i915) >= 12) 180 intel_uncore_write_fw(uncore, GEN12_GUC_TLB_INV_CR, 181 GEN12_GUC_TLB_INV_CR_INVALIDATE); 182 else 183 intel_uncore_write_fw(uncore, GEN8_GTCR, GEN8_GTCR_INVALIDATE); 184 } 185 186 static void gmch_ggtt_invalidate(struct i915_ggtt *ggtt) 187 { 188 intel_gtt_chipset_flush(); 189 } 190 191 u64 gen8_ggtt_pte_encode(dma_addr_t addr, 192 enum i915_cache_level level, 193 u32 flags) 194 { 195 gen8_pte_t pte = addr | _PAGE_PRESENT; 196 197 if (flags & PTE_LM) 198 pte |= GEN12_GGTT_PTE_LM; 199 200 return pte; 201 } 202 203 static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte) 204 { 205 writeq(pte, addr); 206 } 207 208 static void gen8_ggtt_insert_page(struct i915_address_space *vm, 209 dma_addr_t addr, 210 u64 offset, 211 enum i915_cache_level level, 212 u32 flags) 213 { 214 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 215 gen8_pte_t __iomem *pte = 216 (gen8_pte_t __iomem *)ggtt->gsm + offset / I915_GTT_PAGE_SIZE; 217 218 gen8_set_pte(pte, gen8_ggtt_pte_encode(addr, level, flags)); 219 220 ggtt->invalidate(ggtt); 221 } 222 223 static void gen8_ggtt_insert_entries(struct i915_address_space *vm, 224 struct i915_vma *vma, 225 enum i915_cache_level level, 226 u32 flags) 227 { 228 const gen8_pte_t pte_encode = gen8_ggtt_pte_encode(0, level, flags); 229 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 230 gen8_pte_t __iomem *gte; 231 gen8_pte_t __iomem *end; 232 struct sgt_iter iter; 233 dma_addr_t addr; 234 235 /* 236 * Note that we ignore PTE_READ_ONLY here. The caller must be careful 237 * not to allow the user to override access to a read only page. 238 */ 239 240 gte = (gen8_pte_t __iomem *)ggtt->gsm; 241 gte += vma->node.start / I915_GTT_PAGE_SIZE; 242 end = gte + vma->node.size / I915_GTT_PAGE_SIZE; 243 244 for_each_sgt_daddr(addr, iter, vma->pages) 245 gen8_set_pte(gte++, pte_encode | addr); 246 GEM_BUG_ON(gte > end); 247 248 /* Fill the allocated but "unused" space beyond the end of the buffer */ 249 while (gte < end) 250 gen8_set_pte(gte++, vm->scratch[0]->encode); 251 252 /* 253 * We want to flush the TLBs only after we're certain all the PTE 254 * updates have finished. 255 */ 256 ggtt->invalidate(ggtt); 257 } 258 259 static void gen6_ggtt_insert_page(struct i915_address_space *vm, 260 dma_addr_t addr, 261 u64 offset, 262 enum i915_cache_level level, 263 u32 flags) 264 { 265 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 266 gen6_pte_t __iomem *pte = 267 (gen6_pte_t __iomem *)ggtt->gsm + offset / I915_GTT_PAGE_SIZE; 268 269 iowrite32(vm->pte_encode(addr, level, flags), pte); 270 271 ggtt->invalidate(ggtt); 272 } 273 274 /* 275 * Binds an object into the global gtt with the specified cache level. 276 * The object will be accessible to the GPU via commands whose operands 277 * reference offsets within the global GTT as well as accessible by the GPU 278 * through the GMADR mapped BAR (i915->mm.gtt->gtt). 279 */ 280 static void gen6_ggtt_insert_entries(struct i915_address_space *vm, 281 struct i915_vma *vma, 282 enum i915_cache_level level, 283 u32 flags) 284 { 285 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 286 gen6_pte_t __iomem *gte; 287 gen6_pte_t __iomem *end; 288 struct sgt_iter iter; 289 dma_addr_t addr; 290 291 gte = (gen6_pte_t __iomem *)ggtt->gsm; 292 gte += vma->node.start / I915_GTT_PAGE_SIZE; 293 end = gte + vma->node.size / I915_GTT_PAGE_SIZE; 294 295 for_each_sgt_daddr(addr, iter, vma->pages) 296 iowrite32(vm->pte_encode(addr, level, flags), gte++); 297 GEM_BUG_ON(gte > end); 298 299 /* Fill the allocated but "unused" space beyond the end of the buffer */ 300 while (gte < end) 301 iowrite32(vm->scratch[0]->encode, gte++); 302 303 /* 304 * We want to flush the TLBs only after we're certain all the PTE 305 * updates have finished. 306 */ 307 ggtt->invalidate(ggtt); 308 } 309 310 static void nop_clear_range(struct i915_address_space *vm, 311 u64 start, u64 length) 312 { 313 } 314 315 static void gen8_ggtt_clear_range(struct i915_address_space *vm, 316 u64 start, u64 length) 317 { 318 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 319 unsigned int first_entry = start / I915_GTT_PAGE_SIZE; 320 unsigned int num_entries = length / I915_GTT_PAGE_SIZE; 321 const gen8_pte_t scratch_pte = vm->scratch[0]->encode; 322 gen8_pte_t __iomem *gtt_base = 323 (gen8_pte_t __iomem *)ggtt->gsm + first_entry; 324 const int max_entries = ggtt_total_entries(ggtt) - first_entry; 325 int i; 326 327 if (WARN(num_entries > max_entries, 328 "First entry = %d; Num entries = %d (max=%d)\n", 329 first_entry, num_entries, max_entries)) 330 num_entries = max_entries; 331 332 for (i = 0; i < num_entries; i++) 333 gen8_set_pte(>t_base[i], scratch_pte); 334 } 335 336 static void bxt_vtd_ggtt_wa(struct i915_address_space *vm) 337 { 338 /* 339 * Make sure the internal GAM fifo has been cleared of all GTT 340 * writes before exiting stop_machine(). This guarantees that 341 * any aperture accesses waiting to start in another process 342 * cannot back up behind the GTT writes causing a hang. 343 * The register can be any arbitrary GAM register. 344 */ 345 intel_uncore_posting_read_fw(vm->gt->uncore, GFX_FLSH_CNTL_GEN6); 346 } 347 348 struct insert_page { 349 struct i915_address_space *vm; 350 dma_addr_t addr; 351 u64 offset; 352 enum i915_cache_level level; 353 }; 354 355 static int bxt_vtd_ggtt_insert_page__cb(void *_arg) 356 { 357 struct insert_page *arg = _arg; 358 359 gen8_ggtt_insert_page(arg->vm, arg->addr, arg->offset, arg->level, 0); 360 bxt_vtd_ggtt_wa(arg->vm); 361 362 return 0; 363 } 364 365 static void bxt_vtd_ggtt_insert_page__BKL(struct i915_address_space *vm, 366 dma_addr_t addr, 367 u64 offset, 368 enum i915_cache_level level, 369 u32 unused) 370 { 371 struct insert_page arg = { vm, addr, offset, level }; 372 373 stop_machine(bxt_vtd_ggtt_insert_page__cb, &arg, NULL); 374 } 375 376 struct insert_entries { 377 struct i915_address_space *vm; 378 struct i915_vma *vma; 379 enum i915_cache_level level; 380 u32 flags; 381 }; 382 383 static int bxt_vtd_ggtt_insert_entries__cb(void *_arg) 384 { 385 struct insert_entries *arg = _arg; 386 387 gen8_ggtt_insert_entries(arg->vm, arg->vma, arg->level, arg->flags); 388 bxt_vtd_ggtt_wa(arg->vm); 389 390 return 0; 391 } 392 393 static void bxt_vtd_ggtt_insert_entries__BKL(struct i915_address_space *vm, 394 struct i915_vma *vma, 395 enum i915_cache_level level, 396 u32 flags) 397 { 398 struct insert_entries arg = { vm, vma, level, flags }; 399 400 stop_machine(bxt_vtd_ggtt_insert_entries__cb, &arg, NULL); 401 } 402 403 static void gen6_ggtt_clear_range(struct i915_address_space *vm, 404 u64 start, u64 length) 405 { 406 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 407 unsigned int first_entry = start / I915_GTT_PAGE_SIZE; 408 unsigned int num_entries = length / I915_GTT_PAGE_SIZE; 409 gen6_pte_t scratch_pte, __iomem *gtt_base = 410 (gen6_pte_t __iomem *)ggtt->gsm + first_entry; 411 const int max_entries = ggtt_total_entries(ggtt) - first_entry; 412 int i; 413 414 if (WARN(num_entries > max_entries, 415 "First entry = %d; Num entries = %d (max=%d)\n", 416 first_entry, num_entries, max_entries)) 417 num_entries = max_entries; 418 419 scratch_pte = vm->scratch[0]->encode; 420 for (i = 0; i < num_entries; i++) 421 iowrite32(scratch_pte, >t_base[i]); 422 } 423 424 static void i915_ggtt_insert_page(struct i915_address_space *vm, 425 dma_addr_t addr, 426 u64 offset, 427 enum i915_cache_level cache_level, 428 u32 unused) 429 { 430 unsigned int flags = (cache_level == I915_CACHE_NONE) ? 431 AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY; 432 433 intel_gtt_insert_page(addr, offset >> PAGE_SHIFT, flags); 434 } 435 436 static void i915_ggtt_insert_entries(struct i915_address_space *vm, 437 struct i915_vma *vma, 438 enum i915_cache_level cache_level, 439 u32 unused) 440 { 441 unsigned int flags = (cache_level == I915_CACHE_NONE) ? 442 AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY; 443 444 intel_gtt_insert_sg_entries(vma->pages, vma->node.start >> PAGE_SHIFT, 445 flags); 446 } 447 448 static void i915_ggtt_clear_range(struct i915_address_space *vm, 449 u64 start, u64 length) 450 { 451 intel_gtt_clear_range(start >> PAGE_SHIFT, length >> PAGE_SHIFT); 452 } 453 454 static void ggtt_bind_vma(struct i915_address_space *vm, 455 struct i915_vm_pt_stash *stash, 456 struct i915_vma *vma, 457 enum i915_cache_level cache_level, 458 u32 flags) 459 { 460 struct drm_i915_gem_object *obj = vma->obj; 461 u32 pte_flags; 462 463 if (i915_vma_is_bound(vma, ~flags & I915_VMA_BIND_MASK)) 464 return; 465 466 /* Applicable to VLV (gen8+ do not support RO in the GGTT) */ 467 pte_flags = 0; 468 if (i915_gem_object_is_readonly(obj)) 469 pte_flags |= PTE_READ_ONLY; 470 if (i915_gem_object_is_lmem(obj)) 471 pte_flags |= PTE_LM; 472 473 vm->insert_entries(vm, vma, cache_level, pte_flags); 474 vma->page_sizes.gtt = I915_GTT_PAGE_SIZE; 475 } 476 477 static void ggtt_unbind_vma(struct i915_address_space *vm, struct i915_vma *vma) 478 { 479 vm->clear_range(vm, vma->node.start, vma->size); 480 } 481 482 static int ggtt_reserve_guc_top(struct i915_ggtt *ggtt) 483 { 484 u64 size; 485 int ret; 486 487 if (!intel_uc_uses_guc(&ggtt->vm.gt->uc)) 488 return 0; 489 490 GEM_BUG_ON(ggtt->vm.total <= GUC_GGTT_TOP); 491 size = ggtt->vm.total - GUC_GGTT_TOP; 492 493 ret = i915_gem_gtt_reserve(&ggtt->vm, &ggtt->uc_fw, size, 494 GUC_GGTT_TOP, I915_COLOR_UNEVICTABLE, 495 PIN_NOEVICT); 496 if (ret) 497 drm_dbg(&ggtt->vm.i915->drm, 498 "Failed to reserve top of GGTT for GuC\n"); 499 500 return ret; 501 } 502 503 static void ggtt_release_guc_top(struct i915_ggtt *ggtt) 504 { 505 if (drm_mm_node_allocated(&ggtt->uc_fw)) 506 drm_mm_remove_node(&ggtt->uc_fw); 507 } 508 509 static void cleanup_init_ggtt(struct i915_ggtt *ggtt) 510 { 511 ggtt_release_guc_top(ggtt); 512 if (drm_mm_node_allocated(&ggtt->error_capture)) 513 drm_mm_remove_node(&ggtt->error_capture); 514 mutex_destroy(&ggtt->error_mutex); 515 } 516 517 static int init_ggtt(struct i915_ggtt *ggtt) 518 { 519 /* 520 * Let GEM Manage all of the aperture. 521 * 522 * However, leave one page at the end still bound to the scratch page. 523 * There are a number of places where the hardware apparently prefetches 524 * past the end of the object, and we've seen multiple hangs with the 525 * GPU head pointer stuck in a batchbuffer bound at the last page of the 526 * aperture. One page should be enough to keep any prefetching inside 527 * of the aperture. 528 */ 529 unsigned long hole_start, hole_end; 530 struct drm_mm_node *entry; 531 int ret; 532 533 /* 534 * GuC requires all resources that we're sharing with it to be placed in 535 * non-WOPCM memory. If GuC is not present or not in use we still need a 536 * small bias as ring wraparound at offset 0 sometimes hangs. No idea 537 * why. 538 */ 539 ggtt->pin_bias = max_t(u32, I915_GTT_PAGE_SIZE, 540 intel_wopcm_guc_size(&ggtt->vm.i915->wopcm)); 541 542 ret = intel_vgt_balloon(ggtt); 543 if (ret) 544 return ret; 545 546 mutex_init(&ggtt->error_mutex); 547 if (ggtt->mappable_end) { 548 /* 549 * Reserve a mappable slot for our lockless error capture. 550 * 551 * We strongly prefer taking address 0x0 in order to protect 552 * other critical buffers against accidental overwrites, 553 * as writing to address 0 is a very common mistake. 554 * 555 * Since 0 may already be in use by the system (e.g. the BIOS 556 * framebuffer), we let the reservation fail quietly and hope 557 * 0 remains reserved always. 558 * 559 * If we fail to reserve 0, and then fail to find any space 560 * for an error-capture, remain silent. We can afford not 561 * to reserve an error_capture node as we have fallback 562 * paths, and we trust that 0 will remain reserved. However, 563 * the only likely reason for failure to insert is a driver 564 * bug, which we expect to cause other failures... 565 */ 566 ggtt->error_capture.size = I915_GTT_PAGE_SIZE; 567 ggtt->error_capture.color = I915_COLOR_UNEVICTABLE; 568 if (drm_mm_reserve_node(&ggtt->vm.mm, &ggtt->error_capture)) 569 drm_mm_insert_node_in_range(&ggtt->vm.mm, 570 &ggtt->error_capture, 571 ggtt->error_capture.size, 0, 572 ggtt->error_capture.color, 573 0, ggtt->mappable_end, 574 DRM_MM_INSERT_LOW); 575 } 576 if (drm_mm_node_allocated(&ggtt->error_capture)) 577 drm_dbg(&ggtt->vm.i915->drm, 578 "Reserved GGTT:[%llx, %llx] for use by error capture\n", 579 ggtt->error_capture.start, 580 ggtt->error_capture.start + ggtt->error_capture.size); 581 582 /* 583 * The upper portion of the GuC address space has a sizeable hole 584 * (several MB) that is inaccessible by GuC. Reserve this range within 585 * GGTT as it can comfortably hold GuC/HuC firmware images. 586 */ 587 ret = ggtt_reserve_guc_top(ggtt); 588 if (ret) 589 goto err; 590 591 /* Clear any non-preallocated blocks */ 592 drm_mm_for_each_hole(entry, &ggtt->vm.mm, hole_start, hole_end) { 593 drm_dbg(&ggtt->vm.i915->drm, 594 "clearing unused GTT space: [%lx, %lx]\n", 595 hole_start, hole_end); 596 ggtt->vm.clear_range(&ggtt->vm, hole_start, 597 hole_end - hole_start); 598 } 599 600 /* And finally clear the reserved guard page */ 601 ggtt->vm.clear_range(&ggtt->vm, ggtt->vm.total - PAGE_SIZE, PAGE_SIZE); 602 603 return 0; 604 605 err: 606 cleanup_init_ggtt(ggtt); 607 return ret; 608 } 609 610 static void aliasing_gtt_bind_vma(struct i915_address_space *vm, 611 struct i915_vm_pt_stash *stash, 612 struct i915_vma *vma, 613 enum i915_cache_level cache_level, 614 u32 flags) 615 { 616 u32 pte_flags; 617 618 /* Currently applicable only to VLV */ 619 pte_flags = 0; 620 if (i915_gem_object_is_readonly(vma->obj)) 621 pte_flags |= PTE_READ_ONLY; 622 623 if (flags & I915_VMA_LOCAL_BIND) 624 ppgtt_bind_vma(&i915_vm_to_ggtt(vm)->alias->vm, 625 stash, vma, cache_level, flags); 626 627 if (flags & I915_VMA_GLOBAL_BIND) 628 vm->insert_entries(vm, vma, cache_level, pte_flags); 629 } 630 631 static void aliasing_gtt_unbind_vma(struct i915_address_space *vm, 632 struct i915_vma *vma) 633 { 634 if (i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND)) 635 vm->clear_range(vm, vma->node.start, vma->size); 636 637 if (i915_vma_is_bound(vma, I915_VMA_LOCAL_BIND)) 638 ppgtt_unbind_vma(&i915_vm_to_ggtt(vm)->alias->vm, vma); 639 } 640 641 static int init_aliasing_ppgtt(struct i915_ggtt *ggtt) 642 { 643 struct i915_vm_pt_stash stash = {}; 644 struct i915_ppgtt *ppgtt; 645 int err; 646 647 ppgtt = i915_ppgtt_create(ggtt->vm.gt); 648 if (IS_ERR(ppgtt)) 649 return PTR_ERR(ppgtt); 650 651 if (GEM_WARN_ON(ppgtt->vm.total < ggtt->vm.total)) { 652 err = -ENODEV; 653 goto err_ppgtt; 654 } 655 656 err = i915_vm_alloc_pt_stash(&ppgtt->vm, &stash, ggtt->vm.total); 657 if (err) 658 goto err_ppgtt; 659 660 i915_gem_object_lock(ppgtt->vm.scratch[0], NULL); 661 err = i915_vm_map_pt_stash(&ppgtt->vm, &stash); 662 i915_gem_object_unlock(ppgtt->vm.scratch[0]); 663 if (err) 664 goto err_stash; 665 666 /* 667 * Note we only pre-allocate as far as the end of the global 668 * GTT. On 48b / 4-level page-tables, the difference is very, 669 * very significant! We have to preallocate as GVT/vgpu does 670 * not like the page directory disappearing. 671 */ 672 ppgtt->vm.allocate_va_range(&ppgtt->vm, &stash, 0, ggtt->vm.total); 673 674 ggtt->alias = ppgtt; 675 ggtt->vm.bind_async_flags |= ppgtt->vm.bind_async_flags; 676 677 GEM_BUG_ON(ggtt->vm.vma_ops.bind_vma != ggtt_bind_vma); 678 ggtt->vm.vma_ops.bind_vma = aliasing_gtt_bind_vma; 679 680 GEM_BUG_ON(ggtt->vm.vma_ops.unbind_vma != ggtt_unbind_vma); 681 ggtt->vm.vma_ops.unbind_vma = aliasing_gtt_unbind_vma; 682 683 i915_vm_free_pt_stash(&ppgtt->vm, &stash); 684 return 0; 685 686 err_stash: 687 i915_vm_free_pt_stash(&ppgtt->vm, &stash); 688 err_ppgtt: 689 i915_vm_put(&ppgtt->vm); 690 return err; 691 } 692 693 static void fini_aliasing_ppgtt(struct i915_ggtt *ggtt) 694 { 695 struct i915_ppgtt *ppgtt; 696 697 ppgtt = fetch_and_zero(&ggtt->alias); 698 if (!ppgtt) 699 return; 700 701 i915_vm_put(&ppgtt->vm); 702 703 ggtt->vm.vma_ops.bind_vma = ggtt_bind_vma; 704 ggtt->vm.vma_ops.unbind_vma = ggtt_unbind_vma; 705 } 706 707 int i915_init_ggtt(struct drm_i915_private *i915) 708 { 709 int ret; 710 711 ret = init_ggtt(&i915->ggtt); 712 if (ret) 713 return ret; 714 715 if (INTEL_PPGTT(i915) == INTEL_PPGTT_ALIASING) { 716 ret = init_aliasing_ppgtt(&i915->ggtt); 717 if (ret) 718 cleanup_init_ggtt(&i915->ggtt); 719 } 720 721 return 0; 722 } 723 724 static void ggtt_cleanup_hw(struct i915_ggtt *ggtt) 725 { 726 struct i915_vma *vma, *vn; 727 728 atomic_set(&ggtt->vm.open, 0); 729 730 rcu_barrier(); /* flush the RCU'ed__i915_vm_release */ 731 flush_workqueue(ggtt->vm.i915->wq); 732 733 mutex_lock(&ggtt->vm.mutex); 734 735 list_for_each_entry_safe(vma, vn, &ggtt->vm.bound_list, vm_link) 736 WARN_ON(__i915_vma_unbind(vma)); 737 738 if (drm_mm_node_allocated(&ggtt->error_capture)) 739 drm_mm_remove_node(&ggtt->error_capture); 740 mutex_destroy(&ggtt->error_mutex); 741 742 ggtt_release_guc_top(ggtt); 743 intel_vgt_deballoon(ggtt); 744 745 ggtt->vm.cleanup(&ggtt->vm); 746 747 mutex_unlock(&ggtt->vm.mutex); 748 i915_address_space_fini(&ggtt->vm); 749 dma_resv_fini(&ggtt->vm.resv); 750 751 arch_phys_wc_del(ggtt->mtrr); 752 753 if (ggtt->iomap.size) 754 io_mapping_fini(&ggtt->iomap); 755 } 756 757 /** 758 * i915_ggtt_driver_release - Clean up GGTT hardware initialization 759 * @i915: i915 device 760 */ 761 void i915_ggtt_driver_release(struct drm_i915_private *i915) 762 { 763 struct i915_ggtt *ggtt = &i915->ggtt; 764 765 fini_aliasing_ppgtt(ggtt); 766 767 intel_ggtt_fini_fences(ggtt); 768 ggtt_cleanup_hw(ggtt); 769 } 770 771 static unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl) 772 { 773 snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT; 774 snb_gmch_ctl &= SNB_GMCH_GGMS_MASK; 775 return snb_gmch_ctl << 20; 776 } 777 778 static unsigned int gen8_get_total_gtt_size(u16 bdw_gmch_ctl) 779 { 780 bdw_gmch_ctl >>= BDW_GMCH_GGMS_SHIFT; 781 bdw_gmch_ctl &= BDW_GMCH_GGMS_MASK; 782 if (bdw_gmch_ctl) 783 bdw_gmch_ctl = 1 << bdw_gmch_ctl; 784 785 #ifdef CONFIG_X86_32 786 /* Limit 32b platforms to a 2GB GGTT: 4 << 20 / pte size * I915_GTT_PAGE_SIZE */ 787 if (bdw_gmch_ctl > 4) 788 bdw_gmch_ctl = 4; 789 #endif 790 791 return bdw_gmch_ctl << 20; 792 } 793 794 static unsigned int chv_get_total_gtt_size(u16 gmch_ctrl) 795 { 796 gmch_ctrl >>= SNB_GMCH_GGMS_SHIFT; 797 gmch_ctrl &= SNB_GMCH_GGMS_MASK; 798 799 if (gmch_ctrl) 800 return 1 << (20 + gmch_ctrl); 801 802 return 0; 803 } 804 805 static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size) 806 { 807 struct drm_i915_private *i915 = ggtt->vm.i915; 808 struct pci_dev *pdev = to_pci_dev(i915->drm.dev); 809 phys_addr_t phys_addr; 810 u32 pte_flags; 811 int ret; 812 813 /* For Modern GENs the PTEs and register space are split in the BAR */ 814 phys_addr = pci_resource_start(pdev, 0) + pci_resource_len(pdev, 0) / 2; 815 816 /* 817 * On BXT+/CNL+ writes larger than 64 bit to the GTT pagetable range 818 * will be dropped. For WC mappings in general we have 64 byte burst 819 * writes when the WC buffer is flushed, so we can't use it, but have to 820 * resort to an uncached mapping. The WC issue is easily caught by the 821 * readback check when writing GTT PTE entries. 822 */ 823 if (IS_GEN9_LP(i915) || INTEL_GEN(i915) >= 10) 824 ggtt->gsm = ioremap(phys_addr, size); 825 else 826 ggtt->gsm = ioremap_wc(phys_addr, size); 827 if (!ggtt->gsm) { 828 drm_err(&i915->drm, "Failed to map the ggtt page table\n"); 829 return -ENOMEM; 830 } 831 832 ret = setup_scratch_page(&ggtt->vm); 833 if (ret) { 834 drm_err(&i915->drm, "Scratch setup failed\n"); 835 /* iounmap will also get called at remove, but meh */ 836 iounmap(ggtt->gsm); 837 return ret; 838 } 839 840 pte_flags = 0; 841 if (i915_gem_object_is_lmem(ggtt->vm.scratch[0])) 842 pte_flags |= PTE_LM; 843 844 ggtt->vm.scratch[0]->encode = 845 ggtt->vm.pte_encode(px_dma(ggtt->vm.scratch[0]), 846 I915_CACHE_NONE, pte_flags); 847 848 return 0; 849 } 850 851 int ggtt_set_pages(struct i915_vma *vma) 852 { 853 int ret; 854 855 GEM_BUG_ON(vma->pages); 856 857 ret = i915_get_ggtt_vma_pages(vma); 858 if (ret) 859 return ret; 860 861 vma->page_sizes = vma->obj->mm.page_sizes; 862 863 return 0; 864 } 865 866 static void gen6_gmch_remove(struct i915_address_space *vm) 867 { 868 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 869 870 iounmap(ggtt->gsm); 871 free_scratch(vm); 872 } 873 874 static struct resource pci_resource(struct pci_dev *pdev, int bar) 875 { 876 return (struct resource)DEFINE_RES_MEM(pci_resource_start(pdev, bar), 877 pci_resource_len(pdev, bar)); 878 } 879 880 static int gen8_gmch_probe(struct i915_ggtt *ggtt) 881 { 882 struct drm_i915_private *i915 = ggtt->vm.i915; 883 struct pci_dev *pdev = to_pci_dev(i915->drm.dev); 884 unsigned int size; 885 u16 snb_gmch_ctl; 886 887 /* TODO: We're not aware of mappable constraints on gen8 yet */ 888 if (!HAS_LMEM(i915)) { 889 ggtt->gmadr = pci_resource(pdev, 2); 890 ggtt->mappable_end = resource_size(&ggtt->gmadr); 891 } 892 893 pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl); 894 if (IS_CHERRYVIEW(i915)) 895 size = chv_get_total_gtt_size(snb_gmch_ctl); 896 else 897 size = gen8_get_total_gtt_size(snb_gmch_ctl); 898 899 ggtt->vm.alloc_pt_dma = alloc_pt_dma; 900 901 ggtt->vm.total = (size / sizeof(gen8_pte_t)) * I915_GTT_PAGE_SIZE; 902 ggtt->vm.cleanup = gen6_gmch_remove; 903 ggtt->vm.insert_page = gen8_ggtt_insert_page; 904 ggtt->vm.clear_range = nop_clear_range; 905 if (intel_scanout_needs_vtd_wa(i915)) 906 ggtt->vm.clear_range = gen8_ggtt_clear_range; 907 908 ggtt->vm.insert_entries = gen8_ggtt_insert_entries; 909 910 /* 911 * Serialize GTT updates with aperture access on BXT if VT-d is on, 912 * and always on CHV. 913 */ 914 if (intel_vm_no_concurrent_access_wa(i915)) { 915 ggtt->vm.insert_entries = bxt_vtd_ggtt_insert_entries__BKL; 916 ggtt->vm.insert_page = bxt_vtd_ggtt_insert_page__BKL; 917 ggtt->vm.bind_async_flags = 918 I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND; 919 } 920 921 ggtt->invalidate = gen8_ggtt_invalidate; 922 923 ggtt->vm.vma_ops.bind_vma = ggtt_bind_vma; 924 ggtt->vm.vma_ops.unbind_vma = ggtt_unbind_vma; 925 ggtt->vm.vma_ops.set_pages = ggtt_set_pages; 926 ggtt->vm.vma_ops.clear_pages = clear_pages; 927 928 ggtt->vm.pte_encode = gen8_ggtt_pte_encode; 929 930 setup_private_pat(ggtt->vm.gt->uncore); 931 932 return ggtt_probe_common(ggtt, size); 933 } 934 935 static u64 snb_pte_encode(dma_addr_t addr, 936 enum i915_cache_level level, 937 u32 flags) 938 { 939 gen6_pte_t pte = GEN6_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID; 940 941 switch (level) { 942 case I915_CACHE_L3_LLC: 943 case I915_CACHE_LLC: 944 pte |= GEN6_PTE_CACHE_LLC; 945 break; 946 case I915_CACHE_NONE: 947 pte |= GEN6_PTE_UNCACHED; 948 break; 949 default: 950 MISSING_CASE(level); 951 } 952 953 return pte; 954 } 955 956 static u64 ivb_pte_encode(dma_addr_t addr, 957 enum i915_cache_level level, 958 u32 flags) 959 { 960 gen6_pte_t pte = GEN6_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID; 961 962 switch (level) { 963 case I915_CACHE_L3_LLC: 964 pte |= GEN7_PTE_CACHE_L3_LLC; 965 break; 966 case I915_CACHE_LLC: 967 pte |= GEN6_PTE_CACHE_LLC; 968 break; 969 case I915_CACHE_NONE: 970 pte |= GEN6_PTE_UNCACHED; 971 break; 972 default: 973 MISSING_CASE(level); 974 } 975 976 return pte; 977 } 978 979 static u64 byt_pte_encode(dma_addr_t addr, 980 enum i915_cache_level level, 981 u32 flags) 982 { 983 gen6_pte_t pte = GEN6_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID; 984 985 if (!(flags & PTE_READ_ONLY)) 986 pte |= BYT_PTE_WRITEABLE; 987 988 if (level != I915_CACHE_NONE) 989 pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES; 990 991 return pte; 992 } 993 994 static u64 hsw_pte_encode(dma_addr_t addr, 995 enum i915_cache_level level, 996 u32 flags) 997 { 998 gen6_pte_t pte = HSW_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID; 999 1000 if (level != I915_CACHE_NONE) 1001 pte |= HSW_WB_LLC_AGE3; 1002 1003 return pte; 1004 } 1005 1006 static u64 iris_pte_encode(dma_addr_t addr, 1007 enum i915_cache_level level, 1008 u32 flags) 1009 { 1010 gen6_pte_t pte = HSW_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID; 1011 1012 switch (level) { 1013 case I915_CACHE_NONE: 1014 break; 1015 case I915_CACHE_WT: 1016 pte |= HSW_WT_ELLC_LLC_AGE3; 1017 break; 1018 default: 1019 pte |= HSW_WB_ELLC_LLC_AGE3; 1020 break; 1021 } 1022 1023 return pte; 1024 } 1025 1026 static int gen6_gmch_probe(struct i915_ggtt *ggtt) 1027 { 1028 struct drm_i915_private *i915 = ggtt->vm.i915; 1029 struct pci_dev *pdev = to_pci_dev(i915->drm.dev); 1030 unsigned int size; 1031 u16 snb_gmch_ctl; 1032 1033 ggtt->gmadr = pci_resource(pdev, 2); 1034 ggtt->mappable_end = resource_size(&ggtt->gmadr); 1035 1036 /* 1037 * 64/512MB is the current min/max we actually know of, but this is 1038 * just a coarse sanity check. 1039 */ 1040 if (ggtt->mappable_end < (64<<20) || ggtt->mappable_end > (512<<20)) { 1041 drm_err(&i915->drm, "Unknown GMADR size (%pa)\n", 1042 &ggtt->mappable_end); 1043 return -ENXIO; 1044 } 1045 1046 pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl); 1047 1048 size = gen6_get_total_gtt_size(snb_gmch_ctl); 1049 ggtt->vm.total = (size / sizeof(gen6_pte_t)) * I915_GTT_PAGE_SIZE; 1050 1051 ggtt->vm.alloc_pt_dma = alloc_pt_dma; 1052 1053 ggtt->vm.clear_range = nop_clear_range; 1054 if (!HAS_FULL_PPGTT(i915) || intel_scanout_needs_vtd_wa(i915)) 1055 ggtt->vm.clear_range = gen6_ggtt_clear_range; 1056 ggtt->vm.insert_page = gen6_ggtt_insert_page; 1057 ggtt->vm.insert_entries = gen6_ggtt_insert_entries; 1058 ggtt->vm.cleanup = gen6_gmch_remove; 1059 1060 ggtt->invalidate = gen6_ggtt_invalidate; 1061 1062 if (HAS_EDRAM(i915)) 1063 ggtt->vm.pte_encode = iris_pte_encode; 1064 else if (IS_HASWELL(i915)) 1065 ggtt->vm.pte_encode = hsw_pte_encode; 1066 else if (IS_VALLEYVIEW(i915)) 1067 ggtt->vm.pte_encode = byt_pte_encode; 1068 else if (INTEL_GEN(i915) >= 7) 1069 ggtt->vm.pte_encode = ivb_pte_encode; 1070 else 1071 ggtt->vm.pte_encode = snb_pte_encode; 1072 1073 ggtt->vm.vma_ops.bind_vma = ggtt_bind_vma; 1074 ggtt->vm.vma_ops.unbind_vma = ggtt_unbind_vma; 1075 ggtt->vm.vma_ops.set_pages = ggtt_set_pages; 1076 ggtt->vm.vma_ops.clear_pages = clear_pages; 1077 1078 return ggtt_probe_common(ggtt, size); 1079 } 1080 1081 static void i915_gmch_remove(struct i915_address_space *vm) 1082 { 1083 intel_gmch_remove(); 1084 } 1085 1086 static int i915_gmch_probe(struct i915_ggtt *ggtt) 1087 { 1088 struct drm_i915_private *i915 = ggtt->vm.i915; 1089 phys_addr_t gmadr_base; 1090 int ret; 1091 1092 ret = intel_gmch_probe(i915->bridge_dev, to_pci_dev(i915->drm.dev), NULL); 1093 if (!ret) { 1094 drm_err(&i915->drm, "failed to set up gmch\n"); 1095 return -EIO; 1096 } 1097 1098 intel_gtt_get(&ggtt->vm.total, &gmadr_base, &ggtt->mappable_end); 1099 1100 ggtt->gmadr = 1101 (struct resource)DEFINE_RES_MEM(gmadr_base, ggtt->mappable_end); 1102 1103 ggtt->vm.alloc_pt_dma = alloc_pt_dma; 1104 1105 if (needs_idle_maps(i915)) { 1106 drm_notice(&i915->drm, 1107 "Flushing DMA requests before IOMMU unmaps; performance may be degraded\n"); 1108 ggtt->do_idle_maps = true; 1109 } 1110 1111 ggtt->vm.insert_page = i915_ggtt_insert_page; 1112 ggtt->vm.insert_entries = i915_ggtt_insert_entries; 1113 ggtt->vm.clear_range = i915_ggtt_clear_range; 1114 ggtt->vm.cleanup = i915_gmch_remove; 1115 1116 ggtt->invalidate = gmch_ggtt_invalidate; 1117 1118 ggtt->vm.vma_ops.bind_vma = ggtt_bind_vma; 1119 ggtt->vm.vma_ops.unbind_vma = ggtt_unbind_vma; 1120 ggtt->vm.vma_ops.set_pages = ggtt_set_pages; 1121 ggtt->vm.vma_ops.clear_pages = clear_pages; 1122 1123 if (unlikely(ggtt->do_idle_maps)) 1124 drm_notice(&i915->drm, 1125 "Applying Ironlake quirks for intel_iommu\n"); 1126 1127 return 0; 1128 } 1129 1130 static int ggtt_probe_hw(struct i915_ggtt *ggtt, struct intel_gt *gt) 1131 { 1132 struct drm_i915_private *i915 = gt->i915; 1133 int ret; 1134 1135 ggtt->vm.gt = gt; 1136 ggtt->vm.i915 = i915; 1137 ggtt->vm.dma = i915->drm.dev; 1138 dma_resv_init(&ggtt->vm.resv); 1139 1140 if (INTEL_GEN(i915) <= 5) 1141 ret = i915_gmch_probe(ggtt); 1142 else if (INTEL_GEN(i915) < 8) 1143 ret = gen6_gmch_probe(ggtt); 1144 else 1145 ret = gen8_gmch_probe(ggtt); 1146 if (ret) { 1147 dma_resv_fini(&ggtt->vm.resv); 1148 return ret; 1149 } 1150 1151 if ((ggtt->vm.total - 1) >> 32) { 1152 drm_err(&i915->drm, 1153 "We never expected a Global GTT with more than 32bits" 1154 " of address space! Found %lldM!\n", 1155 ggtt->vm.total >> 20); 1156 ggtt->vm.total = 1ULL << 32; 1157 ggtt->mappable_end = 1158 min_t(u64, ggtt->mappable_end, ggtt->vm.total); 1159 } 1160 1161 if (ggtt->mappable_end > ggtt->vm.total) { 1162 drm_err(&i915->drm, 1163 "mappable aperture extends past end of GGTT," 1164 " aperture=%pa, total=%llx\n", 1165 &ggtt->mappable_end, ggtt->vm.total); 1166 ggtt->mappable_end = ggtt->vm.total; 1167 } 1168 1169 /* GMADR is the PCI mmio aperture into the global GTT. */ 1170 drm_dbg(&i915->drm, "GGTT size = %lluM\n", ggtt->vm.total >> 20); 1171 drm_dbg(&i915->drm, "GMADR size = %lluM\n", 1172 (u64)ggtt->mappable_end >> 20); 1173 drm_dbg(&i915->drm, "DSM size = %lluM\n", 1174 (u64)resource_size(&intel_graphics_stolen_res) >> 20); 1175 1176 return 0; 1177 } 1178 1179 /** 1180 * i915_ggtt_probe_hw - Probe GGTT hardware location 1181 * @i915: i915 device 1182 */ 1183 int i915_ggtt_probe_hw(struct drm_i915_private *i915) 1184 { 1185 int ret; 1186 1187 ret = ggtt_probe_hw(&i915->ggtt, &i915->gt); 1188 if (ret) 1189 return ret; 1190 1191 if (intel_vtd_active()) 1192 drm_info(&i915->drm, "VT-d active for gfx access\n"); 1193 1194 return 0; 1195 } 1196 1197 int i915_ggtt_enable_hw(struct drm_i915_private *i915) 1198 { 1199 if (INTEL_GEN(i915) < 6 && !intel_enable_gtt()) 1200 return -EIO; 1201 1202 return 0; 1203 } 1204 1205 void i915_ggtt_enable_guc(struct i915_ggtt *ggtt) 1206 { 1207 GEM_BUG_ON(ggtt->invalidate != gen8_ggtt_invalidate); 1208 1209 ggtt->invalidate = guc_ggtt_invalidate; 1210 1211 ggtt->invalidate(ggtt); 1212 } 1213 1214 void i915_ggtt_disable_guc(struct i915_ggtt *ggtt) 1215 { 1216 /* XXX Temporary pardon for error unload */ 1217 if (ggtt->invalidate == gen8_ggtt_invalidate) 1218 return; 1219 1220 /* We should only be called after i915_ggtt_enable_guc() */ 1221 GEM_BUG_ON(ggtt->invalidate != guc_ggtt_invalidate); 1222 1223 ggtt->invalidate = gen8_ggtt_invalidate; 1224 1225 ggtt->invalidate(ggtt); 1226 } 1227 1228 void i915_ggtt_resume(struct i915_ggtt *ggtt) 1229 { 1230 struct i915_vma *vma; 1231 bool flush = false; 1232 int open; 1233 1234 intel_gt_check_and_clear_faults(ggtt->vm.gt); 1235 1236 /* First fill our portion of the GTT with scratch pages */ 1237 ggtt->vm.clear_range(&ggtt->vm, 0, ggtt->vm.total); 1238 1239 /* Skip rewriting PTE on VMA unbind. */ 1240 open = atomic_xchg(&ggtt->vm.open, 0); 1241 1242 /* clflush objects bound into the GGTT and rebind them. */ 1243 list_for_each_entry(vma, &ggtt->vm.bound_list, vm_link) { 1244 struct drm_i915_gem_object *obj = vma->obj; 1245 unsigned int was_bound = 1246 atomic_read(&vma->flags) & I915_VMA_BIND_MASK; 1247 1248 GEM_BUG_ON(!was_bound); 1249 vma->ops->bind_vma(&ggtt->vm, NULL, vma, 1250 obj ? obj->cache_level : 0, 1251 was_bound); 1252 if (obj) { /* only used during resume => exclusive access */ 1253 flush |= fetch_and_zero(&obj->write_domain); 1254 obj->read_domains |= I915_GEM_DOMAIN_GTT; 1255 } 1256 } 1257 1258 atomic_set(&ggtt->vm.open, open); 1259 ggtt->invalidate(ggtt); 1260 1261 if (flush) 1262 wbinvd_on_all_cpus(); 1263 1264 if (INTEL_GEN(ggtt->vm.i915) >= 8) 1265 setup_private_pat(ggtt->vm.gt->uncore); 1266 1267 intel_ggtt_restore_fences(ggtt); 1268 } 1269 1270 static struct scatterlist * 1271 rotate_pages(struct drm_i915_gem_object *obj, unsigned int offset, 1272 unsigned int width, unsigned int height, 1273 unsigned int src_stride, unsigned int dst_stride, 1274 struct sg_table *st, struct scatterlist *sg) 1275 { 1276 unsigned int column, row; 1277 unsigned int src_idx; 1278 1279 for (column = 0; column < width; column++) { 1280 unsigned int left; 1281 1282 src_idx = src_stride * (height - 1) + column + offset; 1283 for (row = 0; row < height; row++) { 1284 st->nents++; 1285 /* 1286 * We don't need the pages, but need to initialize 1287 * the entries so the sg list can be happily traversed. 1288 * The only thing we need are DMA addresses. 1289 */ 1290 sg_set_page(sg, NULL, I915_GTT_PAGE_SIZE, 0); 1291 sg_dma_address(sg) = 1292 i915_gem_object_get_dma_address(obj, src_idx); 1293 sg_dma_len(sg) = I915_GTT_PAGE_SIZE; 1294 sg = sg_next(sg); 1295 src_idx -= src_stride; 1296 } 1297 1298 left = (dst_stride - height) * I915_GTT_PAGE_SIZE; 1299 1300 if (!left) 1301 continue; 1302 1303 st->nents++; 1304 1305 /* 1306 * The DE ignores the PTEs for the padding tiles, the sg entry 1307 * here is just a conenience to indicate how many padding PTEs 1308 * to insert at this spot. 1309 */ 1310 sg_set_page(sg, NULL, left, 0); 1311 sg_dma_address(sg) = 0; 1312 sg_dma_len(sg) = left; 1313 sg = sg_next(sg); 1314 } 1315 1316 return sg; 1317 } 1318 1319 static noinline struct sg_table * 1320 intel_rotate_pages(struct intel_rotation_info *rot_info, 1321 struct drm_i915_gem_object *obj) 1322 { 1323 unsigned int size = intel_rotation_info_size(rot_info); 1324 struct drm_i915_private *i915 = to_i915(obj->base.dev); 1325 struct sg_table *st; 1326 struct scatterlist *sg; 1327 int ret = -ENOMEM; 1328 int i; 1329 1330 /* Allocate target SG list. */ 1331 st = kmalloc(sizeof(*st), GFP_KERNEL); 1332 if (!st) 1333 goto err_st_alloc; 1334 1335 ret = sg_alloc_table(st, size, GFP_KERNEL); 1336 if (ret) 1337 goto err_sg_alloc; 1338 1339 st->nents = 0; 1340 sg = st->sgl; 1341 1342 for (i = 0 ; i < ARRAY_SIZE(rot_info->plane); i++) 1343 sg = rotate_pages(obj, rot_info->plane[i].offset, 1344 rot_info->plane[i].width, rot_info->plane[i].height, 1345 rot_info->plane[i].src_stride, 1346 rot_info->plane[i].dst_stride, 1347 st, sg); 1348 1349 return st; 1350 1351 err_sg_alloc: 1352 kfree(st); 1353 err_st_alloc: 1354 1355 drm_dbg(&i915->drm, "Failed to create rotated mapping for object size %zu! (%ux%u tiles, %u pages)\n", 1356 obj->base.size, rot_info->plane[0].width, 1357 rot_info->plane[0].height, size); 1358 1359 return ERR_PTR(ret); 1360 } 1361 1362 static struct scatterlist * 1363 remap_pages(struct drm_i915_gem_object *obj, unsigned int offset, 1364 unsigned int width, unsigned int height, 1365 unsigned int src_stride, unsigned int dst_stride, 1366 struct sg_table *st, struct scatterlist *sg) 1367 { 1368 unsigned int row; 1369 1370 for (row = 0; row < height; row++) { 1371 unsigned int left = width * I915_GTT_PAGE_SIZE; 1372 1373 while (left) { 1374 dma_addr_t addr; 1375 unsigned int length; 1376 1377 /* 1378 * We don't need the pages, but need to initialize 1379 * the entries so the sg list can be happily traversed. 1380 * The only thing we need are DMA addresses. 1381 */ 1382 1383 addr = i915_gem_object_get_dma_address_len(obj, offset, &length); 1384 1385 length = min(left, length); 1386 1387 st->nents++; 1388 1389 sg_set_page(sg, NULL, length, 0); 1390 sg_dma_address(sg) = addr; 1391 sg_dma_len(sg) = length; 1392 sg = sg_next(sg); 1393 1394 offset += length / I915_GTT_PAGE_SIZE; 1395 left -= length; 1396 } 1397 1398 offset += src_stride - width; 1399 1400 left = (dst_stride - width) * I915_GTT_PAGE_SIZE; 1401 1402 if (!left) 1403 continue; 1404 1405 st->nents++; 1406 1407 /* 1408 * The DE ignores the PTEs for the padding tiles, the sg entry 1409 * here is just a conenience to indicate how many padding PTEs 1410 * to insert at this spot. 1411 */ 1412 sg_set_page(sg, NULL, left, 0); 1413 sg_dma_address(sg) = 0; 1414 sg_dma_len(sg) = left; 1415 sg = sg_next(sg); 1416 } 1417 1418 return sg; 1419 } 1420 1421 static noinline struct sg_table * 1422 intel_remap_pages(struct intel_remapped_info *rem_info, 1423 struct drm_i915_gem_object *obj) 1424 { 1425 unsigned int size = intel_remapped_info_size(rem_info); 1426 struct drm_i915_private *i915 = to_i915(obj->base.dev); 1427 struct sg_table *st; 1428 struct scatterlist *sg; 1429 int ret = -ENOMEM; 1430 int i; 1431 1432 /* Allocate target SG list. */ 1433 st = kmalloc(sizeof(*st), GFP_KERNEL); 1434 if (!st) 1435 goto err_st_alloc; 1436 1437 ret = sg_alloc_table(st, size, GFP_KERNEL); 1438 if (ret) 1439 goto err_sg_alloc; 1440 1441 st->nents = 0; 1442 sg = st->sgl; 1443 1444 for (i = 0 ; i < ARRAY_SIZE(rem_info->plane); i++) { 1445 sg = remap_pages(obj, rem_info->plane[i].offset, 1446 rem_info->plane[i].width, rem_info->plane[i].height, 1447 rem_info->plane[i].src_stride, rem_info->plane[i].dst_stride, 1448 st, sg); 1449 } 1450 1451 i915_sg_trim(st); 1452 1453 return st; 1454 1455 err_sg_alloc: 1456 kfree(st); 1457 err_st_alloc: 1458 1459 drm_dbg(&i915->drm, "Failed to create remapped mapping for object size %zu! (%ux%u tiles, %u pages)\n", 1460 obj->base.size, rem_info->plane[0].width, 1461 rem_info->plane[0].height, size); 1462 1463 return ERR_PTR(ret); 1464 } 1465 1466 static noinline struct sg_table * 1467 intel_partial_pages(const struct i915_ggtt_view *view, 1468 struct drm_i915_gem_object *obj) 1469 { 1470 struct sg_table *st; 1471 struct scatterlist *sg, *iter; 1472 unsigned int count = view->partial.size; 1473 unsigned int offset; 1474 int ret = -ENOMEM; 1475 1476 st = kmalloc(sizeof(*st), GFP_KERNEL); 1477 if (!st) 1478 goto err_st_alloc; 1479 1480 ret = sg_alloc_table(st, count, GFP_KERNEL); 1481 if (ret) 1482 goto err_sg_alloc; 1483 1484 iter = i915_gem_object_get_sg_dma(obj, view->partial.offset, &offset, true); 1485 GEM_BUG_ON(!iter); 1486 1487 sg = st->sgl; 1488 st->nents = 0; 1489 do { 1490 unsigned int len; 1491 1492 len = min(sg_dma_len(iter) - (offset << PAGE_SHIFT), 1493 count << PAGE_SHIFT); 1494 sg_set_page(sg, NULL, len, 0); 1495 sg_dma_address(sg) = 1496 sg_dma_address(iter) + (offset << PAGE_SHIFT); 1497 sg_dma_len(sg) = len; 1498 1499 st->nents++; 1500 count -= len >> PAGE_SHIFT; 1501 if (count == 0) { 1502 sg_mark_end(sg); 1503 i915_sg_trim(st); /* Drop any unused tail entries. */ 1504 1505 return st; 1506 } 1507 1508 sg = __sg_next(sg); 1509 iter = __sg_next(iter); 1510 offset = 0; 1511 } while (1); 1512 1513 err_sg_alloc: 1514 kfree(st); 1515 err_st_alloc: 1516 return ERR_PTR(ret); 1517 } 1518 1519 static int 1520 i915_get_ggtt_vma_pages(struct i915_vma *vma) 1521 { 1522 int ret; 1523 1524 /* 1525 * The vma->pages are only valid within the lifespan of the borrowed 1526 * obj->mm.pages. When the obj->mm.pages sg_table is regenerated, so 1527 * must be the vma->pages. A simple rule is that vma->pages must only 1528 * be accessed when the obj->mm.pages are pinned. 1529 */ 1530 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(vma->obj)); 1531 1532 switch (vma->ggtt_view.type) { 1533 default: 1534 GEM_BUG_ON(vma->ggtt_view.type); 1535 fallthrough; 1536 case I915_GGTT_VIEW_NORMAL: 1537 vma->pages = vma->obj->mm.pages; 1538 return 0; 1539 1540 case I915_GGTT_VIEW_ROTATED: 1541 vma->pages = 1542 intel_rotate_pages(&vma->ggtt_view.rotated, vma->obj); 1543 break; 1544 1545 case I915_GGTT_VIEW_REMAPPED: 1546 vma->pages = 1547 intel_remap_pages(&vma->ggtt_view.remapped, vma->obj); 1548 break; 1549 1550 case I915_GGTT_VIEW_PARTIAL: 1551 vma->pages = intel_partial_pages(&vma->ggtt_view, vma->obj); 1552 break; 1553 } 1554 1555 ret = 0; 1556 if (IS_ERR(vma->pages)) { 1557 ret = PTR_ERR(vma->pages); 1558 vma->pages = NULL; 1559 drm_err(&vma->vm->i915->drm, 1560 "Failed to get pages for VMA view type %u (%d)!\n", 1561 vma->ggtt_view.type, ret); 1562 } 1563 return ret; 1564 } 1565