1 /* 2 * Copyright 2007 Dave Airlied 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice (including the next 13 * paragraph) shall be included in all copies or substantial portions of the 14 * Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 * OTHER DEALINGS IN THE SOFTWARE. 23 */ 24 /* 25 * Authors: Dave Airlied <airlied@linux.ie> 26 * Ben Skeggs <darktama@iinet.net.au> 27 * Jeremy Kolb <jkolb@brandeis.edu> 28 */ 29 30 #include <linux/dma-mapping.h> 31 #include <drm/ttm/ttm_tt.h> 32 33 #include "nouveau_drv.h" 34 #include "nouveau_chan.h" 35 #include "nouveau_fence.h" 36 37 #include "nouveau_bo.h" 38 #include "nouveau_ttm.h" 39 #include "nouveau_gem.h" 40 #include "nouveau_mem.h" 41 #include "nouveau_vmm.h" 42 43 #include <nvif/class.h> 44 #include <nvif/if500b.h> 45 #include <nvif/if900b.h> 46 47 static int nouveau_ttm_tt_bind(struct ttm_device *bdev, struct ttm_tt *ttm, 48 struct ttm_resource *reg); 49 static void nouveau_ttm_tt_unbind(struct ttm_device *bdev, struct ttm_tt *ttm); 50 51 /* 52 * NV10-NV40 tiling helpers 53 */ 54 55 static void 56 nv10_bo_update_tile_region(struct drm_device *dev, struct nouveau_drm_tile *reg, 57 u32 addr, u32 size, u32 pitch, u32 flags) 58 { 59 struct nouveau_drm *drm = nouveau_drm(dev); 60 int i = reg - drm->tile.reg; 61 struct nvkm_fb *fb = nvxx_fb(&drm->client.device); 62 struct nvkm_fb_tile *tile = &fb->tile.region[i]; 63 64 nouveau_fence_unref(®->fence); 65 66 if (tile->pitch) 67 nvkm_fb_tile_fini(fb, i, tile); 68 69 if (pitch) 70 nvkm_fb_tile_init(fb, i, addr, size, pitch, flags, tile); 71 72 nvkm_fb_tile_prog(fb, i, tile); 73 } 74 75 static struct nouveau_drm_tile * 76 nv10_bo_get_tile_region(struct drm_device *dev, int i) 77 { 78 struct nouveau_drm *drm = nouveau_drm(dev); 79 struct nouveau_drm_tile *tile = &drm->tile.reg[i]; 80 81 spin_lock(&drm->tile.lock); 82 83 if (!tile->used && 84 (!tile->fence || nouveau_fence_done(tile->fence))) 85 tile->used = true; 86 else 87 tile = NULL; 88 89 spin_unlock(&drm->tile.lock); 90 return tile; 91 } 92 93 static void 94 nv10_bo_put_tile_region(struct drm_device *dev, struct nouveau_drm_tile *tile, 95 struct dma_fence *fence) 96 { 97 struct nouveau_drm *drm = nouveau_drm(dev); 98 99 if (tile) { 100 spin_lock(&drm->tile.lock); 101 tile->fence = (struct nouveau_fence *)dma_fence_get(fence); 102 tile->used = false; 103 spin_unlock(&drm->tile.lock); 104 } 105 } 106 107 static struct nouveau_drm_tile * 108 nv10_bo_set_tiling(struct drm_device *dev, u32 addr, 109 u32 size, u32 pitch, u32 zeta) 110 { 111 struct nouveau_drm *drm = nouveau_drm(dev); 112 struct nvkm_fb *fb = nvxx_fb(&drm->client.device); 113 struct nouveau_drm_tile *tile, *found = NULL; 114 int i; 115 116 for (i = 0; i < fb->tile.regions; i++) { 117 tile = nv10_bo_get_tile_region(dev, i); 118 119 if (pitch && !found) { 120 found = tile; 121 continue; 122 123 } else if (tile && fb->tile.region[i].pitch) { 124 /* Kill an unused tile region. */ 125 nv10_bo_update_tile_region(dev, tile, 0, 0, 0, 0); 126 } 127 128 nv10_bo_put_tile_region(dev, tile, NULL); 129 } 130 131 if (found) 132 nv10_bo_update_tile_region(dev, found, addr, size, pitch, zeta); 133 return found; 134 } 135 136 static void 137 nouveau_bo_del_ttm(struct ttm_buffer_object *bo) 138 { 139 struct nouveau_drm *drm = nouveau_bdev(bo->bdev); 140 struct drm_device *dev = drm->dev; 141 struct nouveau_bo *nvbo = nouveau_bo(bo); 142 143 WARN_ON(nvbo->bo.pin_count > 0); 144 nouveau_bo_del_io_reserve_lru(bo); 145 nv10_bo_put_tile_region(dev, nvbo->tile, NULL); 146 147 /* 148 * If nouveau_bo_new() allocated this buffer, the GEM object was never 149 * initialized, so don't attempt to release it. 150 */ 151 if (bo->base.dev) { 152 /* Gem objects not being shared with other VMs get their 153 * dma_resv from a root GEM object. 154 */ 155 if (nvbo->no_share) 156 drm_gem_object_put(nvbo->r_obj); 157 158 drm_gem_object_release(&bo->base); 159 } else { 160 dma_resv_fini(&bo->base._resv); 161 } 162 163 kfree(nvbo); 164 } 165 166 static inline u64 167 roundup_64(u64 x, u32 y) 168 { 169 x += y - 1; 170 do_div(x, y); 171 return x * y; 172 } 173 174 static void 175 nouveau_bo_fixup_align(struct nouveau_bo *nvbo, int *align, u64 *size) 176 { 177 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 178 struct nvif_device *device = &drm->client.device; 179 180 if (device->info.family < NV_DEVICE_INFO_V0_TESLA) { 181 if (nvbo->mode) { 182 if (device->info.chipset >= 0x40) { 183 *align = 65536; 184 *size = roundup_64(*size, 64 * nvbo->mode); 185 186 } else if (device->info.chipset >= 0x30) { 187 *align = 32768; 188 *size = roundup_64(*size, 64 * nvbo->mode); 189 190 } else if (device->info.chipset >= 0x20) { 191 *align = 16384; 192 *size = roundup_64(*size, 64 * nvbo->mode); 193 194 } else if (device->info.chipset >= 0x10) { 195 *align = 16384; 196 *size = roundup_64(*size, 32 * nvbo->mode); 197 } 198 } 199 } else { 200 *size = roundup_64(*size, (1 << nvbo->page)); 201 *align = max((1 << nvbo->page), *align); 202 } 203 204 *size = roundup_64(*size, PAGE_SIZE); 205 } 206 207 struct nouveau_bo * 208 nouveau_bo_alloc(struct nouveau_cli *cli, u64 *size, int *align, u32 domain, 209 u32 tile_mode, u32 tile_flags, bool internal) 210 { 211 struct nouveau_drm *drm = cli->drm; 212 struct nouveau_bo *nvbo; 213 struct nvif_mmu *mmu = &cli->mmu; 214 struct nvif_vmm *vmm = &nouveau_cli_vmm(cli)->vmm; 215 int i, pi = -1; 216 217 if (!*size) { 218 NV_WARN(drm, "skipped size %016llx\n", *size); 219 return ERR_PTR(-EINVAL); 220 } 221 222 nvbo = kzalloc(sizeof(struct nouveau_bo), GFP_KERNEL); 223 if (!nvbo) 224 return ERR_PTR(-ENOMEM); 225 226 INIT_LIST_HEAD(&nvbo->head); 227 INIT_LIST_HEAD(&nvbo->entry); 228 INIT_LIST_HEAD(&nvbo->vma_list); 229 nvbo->bo.bdev = &drm->ttm.bdev; 230 231 /* This is confusing, and doesn't actually mean we want an uncached 232 * mapping, but is what NOUVEAU_GEM_DOMAIN_COHERENT gets translated 233 * into in nouveau_gem_new(). 234 */ 235 if (domain & NOUVEAU_GEM_DOMAIN_COHERENT) { 236 /* Determine if we can get a cache-coherent map, forcing 237 * uncached mapping if we can't. 238 */ 239 if (!nouveau_drm_use_coherent_gpu_mapping(drm)) 240 nvbo->force_coherent = true; 241 } 242 243 nvbo->contig = !(tile_flags & NOUVEAU_GEM_TILE_NONCONTIG); 244 245 if (cli->device.info.family >= NV_DEVICE_INFO_V0_FERMI) { 246 nvbo->kind = (tile_flags & 0x0000ff00) >> 8; 247 if (!nvif_mmu_kind_valid(mmu, nvbo->kind)) { 248 kfree(nvbo); 249 return ERR_PTR(-EINVAL); 250 } 251 252 nvbo->comp = mmu->kind[nvbo->kind] != nvbo->kind; 253 } else if (cli->device.info.family >= NV_DEVICE_INFO_V0_TESLA) { 254 nvbo->kind = (tile_flags & 0x00007f00) >> 8; 255 nvbo->comp = (tile_flags & 0x00030000) >> 16; 256 if (!nvif_mmu_kind_valid(mmu, nvbo->kind)) { 257 kfree(nvbo); 258 return ERR_PTR(-EINVAL); 259 } 260 } else { 261 nvbo->zeta = (tile_flags & 0x00000007); 262 } 263 nvbo->mode = tile_mode; 264 265 if (!nouveau_cli_uvmm(cli) || internal) { 266 /* Determine the desirable target GPU page size for the buffer. */ 267 for (i = 0; i < vmm->page_nr; i++) { 268 /* Because we cannot currently allow VMM maps to fail 269 * during buffer migration, we need to determine page 270 * size for the buffer up-front, and pre-allocate its 271 * page tables. 272 * 273 * Skip page sizes that can't support needed domains. 274 */ 275 if (cli->device.info.family > NV_DEVICE_INFO_V0_CURIE && 276 (domain & NOUVEAU_GEM_DOMAIN_VRAM) && !vmm->page[i].vram) 277 continue; 278 if ((domain & NOUVEAU_GEM_DOMAIN_GART) && 279 (!vmm->page[i].host || vmm->page[i].shift > PAGE_SHIFT)) 280 continue; 281 282 /* Select this page size if it's the first that supports 283 * the potential memory domains, or when it's compatible 284 * with the requested compression settings. 285 */ 286 if (pi < 0 || !nvbo->comp || vmm->page[i].comp) 287 pi = i; 288 289 /* Stop once the buffer is larger than the current page size. */ 290 if (*size >= 1ULL << vmm->page[i].shift) 291 break; 292 } 293 294 if (WARN_ON(pi < 0)) { 295 kfree(nvbo); 296 return ERR_PTR(-EINVAL); 297 } 298 299 /* Disable compression if suitable settings couldn't be found. */ 300 if (nvbo->comp && !vmm->page[pi].comp) { 301 if (mmu->object.oclass >= NVIF_CLASS_MMU_GF100) 302 nvbo->kind = mmu->kind[nvbo->kind]; 303 nvbo->comp = 0; 304 } 305 nvbo->page = vmm->page[pi].shift; 306 } else { 307 /* Determine the desirable target GPU page size for the buffer. */ 308 for (i = 0; i < vmm->page_nr; i++) { 309 /* Because we cannot currently allow VMM maps to fail 310 * during buffer migration, we need to determine page 311 * size for the buffer up-front, and pre-allocate its 312 * page tables. 313 * 314 * Skip page sizes that can't support needed domains. 315 */ 316 if ((domain & NOUVEAU_GEM_DOMAIN_VRAM) && !vmm->page[i].vram) 317 continue; 318 if ((domain & NOUVEAU_GEM_DOMAIN_GART) && 319 (!vmm->page[i].host || vmm->page[i].shift > PAGE_SHIFT)) 320 continue; 321 322 /* pick the last one as it will be smallest. */ 323 pi = i; 324 325 /* Stop once the buffer is larger than the current page size. */ 326 if (*size >= 1ULL << vmm->page[i].shift) 327 break; 328 } 329 if (WARN_ON(pi < 0)) { 330 kfree(nvbo); 331 return ERR_PTR(-EINVAL); 332 } 333 nvbo->page = vmm->page[pi].shift; 334 } 335 336 nouveau_bo_fixup_align(nvbo, align, size); 337 338 return nvbo; 339 } 340 341 int 342 nouveau_bo_init(struct nouveau_bo *nvbo, u64 size, int align, u32 domain, 343 struct sg_table *sg, struct dma_resv *robj) 344 { 345 int type = sg ? ttm_bo_type_sg : ttm_bo_type_device; 346 int ret; 347 struct ttm_operation_ctx ctx = { 348 .interruptible = false, 349 .no_wait_gpu = false, 350 .resv = robj, 351 }; 352 353 nouveau_bo_placement_set(nvbo, domain, 0); 354 INIT_LIST_HEAD(&nvbo->io_reserve_lru); 355 356 ret = ttm_bo_init_reserved(nvbo->bo.bdev, &nvbo->bo, type, 357 &nvbo->placement, align >> PAGE_SHIFT, &ctx, 358 sg, robj, nouveau_bo_del_ttm); 359 if (ret) { 360 /* ttm will call nouveau_bo_del_ttm if it fails.. */ 361 return ret; 362 } 363 364 if (!robj) 365 ttm_bo_unreserve(&nvbo->bo); 366 367 return 0; 368 } 369 370 int 371 nouveau_bo_new(struct nouveau_cli *cli, u64 size, int align, 372 uint32_t domain, uint32_t tile_mode, uint32_t tile_flags, 373 struct sg_table *sg, struct dma_resv *robj, 374 struct nouveau_bo **pnvbo) 375 { 376 struct nouveau_bo *nvbo; 377 int ret; 378 379 nvbo = nouveau_bo_alloc(cli, &size, &align, domain, tile_mode, 380 tile_flags, true); 381 if (IS_ERR(nvbo)) 382 return PTR_ERR(nvbo); 383 384 nvbo->bo.base.size = size; 385 dma_resv_init(&nvbo->bo.base._resv); 386 drm_vma_node_reset(&nvbo->bo.base.vma_node); 387 388 /* This must be called before ttm_bo_init_reserved(). Subsequent 389 * bo_move() callbacks might already iterate the GEMs GPUVA list. 390 */ 391 drm_gem_gpuva_init(&nvbo->bo.base); 392 393 ret = nouveau_bo_init(nvbo, size, align, domain, sg, robj); 394 if (ret) 395 return ret; 396 397 *pnvbo = nvbo; 398 return 0; 399 } 400 401 static void 402 set_placement_range(struct nouveau_bo *nvbo, uint32_t domain) 403 { 404 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 405 u64 vram_size = drm->client.device.info.ram_size; 406 unsigned i, fpfn, lpfn; 407 408 if (drm->client.device.info.family == NV_DEVICE_INFO_V0_CELSIUS && 409 nvbo->mode && (domain & NOUVEAU_GEM_DOMAIN_VRAM) && 410 nvbo->bo.base.size < vram_size / 4) { 411 /* 412 * Make sure that the color and depth buffers are handled 413 * by independent memory controller units. Up to a 9x 414 * speed up when alpha-blending and depth-test are enabled 415 * at the same time. 416 */ 417 if (nvbo->zeta) { 418 fpfn = (vram_size / 2) >> PAGE_SHIFT; 419 lpfn = ~0; 420 } else { 421 fpfn = 0; 422 lpfn = (vram_size / 2) >> PAGE_SHIFT; 423 } 424 for (i = 0; i < nvbo->placement.num_placement; ++i) { 425 nvbo->placements[i].fpfn = fpfn; 426 nvbo->placements[i].lpfn = lpfn; 427 } 428 } 429 } 430 431 void 432 nouveau_bo_placement_set(struct nouveau_bo *nvbo, uint32_t domain, 433 uint32_t busy) 434 { 435 unsigned int *n = &nvbo->placement.num_placement; 436 struct ttm_place *pl = nvbo->placements; 437 438 domain |= busy; 439 440 *n = 0; 441 if (domain & NOUVEAU_GEM_DOMAIN_VRAM) { 442 pl[*n].mem_type = TTM_PL_VRAM; 443 pl[*n].flags = busy & NOUVEAU_GEM_DOMAIN_VRAM ? 444 TTM_PL_FLAG_FALLBACK : 0; 445 (*n)++; 446 } 447 if (domain & NOUVEAU_GEM_DOMAIN_GART) { 448 pl[*n].mem_type = TTM_PL_TT; 449 pl[*n].flags = busy & NOUVEAU_GEM_DOMAIN_GART ? 450 TTM_PL_FLAG_FALLBACK : 0; 451 (*n)++; 452 } 453 if (domain & NOUVEAU_GEM_DOMAIN_CPU) { 454 pl[*n].mem_type = TTM_PL_SYSTEM; 455 pl[*n].flags = busy & NOUVEAU_GEM_DOMAIN_CPU ? 456 TTM_PL_FLAG_FALLBACK : 0; 457 (*n)++; 458 } 459 460 nvbo->placement.placement = nvbo->placements; 461 set_placement_range(nvbo, domain); 462 } 463 464 int nouveau_bo_pin_locked(struct nouveau_bo *nvbo, uint32_t domain, bool contig) 465 { 466 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 467 struct ttm_buffer_object *bo = &nvbo->bo; 468 bool force = false, evict = false; 469 int ret = 0; 470 471 dma_resv_assert_held(bo->base.resv); 472 473 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA && 474 domain == NOUVEAU_GEM_DOMAIN_VRAM && contig) { 475 if (!nvbo->contig) { 476 nvbo->contig = true; 477 force = true; 478 evict = true; 479 } 480 } 481 482 if (nvbo->bo.pin_count) { 483 bool error = evict; 484 485 switch (bo->resource->mem_type) { 486 case TTM_PL_VRAM: 487 error |= !(domain & NOUVEAU_GEM_DOMAIN_VRAM); 488 break; 489 case TTM_PL_TT: 490 error |= !(domain & NOUVEAU_GEM_DOMAIN_GART); 491 break; 492 default: 493 break; 494 } 495 496 if (error) { 497 NV_ERROR(drm, "bo %p pinned elsewhere: " 498 "0x%08x vs 0x%08x\n", bo, 499 bo->resource->mem_type, domain); 500 ret = -EBUSY; 501 } 502 ttm_bo_pin(&nvbo->bo); 503 goto out; 504 } 505 506 if (evict) { 507 nouveau_bo_placement_set(nvbo, NOUVEAU_GEM_DOMAIN_GART, 0); 508 ret = nouveau_bo_validate(nvbo, false, false); 509 if (ret) 510 goto out; 511 } 512 513 nouveau_bo_placement_set(nvbo, domain, 0); 514 ret = nouveau_bo_validate(nvbo, false, false); 515 if (ret) 516 goto out; 517 518 ttm_bo_pin(&nvbo->bo); 519 520 switch (bo->resource->mem_type) { 521 case TTM_PL_VRAM: 522 drm->gem.vram_available -= bo->base.size; 523 break; 524 case TTM_PL_TT: 525 drm->gem.gart_available -= bo->base.size; 526 break; 527 default: 528 break; 529 } 530 531 out: 532 if (force && ret) 533 nvbo->contig = false; 534 return ret; 535 } 536 537 void nouveau_bo_unpin_locked(struct nouveau_bo *nvbo) 538 { 539 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 540 struct ttm_buffer_object *bo = &nvbo->bo; 541 542 dma_resv_assert_held(bo->base.resv); 543 544 ttm_bo_unpin(&nvbo->bo); 545 if (!nvbo->bo.pin_count) { 546 switch (bo->resource->mem_type) { 547 case TTM_PL_VRAM: 548 drm->gem.vram_available += bo->base.size; 549 break; 550 case TTM_PL_TT: 551 drm->gem.gart_available += bo->base.size; 552 break; 553 default: 554 break; 555 } 556 } 557 } 558 559 int nouveau_bo_pin(struct nouveau_bo *nvbo, uint32_t domain, bool contig) 560 { 561 struct ttm_buffer_object *bo = &nvbo->bo; 562 int ret; 563 564 ret = ttm_bo_reserve(bo, false, false, NULL); 565 if (ret) 566 return ret; 567 ret = nouveau_bo_pin_locked(nvbo, domain, contig); 568 ttm_bo_unreserve(bo); 569 570 return ret; 571 } 572 573 int nouveau_bo_unpin(struct nouveau_bo *nvbo) 574 { 575 struct ttm_buffer_object *bo = &nvbo->bo; 576 int ret; 577 578 ret = ttm_bo_reserve(bo, false, false, NULL); 579 if (ret) 580 return ret; 581 nouveau_bo_unpin_locked(nvbo); 582 ttm_bo_unreserve(bo); 583 584 return 0; 585 } 586 587 int 588 nouveau_bo_map(struct nouveau_bo *nvbo) 589 { 590 int ret; 591 592 ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL); 593 if (ret) 594 return ret; 595 596 ret = ttm_bo_kmap(&nvbo->bo, 0, PFN_UP(nvbo->bo.base.size), &nvbo->kmap); 597 598 ttm_bo_unreserve(&nvbo->bo); 599 return ret; 600 } 601 602 void 603 nouveau_bo_unmap(struct nouveau_bo *nvbo) 604 { 605 if (!nvbo) 606 return; 607 608 ttm_bo_kunmap(&nvbo->kmap); 609 } 610 611 void 612 nouveau_bo_sync_for_device(struct nouveau_bo *nvbo) 613 { 614 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 615 struct ttm_tt *ttm_dma = (struct ttm_tt *)nvbo->bo.ttm; 616 int i, j; 617 618 if (!ttm_dma || !ttm_dma->dma_address) 619 return; 620 if (!ttm_dma->pages) { 621 NV_DEBUG(drm, "ttm_dma 0x%p: pages NULL\n", ttm_dma); 622 return; 623 } 624 625 /* Don't waste time looping if the object is coherent */ 626 if (nvbo->force_coherent) 627 return; 628 629 i = 0; 630 while (i < ttm_dma->num_pages) { 631 struct page *p = ttm_dma->pages[i]; 632 size_t num_pages = 1; 633 634 for (j = i + 1; j < ttm_dma->num_pages; ++j) { 635 if (++p != ttm_dma->pages[j]) 636 break; 637 638 ++num_pages; 639 } 640 dma_sync_single_for_device(drm->dev->dev, 641 ttm_dma->dma_address[i], 642 num_pages * PAGE_SIZE, DMA_TO_DEVICE); 643 i += num_pages; 644 } 645 } 646 647 void 648 nouveau_bo_sync_for_cpu(struct nouveau_bo *nvbo) 649 { 650 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 651 struct ttm_tt *ttm_dma = (struct ttm_tt *)nvbo->bo.ttm; 652 int i, j; 653 654 if (!ttm_dma || !ttm_dma->dma_address) 655 return; 656 if (!ttm_dma->pages) { 657 NV_DEBUG(drm, "ttm_dma 0x%p: pages NULL\n", ttm_dma); 658 return; 659 } 660 661 /* Don't waste time looping if the object is coherent */ 662 if (nvbo->force_coherent) 663 return; 664 665 i = 0; 666 while (i < ttm_dma->num_pages) { 667 struct page *p = ttm_dma->pages[i]; 668 size_t num_pages = 1; 669 670 for (j = i + 1; j < ttm_dma->num_pages; ++j) { 671 if (++p != ttm_dma->pages[j]) 672 break; 673 674 ++num_pages; 675 } 676 677 dma_sync_single_for_cpu(drm->dev->dev, ttm_dma->dma_address[i], 678 num_pages * PAGE_SIZE, DMA_FROM_DEVICE); 679 i += num_pages; 680 } 681 } 682 683 void nouveau_bo_add_io_reserve_lru(struct ttm_buffer_object *bo) 684 { 685 struct nouveau_drm *drm = nouveau_bdev(bo->bdev); 686 struct nouveau_bo *nvbo = nouveau_bo(bo); 687 688 mutex_lock(&drm->ttm.io_reserve_mutex); 689 list_move_tail(&nvbo->io_reserve_lru, &drm->ttm.io_reserve_lru); 690 mutex_unlock(&drm->ttm.io_reserve_mutex); 691 } 692 693 void nouveau_bo_del_io_reserve_lru(struct ttm_buffer_object *bo) 694 { 695 struct nouveau_drm *drm = nouveau_bdev(bo->bdev); 696 struct nouveau_bo *nvbo = nouveau_bo(bo); 697 698 mutex_lock(&drm->ttm.io_reserve_mutex); 699 list_del_init(&nvbo->io_reserve_lru); 700 mutex_unlock(&drm->ttm.io_reserve_mutex); 701 } 702 703 int 704 nouveau_bo_validate(struct nouveau_bo *nvbo, bool interruptible, 705 bool no_wait_gpu) 706 { 707 struct ttm_operation_ctx ctx = { interruptible, no_wait_gpu }; 708 int ret; 709 710 ret = ttm_bo_validate(&nvbo->bo, &nvbo->placement, &ctx); 711 if (ret) 712 return ret; 713 714 nouveau_bo_sync_for_device(nvbo); 715 716 return 0; 717 } 718 719 void 720 nouveau_bo_wr16(struct nouveau_bo *nvbo, unsigned index, u16 val) 721 { 722 bool is_iomem; 723 u16 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem); 724 725 mem += index; 726 727 if (is_iomem) 728 iowrite16_native(val, (void __force __iomem *)mem); 729 else 730 *mem = val; 731 } 732 733 u32 734 nouveau_bo_rd32(struct nouveau_bo *nvbo, unsigned index) 735 { 736 bool is_iomem; 737 u32 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem); 738 739 mem += index; 740 741 if (is_iomem) 742 return ioread32_native((void __force __iomem *)mem); 743 else 744 return *mem; 745 } 746 747 void 748 nouveau_bo_wr32(struct nouveau_bo *nvbo, unsigned index, u32 val) 749 { 750 bool is_iomem; 751 u32 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem); 752 753 mem += index; 754 755 if (is_iomem) 756 iowrite32_native(val, (void __force __iomem *)mem); 757 else 758 *mem = val; 759 } 760 761 static struct ttm_tt * 762 nouveau_ttm_tt_create(struct ttm_buffer_object *bo, uint32_t page_flags) 763 { 764 #if IS_ENABLED(CONFIG_AGP) 765 struct nouveau_drm *drm = nouveau_bdev(bo->bdev); 766 767 if (drm->agp.bridge) { 768 return ttm_agp_tt_create(bo, drm->agp.bridge, page_flags); 769 } 770 #endif 771 772 return nouveau_sgdma_create_ttm(bo, page_flags); 773 } 774 775 static int 776 nouveau_ttm_tt_bind(struct ttm_device *bdev, struct ttm_tt *ttm, 777 struct ttm_resource *reg) 778 { 779 #if IS_ENABLED(CONFIG_AGP) 780 struct nouveau_drm *drm = nouveau_bdev(bdev); 781 #endif 782 if (!reg) 783 return -EINVAL; 784 #if IS_ENABLED(CONFIG_AGP) 785 if (drm->agp.bridge) 786 return ttm_agp_bind(ttm, reg); 787 #endif 788 return nouveau_sgdma_bind(bdev, ttm, reg); 789 } 790 791 static void 792 nouveau_ttm_tt_unbind(struct ttm_device *bdev, struct ttm_tt *ttm) 793 { 794 #if IS_ENABLED(CONFIG_AGP) 795 struct nouveau_drm *drm = nouveau_bdev(bdev); 796 797 if (drm->agp.bridge) { 798 ttm_agp_unbind(ttm); 799 return; 800 } 801 #endif 802 nouveau_sgdma_unbind(bdev, ttm); 803 } 804 805 static void 806 nouveau_bo_evict_flags(struct ttm_buffer_object *bo, struct ttm_placement *pl) 807 { 808 struct nouveau_bo *nvbo = nouveau_bo(bo); 809 810 switch (bo->resource->mem_type) { 811 case TTM_PL_VRAM: 812 nouveau_bo_placement_set(nvbo, NOUVEAU_GEM_DOMAIN_GART, 813 NOUVEAU_GEM_DOMAIN_CPU); 814 break; 815 default: 816 nouveau_bo_placement_set(nvbo, NOUVEAU_GEM_DOMAIN_CPU, 0); 817 break; 818 } 819 820 *pl = nvbo->placement; 821 } 822 823 static int 824 nouveau_bo_move_prep(struct nouveau_drm *drm, struct ttm_buffer_object *bo, 825 struct ttm_resource *reg) 826 { 827 struct nouveau_mem *old_mem = nouveau_mem(bo->resource); 828 struct nouveau_mem *new_mem = nouveau_mem(reg); 829 struct nvif_vmm *vmm = &drm->client.vmm.vmm; 830 int ret; 831 832 ret = nvif_vmm_get(vmm, LAZY, false, old_mem->mem.page, 0, 833 old_mem->mem.size, &old_mem->vma[0]); 834 if (ret) 835 return ret; 836 837 ret = nvif_vmm_get(vmm, LAZY, false, new_mem->mem.page, 0, 838 new_mem->mem.size, &old_mem->vma[1]); 839 if (ret) 840 goto done; 841 842 ret = nouveau_mem_map(old_mem, vmm, &old_mem->vma[0]); 843 if (ret) 844 goto done; 845 846 ret = nouveau_mem_map(new_mem, vmm, &old_mem->vma[1]); 847 done: 848 if (ret) { 849 nvif_vmm_put(vmm, &old_mem->vma[1]); 850 nvif_vmm_put(vmm, &old_mem->vma[0]); 851 } 852 return 0; 853 } 854 855 static int 856 nouveau_bo_move_m2mf(struct ttm_buffer_object *bo, int evict, 857 struct ttm_operation_ctx *ctx, 858 struct ttm_resource *new_reg) 859 { 860 struct nouveau_drm *drm = nouveau_bdev(bo->bdev); 861 struct nouveau_channel *chan = drm->ttm.chan; 862 struct nouveau_cli *cli = (void *)chan->user.client; 863 struct nouveau_fence *fence; 864 int ret; 865 866 /* create temporary vmas for the transfer and attach them to the 867 * old nvkm_mem node, these will get cleaned up after ttm has 868 * destroyed the ttm_resource 869 */ 870 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA) { 871 ret = nouveau_bo_move_prep(drm, bo, new_reg); 872 if (ret) 873 return ret; 874 } 875 876 if (drm_drv_uses_atomic_modeset(drm->dev)) 877 mutex_lock(&cli->mutex); 878 else 879 mutex_lock_nested(&cli->mutex, SINGLE_DEPTH_NESTING); 880 881 ret = nouveau_fence_sync(nouveau_bo(bo), chan, true, ctx->interruptible); 882 if (ret) 883 goto out_unlock; 884 885 ret = drm->ttm.move(chan, bo, bo->resource, new_reg); 886 if (ret) 887 goto out_unlock; 888 889 ret = nouveau_fence_new(&fence, chan); 890 if (ret) 891 goto out_unlock; 892 893 /* TODO: figure out a better solution here 894 * 895 * wait on the fence here explicitly as going through 896 * ttm_bo_move_accel_cleanup somehow doesn't seem to do it. 897 * 898 * Without this the operation can timeout and we'll fallback to a 899 * software copy, which might take several minutes to finish. 900 */ 901 nouveau_fence_wait(fence, false, false); 902 ret = ttm_bo_move_accel_cleanup(bo, &fence->base, evict, false, 903 new_reg); 904 nouveau_fence_unref(&fence); 905 906 out_unlock: 907 mutex_unlock(&cli->mutex); 908 return ret; 909 } 910 911 void 912 nouveau_bo_move_init(struct nouveau_drm *drm) 913 { 914 static const struct _method_table { 915 const char *name; 916 int engine; 917 s32 oclass; 918 int (*exec)(struct nouveau_channel *, 919 struct ttm_buffer_object *, 920 struct ttm_resource *, struct ttm_resource *); 921 int (*init)(struct nouveau_channel *, u32 handle); 922 } _methods[] = { 923 { "COPY", 4, 0xc7b5, nve0_bo_move_copy, nve0_bo_move_init }, 924 { "GRCE", 0, 0xc7b5, nve0_bo_move_copy, nvc0_bo_move_init }, 925 { "COPY", 4, 0xc6b5, nve0_bo_move_copy, nve0_bo_move_init }, 926 { "GRCE", 0, 0xc6b5, nve0_bo_move_copy, nvc0_bo_move_init }, 927 { "COPY", 4, 0xc5b5, nve0_bo_move_copy, nve0_bo_move_init }, 928 { "GRCE", 0, 0xc5b5, nve0_bo_move_copy, nvc0_bo_move_init }, 929 { "COPY", 4, 0xc3b5, nve0_bo_move_copy, nve0_bo_move_init }, 930 { "GRCE", 0, 0xc3b5, nve0_bo_move_copy, nvc0_bo_move_init }, 931 { "COPY", 4, 0xc1b5, nve0_bo_move_copy, nve0_bo_move_init }, 932 { "GRCE", 0, 0xc1b5, nve0_bo_move_copy, nvc0_bo_move_init }, 933 { "COPY", 4, 0xc0b5, nve0_bo_move_copy, nve0_bo_move_init }, 934 { "GRCE", 0, 0xc0b5, nve0_bo_move_copy, nvc0_bo_move_init }, 935 { "COPY", 4, 0xb0b5, nve0_bo_move_copy, nve0_bo_move_init }, 936 { "GRCE", 0, 0xb0b5, nve0_bo_move_copy, nvc0_bo_move_init }, 937 { "COPY", 4, 0xa0b5, nve0_bo_move_copy, nve0_bo_move_init }, 938 { "GRCE", 0, 0xa0b5, nve0_bo_move_copy, nvc0_bo_move_init }, 939 { "COPY1", 5, 0x90b8, nvc0_bo_move_copy, nvc0_bo_move_init }, 940 { "COPY0", 4, 0x90b5, nvc0_bo_move_copy, nvc0_bo_move_init }, 941 { "COPY", 0, 0x85b5, nva3_bo_move_copy, nv50_bo_move_init }, 942 { "CRYPT", 0, 0x74c1, nv84_bo_move_exec, nv50_bo_move_init }, 943 { "M2MF", 0, 0x9039, nvc0_bo_move_m2mf, nvc0_bo_move_init }, 944 { "M2MF", 0, 0x5039, nv50_bo_move_m2mf, nv50_bo_move_init }, 945 { "M2MF", 0, 0x0039, nv04_bo_move_m2mf, nv04_bo_move_init }, 946 {}, 947 }; 948 const struct _method_table *mthd = _methods; 949 const char *name = "CPU"; 950 int ret; 951 952 do { 953 struct nouveau_channel *chan; 954 955 if (mthd->engine) 956 chan = drm->cechan; 957 else 958 chan = drm->channel; 959 if (chan == NULL) 960 continue; 961 962 ret = nvif_object_ctor(&chan->user, "ttmBoMove", 963 mthd->oclass | (mthd->engine << 16), 964 mthd->oclass, NULL, 0, 965 &drm->ttm.copy); 966 if (ret == 0) { 967 ret = mthd->init(chan, drm->ttm.copy.handle); 968 if (ret) { 969 nvif_object_dtor(&drm->ttm.copy); 970 continue; 971 } 972 973 drm->ttm.move = mthd->exec; 974 drm->ttm.chan = chan; 975 name = mthd->name; 976 break; 977 } 978 } while ((++mthd)->exec); 979 980 NV_INFO(drm, "MM: using %s for buffer copies\n", name); 981 } 982 983 static void nouveau_bo_move_ntfy(struct ttm_buffer_object *bo, 984 struct ttm_resource *new_reg) 985 { 986 struct nouveau_mem *mem = new_reg ? nouveau_mem(new_reg) : NULL; 987 struct nouveau_bo *nvbo = nouveau_bo(bo); 988 struct nouveau_vma *vma; 989 long ret; 990 991 /* ttm can now (stupidly) pass the driver bos it didn't create... */ 992 if (bo->destroy != nouveau_bo_del_ttm) 993 return; 994 995 nouveau_bo_del_io_reserve_lru(bo); 996 997 if (mem && new_reg->mem_type != TTM_PL_SYSTEM && 998 mem->mem.page == nvbo->page) { 999 list_for_each_entry(vma, &nvbo->vma_list, head) { 1000 nouveau_vma_map(vma, mem); 1001 } 1002 nouveau_uvmm_bo_map_all(nvbo, mem); 1003 } else { 1004 list_for_each_entry(vma, &nvbo->vma_list, head) { 1005 ret = dma_resv_wait_timeout(bo->base.resv, 1006 DMA_RESV_USAGE_BOOKKEEP, 1007 false, 15 * HZ); 1008 WARN_ON(ret <= 0); 1009 nouveau_vma_unmap(vma); 1010 } 1011 nouveau_uvmm_bo_unmap_all(nvbo); 1012 } 1013 1014 if (new_reg) 1015 nvbo->offset = (new_reg->start << PAGE_SHIFT); 1016 1017 } 1018 1019 static int 1020 nouveau_bo_vm_bind(struct ttm_buffer_object *bo, struct ttm_resource *new_reg, 1021 struct nouveau_drm_tile **new_tile) 1022 { 1023 struct nouveau_drm *drm = nouveau_bdev(bo->bdev); 1024 struct drm_device *dev = drm->dev; 1025 struct nouveau_bo *nvbo = nouveau_bo(bo); 1026 u64 offset = new_reg->start << PAGE_SHIFT; 1027 1028 *new_tile = NULL; 1029 if (new_reg->mem_type != TTM_PL_VRAM) 1030 return 0; 1031 1032 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_CELSIUS) { 1033 *new_tile = nv10_bo_set_tiling(dev, offset, bo->base.size, 1034 nvbo->mode, nvbo->zeta); 1035 } 1036 1037 return 0; 1038 } 1039 1040 static void 1041 nouveau_bo_vm_cleanup(struct ttm_buffer_object *bo, 1042 struct nouveau_drm_tile *new_tile, 1043 struct nouveau_drm_tile **old_tile) 1044 { 1045 struct nouveau_drm *drm = nouveau_bdev(bo->bdev); 1046 struct drm_device *dev = drm->dev; 1047 struct dma_fence *fence; 1048 int ret; 1049 1050 ret = dma_resv_get_singleton(bo->base.resv, DMA_RESV_USAGE_WRITE, 1051 &fence); 1052 if (ret) 1053 dma_resv_wait_timeout(bo->base.resv, DMA_RESV_USAGE_WRITE, 1054 false, MAX_SCHEDULE_TIMEOUT); 1055 1056 nv10_bo_put_tile_region(dev, *old_tile, fence); 1057 *old_tile = new_tile; 1058 } 1059 1060 static int 1061 nouveau_bo_move(struct ttm_buffer_object *bo, bool evict, 1062 struct ttm_operation_ctx *ctx, 1063 struct ttm_resource *new_reg, 1064 struct ttm_place *hop) 1065 { 1066 struct nouveau_drm *drm = nouveau_bdev(bo->bdev); 1067 struct nouveau_bo *nvbo = nouveau_bo(bo); 1068 struct drm_gem_object *obj = &bo->base; 1069 struct ttm_resource *old_reg = bo->resource; 1070 struct nouveau_drm_tile *new_tile = NULL; 1071 int ret = 0; 1072 1073 if (new_reg->mem_type == TTM_PL_TT) { 1074 ret = nouveau_ttm_tt_bind(bo->bdev, bo->ttm, new_reg); 1075 if (ret) 1076 return ret; 1077 } 1078 1079 drm_gpuvm_bo_gem_evict(obj, evict); 1080 nouveau_bo_move_ntfy(bo, new_reg); 1081 ret = ttm_bo_wait_ctx(bo, ctx); 1082 if (ret) 1083 goto out_ntfy; 1084 1085 if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA) { 1086 ret = nouveau_bo_vm_bind(bo, new_reg, &new_tile); 1087 if (ret) 1088 goto out_ntfy; 1089 } 1090 1091 /* Fake bo copy. */ 1092 if (!old_reg || (old_reg->mem_type == TTM_PL_SYSTEM && 1093 !bo->ttm)) { 1094 ttm_bo_move_null(bo, new_reg); 1095 goto out; 1096 } 1097 1098 if (old_reg->mem_type == TTM_PL_SYSTEM && 1099 new_reg->mem_type == TTM_PL_TT) { 1100 ttm_bo_move_null(bo, new_reg); 1101 goto out; 1102 } 1103 1104 if (old_reg->mem_type == TTM_PL_TT && 1105 new_reg->mem_type == TTM_PL_SYSTEM) { 1106 nouveau_ttm_tt_unbind(bo->bdev, bo->ttm); 1107 ttm_resource_free(bo, &bo->resource); 1108 ttm_bo_assign_mem(bo, new_reg); 1109 goto out; 1110 } 1111 1112 /* Hardware assisted copy. */ 1113 if (drm->ttm.move) { 1114 if ((old_reg->mem_type == TTM_PL_SYSTEM && 1115 new_reg->mem_type == TTM_PL_VRAM) || 1116 (old_reg->mem_type == TTM_PL_VRAM && 1117 new_reg->mem_type == TTM_PL_SYSTEM)) { 1118 hop->fpfn = 0; 1119 hop->lpfn = 0; 1120 hop->mem_type = TTM_PL_TT; 1121 hop->flags = 0; 1122 return -EMULTIHOP; 1123 } 1124 ret = nouveau_bo_move_m2mf(bo, evict, ctx, 1125 new_reg); 1126 } else 1127 ret = -ENODEV; 1128 1129 if (ret) { 1130 /* Fallback to software copy. */ 1131 ret = ttm_bo_move_memcpy(bo, ctx, new_reg); 1132 } 1133 1134 out: 1135 if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA) { 1136 if (ret) 1137 nouveau_bo_vm_cleanup(bo, NULL, &new_tile); 1138 else 1139 nouveau_bo_vm_cleanup(bo, new_tile, &nvbo->tile); 1140 } 1141 out_ntfy: 1142 if (ret) { 1143 nouveau_bo_move_ntfy(bo, bo->resource); 1144 drm_gpuvm_bo_gem_evict(obj, !evict); 1145 } 1146 return ret; 1147 } 1148 1149 static void 1150 nouveau_ttm_io_mem_free_locked(struct nouveau_drm *drm, 1151 struct ttm_resource *reg) 1152 { 1153 struct nouveau_mem *mem = nouveau_mem(reg); 1154 1155 if (drm->client.mem->oclass >= NVIF_CLASS_MEM_NV50) { 1156 switch (reg->mem_type) { 1157 case TTM_PL_TT: 1158 if (mem->kind) 1159 nvif_object_unmap_handle(&mem->mem.object); 1160 break; 1161 case TTM_PL_VRAM: 1162 nvif_object_unmap_handle(&mem->mem.object); 1163 break; 1164 default: 1165 break; 1166 } 1167 } 1168 } 1169 1170 static int 1171 nouveau_ttm_io_mem_reserve(struct ttm_device *bdev, struct ttm_resource *reg) 1172 { 1173 struct nouveau_drm *drm = nouveau_bdev(bdev); 1174 struct nvkm_device *device = nvxx_device(&drm->client.device); 1175 struct nouveau_mem *mem = nouveau_mem(reg); 1176 struct nvif_mmu *mmu = &drm->client.mmu; 1177 int ret; 1178 1179 mutex_lock(&drm->ttm.io_reserve_mutex); 1180 retry: 1181 switch (reg->mem_type) { 1182 case TTM_PL_SYSTEM: 1183 /* System memory */ 1184 ret = 0; 1185 goto out; 1186 case TTM_PL_TT: 1187 #if IS_ENABLED(CONFIG_AGP) 1188 if (drm->agp.bridge) { 1189 reg->bus.offset = (reg->start << PAGE_SHIFT) + 1190 drm->agp.base; 1191 reg->bus.is_iomem = !drm->agp.cma; 1192 reg->bus.caching = ttm_write_combined; 1193 } 1194 #endif 1195 if (drm->client.mem->oclass < NVIF_CLASS_MEM_NV50 || 1196 !mem->kind) { 1197 /* untiled */ 1198 ret = 0; 1199 break; 1200 } 1201 fallthrough; /* tiled memory */ 1202 case TTM_PL_VRAM: 1203 reg->bus.offset = (reg->start << PAGE_SHIFT) + 1204 device->func->resource_addr(device, 1); 1205 reg->bus.is_iomem = true; 1206 1207 /* Some BARs do not support being ioremapped WC */ 1208 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA && 1209 mmu->type[drm->ttm.type_vram].type & NVIF_MEM_UNCACHED) 1210 reg->bus.caching = ttm_uncached; 1211 else 1212 reg->bus.caching = ttm_write_combined; 1213 1214 if (drm->client.mem->oclass >= NVIF_CLASS_MEM_NV50) { 1215 union { 1216 struct nv50_mem_map_v0 nv50; 1217 struct gf100_mem_map_v0 gf100; 1218 } args; 1219 u64 handle, length; 1220 u32 argc = 0; 1221 1222 switch (mem->mem.object.oclass) { 1223 case NVIF_CLASS_MEM_NV50: 1224 args.nv50.version = 0; 1225 args.nv50.ro = 0; 1226 args.nv50.kind = mem->kind; 1227 args.nv50.comp = mem->comp; 1228 argc = sizeof(args.nv50); 1229 break; 1230 case NVIF_CLASS_MEM_GF100: 1231 args.gf100.version = 0; 1232 args.gf100.ro = 0; 1233 args.gf100.kind = mem->kind; 1234 argc = sizeof(args.gf100); 1235 break; 1236 default: 1237 WARN_ON(1); 1238 break; 1239 } 1240 1241 ret = nvif_object_map_handle(&mem->mem.object, 1242 &args, argc, 1243 &handle, &length); 1244 if (ret != 1) { 1245 if (WARN_ON(ret == 0)) 1246 ret = -EINVAL; 1247 goto out; 1248 } 1249 1250 reg->bus.offset = handle; 1251 } 1252 ret = 0; 1253 break; 1254 default: 1255 ret = -EINVAL; 1256 } 1257 1258 out: 1259 if (ret == -ENOSPC) { 1260 struct nouveau_bo *nvbo; 1261 1262 nvbo = list_first_entry_or_null(&drm->ttm.io_reserve_lru, 1263 typeof(*nvbo), 1264 io_reserve_lru); 1265 if (nvbo) { 1266 list_del_init(&nvbo->io_reserve_lru); 1267 drm_vma_node_unmap(&nvbo->bo.base.vma_node, 1268 bdev->dev_mapping); 1269 nouveau_ttm_io_mem_free_locked(drm, nvbo->bo.resource); 1270 nvbo->bo.resource->bus.offset = 0; 1271 nvbo->bo.resource->bus.addr = NULL; 1272 goto retry; 1273 } 1274 1275 } 1276 mutex_unlock(&drm->ttm.io_reserve_mutex); 1277 return ret; 1278 } 1279 1280 static void 1281 nouveau_ttm_io_mem_free(struct ttm_device *bdev, struct ttm_resource *reg) 1282 { 1283 struct nouveau_drm *drm = nouveau_bdev(bdev); 1284 1285 mutex_lock(&drm->ttm.io_reserve_mutex); 1286 nouveau_ttm_io_mem_free_locked(drm, reg); 1287 mutex_unlock(&drm->ttm.io_reserve_mutex); 1288 } 1289 1290 vm_fault_t nouveau_ttm_fault_reserve_notify(struct ttm_buffer_object *bo) 1291 { 1292 struct nouveau_drm *drm = nouveau_bdev(bo->bdev); 1293 struct nouveau_bo *nvbo = nouveau_bo(bo); 1294 struct nvkm_device *device = nvxx_device(&drm->client.device); 1295 u32 mappable = device->func->resource_size(device, 1) >> PAGE_SHIFT; 1296 int i, ret; 1297 1298 /* as long as the bo isn't in vram, and isn't tiled, we've got 1299 * nothing to do here. 1300 */ 1301 if (bo->resource->mem_type != TTM_PL_VRAM) { 1302 if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA || 1303 !nvbo->kind) 1304 return 0; 1305 1306 if (bo->resource->mem_type != TTM_PL_SYSTEM) 1307 return 0; 1308 1309 nouveau_bo_placement_set(nvbo, NOUVEAU_GEM_DOMAIN_GART, 0); 1310 1311 } else { 1312 /* make sure bo is in mappable vram */ 1313 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA || 1314 bo->resource->start + PFN_UP(bo->resource->size) < mappable) 1315 return 0; 1316 1317 for (i = 0; i < nvbo->placement.num_placement; ++i) { 1318 nvbo->placements[i].fpfn = 0; 1319 nvbo->placements[i].lpfn = mappable; 1320 } 1321 1322 nouveau_bo_placement_set(nvbo, NOUVEAU_GEM_DOMAIN_VRAM, 0); 1323 } 1324 1325 ret = nouveau_bo_validate(nvbo, false, false); 1326 if (unlikely(ret == -EBUSY || ret == -ERESTARTSYS)) 1327 return VM_FAULT_NOPAGE; 1328 else if (unlikely(ret)) 1329 return VM_FAULT_SIGBUS; 1330 1331 ttm_bo_move_to_lru_tail_unlocked(bo); 1332 return 0; 1333 } 1334 1335 static int 1336 nouveau_ttm_tt_populate(struct ttm_device *bdev, 1337 struct ttm_tt *ttm, struct ttm_operation_ctx *ctx) 1338 { 1339 struct ttm_tt *ttm_dma = (void *)ttm; 1340 struct nouveau_drm *drm; 1341 bool slave = !!(ttm->page_flags & TTM_TT_FLAG_EXTERNAL); 1342 1343 if (ttm_tt_is_populated(ttm)) 1344 return 0; 1345 1346 if (slave && ttm->sg) { 1347 drm_prime_sg_to_dma_addr_array(ttm->sg, ttm_dma->dma_address, 1348 ttm->num_pages); 1349 return 0; 1350 } 1351 1352 drm = nouveau_bdev(bdev); 1353 1354 return ttm_pool_alloc(&drm->ttm.bdev.pool, ttm, ctx); 1355 } 1356 1357 static void 1358 nouveau_ttm_tt_unpopulate(struct ttm_device *bdev, 1359 struct ttm_tt *ttm) 1360 { 1361 struct nouveau_drm *drm; 1362 bool slave = !!(ttm->page_flags & TTM_TT_FLAG_EXTERNAL); 1363 1364 if (slave) 1365 return; 1366 1367 nouveau_ttm_tt_unbind(bdev, ttm); 1368 1369 drm = nouveau_bdev(bdev); 1370 1371 return ttm_pool_free(&drm->ttm.bdev.pool, ttm); 1372 } 1373 1374 static void 1375 nouveau_ttm_tt_destroy(struct ttm_device *bdev, 1376 struct ttm_tt *ttm) 1377 { 1378 #if IS_ENABLED(CONFIG_AGP) 1379 struct nouveau_drm *drm = nouveau_bdev(bdev); 1380 if (drm->agp.bridge) { 1381 ttm_agp_destroy(ttm); 1382 return; 1383 } 1384 #endif 1385 nouveau_sgdma_destroy(bdev, ttm); 1386 } 1387 1388 void 1389 nouveau_bo_fence(struct nouveau_bo *nvbo, struct nouveau_fence *fence, bool exclusive) 1390 { 1391 struct dma_resv *resv = nvbo->bo.base.resv; 1392 1393 if (!fence) 1394 return; 1395 1396 dma_resv_add_fence(resv, &fence->base, exclusive ? 1397 DMA_RESV_USAGE_WRITE : DMA_RESV_USAGE_READ); 1398 } 1399 1400 static void 1401 nouveau_bo_delete_mem_notify(struct ttm_buffer_object *bo) 1402 { 1403 nouveau_bo_move_ntfy(bo, NULL); 1404 } 1405 1406 struct ttm_device_funcs nouveau_bo_driver = { 1407 .ttm_tt_create = &nouveau_ttm_tt_create, 1408 .ttm_tt_populate = &nouveau_ttm_tt_populate, 1409 .ttm_tt_unpopulate = &nouveau_ttm_tt_unpopulate, 1410 .ttm_tt_destroy = &nouveau_ttm_tt_destroy, 1411 .eviction_valuable = ttm_bo_eviction_valuable, 1412 .evict_flags = nouveau_bo_evict_flags, 1413 .delete_mem_notify = nouveau_bo_delete_mem_notify, 1414 .move = nouveau_bo_move, 1415 .io_mem_reserve = &nouveau_ttm_io_mem_reserve, 1416 .io_mem_free = &nouveau_ttm_io_mem_free, 1417 }; 1418