1 /* 2 * Copyright 2007 Dave Airlied 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice (including the next 13 * paragraph) shall be included in all copies or substantial portions of the 14 * Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 * OTHER DEALINGS IN THE SOFTWARE. 23 */ 24 /* 25 * Authors: Dave Airlied <airlied@linux.ie> 26 * Ben Skeggs <darktama@iinet.net.au> 27 * Jeremy Kolb <jkolb@brandeis.edu> 28 */ 29 30 #include <linux/dma-mapping.h> 31 #include <drm/ttm/ttm_tt.h> 32 33 #include "nouveau_drv.h" 34 #include "nouveau_chan.h" 35 #include "nouveau_fence.h" 36 37 #include "nouveau_bo.h" 38 #include "nouveau_ttm.h" 39 #include "nouveau_gem.h" 40 #include "nouveau_mem.h" 41 #include "nouveau_vmm.h" 42 43 #include <nvif/class.h> 44 #include <nvif/if500b.h> 45 #include <nvif/if900b.h> 46 47 static int nouveau_ttm_tt_bind(struct ttm_device *bdev, struct ttm_tt *ttm, 48 struct ttm_resource *reg); 49 static void nouveau_ttm_tt_unbind(struct ttm_device *bdev, struct ttm_tt *ttm); 50 51 /* 52 * NV10-NV40 tiling helpers 53 */ 54 55 static void 56 nv10_bo_update_tile_region(struct drm_device *dev, struct nouveau_drm_tile *reg, 57 u32 addr, u32 size, u32 pitch, u32 flags) 58 { 59 struct nouveau_drm *drm = nouveau_drm(dev); 60 int i = reg - drm->tile.reg; 61 struct nvkm_fb *fb = nvxx_fb(&drm->client.device); 62 struct nvkm_fb_tile *tile = &fb->tile.region[i]; 63 64 nouveau_fence_unref(®->fence); 65 66 if (tile->pitch) 67 nvkm_fb_tile_fini(fb, i, tile); 68 69 if (pitch) 70 nvkm_fb_tile_init(fb, i, addr, size, pitch, flags, tile); 71 72 nvkm_fb_tile_prog(fb, i, tile); 73 } 74 75 static struct nouveau_drm_tile * 76 nv10_bo_get_tile_region(struct drm_device *dev, int i) 77 { 78 struct nouveau_drm *drm = nouveau_drm(dev); 79 struct nouveau_drm_tile *tile = &drm->tile.reg[i]; 80 81 spin_lock(&drm->tile.lock); 82 83 if (!tile->used && 84 (!tile->fence || nouveau_fence_done(tile->fence))) 85 tile->used = true; 86 else 87 tile = NULL; 88 89 spin_unlock(&drm->tile.lock); 90 return tile; 91 } 92 93 static void 94 nv10_bo_put_tile_region(struct drm_device *dev, struct nouveau_drm_tile *tile, 95 struct dma_fence *fence) 96 { 97 struct nouveau_drm *drm = nouveau_drm(dev); 98 99 if (tile) { 100 spin_lock(&drm->tile.lock); 101 tile->fence = (struct nouveau_fence *)dma_fence_get(fence); 102 tile->used = false; 103 spin_unlock(&drm->tile.lock); 104 } 105 } 106 107 static struct nouveau_drm_tile * 108 nv10_bo_set_tiling(struct drm_device *dev, u32 addr, 109 u32 size, u32 pitch, u32 zeta) 110 { 111 struct nouveau_drm *drm = nouveau_drm(dev); 112 struct nvkm_fb *fb = nvxx_fb(&drm->client.device); 113 struct nouveau_drm_tile *tile, *found = NULL; 114 int i; 115 116 for (i = 0; i < fb->tile.regions; i++) { 117 tile = nv10_bo_get_tile_region(dev, i); 118 119 if (pitch && !found) { 120 found = tile; 121 continue; 122 123 } else if (tile && fb->tile.region[i].pitch) { 124 /* Kill an unused tile region. */ 125 nv10_bo_update_tile_region(dev, tile, 0, 0, 0, 0); 126 } 127 128 nv10_bo_put_tile_region(dev, tile, NULL); 129 } 130 131 if (found) 132 nv10_bo_update_tile_region(dev, found, addr, size, pitch, zeta); 133 return found; 134 } 135 136 static void 137 nouveau_bo_del_ttm(struct ttm_buffer_object *bo) 138 { 139 struct nouveau_drm *drm = nouveau_bdev(bo->bdev); 140 struct drm_device *dev = drm->dev; 141 struct nouveau_bo *nvbo = nouveau_bo(bo); 142 143 WARN_ON(nvbo->bo.pin_count > 0); 144 nouveau_bo_del_io_reserve_lru(bo); 145 nv10_bo_put_tile_region(dev, nvbo->tile, NULL); 146 147 /* 148 * If nouveau_bo_new() allocated this buffer, the GEM object was never 149 * initialized, so don't attempt to release it. 150 */ 151 if (bo->base.dev) 152 drm_gem_object_release(&bo->base); 153 else 154 dma_resv_fini(&bo->base._resv); 155 156 kfree(nvbo); 157 } 158 159 static inline u64 160 roundup_64(u64 x, u32 y) 161 { 162 x += y - 1; 163 do_div(x, y); 164 return x * y; 165 } 166 167 static void 168 nouveau_bo_fixup_align(struct nouveau_bo *nvbo, int *align, u64 *size) 169 { 170 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 171 struct nvif_device *device = &drm->client.device; 172 173 if (device->info.family < NV_DEVICE_INFO_V0_TESLA) { 174 if (nvbo->mode) { 175 if (device->info.chipset >= 0x40) { 176 *align = 65536; 177 *size = roundup_64(*size, 64 * nvbo->mode); 178 179 } else if (device->info.chipset >= 0x30) { 180 *align = 32768; 181 *size = roundup_64(*size, 64 * nvbo->mode); 182 183 } else if (device->info.chipset >= 0x20) { 184 *align = 16384; 185 *size = roundup_64(*size, 64 * nvbo->mode); 186 187 } else if (device->info.chipset >= 0x10) { 188 *align = 16384; 189 *size = roundup_64(*size, 32 * nvbo->mode); 190 } 191 } 192 } else { 193 *size = roundup_64(*size, (1 << nvbo->page)); 194 *align = max((1 << nvbo->page), *align); 195 } 196 197 *size = roundup_64(*size, PAGE_SIZE); 198 } 199 200 struct nouveau_bo * 201 nouveau_bo_alloc(struct nouveau_cli *cli, u64 *size, int *align, u32 domain, 202 u32 tile_mode, u32 tile_flags) 203 { 204 struct nouveau_drm *drm = cli->drm; 205 struct nouveau_bo *nvbo; 206 struct nvif_mmu *mmu = &cli->mmu; 207 struct nvif_vmm *vmm = cli->svm.cli ? &cli->svm.vmm : &cli->vmm.vmm; 208 int i, pi = -1; 209 210 if (!*size) { 211 NV_WARN(drm, "skipped size %016llx\n", *size); 212 return ERR_PTR(-EINVAL); 213 } 214 215 nvbo = kzalloc(sizeof(struct nouveau_bo), GFP_KERNEL); 216 if (!nvbo) 217 return ERR_PTR(-ENOMEM); 218 INIT_LIST_HEAD(&nvbo->head); 219 INIT_LIST_HEAD(&nvbo->entry); 220 INIT_LIST_HEAD(&nvbo->vma_list); 221 nvbo->bo.bdev = &drm->ttm.bdev; 222 223 /* This is confusing, and doesn't actually mean we want an uncached 224 * mapping, but is what NOUVEAU_GEM_DOMAIN_COHERENT gets translated 225 * into in nouveau_gem_new(). 226 */ 227 if (domain & NOUVEAU_GEM_DOMAIN_COHERENT) { 228 /* Determine if we can get a cache-coherent map, forcing 229 * uncached mapping if we can't. 230 */ 231 if (!nouveau_drm_use_coherent_gpu_mapping(drm)) 232 nvbo->force_coherent = true; 233 } 234 235 if (cli->device.info.family >= NV_DEVICE_INFO_V0_FERMI) { 236 nvbo->kind = (tile_flags & 0x0000ff00) >> 8; 237 if (!nvif_mmu_kind_valid(mmu, nvbo->kind)) { 238 kfree(nvbo); 239 return ERR_PTR(-EINVAL); 240 } 241 242 nvbo->comp = mmu->kind[nvbo->kind] != nvbo->kind; 243 } else 244 if (cli->device.info.family >= NV_DEVICE_INFO_V0_TESLA) { 245 nvbo->kind = (tile_flags & 0x00007f00) >> 8; 246 nvbo->comp = (tile_flags & 0x00030000) >> 16; 247 if (!nvif_mmu_kind_valid(mmu, nvbo->kind)) { 248 kfree(nvbo); 249 return ERR_PTR(-EINVAL); 250 } 251 } else { 252 nvbo->zeta = (tile_flags & 0x00000007); 253 } 254 nvbo->mode = tile_mode; 255 nvbo->contig = !(tile_flags & NOUVEAU_GEM_TILE_NONCONTIG); 256 257 /* Determine the desirable target GPU page size for the buffer. */ 258 for (i = 0; i < vmm->page_nr; i++) { 259 /* Because we cannot currently allow VMM maps to fail 260 * during buffer migration, we need to determine page 261 * size for the buffer up-front, and pre-allocate its 262 * page tables. 263 * 264 * Skip page sizes that can't support needed domains. 265 */ 266 if (cli->device.info.family > NV_DEVICE_INFO_V0_CURIE && 267 (domain & NOUVEAU_GEM_DOMAIN_VRAM) && !vmm->page[i].vram) 268 continue; 269 if ((domain & NOUVEAU_GEM_DOMAIN_GART) && 270 (!vmm->page[i].host || vmm->page[i].shift > PAGE_SHIFT)) 271 continue; 272 273 /* Select this page size if it's the first that supports 274 * the potential memory domains, or when it's compatible 275 * with the requested compression settings. 276 */ 277 if (pi < 0 || !nvbo->comp || vmm->page[i].comp) 278 pi = i; 279 280 /* Stop once the buffer is larger than the current page size. */ 281 if (*size >= 1ULL << vmm->page[i].shift) 282 break; 283 } 284 285 if (WARN_ON(pi < 0)) { 286 kfree(nvbo); 287 return ERR_PTR(-EINVAL); 288 } 289 290 /* Disable compression if suitable settings couldn't be found. */ 291 if (nvbo->comp && !vmm->page[pi].comp) { 292 if (mmu->object.oclass >= NVIF_CLASS_MMU_GF100) 293 nvbo->kind = mmu->kind[nvbo->kind]; 294 nvbo->comp = 0; 295 } 296 nvbo->page = vmm->page[pi].shift; 297 298 nouveau_bo_fixup_align(nvbo, align, size); 299 300 return nvbo; 301 } 302 303 int 304 nouveau_bo_init(struct nouveau_bo *nvbo, u64 size, int align, u32 domain, 305 struct sg_table *sg, struct dma_resv *robj) 306 { 307 int type = sg ? ttm_bo_type_sg : ttm_bo_type_device; 308 int ret; 309 310 nouveau_bo_placement_set(nvbo, domain, 0); 311 INIT_LIST_HEAD(&nvbo->io_reserve_lru); 312 313 ret = ttm_bo_init_validate(nvbo->bo.bdev, &nvbo->bo, type, 314 &nvbo->placement, align >> PAGE_SHIFT, false, 315 sg, robj, nouveau_bo_del_ttm); 316 if (ret) { 317 /* ttm will call nouveau_bo_del_ttm if it fails.. */ 318 return ret; 319 } 320 321 return 0; 322 } 323 324 int 325 nouveau_bo_new(struct nouveau_cli *cli, u64 size, int align, 326 uint32_t domain, uint32_t tile_mode, uint32_t tile_flags, 327 struct sg_table *sg, struct dma_resv *robj, 328 struct nouveau_bo **pnvbo) 329 { 330 struct nouveau_bo *nvbo; 331 int ret; 332 333 nvbo = nouveau_bo_alloc(cli, &size, &align, domain, tile_mode, 334 tile_flags); 335 if (IS_ERR(nvbo)) 336 return PTR_ERR(nvbo); 337 338 nvbo->bo.base.size = size; 339 dma_resv_init(&nvbo->bo.base._resv); 340 drm_vma_node_reset(&nvbo->bo.base.vma_node); 341 342 ret = nouveau_bo_init(nvbo, size, align, domain, sg, robj); 343 if (ret) 344 return ret; 345 346 *pnvbo = nvbo; 347 return 0; 348 } 349 350 static void 351 set_placement_list(struct ttm_place *pl, unsigned *n, uint32_t domain) 352 { 353 *n = 0; 354 355 if (domain & NOUVEAU_GEM_DOMAIN_VRAM) { 356 pl[*n].mem_type = TTM_PL_VRAM; 357 pl[*n].flags = 0; 358 (*n)++; 359 } 360 if (domain & NOUVEAU_GEM_DOMAIN_GART) { 361 pl[*n].mem_type = TTM_PL_TT; 362 pl[*n].flags = 0; 363 (*n)++; 364 } 365 if (domain & NOUVEAU_GEM_DOMAIN_CPU) { 366 pl[*n].mem_type = TTM_PL_SYSTEM; 367 pl[(*n)++].flags = 0; 368 } 369 } 370 371 static void 372 set_placement_range(struct nouveau_bo *nvbo, uint32_t domain) 373 { 374 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 375 u64 vram_size = drm->client.device.info.ram_size; 376 unsigned i, fpfn, lpfn; 377 378 if (drm->client.device.info.family == NV_DEVICE_INFO_V0_CELSIUS && 379 nvbo->mode && (domain & NOUVEAU_GEM_DOMAIN_VRAM) && 380 nvbo->bo.base.size < vram_size / 4) { 381 /* 382 * Make sure that the color and depth buffers are handled 383 * by independent memory controller units. Up to a 9x 384 * speed up when alpha-blending and depth-test are enabled 385 * at the same time. 386 */ 387 if (nvbo->zeta) { 388 fpfn = (vram_size / 2) >> PAGE_SHIFT; 389 lpfn = ~0; 390 } else { 391 fpfn = 0; 392 lpfn = (vram_size / 2) >> PAGE_SHIFT; 393 } 394 for (i = 0; i < nvbo->placement.num_placement; ++i) { 395 nvbo->placements[i].fpfn = fpfn; 396 nvbo->placements[i].lpfn = lpfn; 397 } 398 for (i = 0; i < nvbo->placement.num_busy_placement; ++i) { 399 nvbo->busy_placements[i].fpfn = fpfn; 400 nvbo->busy_placements[i].lpfn = lpfn; 401 } 402 } 403 } 404 405 void 406 nouveau_bo_placement_set(struct nouveau_bo *nvbo, uint32_t domain, 407 uint32_t busy) 408 { 409 struct ttm_placement *pl = &nvbo->placement; 410 411 pl->placement = nvbo->placements; 412 set_placement_list(nvbo->placements, &pl->num_placement, domain); 413 414 pl->busy_placement = nvbo->busy_placements; 415 set_placement_list(nvbo->busy_placements, &pl->num_busy_placement, 416 domain | busy); 417 418 set_placement_range(nvbo, domain); 419 } 420 421 int 422 nouveau_bo_pin(struct nouveau_bo *nvbo, uint32_t domain, bool contig) 423 { 424 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 425 struct ttm_buffer_object *bo = &nvbo->bo; 426 bool force = false, evict = false; 427 int ret; 428 429 ret = ttm_bo_reserve(bo, false, false, NULL); 430 if (ret) 431 return ret; 432 433 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA && 434 domain == NOUVEAU_GEM_DOMAIN_VRAM && contig) { 435 if (!nvbo->contig) { 436 nvbo->contig = true; 437 force = true; 438 evict = true; 439 } 440 } 441 442 if (nvbo->bo.pin_count) { 443 bool error = evict; 444 445 switch (bo->resource->mem_type) { 446 case TTM_PL_VRAM: 447 error |= !(domain & NOUVEAU_GEM_DOMAIN_VRAM); 448 break; 449 case TTM_PL_TT: 450 error |= !(domain & NOUVEAU_GEM_DOMAIN_GART); 451 break; 452 default: 453 break; 454 } 455 456 if (error) { 457 NV_ERROR(drm, "bo %p pinned elsewhere: " 458 "0x%08x vs 0x%08x\n", bo, 459 bo->resource->mem_type, domain); 460 ret = -EBUSY; 461 } 462 ttm_bo_pin(&nvbo->bo); 463 goto out; 464 } 465 466 if (evict) { 467 nouveau_bo_placement_set(nvbo, NOUVEAU_GEM_DOMAIN_GART, 0); 468 ret = nouveau_bo_validate(nvbo, false, false); 469 if (ret) 470 goto out; 471 } 472 473 nouveau_bo_placement_set(nvbo, domain, 0); 474 ret = nouveau_bo_validate(nvbo, false, false); 475 if (ret) 476 goto out; 477 478 ttm_bo_pin(&nvbo->bo); 479 480 switch (bo->resource->mem_type) { 481 case TTM_PL_VRAM: 482 drm->gem.vram_available -= bo->base.size; 483 break; 484 case TTM_PL_TT: 485 drm->gem.gart_available -= bo->base.size; 486 break; 487 default: 488 break; 489 } 490 491 out: 492 if (force && ret) 493 nvbo->contig = false; 494 ttm_bo_unreserve(bo); 495 return ret; 496 } 497 498 int 499 nouveau_bo_unpin(struct nouveau_bo *nvbo) 500 { 501 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 502 struct ttm_buffer_object *bo = &nvbo->bo; 503 int ret; 504 505 ret = ttm_bo_reserve(bo, false, false, NULL); 506 if (ret) 507 return ret; 508 509 ttm_bo_unpin(&nvbo->bo); 510 if (!nvbo->bo.pin_count) { 511 switch (bo->resource->mem_type) { 512 case TTM_PL_VRAM: 513 drm->gem.vram_available += bo->base.size; 514 break; 515 case TTM_PL_TT: 516 drm->gem.gart_available += bo->base.size; 517 break; 518 default: 519 break; 520 } 521 } 522 523 ttm_bo_unreserve(bo); 524 return 0; 525 } 526 527 int 528 nouveau_bo_map(struct nouveau_bo *nvbo) 529 { 530 int ret; 531 532 ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL); 533 if (ret) 534 return ret; 535 536 ret = ttm_bo_kmap(&nvbo->bo, 0, PFN_UP(nvbo->bo.base.size), &nvbo->kmap); 537 538 ttm_bo_unreserve(&nvbo->bo); 539 return ret; 540 } 541 542 void 543 nouveau_bo_unmap(struct nouveau_bo *nvbo) 544 { 545 if (!nvbo) 546 return; 547 548 ttm_bo_kunmap(&nvbo->kmap); 549 } 550 551 void 552 nouveau_bo_sync_for_device(struct nouveau_bo *nvbo) 553 { 554 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 555 struct ttm_tt *ttm_dma = (struct ttm_tt *)nvbo->bo.ttm; 556 int i, j; 557 558 if (!ttm_dma || !ttm_dma->dma_address) 559 return; 560 if (!ttm_dma->pages) { 561 NV_DEBUG(drm, "ttm_dma 0x%p: pages NULL\n", ttm_dma); 562 return; 563 } 564 565 /* Don't waste time looping if the object is coherent */ 566 if (nvbo->force_coherent) 567 return; 568 569 i = 0; 570 while (i < ttm_dma->num_pages) { 571 struct page *p = ttm_dma->pages[i]; 572 size_t num_pages = 1; 573 574 for (j = i + 1; j < ttm_dma->num_pages; ++j) { 575 if (++p != ttm_dma->pages[j]) 576 break; 577 578 ++num_pages; 579 } 580 dma_sync_single_for_device(drm->dev->dev, 581 ttm_dma->dma_address[i], 582 num_pages * PAGE_SIZE, DMA_TO_DEVICE); 583 i += num_pages; 584 } 585 } 586 587 void 588 nouveau_bo_sync_for_cpu(struct nouveau_bo *nvbo) 589 { 590 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 591 struct ttm_tt *ttm_dma = (struct ttm_tt *)nvbo->bo.ttm; 592 int i, j; 593 594 if (!ttm_dma || !ttm_dma->dma_address) 595 return; 596 if (!ttm_dma->pages) { 597 NV_DEBUG(drm, "ttm_dma 0x%p: pages NULL\n", ttm_dma); 598 return; 599 } 600 601 /* Don't waste time looping if the object is coherent */ 602 if (nvbo->force_coherent) 603 return; 604 605 i = 0; 606 while (i < ttm_dma->num_pages) { 607 struct page *p = ttm_dma->pages[i]; 608 size_t num_pages = 1; 609 610 for (j = i + 1; j < ttm_dma->num_pages; ++j) { 611 if (++p != ttm_dma->pages[j]) 612 break; 613 614 ++num_pages; 615 } 616 617 dma_sync_single_for_cpu(drm->dev->dev, ttm_dma->dma_address[i], 618 num_pages * PAGE_SIZE, DMA_FROM_DEVICE); 619 i += num_pages; 620 } 621 } 622 623 void nouveau_bo_add_io_reserve_lru(struct ttm_buffer_object *bo) 624 { 625 struct nouveau_drm *drm = nouveau_bdev(bo->bdev); 626 struct nouveau_bo *nvbo = nouveau_bo(bo); 627 628 mutex_lock(&drm->ttm.io_reserve_mutex); 629 list_move_tail(&nvbo->io_reserve_lru, &drm->ttm.io_reserve_lru); 630 mutex_unlock(&drm->ttm.io_reserve_mutex); 631 } 632 633 void nouveau_bo_del_io_reserve_lru(struct ttm_buffer_object *bo) 634 { 635 struct nouveau_drm *drm = nouveau_bdev(bo->bdev); 636 struct nouveau_bo *nvbo = nouveau_bo(bo); 637 638 mutex_lock(&drm->ttm.io_reserve_mutex); 639 list_del_init(&nvbo->io_reserve_lru); 640 mutex_unlock(&drm->ttm.io_reserve_mutex); 641 } 642 643 int 644 nouveau_bo_validate(struct nouveau_bo *nvbo, bool interruptible, 645 bool no_wait_gpu) 646 { 647 struct ttm_operation_ctx ctx = { interruptible, no_wait_gpu }; 648 int ret; 649 650 ret = ttm_bo_validate(&nvbo->bo, &nvbo->placement, &ctx); 651 if (ret) 652 return ret; 653 654 nouveau_bo_sync_for_device(nvbo); 655 656 return 0; 657 } 658 659 void 660 nouveau_bo_wr16(struct nouveau_bo *nvbo, unsigned index, u16 val) 661 { 662 bool is_iomem; 663 u16 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem); 664 665 mem += index; 666 667 if (is_iomem) 668 iowrite16_native(val, (void __force __iomem *)mem); 669 else 670 *mem = val; 671 } 672 673 u32 674 nouveau_bo_rd32(struct nouveau_bo *nvbo, unsigned index) 675 { 676 bool is_iomem; 677 u32 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem); 678 679 mem += index; 680 681 if (is_iomem) 682 return ioread32_native((void __force __iomem *)mem); 683 else 684 return *mem; 685 } 686 687 void 688 nouveau_bo_wr32(struct nouveau_bo *nvbo, unsigned index, u32 val) 689 { 690 bool is_iomem; 691 u32 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem); 692 693 mem += index; 694 695 if (is_iomem) 696 iowrite32_native(val, (void __force __iomem *)mem); 697 else 698 *mem = val; 699 } 700 701 static struct ttm_tt * 702 nouveau_ttm_tt_create(struct ttm_buffer_object *bo, uint32_t page_flags) 703 { 704 #if IS_ENABLED(CONFIG_AGP) 705 struct nouveau_drm *drm = nouveau_bdev(bo->bdev); 706 707 if (drm->agp.bridge) { 708 return ttm_agp_tt_create(bo, drm->agp.bridge, page_flags); 709 } 710 #endif 711 712 return nouveau_sgdma_create_ttm(bo, page_flags); 713 } 714 715 static int 716 nouveau_ttm_tt_bind(struct ttm_device *bdev, struct ttm_tt *ttm, 717 struct ttm_resource *reg) 718 { 719 #if IS_ENABLED(CONFIG_AGP) 720 struct nouveau_drm *drm = nouveau_bdev(bdev); 721 #endif 722 if (!reg) 723 return -EINVAL; 724 #if IS_ENABLED(CONFIG_AGP) 725 if (drm->agp.bridge) 726 return ttm_agp_bind(ttm, reg); 727 #endif 728 return nouveau_sgdma_bind(bdev, ttm, reg); 729 } 730 731 static void 732 nouveau_ttm_tt_unbind(struct ttm_device *bdev, struct ttm_tt *ttm) 733 { 734 #if IS_ENABLED(CONFIG_AGP) 735 struct nouveau_drm *drm = nouveau_bdev(bdev); 736 737 if (drm->agp.bridge) { 738 ttm_agp_unbind(ttm); 739 return; 740 } 741 #endif 742 nouveau_sgdma_unbind(bdev, ttm); 743 } 744 745 static void 746 nouveau_bo_evict_flags(struct ttm_buffer_object *bo, struct ttm_placement *pl) 747 { 748 struct nouveau_bo *nvbo = nouveau_bo(bo); 749 750 switch (bo->resource->mem_type) { 751 case TTM_PL_VRAM: 752 nouveau_bo_placement_set(nvbo, NOUVEAU_GEM_DOMAIN_GART, 753 NOUVEAU_GEM_DOMAIN_CPU); 754 break; 755 default: 756 nouveau_bo_placement_set(nvbo, NOUVEAU_GEM_DOMAIN_CPU, 0); 757 break; 758 } 759 760 *pl = nvbo->placement; 761 } 762 763 static int 764 nouveau_bo_move_prep(struct nouveau_drm *drm, struct ttm_buffer_object *bo, 765 struct ttm_resource *reg) 766 { 767 struct nouveau_mem *old_mem = nouveau_mem(bo->resource); 768 struct nouveau_mem *new_mem = nouveau_mem(reg); 769 struct nvif_vmm *vmm = &drm->client.vmm.vmm; 770 int ret; 771 772 ret = nvif_vmm_get(vmm, LAZY, false, old_mem->mem.page, 0, 773 old_mem->mem.size, &old_mem->vma[0]); 774 if (ret) 775 return ret; 776 777 ret = nvif_vmm_get(vmm, LAZY, false, new_mem->mem.page, 0, 778 new_mem->mem.size, &old_mem->vma[1]); 779 if (ret) 780 goto done; 781 782 ret = nouveau_mem_map(old_mem, vmm, &old_mem->vma[0]); 783 if (ret) 784 goto done; 785 786 ret = nouveau_mem_map(new_mem, vmm, &old_mem->vma[1]); 787 done: 788 if (ret) { 789 nvif_vmm_put(vmm, &old_mem->vma[1]); 790 nvif_vmm_put(vmm, &old_mem->vma[0]); 791 } 792 return 0; 793 } 794 795 static int 796 nouveau_bo_move_m2mf(struct ttm_buffer_object *bo, int evict, 797 struct ttm_operation_ctx *ctx, 798 struct ttm_resource *new_reg) 799 { 800 struct nouveau_drm *drm = nouveau_bdev(bo->bdev); 801 struct nouveau_channel *chan = drm->ttm.chan; 802 struct nouveau_cli *cli = (void *)chan->user.client; 803 struct nouveau_fence *fence; 804 int ret; 805 806 /* create temporary vmas for the transfer and attach them to the 807 * old nvkm_mem node, these will get cleaned up after ttm has 808 * destroyed the ttm_resource 809 */ 810 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA) { 811 ret = nouveau_bo_move_prep(drm, bo, new_reg); 812 if (ret) 813 return ret; 814 } 815 816 if (drm_drv_uses_atomic_modeset(drm->dev)) 817 mutex_lock(&cli->mutex); 818 else 819 mutex_lock_nested(&cli->mutex, SINGLE_DEPTH_NESTING); 820 ret = nouveau_fence_sync(nouveau_bo(bo), chan, true, ctx->interruptible); 821 if (ret == 0) { 822 ret = drm->ttm.move(chan, bo, bo->resource, new_reg); 823 if (ret == 0) { 824 ret = nouveau_fence_new(chan, false, &fence); 825 if (ret == 0) { 826 /* TODO: figure out a better solution here 827 * 828 * wait on the fence here explicitly as going through 829 * ttm_bo_move_accel_cleanup somehow doesn't seem to do it. 830 * 831 * Without this the operation can timeout and we'll fallback to a 832 * software copy, which might take several minutes to finish. 833 */ 834 nouveau_fence_wait(fence, false, false); 835 ret = ttm_bo_move_accel_cleanup(bo, 836 &fence->base, 837 evict, false, 838 new_reg); 839 nouveau_fence_unref(&fence); 840 } 841 } 842 } 843 mutex_unlock(&cli->mutex); 844 return ret; 845 } 846 847 void 848 nouveau_bo_move_init(struct nouveau_drm *drm) 849 { 850 static const struct _method_table { 851 const char *name; 852 int engine; 853 s32 oclass; 854 int (*exec)(struct nouveau_channel *, 855 struct ttm_buffer_object *, 856 struct ttm_resource *, struct ttm_resource *); 857 int (*init)(struct nouveau_channel *, u32 handle); 858 } _methods[] = { 859 { "COPY", 4, 0xc7b5, nve0_bo_move_copy, nve0_bo_move_init }, 860 { "GRCE", 0, 0xc7b5, nve0_bo_move_copy, nvc0_bo_move_init }, 861 { "COPY", 4, 0xc6b5, nve0_bo_move_copy, nve0_bo_move_init }, 862 { "GRCE", 0, 0xc6b5, nve0_bo_move_copy, nvc0_bo_move_init }, 863 { "COPY", 4, 0xc5b5, nve0_bo_move_copy, nve0_bo_move_init }, 864 { "GRCE", 0, 0xc5b5, nve0_bo_move_copy, nvc0_bo_move_init }, 865 { "COPY", 4, 0xc3b5, nve0_bo_move_copy, nve0_bo_move_init }, 866 { "GRCE", 0, 0xc3b5, nve0_bo_move_copy, nvc0_bo_move_init }, 867 { "COPY", 4, 0xc1b5, nve0_bo_move_copy, nve0_bo_move_init }, 868 { "GRCE", 0, 0xc1b5, nve0_bo_move_copy, nvc0_bo_move_init }, 869 { "COPY", 4, 0xc0b5, nve0_bo_move_copy, nve0_bo_move_init }, 870 { "GRCE", 0, 0xc0b5, nve0_bo_move_copy, nvc0_bo_move_init }, 871 { "COPY", 4, 0xb0b5, nve0_bo_move_copy, nve0_bo_move_init }, 872 { "GRCE", 0, 0xb0b5, nve0_bo_move_copy, nvc0_bo_move_init }, 873 { "COPY", 4, 0xa0b5, nve0_bo_move_copy, nve0_bo_move_init }, 874 { "GRCE", 0, 0xa0b5, nve0_bo_move_copy, nvc0_bo_move_init }, 875 { "COPY1", 5, 0x90b8, nvc0_bo_move_copy, nvc0_bo_move_init }, 876 { "COPY0", 4, 0x90b5, nvc0_bo_move_copy, nvc0_bo_move_init }, 877 { "COPY", 0, 0x85b5, nva3_bo_move_copy, nv50_bo_move_init }, 878 { "CRYPT", 0, 0x74c1, nv84_bo_move_exec, nv50_bo_move_init }, 879 { "M2MF", 0, 0x9039, nvc0_bo_move_m2mf, nvc0_bo_move_init }, 880 { "M2MF", 0, 0x5039, nv50_bo_move_m2mf, nv50_bo_move_init }, 881 { "M2MF", 0, 0x0039, nv04_bo_move_m2mf, nv04_bo_move_init }, 882 {}, 883 }; 884 const struct _method_table *mthd = _methods; 885 const char *name = "CPU"; 886 int ret; 887 888 do { 889 struct nouveau_channel *chan; 890 891 if (mthd->engine) 892 chan = drm->cechan; 893 else 894 chan = drm->channel; 895 if (chan == NULL) 896 continue; 897 898 ret = nvif_object_ctor(&chan->user, "ttmBoMove", 899 mthd->oclass | (mthd->engine << 16), 900 mthd->oclass, NULL, 0, 901 &drm->ttm.copy); 902 if (ret == 0) { 903 ret = mthd->init(chan, drm->ttm.copy.handle); 904 if (ret) { 905 nvif_object_dtor(&drm->ttm.copy); 906 continue; 907 } 908 909 drm->ttm.move = mthd->exec; 910 drm->ttm.chan = chan; 911 name = mthd->name; 912 break; 913 } 914 } while ((++mthd)->exec); 915 916 NV_INFO(drm, "MM: using %s for buffer copies\n", name); 917 } 918 919 static void nouveau_bo_move_ntfy(struct ttm_buffer_object *bo, 920 struct ttm_resource *new_reg) 921 { 922 struct nouveau_mem *mem = new_reg ? nouveau_mem(new_reg) : NULL; 923 struct nouveau_bo *nvbo = nouveau_bo(bo); 924 struct nouveau_vma *vma; 925 long ret; 926 927 /* ttm can now (stupidly) pass the driver bos it didn't create... */ 928 if (bo->destroy != nouveau_bo_del_ttm) 929 return; 930 931 nouveau_bo_del_io_reserve_lru(bo); 932 933 if (mem && new_reg->mem_type != TTM_PL_SYSTEM && 934 mem->mem.page == nvbo->page) { 935 list_for_each_entry(vma, &nvbo->vma_list, head) { 936 nouveau_vma_map(vma, mem); 937 } 938 } else { 939 list_for_each_entry(vma, &nvbo->vma_list, head) { 940 ret = dma_resv_wait_timeout(bo->base.resv, 941 DMA_RESV_USAGE_BOOKKEEP, 942 false, 15 * HZ); 943 WARN_ON(ret <= 0); 944 nouveau_vma_unmap(vma); 945 } 946 } 947 948 if (new_reg) 949 nvbo->offset = (new_reg->start << PAGE_SHIFT); 950 951 } 952 953 static int 954 nouveau_bo_vm_bind(struct ttm_buffer_object *bo, struct ttm_resource *new_reg, 955 struct nouveau_drm_tile **new_tile) 956 { 957 struct nouveau_drm *drm = nouveau_bdev(bo->bdev); 958 struct drm_device *dev = drm->dev; 959 struct nouveau_bo *nvbo = nouveau_bo(bo); 960 u64 offset = new_reg->start << PAGE_SHIFT; 961 962 *new_tile = NULL; 963 if (new_reg->mem_type != TTM_PL_VRAM) 964 return 0; 965 966 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_CELSIUS) { 967 *new_tile = nv10_bo_set_tiling(dev, offset, bo->base.size, 968 nvbo->mode, nvbo->zeta); 969 } 970 971 return 0; 972 } 973 974 static void 975 nouveau_bo_vm_cleanup(struct ttm_buffer_object *bo, 976 struct nouveau_drm_tile *new_tile, 977 struct nouveau_drm_tile **old_tile) 978 { 979 struct nouveau_drm *drm = nouveau_bdev(bo->bdev); 980 struct drm_device *dev = drm->dev; 981 struct dma_fence *fence; 982 int ret; 983 984 ret = dma_resv_get_singleton(bo->base.resv, DMA_RESV_USAGE_WRITE, 985 &fence); 986 if (ret) 987 dma_resv_wait_timeout(bo->base.resv, DMA_RESV_USAGE_WRITE, 988 false, MAX_SCHEDULE_TIMEOUT); 989 990 nv10_bo_put_tile_region(dev, *old_tile, fence); 991 *old_tile = new_tile; 992 } 993 994 static int 995 nouveau_bo_move(struct ttm_buffer_object *bo, bool evict, 996 struct ttm_operation_ctx *ctx, 997 struct ttm_resource *new_reg, 998 struct ttm_place *hop) 999 { 1000 struct nouveau_drm *drm = nouveau_bdev(bo->bdev); 1001 struct nouveau_bo *nvbo = nouveau_bo(bo); 1002 struct ttm_resource *old_reg = bo->resource; 1003 struct nouveau_drm_tile *new_tile = NULL; 1004 int ret = 0; 1005 1006 1007 if (new_reg->mem_type == TTM_PL_TT) { 1008 ret = nouveau_ttm_tt_bind(bo->bdev, bo->ttm, new_reg); 1009 if (ret) 1010 return ret; 1011 } 1012 1013 nouveau_bo_move_ntfy(bo, new_reg); 1014 ret = ttm_bo_wait_ctx(bo, ctx); 1015 if (ret) 1016 goto out_ntfy; 1017 1018 if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA) { 1019 ret = nouveau_bo_vm_bind(bo, new_reg, &new_tile); 1020 if (ret) 1021 goto out_ntfy; 1022 } 1023 1024 /* Fake bo copy. */ 1025 if (!old_reg || (old_reg->mem_type == TTM_PL_SYSTEM && 1026 !bo->ttm)) { 1027 ttm_bo_move_null(bo, new_reg); 1028 goto out; 1029 } 1030 1031 if (old_reg->mem_type == TTM_PL_SYSTEM && 1032 new_reg->mem_type == TTM_PL_TT) { 1033 ttm_bo_move_null(bo, new_reg); 1034 goto out; 1035 } 1036 1037 if (old_reg->mem_type == TTM_PL_TT && 1038 new_reg->mem_type == TTM_PL_SYSTEM) { 1039 nouveau_ttm_tt_unbind(bo->bdev, bo->ttm); 1040 ttm_resource_free(bo, &bo->resource); 1041 ttm_bo_assign_mem(bo, new_reg); 1042 goto out; 1043 } 1044 1045 /* Hardware assisted copy. */ 1046 if (drm->ttm.move) { 1047 if ((old_reg->mem_type == TTM_PL_SYSTEM && 1048 new_reg->mem_type == TTM_PL_VRAM) || 1049 (old_reg->mem_type == TTM_PL_VRAM && 1050 new_reg->mem_type == TTM_PL_SYSTEM)) { 1051 hop->fpfn = 0; 1052 hop->lpfn = 0; 1053 hop->mem_type = TTM_PL_TT; 1054 hop->flags = 0; 1055 return -EMULTIHOP; 1056 } 1057 ret = nouveau_bo_move_m2mf(bo, evict, ctx, 1058 new_reg); 1059 } else 1060 ret = -ENODEV; 1061 1062 if (ret) { 1063 /* Fallback to software copy. */ 1064 ret = ttm_bo_move_memcpy(bo, ctx, new_reg); 1065 } 1066 1067 out: 1068 if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA) { 1069 if (ret) 1070 nouveau_bo_vm_cleanup(bo, NULL, &new_tile); 1071 else 1072 nouveau_bo_vm_cleanup(bo, new_tile, &nvbo->tile); 1073 } 1074 out_ntfy: 1075 if (ret) { 1076 nouveau_bo_move_ntfy(bo, bo->resource); 1077 } 1078 return ret; 1079 } 1080 1081 static void 1082 nouveau_ttm_io_mem_free_locked(struct nouveau_drm *drm, 1083 struct ttm_resource *reg) 1084 { 1085 struct nouveau_mem *mem = nouveau_mem(reg); 1086 1087 if (drm->client.mem->oclass >= NVIF_CLASS_MEM_NV50) { 1088 switch (reg->mem_type) { 1089 case TTM_PL_TT: 1090 if (mem->kind) 1091 nvif_object_unmap_handle(&mem->mem.object); 1092 break; 1093 case TTM_PL_VRAM: 1094 nvif_object_unmap_handle(&mem->mem.object); 1095 break; 1096 default: 1097 break; 1098 } 1099 } 1100 } 1101 1102 static int 1103 nouveau_ttm_io_mem_reserve(struct ttm_device *bdev, struct ttm_resource *reg) 1104 { 1105 struct nouveau_drm *drm = nouveau_bdev(bdev); 1106 struct nvkm_device *device = nvxx_device(&drm->client.device); 1107 struct nouveau_mem *mem = nouveau_mem(reg); 1108 struct nvif_mmu *mmu = &drm->client.mmu; 1109 int ret; 1110 1111 mutex_lock(&drm->ttm.io_reserve_mutex); 1112 retry: 1113 switch (reg->mem_type) { 1114 case TTM_PL_SYSTEM: 1115 /* System memory */ 1116 ret = 0; 1117 goto out; 1118 case TTM_PL_TT: 1119 #if IS_ENABLED(CONFIG_AGP) 1120 if (drm->agp.bridge) { 1121 reg->bus.offset = (reg->start << PAGE_SHIFT) + 1122 drm->agp.base; 1123 reg->bus.is_iomem = !drm->agp.cma; 1124 reg->bus.caching = ttm_write_combined; 1125 } 1126 #endif 1127 if (drm->client.mem->oclass < NVIF_CLASS_MEM_NV50 || 1128 !mem->kind) { 1129 /* untiled */ 1130 ret = 0; 1131 break; 1132 } 1133 fallthrough; /* tiled memory */ 1134 case TTM_PL_VRAM: 1135 reg->bus.offset = (reg->start << PAGE_SHIFT) + 1136 device->func->resource_addr(device, 1); 1137 reg->bus.is_iomem = true; 1138 1139 /* Some BARs do not support being ioremapped WC */ 1140 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA && 1141 mmu->type[drm->ttm.type_vram].type & NVIF_MEM_UNCACHED) 1142 reg->bus.caching = ttm_uncached; 1143 else 1144 reg->bus.caching = ttm_write_combined; 1145 1146 if (drm->client.mem->oclass >= NVIF_CLASS_MEM_NV50) { 1147 union { 1148 struct nv50_mem_map_v0 nv50; 1149 struct gf100_mem_map_v0 gf100; 1150 } args; 1151 u64 handle, length; 1152 u32 argc = 0; 1153 1154 switch (mem->mem.object.oclass) { 1155 case NVIF_CLASS_MEM_NV50: 1156 args.nv50.version = 0; 1157 args.nv50.ro = 0; 1158 args.nv50.kind = mem->kind; 1159 args.nv50.comp = mem->comp; 1160 argc = sizeof(args.nv50); 1161 break; 1162 case NVIF_CLASS_MEM_GF100: 1163 args.gf100.version = 0; 1164 args.gf100.ro = 0; 1165 args.gf100.kind = mem->kind; 1166 argc = sizeof(args.gf100); 1167 break; 1168 default: 1169 WARN_ON(1); 1170 break; 1171 } 1172 1173 ret = nvif_object_map_handle(&mem->mem.object, 1174 &args, argc, 1175 &handle, &length); 1176 if (ret != 1) { 1177 if (WARN_ON(ret == 0)) 1178 ret = -EINVAL; 1179 goto out; 1180 } 1181 1182 reg->bus.offset = handle; 1183 } 1184 ret = 0; 1185 break; 1186 default: 1187 ret = -EINVAL; 1188 } 1189 1190 out: 1191 if (ret == -ENOSPC) { 1192 struct nouveau_bo *nvbo; 1193 1194 nvbo = list_first_entry_or_null(&drm->ttm.io_reserve_lru, 1195 typeof(*nvbo), 1196 io_reserve_lru); 1197 if (nvbo) { 1198 list_del_init(&nvbo->io_reserve_lru); 1199 drm_vma_node_unmap(&nvbo->bo.base.vma_node, 1200 bdev->dev_mapping); 1201 nouveau_ttm_io_mem_free_locked(drm, nvbo->bo.resource); 1202 goto retry; 1203 } 1204 1205 } 1206 mutex_unlock(&drm->ttm.io_reserve_mutex); 1207 return ret; 1208 } 1209 1210 static void 1211 nouveau_ttm_io_mem_free(struct ttm_device *bdev, struct ttm_resource *reg) 1212 { 1213 struct nouveau_drm *drm = nouveau_bdev(bdev); 1214 1215 mutex_lock(&drm->ttm.io_reserve_mutex); 1216 nouveau_ttm_io_mem_free_locked(drm, reg); 1217 mutex_unlock(&drm->ttm.io_reserve_mutex); 1218 } 1219 1220 vm_fault_t nouveau_ttm_fault_reserve_notify(struct ttm_buffer_object *bo) 1221 { 1222 struct nouveau_drm *drm = nouveau_bdev(bo->bdev); 1223 struct nouveau_bo *nvbo = nouveau_bo(bo); 1224 struct nvkm_device *device = nvxx_device(&drm->client.device); 1225 u32 mappable = device->func->resource_size(device, 1) >> PAGE_SHIFT; 1226 int i, ret; 1227 1228 /* as long as the bo isn't in vram, and isn't tiled, we've got 1229 * nothing to do here. 1230 */ 1231 if (bo->resource->mem_type != TTM_PL_VRAM) { 1232 if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA || 1233 !nvbo->kind) 1234 return 0; 1235 1236 if (bo->resource->mem_type != TTM_PL_SYSTEM) 1237 return 0; 1238 1239 nouveau_bo_placement_set(nvbo, NOUVEAU_GEM_DOMAIN_GART, 0); 1240 1241 } else { 1242 /* make sure bo is in mappable vram */ 1243 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA || 1244 bo->resource->start + PFN_UP(bo->resource->size) < mappable) 1245 return 0; 1246 1247 for (i = 0; i < nvbo->placement.num_placement; ++i) { 1248 nvbo->placements[i].fpfn = 0; 1249 nvbo->placements[i].lpfn = mappable; 1250 } 1251 1252 for (i = 0; i < nvbo->placement.num_busy_placement; ++i) { 1253 nvbo->busy_placements[i].fpfn = 0; 1254 nvbo->busy_placements[i].lpfn = mappable; 1255 } 1256 1257 nouveau_bo_placement_set(nvbo, NOUVEAU_GEM_DOMAIN_VRAM, 0); 1258 } 1259 1260 ret = nouveau_bo_validate(nvbo, false, false); 1261 if (unlikely(ret == -EBUSY || ret == -ERESTARTSYS)) 1262 return VM_FAULT_NOPAGE; 1263 else if (unlikely(ret)) 1264 return VM_FAULT_SIGBUS; 1265 1266 ttm_bo_move_to_lru_tail_unlocked(bo); 1267 return 0; 1268 } 1269 1270 static int 1271 nouveau_ttm_tt_populate(struct ttm_device *bdev, 1272 struct ttm_tt *ttm, struct ttm_operation_ctx *ctx) 1273 { 1274 struct ttm_tt *ttm_dma = (void *)ttm; 1275 struct nouveau_drm *drm; 1276 bool slave = !!(ttm->page_flags & TTM_TT_FLAG_EXTERNAL); 1277 1278 if (ttm_tt_is_populated(ttm)) 1279 return 0; 1280 1281 if (slave && ttm->sg) { 1282 drm_prime_sg_to_dma_addr_array(ttm->sg, ttm_dma->dma_address, 1283 ttm->num_pages); 1284 return 0; 1285 } 1286 1287 drm = nouveau_bdev(bdev); 1288 1289 return ttm_pool_alloc(&drm->ttm.bdev.pool, ttm, ctx); 1290 } 1291 1292 static void 1293 nouveau_ttm_tt_unpopulate(struct ttm_device *bdev, 1294 struct ttm_tt *ttm) 1295 { 1296 struct nouveau_drm *drm; 1297 bool slave = !!(ttm->page_flags & TTM_TT_FLAG_EXTERNAL); 1298 1299 if (slave) 1300 return; 1301 1302 nouveau_ttm_tt_unbind(bdev, ttm); 1303 1304 drm = nouveau_bdev(bdev); 1305 1306 return ttm_pool_free(&drm->ttm.bdev.pool, ttm); 1307 } 1308 1309 static void 1310 nouveau_ttm_tt_destroy(struct ttm_device *bdev, 1311 struct ttm_tt *ttm) 1312 { 1313 #if IS_ENABLED(CONFIG_AGP) 1314 struct nouveau_drm *drm = nouveau_bdev(bdev); 1315 if (drm->agp.bridge) { 1316 ttm_agp_destroy(ttm); 1317 return; 1318 } 1319 #endif 1320 nouveau_sgdma_destroy(bdev, ttm); 1321 } 1322 1323 void 1324 nouveau_bo_fence(struct nouveau_bo *nvbo, struct nouveau_fence *fence, bool exclusive) 1325 { 1326 struct dma_resv *resv = nvbo->bo.base.resv; 1327 1328 if (!fence) 1329 return; 1330 1331 dma_resv_add_fence(resv, &fence->base, exclusive ? 1332 DMA_RESV_USAGE_WRITE : DMA_RESV_USAGE_READ); 1333 } 1334 1335 static void 1336 nouveau_bo_delete_mem_notify(struct ttm_buffer_object *bo) 1337 { 1338 nouveau_bo_move_ntfy(bo, NULL); 1339 } 1340 1341 struct ttm_device_funcs nouveau_bo_driver = { 1342 .ttm_tt_create = &nouveau_ttm_tt_create, 1343 .ttm_tt_populate = &nouveau_ttm_tt_populate, 1344 .ttm_tt_unpopulate = &nouveau_ttm_tt_unpopulate, 1345 .ttm_tt_destroy = &nouveau_ttm_tt_destroy, 1346 .eviction_valuable = ttm_bo_eviction_valuable, 1347 .evict_flags = nouveau_bo_evict_flags, 1348 .delete_mem_notify = nouveau_bo_delete_mem_notify, 1349 .move = nouveau_bo_move, 1350 .io_mem_reserve = &nouveau_ttm_io_mem_reserve, 1351 .io_mem_free = &nouveau_ttm_io_mem_free, 1352 }; 1353