1 /* 2 * Copyright 2007 Dave Airlied 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice (including the next 13 * paragraph) shall be included in all copies or substantial portions of the 14 * Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 * OTHER DEALINGS IN THE SOFTWARE. 23 */ 24 /* 25 * Authors: Dave Airlied <airlied@linux.ie> 26 * Ben Skeggs <darktama@iinet.net.au> 27 * Jeremy Kolb <jkolb@brandeis.edu> 28 */ 29 30 #include <linux/dma-mapping.h> 31 #include <drm/ttm/ttm_tt.h> 32 33 #include "nouveau_drv.h" 34 #include "nouveau_chan.h" 35 #include "nouveau_fence.h" 36 37 #include "nouveau_bo.h" 38 #include "nouveau_ttm.h" 39 #include "nouveau_gem.h" 40 #include "nouveau_mem.h" 41 #include "nouveau_vmm.h" 42 43 #include <nvif/class.h> 44 #include <nvif/if500b.h> 45 #include <nvif/if900b.h> 46 47 static int nouveau_ttm_tt_bind(struct ttm_device *bdev, struct ttm_tt *ttm, 48 struct ttm_resource *reg); 49 static void nouveau_ttm_tt_unbind(struct ttm_device *bdev, struct ttm_tt *ttm); 50 51 /* 52 * NV10-NV40 tiling helpers 53 */ 54 55 static void 56 nv10_bo_update_tile_region(struct drm_device *dev, struct nouveau_drm_tile *reg, 57 u32 addr, u32 size, u32 pitch, u32 flags) 58 { 59 struct nouveau_drm *drm = nouveau_drm(dev); 60 int i = reg - drm->tile.reg; 61 struct nvkm_fb *fb = nvxx_fb(&drm->client.device); 62 struct nvkm_fb_tile *tile = &fb->tile.region[i]; 63 64 nouveau_fence_unref(®->fence); 65 66 if (tile->pitch) 67 nvkm_fb_tile_fini(fb, i, tile); 68 69 if (pitch) 70 nvkm_fb_tile_init(fb, i, addr, size, pitch, flags, tile); 71 72 nvkm_fb_tile_prog(fb, i, tile); 73 } 74 75 static struct nouveau_drm_tile * 76 nv10_bo_get_tile_region(struct drm_device *dev, int i) 77 { 78 struct nouveau_drm *drm = nouveau_drm(dev); 79 struct nouveau_drm_tile *tile = &drm->tile.reg[i]; 80 81 spin_lock(&drm->tile.lock); 82 83 if (!tile->used && 84 (!tile->fence || nouveau_fence_done(tile->fence))) 85 tile->used = true; 86 else 87 tile = NULL; 88 89 spin_unlock(&drm->tile.lock); 90 return tile; 91 } 92 93 static void 94 nv10_bo_put_tile_region(struct drm_device *dev, struct nouveau_drm_tile *tile, 95 struct dma_fence *fence) 96 { 97 struct nouveau_drm *drm = nouveau_drm(dev); 98 99 if (tile) { 100 spin_lock(&drm->tile.lock); 101 tile->fence = (struct nouveau_fence *)dma_fence_get(fence); 102 tile->used = false; 103 spin_unlock(&drm->tile.lock); 104 } 105 } 106 107 static struct nouveau_drm_tile * 108 nv10_bo_set_tiling(struct drm_device *dev, u32 addr, 109 u32 size, u32 pitch, u32 zeta) 110 { 111 struct nouveau_drm *drm = nouveau_drm(dev); 112 struct nvkm_fb *fb = nvxx_fb(&drm->client.device); 113 struct nouveau_drm_tile *tile, *found = NULL; 114 int i; 115 116 for (i = 0; i < fb->tile.regions; i++) { 117 tile = nv10_bo_get_tile_region(dev, i); 118 119 if (pitch && !found) { 120 found = tile; 121 continue; 122 123 } else if (tile && fb->tile.region[i].pitch) { 124 /* Kill an unused tile region. */ 125 nv10_bo_update_tile_region(dev, tile, 0, 0, 0, 0); 126 } 127 128 nv10_bo_put_tile_region(dev, tile, NULL); 129 } 130 131 if (found) 132 nv10_bo_update_tile_region(dev, found, addr, size, pitch, zeta); 133 return found; 134 } 135 136 static void 137 nouveau_bo_del_ttm(struct ttm_buffer_object *bo) 138 { 139 struct nouveau_drm *drm = nouveau_bdev(bo->bdev); 140 struct drm_device *dev = drm->dev; 141 struct nouveau_bo *nvbo = nouveau_bo(bo); 142 143 WARN_ON(nvbo->bo.pin_count > 0); 144 nouveau_bo_del_io_reserve_lru(bo); 145 nv10_bo_put_tile_region(dev, nvbo->tile, NULL); 146 147 /* 148 * If nouveau_bo_new() allocated this buffer, the GEM object was never 149 * initialized, so don't attempt to release it. 150 */ 151 if (bo->base.dev) 152 drm_gem_object_release(&bo->base); 153 else 154 dma_resv_fini(&bo->base._resv); 155 156 kfree(nvbo); 157 } 158 159 static inline u64 160 roundup_64(u64 x, u32 y) 161 { 162 x += y - 1; 163 do_div(x, y); 164 return x * y; 165 } 166 167 static void 168 nouveau_bo_fixup_align(struct nouveau_bo *nvbo, int *align, u64 *size) 169 { 170 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 171 struct nvif_device *device = &drm->client.device; 172 173 if (device->info.family < NV_DEVICE_INFO_V0_TESLA) { 174 if (nvbo->mode) { 175 if (device->info.chipset >= 0x40) { 176 *align = 65536; 177 *size = roundup_64(*size, 64 * nvbo->mode); 178 179 } else if (device->info.chipset >= 0x30) { 180 *align = 32768; 181 *size = roundup_64(*size, 64 * nvbo->mode); 182 183 } else if (device->info.chipset >= 0x20) { 184 *align = 16384; 185 *size = roundup_64(*size, 64 * nvbo->mode); 186 187 } else if (device->info.chipset >= 0x10) { 188 *align = 16384; 189 *size = roundup_64(*size, 32 * nvbo->mode); 190 } 191 } 192 } else { 193 *size = roundup_64(*size, (1 << nvbo->page)); 194 *align = max((1 << nvbo->page), *align); 195 } 196 197 *size = roundup_64(*size, PAGE_SIZE); 198 } 199 200 struct nouveau_bo * 201 nouveau_bo_alloc(struct nouveau_cli *cli, u64 *size, int *align, u32 domain, 202 u32 tile_mode, u32 tile_flags) 203 { 204 struct nouveau_drm *drm = cli->drm; 205 struct nouveau_bo *nvbo; 206 struct nvif_mmu *mmu = &cli->mmu; 207 struct nvif_vmm *vmm = cli->svm.cli ? &cli->svm.vmm : &cli->vmm.vmm; 208 int i, pi = -1; 209 210 if (!*size) { 211 NV_WARN(drm, "skipped size %016llx\n", *size); 212 return ERR_PTR(-EINVAL); 213 } 214 215 nvbo = kzalloc(sizeof(struct nouveau_bo), GFP_KERNEL); 216 if (!nvbo) 217 return ERR_PTR(-ENOMEM); 218 INIT_LIST_HEAD(&nvbo->head); 219 INIT_LIST_HEAD(&nvbo->entry); 220 INIT_LIST_HEAD(&nvbo->vma_list); 221 nvbo->bo.bdev = &drm->ttm.bdev; 222 223 /* This is confusing, and doesn't actually mean we want an uncached 224 * mapping, but is what NOUVEAU_GEM_DOMAIN_COHERENT gets translated 225 * into in nouveau_gem_new(). 226 */ 227 if (domain & NOUVEAU_GEM_DOMAIN_COHERENT) { 228 /* Determine if we can get a cache-coherent map, forcing 229 * uncached mapping if we can't. 230 */ 231 if (!nouveau_drm_use_coherent_gpu_mapping(drm)) 232 nvbo->force_coherent = true; 233 } 234 235 if (cli->device.info.family >= NV_DEVICE_INFO_V0_FERMI) { 236 nvbo->kind = (tile_flags & 0x0000ff00) >> 8; 237 if (!nvif_mmu_kind_valid(mmu, nvbo->kind)) { 238 kfree(nvbo); 239 return ERR_PTR(-EINVAL); 240 } 241 242 nvbo->comp = mmu->kind[nvbo->kind] != nvbo->kind; 243 } else 244 if (cli->device.info.family >= NV_DEVICE_INFO_V0_TESLA) { 245 nvbo->kind = (tile_flags & 0x00007f00) >> 8; 246 nvbo->comp = (tile_flags & 0x00030000) >> 16; 247 if (!nvif_mmu_kind_valid(mmu, nvbo->kind)) { 248 kfree(nvbo); 249 return ERR_PTR(-EINVAL); 250 } 251 } else { 252 nvbo->zeta = (tile_flags & 0x00000007); 253 } 254 nvbo->mode = tile_mode; 255 nvbo->contig = !(tile_flags & NOUVEAU_GEM_TILE_NONCONTIG); 256 257 /* Determine the desirable target GPU page size for the buffer. */ 258 for (i = 0; i < vmm->page_nr; i++) { 259 /* Because we cannot currently allow VMM maps to fail 260 * during buffer migration, we need to determine page 261 * size for the buffer up-front, and pre-allocate its 262 * page tables. 263 * 264 * Skip page sizes that can't support needed domains. 265 */ 266 if (cli->device.info.family > NV_DEVICE_INFO_V0_CURIE && 267 (domain & NOUVEAU_GEM_DOMAIN_VRAM) && !vmm->page[i].vram) 268 continue; 269 if ((domain & NOUVEAU_GEM_DOMAIN_GART) && 270 (!vmm->page[i].host || vmm->page[i].shift > PAGE_SHIFT)) 271 continue; 272 273 /* Select this page size if it's the first that supports 274 * the potential memory domains, or when it's compatible 275 * with the requested compression settings. 276 */ 277 if (pi < 0 || !nvbo->comp || vmm->page[i].comp) 278 pi = i; 279 280 /* Stop once the buffer is larger than the current page size. */ 281 if (*size >= 1ULL << vmm->page[i].shift) 282 break; 283 } 284 285 if (WARN_ON(pi < 0)) { 286 kfree(nvbo); 287 return ERR_PTR(-EINVAL); 288 } 289 290 /* Disable compression if suitable settings couldn't be found. */ 291 if (nvbo->comp && !vmm->page[pi].comp) { 292 if (mmu->object.oclass >= NVIF_CLASS_MMU_GF100) 293 nvbo->kind = mmu->kind[nvbo->kind]; 294 nvbo->comp = 0; 295 } 296 nvbo->page = vmm->page[pi].shift; 297 298 nouveau_bo_fixup_align(nvbo, align, size); 299 300 return nvbo; 301 } 302 303 int 304 nouveau_bo_init(struct nouveau_bo *nvbo, u64 size, int align, u32 domain, 305 struct sg_table *sg, struct dma_resv *robj) 306 { 307 int type = sg ? ttm_bo_type_sg : ttm_bo_type_device; 308 int ret; 309 310 nouveau_bo_placement_set(nvbo, domain, 0); 311 INIT_LIST_HEAD(&nvbo->io_reserve_lru); 312 313 ret = ttm_bo_init_validate(nvbo->bo.bdev, &nvbo->bo, type, 314 &nvbo->placement, align >> PAGE_SHIFT, false, 315 sg, robj, nouveau_bo_del_ttm); 316 if (ret) { 317 /* ttm will call nouveau_bo_del_ttm if it fails.. */ 318 return ret; 319 } 320 321 return 0; 322 } 323 324 int 325 nouveau_bo_new(struct nouveau_cli *cli, u64 size, int align, 326 uint32_t domain, uint32_t tile_mode, uint32_t tile_flags, 327 struct sg_table *sg, struct dma_resv *robj, 328 struct nouveau_bo **pnvbo) 329 { 330 struct nouveau_bo *nvbo; 331 int ret; 332 333 nvbo = nouveau_bo_alloc(cli, &size, &align, domain, tile_mode, 334 tile_flags); 335 if (IS_ERR(nvbo)) 336 return PTR_ERR(nvbo); 337 338 nvbo->bo.base.size = size; 339 dma_resv_init(&nvbo->bo.base._resv); 340 drm_vma_node_reset(&nvbo->bo.base.vma_node); 341 342 ret = nouveau_bo_init(nvbo, size, align, domain, sg, robj); 343 if (ret) 344 return ret; 345 346 *pnvbo = nvbo; 347 return 0; 348 } 349 350 static void 351 set_placement_list(struct ttm_place *pl, unsigned *n, uint32_t domain) 352 { 353 *n = 0; 354 355 if (domain & NOUVEAU_GEM_DOMAIN_VRAM) { 356 pl[*n].mem_type = TTM_PL_VRAM; 357 pl[*n].flags = 0; 358 (*n)++; 359 } 360 if (domain & NOUVEAU_GEM_DOMAIN_GART) { 361 pl[*n].mem_type = TTM_PL_TT; 362 pl[*n].flags = 0; 363 (*n)++; 364 } 365 if (domain & NOUVEAU_GEM_DOMAIN_CPU) { 366 pl[*n].mem_type = TTM_PL_SYSTEM; 367 pl[(*n)++].flags = 0; 368 } 369 } 370 371 static void 372 set_placement_range(struct nouveau_bo *nvbo, uint32_t domain) 373 { 374 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 375 u64 vram_size = drm->client.device.info.ram_size; 376 unsigned i, fpfn, lpfn; 377 378 if (drm->client.device.info.family == NV_DEVICE_INFO_V0_CELSIUS && 379 nvbo->mode && (domain & NOUVEAU_GEM_DOMAIN_VRAM) && 380 nvbo->bo.base.size < vram_size / 4) { 381 /* 382 * Make sure that the color and depth buffers are handled 383 * by independent memory controller units. Up to a 9x 384 * speed up when alpha-blending and depth-test are enabled 385 * at the same time. 386 */ 387 if (nvbo->zeta) { 388 fpfn = (vram_size / 2) >> PAGE_SHIFT; 389 lpfn = ~0; 390 } else { 391 fpfn = 0; 392 lpfn = (vram_size / 2) >> PAGE_SHIFT; 393 } 394 for (i = 0; i < nvbo->placement.num_placement; ++i) { 395 nvbo->placements[i].fpfn = fpfn; 396 nvbo->placements[i].lpfn = lpfn; 397 } 398 for (i = 0; i < nvbo->placement.num_busy_placement; ++i) { 399 nvbo->busy_placements[i].fpfn = fpfn; 400 nvbo->busy_placements[i].lpfn = lpfn; 401 } 402 } 403 } 404 405 void 406 nouveau_bo_placement_set(struct nouveau_bo *nvbo, uint32_t domain, 407 uint32_t busy) 408 { 409 struct ttm_placement *pl = &nvbo->placement; 410 411 pl->placement = nvbo->placements; 412 set_placement_list(nvbo->placements, &pl->num_placement, domain); 413 414 pl->busy_placement = nvbo->busy_placements; 415 set_placement_list(nvbo->busy_placements, &pl->num_busy_placement, 416 domain | busy); 417 418 set_placement_range(nvbo, domain); 419 } 420 421 int 422 nouveau_bo_pin(struct nouveau_bo *nvbo, uint32_t domain, bool contig) 423 { 424 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 425 struct ttm_buffer_object *bo = &nvbo->bo; 426 bool force = false, evict = false; 427 int ret; 428 429 ret = ttm_bo_reserve(bo, false, false, NULL); 430 if (ret) 431 return ret; 432 433 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA && 434 domain == NOUVEAU_GEM_DOMAIN_VRAM && contig) { 435 if (!nvbo->contig) { 436 nvbo->contig = true; 437 force = true; 438 evict = true; 439 } 440 } 441 442 if (nvbo->bo.pin_count) { 443 bool error = evict; 444 445 switch (bo->resource->mem_type) { 446 case TTM_PL_VRAM: 447 error |= !(domain & NOUVEAU_GEM_DOMAIN_VRAM); 448 break; 449 case TTM_PL_TT: 450 error |= !(domain & NOUVEAU_GEM_DOMAIN_GART); 451 break; 452 default: 453 break; 454 } 455 456 if (error) { 457 NV_ERROR(drm, "bo %p pinned elsewhere: " 458 "0x%08x vs 0x%08x\n", bo, 459 bo->resource->mem_type, domain); 460 ret = -EBUSY; 461 } 462 ttm_bo_pin(&nvbo->bo); 463 goto out; 464 } 465 466 if (evict) { 467 nouveau_bo_placement_set(nvbo, NOUVEAU_GEM_DOMAIN_GART, 0); 468 ret = nouveau_bo_validate(nvbo, false, false); 469 if (ret) 470 goto out; 471 } 472 473 nouveau_bo_placement_set(nvbo, domain, 0); 474 ret = nouveau_bo_validate(nvbo, false, false); 475 if (ret) 476 goto out; 477 478 ttm_bo_pin(&nvbo->bo); 479 480 switch (bo->resource->mem_type) { 481 case TTM_PL_VRAM: 482 drm->gem.vram_available -= bo->base.size; 483 break; 484 case TTM_PL_TT: 485 drm->gem.gart_available -= bo->base.size; 486 break; 487 default: 488 break; 489 } 490 491 out: 492 if (force && ret) 493 nvbo->contig = false; 494 ttm_bo_unreserve(bo); 495 return ret; 496 } 497 498 int 499 nouveau_bo_unpin(struct nouveau_bo *nvbo) 500 { 501 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 502 struct ttm_buffer_object *bo = &nvbo->bo; 503 int ret; 504 505 ret = ttm_bo_reserve(bo, false, false, NULL); 506 if (ret) 507 return ret; 508 509 ttm_bo_unpin(&nvbo->bo); 510 if (!nvbo->bo.pin_count) { 511 switch (bo->resource->mem_type) { 512 case TTM_PL_VRAM: 513 drm->gem.vram_available += bo->base.size; 514 break; 515 case TTM_PL_TT: 516 drm->gem.gart_available += bo->base.size; 517 break; 518 default: 519 break; 520 } 521 } 522 523 ttm_bo_unreserve(bo); 524 return 0; 525 } 526 527 int 528 nouveau_bo_map(struct nouveau_bo *nvbo) 529 { 530 int ret; 531 532 ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL); 533 if (ret) 534 return ret; 535 536 ret = ttm_bo_kmap(&nvbo->bo, 0, PFN_UP(nvbo->bo.base.size), &nvbo->kmap); 537 538 ttm_bo_unreserve(&nvbo->bo); 539 return ret; 540 } 541 542 void 543 nouveau_bo_unmap(struct nouveau_bo *nvbo) 544 { 545 if (!nvbo) 546 return; 547 548 ttm_bo_kunmap(&nvbo->kmap); 549 } 550 551 void 552 nouveau_bo_sync_for_device(struct nouveau_bo *nvbo) 553 { 554 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 555 struct ttm_tt *ttm_dma = (struct ttm_tt *)nvbo->bo.ttm; 556 int i, j; 557 558 if (!ttm_dma || !ttm_dma->dma_address) 559 return; 560 if (!ttm_dma->pages) { 561 NV_DEBUG(drm, "ttm_dma 0x%p: pages NULL\n", ttm_dma); 562 return; 563 } 564 565 /* Don't waste time looping if the object is coherent */ 566 if (nvbo->force_coherent) 567 return; 568 569 i = 0; 570 while (i < ttm_dma->num_pages) { 571 struct page *p = ttm_dma->pages[i]; 572 size_t num_pages = 1; 573 574 for (j = i + 1; j < ttm_dma->num_pages; ++j) { 575 if (++p != ttm_dma->pages[j]) 576 break; 577 578 ++num_pages; 579 } 580 dma_sync_single_for_device(drm->dev->dev, 581 ttm_dma->dma_address[i], 582 num_pages * PAGE_SIZE, DMA_TO_DEVICE); 583 i += num_pages; 584 } 585 } 586 587 void 588 nouveau_bo_sync_for_cpu(struct nouveau_bo *nvbo) 589 { 590 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 591 struct ttm_tt *ttm_dma = (struct ttm_tt *)nvbo->bo.ttm; 592 int i, j; 593 594 if (!ttm_dma || !ttm_dma->dma_address) 595 return; 596 if (!ttm_dma->pages) { 597 NV_DEBUG(drm, "ttm_dma 0x%p: pages NULL\n", ttm_dma); 598 return; 599 } 600 601 /* Don't waste time looping if the object is coherent */ 602 if (nvbo->force_coherent) 603 return; 604 605 i = 0; 606 while (i < ttm_dma->num_pages) { 607 struct page *p = ttm_dma->pages[i]; 608 size_t num_pages = 1; 609 610 for (j = i + 1; j < ttm_dma->num_pages; ++j) { 611 if (++p != ttm_dma->pages[j]) 612 break; 613 614 ++num_pages; 615 } 616 617 dma_sync_single_for_cpu(drm->dev->dev, ttm_dma->dma_address[i], 618 num_pages * PAGE_SIZE, DMA_FROM_DEVICE); 619 i += num_pages; 620 } 621 } 622 623 void nouveau_bo_add_io_reserve_lru(struct ttm_buffer_object *bo) 624 { 625 struct nouveau_drm *drm = nouveau_bdev(bo->bdev); 626 struct nouveau_bo *nvbo = nouveau_bo(bo); 627 628 mutex_lock(&drm->ttm.io_reserve_mutex); 629 list_move_tail(&nvbo->io_reserve_lru, &drm->ttm.io_reserve_lru); 630 mutex_unlock(&drm->ttm.io_reserve_mutex); 631 } 632 633 void nouveau_bo_del_io_reserve_lru(struct ttm_buffer_object *bo) 634 { 635 struct nouveau_drm *drm = nouveau_bdev(bo->bdev); 636 struct nouveau_bo *nvbo = nouveau_bo(bo); 637 638 mutex_lock(&drm->ttm.io_reserve_mutex); 639 list_del_init(&nvbo->io_reserve_lru); 640 mutex_unlock(&drm->ttm.io_reserve_mutex); 641 } 642 643 int 644 nouveau_bo_validate(struct nouveau_bo *nvbo, bool interruptible, 645 bool no_wait_gpu) 646 { 647 struct ttm_operation_ctx ctx = { interruptible, no_wait_gpu }; 648 int ret; 649 650 ret = ttm_bo_validate(&nvbo->bo, &nvbo->placement, &ctx); 651 if (ret) 652 return ret; 653 654 nouveau_bo_sync_for_device(nvbo); 655 656 return 0; 657 } 658 659 void 660 nouveau_bo_wr16(struct nouveau_bo *nvbo, unsigned index, u16 val) 661 { 662 bool is_iomem; 663 u16 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem); 664 665 mem += index; 666 667 if (is_iomem) 668 iowrite16_native(val, (void __force __iomem *)mem); 669 else 670 *mem = val; 671 } 672 673 u32 674 nouveau_bo_rd32(struct nouveau_bo *nvbo, unsigned index) 675 { 676 bool is_iomem; 677 u32 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem); 678 679 mem += index; 680 681 if (is_iomem) 682 return ioread32_native((void __force __iomem *)mem); 683 else 684 return *mem; 685 } 686 687 void 688 nouveau_bo_wr32(struct nouveau_bo *nvbo, unsigned index, u32 val) 689 { 690 bool is_iomem; 691 u32 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem); 692 693 mem += index; 694 695 if (is_iomem) 696 iowrite32_native(val, (void __force __iomem *)mem); 697 else 698 *mem = val; 699 } 700 701 static struct ttm_tt * 702 nouveau_ttm_tt_create(struct ttm_buffer_object *bo, uint32_t page_flags) 703 { 704 #if IS_ENABLED(CONFIG_AGP) 705 struct nouveau_drm *drm = nouveau_bdev(bo->bdev); 706 707 if (drm->agp.bridge) { 708 return ttm_agp_tt_create(bo, drm->agp.bridge, page_flags); 709 } 710 #endif 711 712 return nouveau_sgdma_create_ttm(bo, page_flags); 713 } 714 715 static int 716 nouveau_ttm_tt_bind(struct ttm_device *bdev, struct ttm_tt *ttm, 717 struct ttm_resource *reg) 718 { 719 #if IS_ENABLED(CONFIG_AGP) 720 struct nouveau_drm *drm = nouveau_bdev(bdev); 721 #endif 722 if (!reg) 723 return -EINVAL; 724 #if IS_ENABLED(CONFIG_AGP) 725 if (drm->agp.bridge) 726 return ttm_agp_bind(ttm, reg); 727 #endif 728 return nouveau_sgdma_bind(bdev, ttm, reg); 729 } 730 731 static void 732 nouveau_ttm_tt_unbind(struct ttm_device *bdev, struct ttm_tt *ttm) 733 { 734 #if IS_ENABLED(CONFIG_AGP) 735 struct nouveau_drm *drm = nouveau_bdev(bdev); 736 737 if (drm->agp.bridge) { 738 ttm_agp_unbind(ttm); 739 return; 740 } 741 #endif 742 nouveau_sgdma_unbind(bdev, ttm); 743 } 744 745 static void 746 nouveau_bo_evict_flags(struct ttm_buffer_object *bo, struct ttm_placement *pl) 747 { 748 struct nouveau_bo *nvbo = nouveau_bo(bo); 749 750 switch (bo->resource->mem_type) { 751 case TTM_PL_VRAM: 752 nouveau_bo_placement_set(nvbo, NOUVEAU_GEM_DOMAIN_GART, 753 NOUVEAU_GEM_DOMAIN_CPU); 754 break; 755 default: 756 nouveau_bo_placement_set(nvbo, NOUVEAU_GEM_DOMAIN_CPU, 0); 757 break; 758 } 759 760 *pl = nvbo->placement; 761 } 762 763 static int 764 nouveau_bo_move_prep(struct nouveau_drm *drm, struct ttm_buffer_object *bo, 765 struct ttm_resource *reg) 766 { 767 struct nouveau_mem *old_mem = nouveau_mem(bo->resource); 768 struct nouveau_mem *new_mem = nouveau_mem(reg); 769 struct nvif_vmm *vmm = &drm->client.vmm.vmm; 770 int ret; 771 772 ret = nvif_vmm_get(vmm, LAZY, false, old_mem->mem.page, 0, 773 old_mem->mem.size, &old_mem->vma[0]); 774 if (ret) 775 return ret; 776 777 ret = nvif_vmm_get(vmm, LAZY, false, new_mem->mem.page, 0, 778 new_mem->mem.size, &old_mem->vma[1]); 779 if (ret) 780 goto done; 781 782 ret = nouveau_mem_map(old_mem, vmm, &old_mem->vma[0]); 783 if (ret) 784 goto done; 785 786 ret = nouveau_mem_map(new_mem, vmm, &old_mem->vma[1]); 787 done: 788 if (ret) { 789 nvif_vmm_put(vmm, &old_mem->vma[1]); 790 nvif_vmm_put(vmm, &old_mem->vma[0]); 791 } 792 return 0; 793 } 794 795 static int 796 nouveau_bo_move_m2mf(struct ttm_buffer_object *bo, int evict, 797 struct ttm_operation_ctx *ctx, 798 struct ttm_resource *new_reg) 799 { 800 struct nouveau_drm *drm = nouveau_bdev(bo->bdev); 801 struct nouveau_channel *chan = drm->ttm.chan; 802 struct nouveau_cli *cli = (void *)chan->user.client; 803 struct nouveau_fence *fence; 804 int ret; 805 806 /* create temporary vmas for the transfer and attach them to the 807 * old nvkm_mem node, these will get cleaned up after ttm has 808 * destroyed the ttm_resource 809 */ 810 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA) { 811 ret = nouveau_bo_move_prep(drm, bo, new_reg); 812 if (ret) 813 return ret; 814 } 815 816 if (drm_drv_uses_atomic_modeset(drm->dev)) 817 mutex_lock(&cli->mutex); 818 else 819 mutex_lock_nested(&cli->mutex, SINGLE_DEPTH_NESTING); 820 ret = nouveau_fence_sync(nouveau_bo(bo), chan, true, ctx->interruptible); 821 if (ret == 0) { 822 ret = drm->ttm.move(chan, bo, bo->resource, new_reg); 823 if (ret == 0) { 824 ret = nouveau_fence_new(chan, false, &fence); 825 if (ret == 0) { 826 /* TODO: figure out a better solution here 827 * 828 * wait on the fence here explicitly as going through 829 * ttm_bo_move_accel_cleanup somehow doesn't seem to do it. 830 * 831 * Without this the operation can timeout and we'll fallback to a 832 * software copy, which might take several minutes to finish. 833 */ 834 nouveau_fence_wait(fence, false, false); 835 ret = ttm_bo_move_accel_cleanup(bo, 836 &fence->base, 837 evict, false, 838 new_reg); 839 nouveau_fence_unref(&fence); 840 } 841 } 842 } 843 mutex_unlock(&cli->mutex); 844 return ret; 845 } 846 847 void 848 nouveau_bo_move_init(struct nouveau_drm *drm) 849 { 850 static const struct _method_table { 851 const char *name; 852 int engine; 853 s32 oclass; 854 int (*exec)(struct nouveau_channel *, 855 struct ttm_buffer_object *, 856 struct ttm_resource *, struct ttm_resource *); 857 int (*init)(struct nouveau_channel *, u32 handle); 858 } _methods[] = { 859 { "COPY", 4, 0xc7b5, nve0_bo_move_copy, nve0_bo_move_init }, 860 { "GRCE", 0, 0xc7b5, nve0_bo_move_copy, nvc0_bo_move_init }, 861 { "COPY", 4, 0xc6b5, nve0_bo_move_copy, nve0_bo_move_init }, 862 { "GRCE", 0, 0xc6b5, nve0_bo_move_copy, nvc0_bo_move_init }, 863 { "COPY", 4, 0xc5b5, nve0_bo_move_copy, nve0_bo_move_init }, 864 { "GRCE", 0, 0xc5b5, nve0_bo_move_copy, nvc0_bo_move_init }, 865 { "COPY", 4, 0xc3b5, nve0_bo_move_copy, nve0_bo_move_init }, 866 { "GRCE", 0, 0xc3b5, nve0_bo_move_copy, nvc0_bo_move_init }, 867 { "COPY", 4, 0xc1b5, nve0_bo_move_copy, nve0_bo_move_init }, 868 { "GRCE", 0, 0xc1b5, nve0_bo_move_copy, nvc0_bo_move_init }, 869 { "COPY", 4, 0xc0b5, nve0_bo_move_copy, nve0_bo_move_init }, 870 { "GRCE", 0, 0xc0b5, nve0_bo_move_copy, nvc0_bo_move_init }, 871 { "COPY", 4, 0xb0b5, nve0_bo_move_copy, nve0_bo_move_init }, 872 { "GRCE", 0, 0xb0b5, nve0_bo_move_copy, nvc0_bo_move_init }, 873 { "COPY", 4, 0xa0b5, nve0_bo_move_copy, nve0_bo_move_init }, 874 { "GRCE", 0, 0xa0b5, nve0_bo_move_copy, nvc0_bo_move_init }, 875 { "COPY1", 5, 0x90b8, nvc0_bo_move_copy, nvc0_bo_move_init }, 876 { "COPY0", 4, 0x90b5, nvc0_bo_move_copy, nvc0_bo_move_init }, 877 { "COPY", 0, 0x85b5, nva3_bo_move_copy, nv50_bo_move_init }, 878 { "CRYPT", 0, 0x74c1, nv84_bo_move_exec, nv50_bo_move_init }, 879 { "M2MF", 0, 0x9039, nvc0_bo_move_m2mf, nvc0_bo_move_init }, 880 { "M2MF", 0, 0x5039, nv50_bo_move_m2mf, nv50_bo_move_init }, 881 { "M2MF", 0, 0x0039, nv04_bo_move_m2mf, nv04_bo_move_init }, 882 {}, 883 }; 884 const struct _method_table *mthd = _methods; 885 const char *name = "CPU"; 886 int ret; 887 888 do { 889 struct nouveau_channel *chan; 890 891 if (mthd->engine) 892 chan = drm->cechan; 893 else 894 chan = drm->channel; 895 if (chan == NULL) 896 continue; 897 898 ret = nvif_object_ctor(&chan->user, "ttmBoMove", 899 mthd->oclass | (mthd->engine << 16), 900 mthd->oclass, NULL, 0, 901 &drm->ttm.copy); 902 if (ret == 0) { 903 ret = mthd->init(chan, drm->ttm.copy.handle); 904 if (ret) { 905 nvif_object_dtor(&drm->ttm.copy); 906 continue; 907 } 908 909 drm->ttm.move = mthd->exec; 910 drm->ttm.chan = chan; 911 name = mthd->name; 912 break; 913 } 914 } while ((++mthd)->exec); 915 916 NV_INFO(drm, "MM: using %s for buffer copies\n", name); 917 } 918 919 static void nouveau_bo_move_ntfy(struct ttm_buffer_object *bo, 920 struct ttm_resource *new_reg) 921 { 922 struct nouveau_mem *mem = new_reg ? nouveau_mem(new_reg) : NULL; 923 struct nouveau_bo *nvbo = nouveau_bo(bo); 924 struct nouveau_vma *vma; 925 926 /* ttm can now (stupidly) pass the driver bos it didn't create... */ 927 if (bo->destroy != nouveau_bo_del_ttm) 928 return; 929 930 nouveau_bo_del_io_reserve_lru(bo); 931 932 if (mem && new_reg->mem_type != TTM_PL_SYSTEM && 933 mem->mem.page == nvbo->page) { 934 list_for_each_entry(vma, &nvbo->vma_list, head) { 935 nouveau_vma_map(vma, mem); 936 } 937 } else { 938 list_for_each_entry(vma, &nvbo->vma_list, head) { 939 WARN_ON(ttm_bo_wait(bo, false, false)); 940 nouveau_vma_unmap(vma); 941 } 942 } 943 944 if (new_reg) 945 nvbo->offset = (new_reg->start << PAGE_SHIFT); 946 947 } 948 949 static int 950 nouveau_bo_vm_bind(struct ttm_buffer_object *bo, struct ttm_resource *new_reg, 951 struct nouveau_drm_tile **new_tile) 952 { 953 struct nouveau_drm *drm = nouveau_bdev(bo->bdev); 954 struct drm_device *dev = drm->dev; 955 struct nouveau_bo *nvbo = nouveau_bo(bo); 956 u64 offset = new_reg->start << PAGE_SHIFT; 957 958 *new_tile = NULL; 959 if (new_reg->mem_type != TTM_PL_VRAM) 960 return 0; 961 962 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_CELSIUS) { 963 *new_tile = nv10_bo_set_tiling(dev, offset, bo->base.size, 964 nvbo->mode, nvbo->zeta); 965 } 966 967 return 0; 968 } 969 970 static void 971 nouveau_bo_vm_cleanup(struct ttm_buffer_object *bo, 972 struct nouveau_drm_tile *new_tile, 973 struct nouveau_drm_tile **old_tile) 974 { 975 struct nouveau_drm *drm = nouveau_bdev(bo->bdev); 976 struct drm_device *dev = drm->dev; 977 struct dma_fence *fence; 978 int ret; 979 980 ret = dma_resv_get_singleton(bo->base.resv, DMA_RESV_USAGE_WRITE, 981 &fence); 982 if (ret) 983 dma_resv_wait_timeout(bo->base.resv, DMA_RESV_USAGE_WRITE, 984 false, MAX_SCHEDULE_TIMEOUT); 985 986 nv10_bo_put_tile_region(dev, *old_tile, fence); 987 *old_tile = new_tile; 988 } 989 990 static int 991 nouveau_bo_move(struct ttm_buffer_object *bo, bool evict, 992 struct ttm_operation_ctx *ctx, 993 struct ttm_resource *new_reg, 994 struct ttm_place *hop) 995 { 996 struct nouveau_drm *drm = nouveau_bdev(bo->bdev); 997 struct nouveau_bo *nvbo = nouveau_bo(bo); 998 struct ttm_resource *old_reg = bo->resource; 999 struct nouveau_drm_tile *new_tile = NULL; 1000 int ret = 0; 1001 1002 1003 if (new_reg->mem_type == TTM_PL_TT) { 1004 ret = nouveau_ttm_tt_bind(bo->bdev, bo->ttm, new_reg); 1005 if (ret) 1006 return ret; 1007 } 1008 1009 nouveau_bo_move_ntfy(bo, new_reg); 1010 ret = ttm_bo_wait_ctx(bo, ctx); 1011 if (ret) 1012 goto out_ntfy; 1013 1014 if (nvbo->bo.pin_count) 1015 NV_WARN(drm, "Moving pinned object %p!\n", nvbo); 1016 1017 if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA) { 1018 ret = nouveau_bo_vm_bind(bo, new_reg, &new_tile); 1019 if (ret) 1020 goto out_ntfy; 1021 } 1022 1023 /* Fake bo copy. */ 1024 if (!old_reg || (old_reg->mem_type == TTM_PL_SYSTEM && 1025 !bo->ttm)) { 1026 ttm_bo_move_null(bo, new_reg); 1027 goto out; 1028 } 1029 1030 if (old_reg->mem_type == TTM_PL_SYSTEM && 1031 new_reg->mem_type == TTM_PL_TT) { 1032 ttm_bo_move_null(bo, new_reg); 1033 goto out; 1034 } 1035 1036 if (old_reg->mem_type == TTM_PL_TT && 1037 new_reg->mem_type == TTM_PL_SYSTEM) { 1038 nouveau_ttm_tt_unbind(bo->bdev, bo->ttm); 1039 ttm_resource_free(bo, &bo->resource); 1040 ttm_bo_assign_mem(bo, new_reg); 1041 goto out; 1042 } 1043 1044 /* Hardware assisted copy. */ 1045 if (drm->ttm.move) { 1046 if ((old_reg->mem_type == TTM_PL_SYSTEM && 1047 new_reg->mem_type == TTM_PL_VRAM) || 1048 (old_reg->mem_type == TTM_PL_VRAM && 1049 new_reg->mem_type == TTM_PL_SYSTEM)) { 1050 hop->fpfn = 0; 1051 hop->lpfn = 0; 1052 hop->mem_type = TTM_PL_TT; 1053 hop->flags = 0; 1054 return -EMULTIHOP; 1055 } 1056 ret = nouveau_bo_move_m2mf(bo, evict, ctx, 1057 new_reg); 1058 } else 1059 ret = -ENODEV; 1060 1061 if (ret) { 1062 /* Fallback to software copy. */ 1063 ret = ttm_bo_move_memcpy(bo, ctx, new_reg); 1064 } 1065 1066 out: 1067 if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA) { 1068 if (ret) 1069 nouveau_bo_vm_cleanup(bo, NULL, &new_tile); 1070 else 1071 nouveau_bo_vm_cleanup(bo, new_tile, &nvbo->tile); 1072 } 1073 out_ntfy: 1074 if (ret) { 1075 nouveau_bo_move_ntfy(bo, bo->resource); 1076 } 1077 return ret; 1078 } 1079 1080 static void 1081 nouveau_ttm_io_mem_free_locked(struct nouveau_drm *drm, 1082 struct ttm_resource *reg) 1083 { 1084 struct nouveau_mem *mem = nouveau_mem(reg); 1085 1086 if (drm->client.mem->oclass >= NVIF_CLASS_MEM_NV50) { 1087 switch (reg->mem_type) { 1088 case TTM_PL_TT: 1089 if (mem->kind) 1090 nvif_object_unmap_handle(&mem->mem.object); 1091 break; 1092 case TTM_PL_VRAM: 1093 nvif_object_unmap_handle(&mem->mem.object); 1094 break; 1095 default: 1096 break; 1097 } 1098 } 1099 } 1100 1101 static int 1102 nouveau_ttm_io_mem_reserve(struct ttm_device *bdev, struct ttm_resource *reg) 1103 { 1104 struct nouveau_drm *drm = nouveau_bdev(bdev); 1105 struct nvkm_device *device = nvxx_device(&drm->client.device); 1106 struct nouveau_mem *mem = nouveau_mem(reg); 1107 struct nvif_mmu *mmu = &drm->client.mmu; 1108 int ret; 1109 1110 mutex_lock(&drm->ttm.io_reserve_mutex); 1111 retry: 1112 switch (reg->mem_type) { 1113 case TTM_PL_SYSTEM: 1114 /* System memory */ 1115 ret = 0; 1116 goto out; 1117 case TTM_PL_TT: 1118 #if IS_ENABLED(CONFIG_AGP) 1119 if (drm->agp.bridge) { 1120 reg->bus.offset = (reg->start << PAGE_SHIFT) + 1121 drm->agp.base; 1122 reg->bus.is_iomem = !drm->agp.cma; 1123 reg->bus.caching = ttm_write_combined; 1124 } 1125 #endif 1126 if (drm->client.mem->oclass < NVIF_CLASS_MEM_NV50 || 1127 !mem->kind) { 1128 /* untiled */ 1129 ret = 0; 1130 break; 1131 } 1132 fallthrough; /* tiled memory */ 1133 case TTM_PL_VRAM: 1134 reg->bus.offset = (reg->start << PAGE_SHIFT) + 1135 device->func->resource_addr(device, 1); 1136 reg->bus.is_iomem = true; 1137 1138 /* Some BARs do not support being ioremapped WC */ 1139 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA && 1140 mmu->type[drm->ttm.type_vram].type & NVIF_MEM_UNCACHED) 1141 reg->bus.caching = ttm_uncached; 1142 else 1143 reg->bus.caching = ttm_write_combined; 1144 1145 if (drm->client.mem->oclass >= NVIF_CLASS_MEM_NV50) { 1146 union { 1147 struct nv50_mem_map_v0 nv50; 1148 struct gf100_mem_map_v0 gf100; 1149 } args; 1150 u64 handle, length; 1151 u32 argc = 0; 1152 1153 switch (mem->mem.object.oclass) { 1154 case NVIF_CLASS_MEM_NV50: 1155 args.nv50.version = 0; 1156 args.nv50.ro = 0; 1157 args.nv50.kind = mem->kind; 1158 args.nv50.comp = mem->comp; 1159 argc = sizeof(args.nv50); 1160 break; 1161 case NVIF_CLASS_MEM_GF100: 1162 args.gf100.version = 0; 1163 args.gf100.ro = 0; 1164 args.gf100.kind = mem->kind; 1165 argc = sizeof(args.gf100); 1166 break; 1167 default: 1168 WARN_ON(1); 1169 break; 1170 } 1171 1172 ret = nvif_object_map_handle(&mem->mem.object, 1173 &args, argc, 1174 &handle, &length); 1175 if (ret != 1) { 1176 if (WARN_ON(ret == 0)) 1177 ret = -EINVAL; 1178 goto out; 1179 } 1180 1181 reg->bus.offset = handle; 1182 } 1183 ret = 0; 1184 break; 1185 default: 1186 ret = -EINVAL; 1187 } 1188 1189 out: 1190 if (ret == -ENOSPC) { 1191 struct nouveau_bo *nvbo; 1192 1193 nvbo = list_first_entry_or_null(&drm->ttm.io_reserve_lru, 1194 typeof(*nvbo), 1195 io_reserve_lru); 1196 if (nvbo) { 1197 list_del_init(&nvbo->io_reserve_lru); 1198 drm_vma_node_unmap(&nvbo->bo.base.vma_node, 1199 bdev->dev_mapping); 1200 nouveau_ttm_io_mem_free_locked(drm, nvbo->bo.resource); 1201 goto retry; 1202 } 1203 1204 } 1205 mutex_unlock(&drm->ttm.io_reserve_mutex); 1206 return ret; 1207 } 1208 1209 static void 1210 nouveau_ttm_io_mem_free(struct ttm_device *bdev, struct ttm_resource *reg) 1211 { 1212 struct nouveau_drm *drm = nouveau_bdev(bdev); 1213 1214 mutex_lock(&drm->ttm.io_reserve_mutex); 1215 nouveau_ttm_io_mem_free_locked(drm, reg); 1216 mutex_unlock(&drm->ttm.io_reserve_mutex); 1217 } 1218 1219 vm_fault_t nouveau_ttm_fault_reserve_notify(struct ttm_buffer_object *bo) 1220 { 1221 struct nouveau_drm *drm = nouveau_bdev(bo->bdev); 1222 struct nouveau_bo *nvbo = nouveau_bo(bo); 1223 struct nvkm_device *device = nvxx_device(&drm->client.device); 1224 u32 mappable = device->func->resource_size(device, 1) >> PAGE_SHIFT; 1225 int i, ret; 1226 1227 /* as long as the bo isn't in vram, and isn't tiled, we've got 1228 * nothing to do here. 1229 */ 1230 if (bo->resource->mem_type != TTM_PL_VRAM) { 1231 if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA || 1232 !nvbo->kind) 1233 return 0; 1234 1235 if (bo->resource->mem_type != TTM_PL_SYSTEM) 1236 return 0; 1237 1238 nouveau_bo_placement_set(nvbo, NOUVEAU_GEM_DOMAIN_GART, 0); 1239 1240 } else { 1241 /* make sure bo is in mappable vram */ 1242 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA || 1243 bo->resource->start + PFN_UP(bo->resource->size) < mappable) 1244 return 0; 1245 1246 for (i = 0; i < nvbo->placement.num_placement; ++i) { 1247 nvbo->placements[i].fpfn = 0; 1248 nvbo->placements[i].lpfn = mappable; 1249 } 1250 1251 for (i = 0; i < nvbo->placement.num_busy_placement; ++i) { 1252 nvbo->busy_placements[i].fpfn = 0; 1253 nvbo->busy_placements[i].lpfn = mappable; 1254 } 1255 1256 nouveau_bo_placement_set(nvbo, NOUVEAU_GEM_DOMAIN_VRAM, 0); 1257 } 1258 1259 ret = nouveau_bo_validate(nvbo, false, false); 1260 if (unlikely(ret == -EBUSY || ret == -ERESTARTSYS)) 1261 return VM_FAULT_NOPAGE; 1262 else if (unlikely(ret)) 1263 return VM_FAULT_SIGBUS; 1264 1265 ttm_bo_move_to_lru_tail_unlocked(bo); 1266 return 0; 1267 } 1268 1269 static int 1270 nouveau_ttm_tt_populate(struct ttm_device *bdev, 1271 struct ttm_tt *ttm, struct ttm_operation_ctx *ctx) 1272 { 1273 struct ttm_tt *ttm_dma = (void *)ttm; 1274 struct nouveau_drm *drm; 1275 bool slave = !!(ttm->page_flags & TTM_TT_FLAG_EXTERNAL); 1276 1277 if (ttm_tt_is_populated(ttm)) 1278 return 0; 1279 1280 if (slave && ttm->sg) { 1281 drm_prime_sg_to_dma_addr_array(ttm->sg, ttm_dma->dma_address, 1282 ttm->num_pages); 1283 return 0; 1284 } 1285 1286 drm = nouveau_bdev(bdev); 1287 1288 return ttm_pool_alloc(&drm->ttm.bdev.pool, ttm, ctx); 1289 } 1290 1291 static void 1292 nouveau_ttm_tt_unpopulate(struct ttm_device *bdev, 1293 struct ttm_tt *ttm) 1294 { 1295 struct nouveau_drm *drm; 1296 bool slave = !!(ttm->page_flags & TTM_TT_FLAG_EXTERNAL); 1297 1298 if (slave) 1299 return; 1300 1301 nouveau_ttm_tt_unbind(bdev, ttm); 1302 1303 drm = nouveau_bdev(bdev); 1304 1305 return ttm_pool_free(&drm->ttm.bdev.pool, ttm); 1306 } 1307 1308 static void 1309 nouveau_ttm_tt_destroy(struct ttm_device *bdev, 1310 struct ttm_tt *ttm) 1311 { 1312 #if IS_ENABLED(CONFIG_AGP) 1313 struct nouveau_drm *drm = nouveau_bdev(bdev); 1314 if (drm->agp.bridge) { 1315 ttm_agp_destroy(ttm); 1316 return; 1317 } 1318 #endif 1319 nouveau_sgdma_destroy(bdev, ttm); 1320 } 1321 1322 void 1323 nouveau_bo_fence(struct nouveau_bo *nvbo, struct nouveau_fence *fence, bool exclusive) 1324 { 1325 struct dma_resv *resv = nvbo->bo.base.resv; 1326 1327 if (!fence) 1328 return; 1329 1330 dma_resv_add_fence(resv, &fence->base, exclusive ? 1331 DMA_RESV_USAGE_WRITE : DMA_RESV_USAGE_READ); 1332 } 1333 1334 static void 1335 nouveau_bo_delete_mem_notify(struct ttm_buffer_object *bo) 1336 { 1337 nouveau_bo_move_ntfy(bo, NULL); 1338 } 1339 1340 struct ttm_device_funcs nouveau_bo_driver = { 1341 .ttm_tt_create = &nouveau_ttm_tt_create, 1342 .ttm_tt_populate = &nouveau_ttm_tt_populate, 1343 .ttm_tt_unpopulate = &nouveau_ttm_tt_unpopulate, 1344 .ttm_tt_destroy = &nouveau_ttm_tt_destroy, 1345 .eviction_valuable = ttm_bo_eviction_valuable, 1346 .evict_flags = nouveau_bo_evict_flags, 1347 .delete_mem_notify = nouveau_bo_delete_mem_notify, 1348 .move = nouveau_bo_move, 1349 .io_mem_reserve = &nouveau_ttm_io_mem_reserve, 1350 .io_mem_free = &nouveau_ttm_io_mem_free, 1351 }; 1352