1 /* 2 * Copyright 2007 Dave Airlied 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice (including the next 13 * paragraph) shall be included in all copies or substantial portions of the 14 * Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 * OTHER DEALINGS IN THE SOFTWARE. 23 */ 24 /* 25 * Authors: Dave Airlied <airlied@linux.ie> 26 * Ben Skeggs <darktama@iinet.net.au> 27 * Jeremy Kolb <jkolb@brandeis.edu> 28 */ 29 30 #include <linux/dma-mapping.h> 31 #include <drm/ttm/ttm_tt.h> 32 33 #include "nouveau_drv.h" 34 #include "nouveau_chan.h" 35 #include "nouveau_fence.h" 36 37 #include "nouveau_bo.h" 38 #include "nouveau_ttm.h" 39 #include "nouveau_gem.h" 40 #include "nouveau_mem.h" 41 #include "nouveau_vmm.h" 42 43 #include <nvif/class.h> 44 #include <nvif/if500b.h> 45 #include <nvif/if900b.h> 46 47 static int nouveau_ttm_tt_bind(struct ttm_device *bdev, struct ttm_tt *ttm, 48 struct ttm_resource *reg); 49 static void nouveau_ttm_tt_unbind(struct ttm_device *bdev, struct ttm_tt *ttm); 50 51 /* 52 * NV10-NV40 tiling helpers 53 */ 54 55 static void 56 nv10_bo_update_tile_region(struct drm_device *dev, struct nouveau_drm_tile *reg, 57 u32 addr, u32 size, u32 pitch, u32 flags) 58 { 59 struct nouveau_drm *drm = nouveau_drm(dev); 60 int i = reg - drm->tile.reg; 61 struct nvkm_fb *fb = nvxx_fb(drm); 62 struct nvkm_fb_tile *tile = &fb->tile.region[i]; 63 64 nouveau_fence_unref(®->fence); 65 66 if (tile->pitch) 67 nvkm_fb_tile_fini(fb, i, tile); 68 69 if (pitch) 70 nvkm_fb_tile_init(fb, i, addr, size, pitch, flags, tile); 71 72 nvkm_fb_tile_prog(fb, i, tile); 73 } 74 75 static struct nouveau_drm_tile * 76 nv10_bo_get_tile_region(struct drm_device *dev, int i) 77 { 78 struct nouveau_drm *drm = nouveau_drm(dev); 79 struct nouveau_drm_tile *tile = &drm->tile.reg[i]; 80 81 spin_lock(&drm->tile.lock); 82 83 if (!tile->used && 84 (!tile->fence || nouveau_fence_done(tile->fence))) 85 tile->used = true; 86 else 87 tile = NULL; 88 89 spin_unlock(&drm->tile.lock); 90 return tile; 91 } 92 93 static void 94 nv10_bo_put_tile_region(struct drm_device *dev, struct nouveau_drm_tile *tile, 95 struct dma_fence *fence) 96 { 97 struct nouveau_drm *drm = nouveau_drm(dev); 98 99 if (tile) { 100 spin_lock(&drm->tile.lock); 101 tile->fence = (struct nouveau_fence *)dma_fence_get(fence); 102 tile->used = false; 103 spin_unlock(&drm->tile.lock); 104 } 105 } 106 107 static struct nouveau_drm_tile * 108 nv10_bo_set_tiling(struct drm_device *dev, u32 addr, 109 u32 size, u32 pitch, u32 zeta) 110 { 111 struct nouveau_drm *drm = nouveau_drm(dev); 112 struct nvkm_fb *fb = nvxx_fb(drm); 113 struct nouveau_drm_tile *tile, *found = NULL; 114 int i; 115 116 for (i = 0; i < fb->tile.regions; i++) { 117 tile = nv10_bo_get_tile_region(dev, i); 118 119 if (pitch && !found) { 120 found = tile; 121 continue; 122 123 } else if (tile && fb->tile.region[i].pitch) { 124 /* Kill an unused tile region. */ 125 nv10_bo_update_tile_region(dev, tile, 0, 0, 0, 0); 126 } 127 128 nv10_bo_put_tile_region(dev, tile, NULL); 129 } 130 131 if (found) 132 nv10_bo_update_tile_region(dev, found, addr, size, pitch, zeta); 133 return found; 134 } 135 136 static void 137 nouveau_bo_del_ttm(struct ttm_buffer_object *bo) 138 { 139 struct nouveau_drm *drm = nouveau_bdev(bo->bdev); 140 struct drm_device *dev = drm->dev; 141 struct nouveau_bo *nvbo = nouveau_bo(bo); 142 143 WARN_ON(nvbo->bo.pin_count > 0); 144 nouveau_bo_del_io_reserve_lru(bo); 145 nv10_bo_put_tile_region(dev, nvbo->tile, NULL); 146 147 if (bo->base.import_attach) 148 drm_prime_gem_destroy(&bo->base, bo->sg); 149 150 /* 151 * If nouveau_bo_new() allocated this buffer, the GEM object was never 152 * initialized, so don't attempt to release it. 153 */ 154 if (bo->base.dev) { 155 /* Gem objects not being shared with other VMs get their 156 * dma_resv from a root GEM object. 157 */ 158 if (nvbo->no_share) 159 drm_gem_object_put(nvbo->r_obj); 160 161 drm_gem_object_release(&bo->base); 162 } else { 163 dma_resv_fini(&bo->base._resv); 164 } 165 166 kfree(nvbo); 167 } 168 169 static inline u64 170 roundup_64(u64 x, u32 y) 171 { 172 x += y - 1; 173 do_div(x, y); 174 return x * y; 175 } 176 177 static void 178 nouveau_bo_fixup_align(struct nouveau_bo *nvbo, int *align, u64 *size) 179 { 180 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 181 struct nvif_device *device = &drm->client.device; 182 183 if (device->info.family < NV_DEVICE_INFO_V0_TESLA) { 184 if (nvbo->mode) { 185 if (device->info.chipset >= 0x40) { 186 *align = 65536; 187 *size = roundup_64(*size, 64 * nvbo->mode); 188 189 } else if (device->info.chipset >= 0x30) { 190 *align = 32768; 191 *size = roundup_64(*size, 64 * nvbo->mode); 192 193 } else if (device->info.chipset >= 0x20) { 194 *align = 16384; 195 *size = roundup_64(*size, 64 * nvbo->mode); 196 197 } else if (device->info.chipset >= 0x10) { 198 *align = 16384; 199 *size = roundup_64(*size, 32 * nvbo->mode); 200 } 201 } 202 } else { 203 *size = roundup_64(*size, (1 << nvbo->page)); 204 *align = max((1 << nvbo->page), *align); 205 } 206 207 *size = roundup_64(*size, PAGE_SIZE); 208 } 209 210 struct nouveau_bo * 211 nouveau_bo_alloc(struct nouveau_cli *cli, u64 *size, int *align, u32 domain, 212 u32 tile_mode, u32 tile_flags, bool internal) 213 { 214 struct nouveau_drm *drm = cli->drm; 215 struct nouveau_bo *nvbo; 216 struct nvif_mmu *mmu = &cli->mmu; 217 struct nvif_vmm *vmm = &nouveau_cli_vmm(cli)->vmm; 218 int i, pi = -1; 219 220 if (!*size) { 221 NV_WARN(drm, "skipped size %016llx\n", *size); 222 return ERR_PTR(-EINVAL); 223 } 224 225 nvbo = kzalloc(sizeof(struct nouveau_bo), GFP_KERNEL); 226 if (!nvbo) 227 return ERR_PTR(-ENOMEM); 228 229 INIT_LIST_HEAD(&nvbo->head); 230 INIT_LIST_HEAD(&nvbo->entry); 231 INIT_LIST_HEAD(&nvbo->vma_list); 232 nvbo->bo.bdev = &drm->ttm.bdev; 233 234 /* This is confusing, and doesn't actually mean we want an uncached 235 * mapping, but is what NOUVEAU_GEM_DOMAIN_COHERENT gets translated 236 * into in nouveau_gem_new(). 237 */ 238 if (domain & NOUVEAU_GEM_DOMAIN_COHERENT) { 239 /* Determine if we can get a cache-coherent map, forcing 240 * uncached mapping if we can't. 241 */ 242 if (!nouveau_drm_use_coherent_gpu_mapping(drm)) 243 nvbo->force_coherent = true; 244 } 245 246 nvbo->contig = !(tile_flags & NOUVEAU_GEM_TILE_NONCONTIG); 247 248 if (cli->device.info.family >= NV_DEVICE_INFO_V0_FERMI) { 249 nvbo->kind = (tile_flags & 0x0000ff00) >> 8; 250 if (!nvif_mmu_kind_valid(mmu, nvbo->kind)) { 251 kfree(nvbo); 252 return ERR_PTR(-EINVAL); 253 } 254 255 nvbo->comp = mmu->kind[nvbo->kind] != nvbo->kind; 256 } else if (cli->device.info.family >= NV_DEVICE_INFO_V0_TESLA) { 257 nvbo->kind = (tile_flags & 0x00007f00) >> 8; 258 nvbo->comp = (tile_flags & 0x00030000) >> 16; 259 if (!nvif_mmu_kind_valid(mmu, nvbo->kind)) { 260 kfree(nvbo); 261 return ERR_PTR(-EINVAL); 262 } 263 } else { 264 nvbo->zeta = (tile_flags & 0x00000007); 265 } 266 nvbo->mode = tile_mode; 267 268 if (!nouveau_cli_uvmm(cli) || internal) { 269 /* Determine the desirable target GPU page size for the buffer. */ 270 for (i = 0; i < vmm->page_nr; i++) { 271 /* Because we cannot currently allow VMM maps to fail 272 * during buffer migration, we need to determine page 273 * size for the buffer up-front, and pre-allocate its 274 * page tables. 275 * 276 * Skip page sizes that can't support needed domains. 277 */ 278 if (cli->device.info.family > NV_DEVICE_INFO_V0_CURIE && 279 (domain & NOUVEAU_GEM_DOMAIN_VRAM) && !vmm->page[i].vram) 280 continue; 281 if ((domain & NOUVEAU_GEM_DOMAIN_GART) && 282 (!vmm->page[i].host || vmm->page[i].shift > PAGE_SHIFT)) 283 continue; 284 285 /* Select this page size if it's the first that supports 286 * the potential memory domains, or when it's compatible 287 * with the requested compression settings. 288 */ 289 if (pi < 0 || !nvbo->comp || vmm->page[i].comp) 290 pi = i; 291 292 /* Stop once the buffer is larger than the current page size. */ 293 if (*size >= 1ULL << vmm->page[i].shift) 294 break; 295 } 296 297 if (WARN_ON(pi < 0)) { 298 kfree(nvbo); 299 return ERR_PTR(-EINVAL); 300 } 301 302 /* Disable compression if suitable settings couldn't be found. */ 303 if (nvbo->comp && !vmm->page[pi].comp) { 304 if (mmu->object.oclass >= NVIF_CLASS_MMU_GF100) 305 nvbo->kind = mmu->kind[nvbo->kind]; 306 nvbo->comp = 0; 307 } 308 nvbo->page = vmm->page[pi].shift; 309 } else { 310 /* Determine the desirable target GPU page size for the buffer. */ 311 for (i = 0; i < vmm->page_nr; i++) { 312 /* Because we cannot currently allow VMM maps to fail 313 * during buffer migration, we need to determine page 314 * size for the buffer up-front, and pre-allocate its 315 * page tables. 316 * 317 * Skip page sizes that can't support needed domains. 318 */ 319 if ((domain & NOUVEAU_GEM_DOMAIN_VRAM) && !vmm->page[i].vram) 320 continue; 321 if ((domain & NOUVEAU_GEM_DOMAIN_GART) && 322 (!vmm->page[i].host || vmm->page[i].shift > PAGE_SHIFT)) 323 continue; 324 325 /* pick the last one as it will be smallest. */ 326 pi = i; 327 328 /* Stop once the buffer is larger than the current page size. */ 329 if (*size >= 1ULL << vmm->page[i].shift) 330 break; 331 } 332 if (WARN_ON(pi < 0)) { 333 kfree(nvbo); 334 return ERR_PTR(-EINVAL); 335 } 336 nvbo->page = vmm->page[pi].shift; 337 } 338 339 nouveau_bo_fixup_align(nvbo, align, size); 340 341 return nvbo; 342 } 343 344 int 345 nouveau_bo_init(struct nouveau_bo *nvbo, u64 size, int align, u32 domain, 346 struct sg_table *sg, struct dma_resv *robj) 347 { 348 int type = sg ? ttm_bo_type_sg : ttm_bo_type_device; 349 int ret; 350 struct ttm_operation_ctx ctx = { 351 .interruptible = false, 352 .no_wait_gpu = false, 353 .resv = robj, 354 }; 355 356 nouveau_bo_placement_set(nvbo, domain, 0); 357 INIT_LIST_HEAD(&nvbo->io_reserve_lru); 358 359 ret = ttm_bo_init_reserved(nvbo->bo.bdev, &nvbo->bo, type, 360 &nvbo->placement, align >> PAGE_SHIFT, &ctx, 361 sg, robj, nouveau_bo_del_ttm); 362 if (ret) { 363 /* ttm will call nouveau_bo_del_ttm if it fails.. */ 364 return ret; 365 } 366 367 if (!robj) 368 ttm_bo_unreserve(&nvbo->bo); 369 370 return 0; 371 } 372 373 int 374 nouveau_bo_new(struct nouveau_cli *cli, u64 size, int align, 375 uint32_t domain, uint32_t tile_mode, uint32_t tile_flags, 376 struct sg_table *sg, struct dma_resv *robj, 377 struct nouveau_bo **pnvbo) 378 { 379 struct nouveau_bo *nvbo; 380 int ret; 381 382 nvbo = nouveau_bo_alloc(cli, &size, &align, domain, tile_mode, 383 tile_flags, true); 384 if (IS_ERR(nvbo)) 385 return PTR_ERR(nvbo); 386 387 nvbo->bo.base.size = size; 388 dma_resv_init(&nvbo->bo.base._resv); 389 drm_vma_node_reset(&nvbo->bo.base.vma_node); 390 391 /* This must be called before ttm_bo_init_reserved(). Subsequent 392 * bo_move() callbacks might already iterate the GEMs GPUVA list. 393 */ 394 drm_gem_gpuva_init(&nvbo->bo.base); 395 396 ret = nouveau_bo_init(nvbo, size, align, domain, sg, robj); 397 if (ret) 398 return ret; 399 400 *pnvbo = nvbo; 401 return 0; 402 } 403 404 void 405 nouveau_bo_unpin_del(struct nouveau_bo **pnvbo) 406 { 407 struct nouveau_bo *nvbo = *pnvbo; 408 409 if (!nvbo) 410 return; 411 412 nouveau_bo_unmap(nvbo); 413 nouveau_bo_unpin(nvbo); 414 nouveau_bo_fini(nvbo); 415 416 *pnvbo = NULL; 417 } 418 419 int 420 nouveau_bo_new_pin(struct nouveau_cli *cli, u32 domain, u32 size, struct nouveau_bo **pnvbo) 421 { 422 struct nouveau_bo *nvbo; 423 int ret; 424 425 ret = nouveau_bo_new(cli, size, 0, domain, 0, 0, NULL, NULL, &nvbo); 426 if (ret) 427 return ret; 428 429 ret = nouveau_bo_pin(nvbo, domain, false); 430 if (ret) { 431 nouveau_bo_fini(nvbo); 432 return ret; 433 } 434 435 *pnvbo = nvbo; 436 return 0; 437 } 438 439 int 440 nouveau_bo_new_map(struct nouveau_cli *cli, u32 domain, u32 size, struct nouveau_bo **pnvbo) 441 { 442 struct nouveau_bo *nvbo; 443 int ret; 444 445 ret = nouveau_bo_new_pin(cli, domain, size, &nvbo); 446 if (ret) 447 return ret; 448 449 ret = nouveau_bo_map(nvbo); 450 if (ret) { 451 nouveau_bo_unpin_del(&nvbo); 452 return ret; 453 } 454 455 *pnvbo = nvbo; 456 return 0; 457 } 458 459 int 460 nouveau_bo_new_map_gpu(struct nouveau_cli *cli, u32 domain, u32 size, 461 struct nouveau_bo **pnvbo, struct nouveau_vma **pvma) 462 { 463 struct nouveau_vmm *vmm = nouveau_cli_vmm(cli); 464 struct nouveau_bo *nvbo; 465 int ret; 466 467 ret = nouveau_bo_new_map(cli, domain, size, &nvbo); 468 if (ret) 469 return ret; 470 471 ret = nouveau_vma_new(nvbo, vmm, pvma); 472 if (ret) { 473 nouveau_bo_unpin_del(&nvbo); 474 return ret; 475 } 476 477 *pnvbo = nvbo; 478 return 0; 479 } 480 481 static void 482 set_placement_range(struct nouveau_bo *nvbo, uint32_t domain) 483 { 484 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 485 u64 vram_size = drm->client.device.info.ram_size; 486 unsigned i, fpfn, lpfn; 487 488 if (drm->client.device.info.family == NV_DEVICE_INFO_V0_CELSIUS && 489 nvbo->mode && (domain & NOUVEAU_GEM_DOMAIN_VRAM) && 490 nvbo->bo.base.size < vram_size / 4) { 491 /* 492 * Make sure that the color and depth buffers are handled 493 * by independent memory controller units. Up to a 9x 494 * speed up when alpha-blending and depth-test are enabled 495 * at the same time. 496 */ 497 if (nvbo->zeta) { 498 fpfn = (vram_size / 2) >> PAGE_SHIFT; 499 lpfn = ~0; 500 } else { 501 fpfn = 0; 502 lpfn = (vram_size / 2) >> PAGE_SHIFT; 503 } 504 for (i = 0; i < nvbo->placement.num_placement; ++i) { 505 nvbo->placements[i].fpfn = fpfn; 506 nvbo->placements[i].lpfn = lpfn; 507 } 508 } 509 } 510 511 void 512 nouveau_bo_placement_set(struct nouveau_bo *nvbo, uint32_t domain, 513 uint32_t busy) 514 { 515 unsigned int *n = &nvbo->placement.num_placement; 516 struct ttm_place *pl = nvbo->placements; 517 518 domain |= busy; 519 520 *n = 0; 521 if (domain & NOUVEAU_GEM_DOMAIN_VRAM) { 522 pl[*n].mem_type = TTM_PL_VRAM; 523 pl[*n].flags = busy & NOUVEAU_GEM_DOMAIN_VRAM ? 524 TTM_PL_FLAG_FALLBACK : 0; 525 (*n)++; 526 } 527 if (domain & NOUVEAU_GEM_DOMAIN_GART) { 528 pl[*n].mem_type = TTM_PL_TT; 529 pl[*n].flags = busy & NOUVEAU_GEM_DOMAIN_GART ? 530 TTM_PL_FLAG_FALLBACK : 0; 531 (*n)++; 532 } 533 if (domain & NOUVEAU_GEM_DOMAIN_CPU) { 534 pl[*n].mem_type = TTM_PL_SYSTEM; 535 pl[*n].flags = busy & NOUVEAU_GEM_DOMAIN_CPU ? 536 TTM_PL_FLAG_FALLBACK : 0; 537 (*n)++; 538 } 539 540 nvbo->placement.placement = nvbo->placements; 541 set_placement_range(nvbo, domain); 542 } 543 544 int nouveau_bo_pin_locked(struct nouveau_bo *nvbo, uint32_t domain, bool contig) 545 { 546 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 547 struct ttm_buffer_object *bo = &nvbo->bo; 548 bool force = false, evict = false; 549 int ret = 0; 550 551 dma_resv_assert_held(bo->base.resv); 552 553 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA && 554 domain == NOUVEAU_GEM_DOMAIN_VRAM && contig) { 555 if (!nvbo->contig) { 556 nvbo->contig = true; 557 force = true; 558 evict = true; 559 } 560 } 561 562 if (nvbo->bo.pin_count) { 563 bool error = evict; 564 565 switch (bo->resource->mem_type) { 566 case TTM_PL_VRAM: 567 error |= !(domain & NOUVEAU_GEM_DOMAIN_VRAM); 568 break; 569 case TTM_PL_TT: 570 error |= !(domain & NOUVEAU_GEM_DOMAIN_GART); 571 break; 572 default: 573 break; 574 } 575 576 if (error) { 577 NV_ERROR(drm, "bo %p pinned elsewhere: " 578 "0x%08x vs 0x%08x\n", bo, 579 bo->resource->mem_type, domain); 580 ret = -EBUSY; 581 } 582 ttm_bo_pin(&nvbo->bo); 583 goto out; 584 } 585 586 if (evict) { 587 nouveau_bo_placement_set(nvbo, NOUVEAU_GEM_DOMAIN_GART, 0); 588 ret = nouveau_bo_validate(nvbo, false, false); 589 if (ret) 590 goto out; 591 } 592 593 nouveau_bo_placement_set(nvbo, domain, 0); 594 ret = nouveau_bo_validate(nvbo, false, false); 595 if (ret) 596 goto out; 597 598 ttm_bo_pin(&nvbo->bo); 599 600 switch (bo->resource->mem_type) { 601 case TTM_PL_VRAM: 602 drm->gem.vram_available -= bo->base.size; 603 break; 604 case TTM_PL_TT: 605 drm->gem.gart_available -= bo->base.size; 606 break; 607 default: 608 break; 609 } 610 611 out: 612 if (force && ret) 613 nvbo->contig = false; 614 return ret; 615 } 616 617 void nouveau_bo_unpin_locked(struct nouveau_bo *nvbo) 618 { 619 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 620 struct ttm_buffer_object *bo = &nvbo->bo; 621 622 dma_resv_assert_held(bo->base.resv); 623 624 ttm_bo_unpin(&nvbo->bo); 625 if (!nvbo->bo.pin_count) { 626 switch (bo->resource->mem_type) { 627 case TTM_PL_VRAM: 628 drm->gem.vram_available += bo->base.size; 629 break; 630 case TTM_PL_TT: 631 drm->gem.gart_available += bo->base.size; 632 break; 633 default: 634 break; 635 } 636 } 637 } 638 639 int nouveau_bo_pin(struct nouveau_bo *nvbo, uint32_t domain, bool contig) 640 { 641 struct ttm_buffer_object *bo = &nvbo->bo; 642 int ret; 643 644 ret = ttm_bo_reserve(bo, false, false, NULL); 645 if (ret) 646 return ret; 647 ret = nouveau_bo_pin_locked(nvbo, domain, contig); 648 ttm_bo_unreserve(bo); 649 650 return ret; 651 } 652 653 int nouveau_bo_unpin(struct nouveau_bo *nvbo) 654 { 655 struct ttm_buffer_object *bo = &nvbo->bo; 656 int ret; 657 658 ret = ttm_bo_reserve(bo, false, false, NULL); 659 if (ret) 660 return ret; 661 nouveau_bo_unpin_locked(nvbo); 662 ttm_bo_unreserve(bo); 663 664 return 0; 665 } 666 667 int 668 nouveau_bo_map(struct nouveau_bo *nvbo) 669 { 670 int ret; 671 672 ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL); 673 if (ret) 674 return ret; 675 676 ret = ttm_bo_kmap(&nvbo->bo, 0, PFN_UP(nvbo->bo.base.size), &nvbo->kmap); 677 678 ttm_bo_unreserve(&nvbo->bo); 679 return ret; 680 } 681 682 void 683 nouveau_bo_unmap(struct nouveau_bo *nvbo) 684 { 685 if (!nvbo) 686 return; 687 688 ttm_bo_kunmap(&nvbo->kmap); 689 } 690 691 void 692 nouveau_bo_sync_for_device(struct nouveau_bo *nvbo) 693 { 694 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 695 struct ttm_tt *ttm_dma = (struct ttm_tt *)nvbo->bo.ttm; 696 int i, j; 697 698 if (!ttm_dma || !ttm_dma->dma_address) 699 return; 700 if (!ttm_dma->pages) { 701 NV_DEBUG(drm, "ttm_dma 0x%p: pages NULL\n", ttm_dma); 702 return; 703 } 704 705 /* Don't waste time looping if the object is coherent */ 706 if (nvbo->force_coherent) 707 return; 708 709 i = 0; 710 while (i < ttm_dma->num_pages) { 711 struct page *p = ttm_dma->pages[i]; 712 size_t num_pages = 1; 713 714 for (j = i + 1; j < ttm_dma->num_pages; ++j) { 715 if (++p != ttm_dma->pages[j]) 716 break; 717 718 ++num_pages; 719 } 720 dma_sync_single_for_device(drm->dev->dev, 721 ttm_dma->dma_address[i], 722 num_pages * PAGE_SIZE, DMA_TO_DEVICE); 723 i += num_pages; 724 } 725 } 726 727 void 728 nouveau_bo_sync_for_cpu(struct nouveau_bo *nvbo) 729 { 730 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 731 struct ttm_tt *ttm_dma = (struct ttm_tt *)nvbo->bo.ttm; 732 int i, j; 733 734 if (!ttm_dma || !ttm_dma->dma_address) 735 return; 736 if (!ttm_dma->pages) { 737 NV_DEBUG(drm, "ttm_dma 0x%p: pages NULL\n", ttm_dma); 738 return; 739 } 740 741 /* Don't waste time looping if the object is coherent */ 742 if (nvbo->force_coherent) 743 return; 744 745 i = 0; 746 while (i < ttm_dma->num_pages) { 747 struct page *p = ttm_dma->pages[i]; 748 size_t num_pages = 1; 749 750 for (j = i + 1; j < ttm_dma->num_pages; ++j) { 751 if (++p != ttm_dma->pages[j]) 752 break; 753 754 ++num_pages; 755 } 756 757 dma_sync_single_for_cpu(drm->dev->dev, ttm_dma->dma_address[i], 758 num_pages * PAGE_SIZE, DMA_FROM_DEVICE); 759 i += num_pages; 760 } 761 } 762 763 void nouveau_bo_add_io_reserve_lru(struct ttm_buffer_object *bo) 764 { 765 struct nouveau_drm *drm = nouveau_bdev(bo->bdev); 766 struct nouveau_bo *nvbo = nouveau_bo(bo); 767 768 mutex_lock(&drm->ttm.io_reserve_mutex); 769 list_move_tail(&nvbo->io_reserve_lru, &drm->ttm.io_reserve_lru); 770 mutex_unlock(&drm->ttm.io_reserve_mutex); 771 } 772 773 void nouveau_bo_del_io_reserve_lru(struct ttm_buffer_object *bo) 774 { 775 struct nouveau_drm *drm = nouveau_bdev(bo->bdev); 776 struct nouveau_bo *nvbo = nouveau_bo(bo); 777 778 mutex_lock(&drm->ttm.io_reserve_mutex); 779 list_del_init(&nvbo->io_reserve_lru); 780 mutex_unlock(&drm->ttm.io_reserve_mutex); 781 } 782 783 int 784 nouveau_bo_validate(struct nouveau_bo *nvbo, bool interruptible, 785 bool no_wait_gpu) 786 { 787 struct ttm_operation_ctx ctx = { interruptible, no_wait_gpu }; 788 int ret; 789 790 ret = ttm_bo_validate(&nvbo->bo, &nvbo->placement, &ctx); 791 if (ret) 792 return ret; 793 794 nouveau_bo_sync_for_device(nvbo); 795 796 return 0; 797 } 798 799 void 800 nouveau_bo_wr16(struct nouveau_bo *nvbo, unsigned index, u16 val) 801 { 802 bool is_iomem; 803 u16 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem); 804 805 mem += index; 806 807 if (is_iomem) 808 iowrite16_native(val, (void __force __iomem *)mem); 809 else 810 *mem = val; 811 } 812 813 u32 814 nouveau_bo_rd32(struct nouveau_bo *nvbo, unsigned index) 815 { 816 bool is_iomem; 817 u32 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem); 818 819 mem += index; 820 821 if (is_iomem) 822 return ioread32_native((void __force __iomem *)mem); 823 else 824 return *mem; 825 } 826 827 void 828 nouveau_bo_wr32(struct nouveau_bo *nvbo, unsigned index, u32 val) 829 { 830 bool is_iomem; 831 u32 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem); 832 833 mem += index; 834 835 if (is_iomem) 836 iowrite32_native(val, (void __force __iomem *)mem); 837 else 838 *mem = val; 839 } 840 841 static struct ttm_tt * 842 nouveau_ttm_tt_create(struct ttm_buffer_object *bo, uint32_t page_flags) 843 { 844 #if IS_ENABLED(CONFIG_AGP) 845 struct nouveau_drm *drm = nouveau_bdev(bo->bdev); 846 847 if (drm->agp.bridge) { 848 return ttm_agp_tt_create(bo, drm->agp.bridge, page_flags); 849 } 850 #endif 851 852 return nouveau_sgdma_create_ttm(bo, page_flags); 853 } 854 855 static int 856 nouveau_ttm_tt_bind(struct ttm_device *bdev, struct ttm_tt *ttm, 857 struct ttm_resource *reg) 858 { 859 #if IS_ENABLED(CONFIG_AGP) 860 struct nouveau_drm *drm = nouveau_bdev(bdev); 861 #endif 862 if (!reg) 863 return -EINVAL; 864 #if IS_ENABLED(CONFIG_AGP) 865 if (drm->agp.bridge) 866 return ttm_agp_bind(ttm, reg); 867 #endif 868 return nouveau_sgdma_bind(bdev, ttm, reg); 869 } 870 871 static void 872 nouveau_ttm_tt_unbind(struct ttm_device *bdev, struct ttm_tt *ttm) 873 { 874 #if IS_ENABLED(CONFIG_AGP) 875 struct nouveau_drm *drm = nouveau_bdev(bdev); 876 877 if (drm->agp.bridge) { 878 ttm_agp_unbind(ttm); 879 return; 880 } 881 #endif 882 nouveau_sgdma_unbind(bdev, ttm); 883 } 884 885 static void 886 nouveau_bo_evict_flags(struct ttm_buffer_object *bo, struct ttm_placement *pl) 887 { 888 struct nouveau_bo *nvbo = nouveau_bo(bo); 889 890 switch (bo->resource->mem_type) { 891 case TTM_PL_VRAM: 892 nouveau_bo_placement_set(nvbo, NOUVEAU_GEM_DOMAIN_GART, 893 NOUVEAU_GEM_DOMAIN_CPU); 894 break; 895 default: 896 nouveau_bo_placement_set(nvbo, NOUVEAU_GEM_DOMAIN_CPU, 0); 897 break; 898 } 899 900 *pl = nvbo->placement; 901 } 902 903 static int 904 nouveau_bo_move_prep(struct nouveau_drm *drm, struct ttm_buffer_object *bo, 905 struct ttm_resource *reg) 906 { 907 struct nouveau_mem *old_mem = nouveau_mem(bo->resource); 908 struct nouveau_mem *new_mem = nouveau_mem(reg); 909 struct nvif_vmm *vmm = &drm->client.vmm.vmm; 910 int ret; 911 912 ret = nvif_vmm_get(vmm, LAZY, false, old_mem->mem.page, 0, 913 old_mem->mem.size, &old_mem->vma[0]); 914 if (ret) 915 return ret; 916 917 ret = nvif_vmm_get(vmm, LAZY, false, new_mem->mem.page, 0, 918 new_mem->mem.size, &old_mem->vma[1]); 919 if (ret) 920 goto done; 921 922 ret = nouveau_mem_map(old_mem, vmm, &old_mem->vma[0]); 923 if (ret) 924 goto done; 925 926 ret = nouveau_mem_map(new_mem, vmm, &old_mem->vma[1]); 927 done: 928 if (ret) { 929 nvif_vmm_put(vmm, &old_mem->vma[1]); 930 nvif_vmm_put(vmm, &old_mem->vma[0]); 931 } 932 return 0; 933 } 934 935 static int 936 nouveau_bo_move_m2mf(struct ttm_buffer_object *bo, int evict, 937 struct ttm_operation_ctx *ctx, 938 struct ttm_resource *new_reg) 939 { 940 struct nouveau_drm *drm = nouveau_bdev(bo->bdev); 941 struct nouveau_channel *chan = drm->ttm.chan; 942 struct nouveau_cli *cli = chan->cli; 943 struct nouveau_fence *fence; 944 int ret; 945 946 /* create temporary vmas for the transfer and attach them to the 947 * old nvkm_mem node, these will get cleaned up after ttm has 948 * destroyed the ttm_resource 949 */ 950 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA) { 951 ret = nouveau_bo_move_prep(drm, bo, new_reg); 952 if (ret) 953 return ret; 954 } 955 956 if (drm_drv_uses_atomic_modeset(drm->dev)) 957 mutex_lock(&cli->mutex); 958 else 959 mutex_lock_nested(&cli->mutex, SINGLE_DEPTH_NESTING); 960 961 ret = nouveau_fence_sync(nouveau_bo(bo), chan, true, ctx->interruptible); 962 if (ret) 963 goto out_unlock; 964 965 ret = drm->ttm.move(chan, bo, bo->resource, new_reg); 966 if (ret) 967 goto out_unlock; 968 969 ret = nouveau_fence_new(&fence, chan); 970 if (ret) 971 goto out_unlock; 972 973 /* TODO: figure out a better solution here 974 * 975 * wait on the fence here explicitly as going through 976 * ttm_bo_move_accel_cleanup somehow doesn't seem to do it. 977 * 978 * Without this the operation can timeout and we'll fallback to a 979 * software copy, which might take several minutes to finish. 980 */ 981 nouveau_fence_wait(fence, false, false); 982 ret = ttm_bo_move_accel_cleanup(bo, &fence->base, evict, false, 983 new_reg); 984 nouveau_fence_unref(&fence); 985 986 out_unlock: 987 mutex_unlock(&cli->mutex); 988 return ret; 989 } 990 991 void 992 nouveau_bo_move_init(struct nouveau_drm *drm) 993 { 994 static const struct _method_table { 995 const char *name; 996 int engine; 997 s32 oclass; 998 int (*exec)(struct nouveau_channel *, 999 struct ttm_buffer_object *, 1000 struct ttm_resource *, struct ttm_resource *); 1001 int (*init)(struct nouveau_channel *, u32 handle); 1002 } _methods[] = { 1003 { "COPY", 4, 0xcab5, nve0_bo_move_copy, nve0_bo_move_init }, 1004 { "COPY", 4, 0xc9b5, nve0_bo_move_copy, nve0_bo_move_init }, 1005 { "COPY", 4, 0xc8b5, nve0_bo_move_copy, nve0_bo_move_init }, 1006 { "COPY", 4, 0xc7b5, nve0_bo_move_copy, nve0_bo_move_init }, 1007 { "GRCE", 0, 0xc7b5, nve0_bo_move_copy, nvc0_bo_move_init }, 1008 { "COPY", 4, 0xc6b5, nve0_bo_move_copy, nve0_bo_move_init }, 1009 { "GRCE", 0, 0xc6b5, nve0_bo_move_copy, nvc0_bo_move_init }, 1010 { "COPY", 4, 0xc5b5, nve0_bo_move_copy, nve0_bo_move_init }, 1011 { "GRCE", 0, 0xc5b5, nve0_bo_move_copy, nvc0_bo_move_init }, 1012 { "COPY", 4, 0xc3b5, nve0_bo_move_copy, nve0_bo_move_init }, 1013 { "GRCE", 0, 0xc3b5, nve0_bo_move_copy, nvc0_bo_move_init }, 1014 { "COPY", 4, 0xc1b5, nve0_bo_move_copy, nve0_bo_move_init }, 1015 { "GRCE", 0, 0xc1b5, nve0_bo_move_copy, nvc0_bo_move_init }, 1016 { "COPY", 4, 0xc0b5, nve0_bo_move_copy, nve0_bo_move_init }, 1017 { "GRCE", 0, 0xc0b5, nve0_bo_move_copy, nvc0_bo_move_init }, 1018 { "COPY", 4, 0xb0b5, nve0_bo_move_copy, nve0_bo_move_init }, 1019 { "GRCE", 0, 0xb0b5, nve0_bo_move_copy, nvc0_bo_move_init }, 1020 { "COPY", 4, 0xa0b5, nve0_bo_move_copy, nve0_bo_move_init }, 1021 { "GRCE", 0, 0xa0b5, nve0_bo_move_copy, nvc0_bo_move_init }, 1022 { "COPY1", 5, 0x90b8, nvc0_bo_move_copy, nvc0_bo_move_init }, 1023 { "COPY0", 4, 0x90b5, nvc0_bo_move_copy, nvc0_bo_move_init }, 1024 { "COPY", 0, 0x85b5, nva3_bo_move_copy, nv50_bo_move_init }, 1025 { "CRYPT", 0, 0x74c1, nv84_bo_move_exec, nv50_bo_move_init }, 1026 { "M2MF", 0, 0x9039, nvc0_bo_move_m2mf, nvc0_bo_move_init }, 1027 { "M2MF", 0, 0x5039, nv50_bo_move_m2mf, nv50_bo_move_init }, 1028 { "M2MF", 0, 0x0039, nv04_bo_move_m2mf, nv04_bo_move_init }, 1029 {}, 1030 }; 1031 const struct _method_table *mthd = _methods; 1032 const char *name = "CPU"; 1033 int ret; 1034 1035 do { 1036 struct nouveau_channel *chan; 1037 1038 if (mthd->engine) 1039 chan = drm->cechan; 1040 else 1041 chan = drm->channel; 1042 if (chan == NULL) 1043 continue; 1044 1045 ret = nvif_object_ctor(&chan->user, "ttmBoMove", 1046 mthd->oclass | (mthd->engine << 16), 1047 mthd->oclass, NULL, 0, 1048 &drm->ttm.copy); 1049 if (ret == 0) { 1050 ret = mthd->init(chan, drm->ttm.copy.handle); 1051 if (ret) { 1052 nvif_object_dtor(&drm->ttm.copy); 1053 continue; 1054 } 1055 1056 drm->ttm.move = mthd->exec; 1057 drm->ttm.chan = chan; 1058 name = mthd->name; 1059 break; 1060 } 1061 } while ((++mthd)->exec); 1062 1063 NV_INFO(drm, "MM: using %s for buffer copies\n", name); 1064 } 1065 1066 static void nouveau_bo_move_ntfy(struct ttm_buffer_object *bo, 1067 struct ttm_resource *new_reg) 1068 { 1069 struct nouveau_mem *mem = new_reg ? nouveau_mem(new_reg) : NULL; 1070 struct nouveau_bo *nvbo = nouveau_bo(bo); 1071 struct nouveau_vma *vma; 1072 long ret; 1073 1074 /* ttm can now (stupidly) pass the driver bos it didn't create... */ 1075 if (bo->destroy != nouveau_bo_del_ttm) 1076 return; 1077 1078 nouveau_bo_del_io_reserve_lru(bo); 1079 1080 if (mem && new_reg->mem_type != TTM_PL_SYSTEM && 1081 mem->mem.page == nvbo->page) { 1082 list_for_each_entry(vma, &nvbo->vma_list, head) { 1083 nouveau_vma_map(vma, mem); 1084 } 1085 nouveau_uvmm_bo_map_all(nvbo, mem); 1086 } else { 1087 list_for_each_entry(vma, &nvbo->vma_list, head) { 1088 ret = dma_resv_wait_timeout(bo->base.resv, 1089 DMA_RESV_USAGE_BOOKKEEP, 1090 false, 15 * HZ); 1091 WARN_ON(ret <= 0); 1092 nouveau_vma_unmap(vma); 1093 } 1094 nouveau_uvmm_bo_unmap_all(nvbo); 1095 } 1096 1097 if (new_reg) 1098 nvbo->offset = (new_reg->start << PAGE_SHIFT); 1099 1100 } 1101 1102 static int 1103 nouveau_bo_vm_bind(struct ttm_buffer_object *bo, struct ttm_resource *new_reg, 1104 struct nouveau_drm_tile **new_tile) 1105 { 1106 struct nouveau_drm *drm = nouveau_bdev(bo->bdev); 1107 struct drm_device *dev = drm->dev; 1108 struct nouveau_bo *nvbo = nouveau_bo(bo); 1109 u64 offset = new_reg->start << PAGE_SHIFT; 1110 1111 *new_tile = NULL; 1112 if (new_reg->mem_type != TTM_PL_VRAM) 1113 return 0; 1114 1115 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_CELSIUS) { 1116 *new_tile = nv10_bo_set_tiling(dev, offset, bo->base.size, 1117 nvbo->mode, nvbo->zeta); 1118 } 1119 1120 return 0; 1121 } 1122 1123 static void 1124 nouveau_bo_vm_cleanup(struct ttm_buffer_object *bo, 1125 struct nouveau_drm_tile *new_tile, 1126 struct nouveau_drm_tile **old_tile) 1127 { 1128 struct nouveau_drm *drm = nouveau_bdev(bo->bdev); 1129 struct drm_device *dev = drm->dev; 1130 struct dma_fence *fence; 1131 int ret; 1132 1133 ret = dma_resv_get_singleton(bo->base.resv, DMA_RESV_USAGE_WRITE, 1134 &fence); 1135 if (ret) 1136 dma_resv_wait_timeout(bo->base.resv, DMA_RESV_USAGE_WRITE, 1137 false, MAX_SCHEDULE_TIMEOUT); 1138 1139 nv10_bo_put_tile_region(dev, *old_tile, fence); 1140 *old_tile = new_tile; 1141 } 1142 1143 static int 1144 nouveau_bo_move(struct ttm_buffer_object *bo, bool evict, 1145 struct ttm_operation_ctx *ctx, 1146 struct ttm_resource *new_reg, 1147 struct ttm_place *hop) 1148 { 1149 struct nouveau_drm *drm = nouveau_bdev(bo->bdev); 1150 struct nouveau_bo *nvbo = nouveau_bo(bo); 1151 struct drm_gem_object *obj = &bo->base; 1152 struct ttm_resource *old_reg = bo->resource; 1153 struct nouveau_drm_tile *new_tile = NULL; 1154 int ret = 0; 1155 1156 if (new_reg->mem_type == TTM_PL_TT) { 1157 ret = nouveau_ttm_tt_bind(bo->bdev, bo->ttm, new_reg); 1158 if (ret) 1159 return ret; 1160 } 1161 1162 drm_gpuvm_bo_gem_evict(obj, evict); 1163 nouveau_bo_move_ntfy(bo, new_reg); 1164 ret = ttm_bo_wait_ctx(bo, ctx); 1165 if (ret) 1166 goto out_ntfy; 1167 1168 if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA) { 1169 ret = nouveau_bo_vm_bind(bo, new_reg, &new_tile); 1170 if (ret) 1171 goto out_ntfy; 1172 } 1173 1174 /* Fake bo copy. */ 1175 if (!old_reg || (old_reg->mem_type == TTM_PL_SYSTEM && 1176 !bo->ttm)) { 1177 ttm_bo_move_null(bo, new_reg); 1178 goto out; 1179 } 1180 1181 if (old_reg->mem_type == TTM_PL_SYSTEM && 1182 new_reg->mem_type == TTM_PL_TT) { 1183 ttm_bo_move_null(bo, new_reg); 1184 goto out; 1185 } 1186 1187 if (old_reg->mem_type == TTM_PL_TT && 1188 new_reg->mem_type == TTM_PL_SYSTEM) { 1189 nouveau_ttm_tt_unbind(bo->bdev, bo->ttm); 1190 ttm_resource_free(bo, &bo->resource); 1191 ttm_bo_assign_mem(bo, new_reg); 1192 goto out; 1193 } 1194 1195 /* Hardware assisted copy. */ 1196 if (drm->ttm.move) { 1197 if ((old_reg->mem_type == TTM_PL_SYSTEM && 1198 new_reg->mem_type == TTM_PL_VRAM) || 1199 (old_reg->mem_type == TTM_PL_VRAM && 1200 new_reg->mem_type == TTM_PL_SYSTEM)) { 1201 hop->fpfn = 0; 1202 hop->lpfn = 0; 1203 hop->mem_type = TTM_PL_TT; 1204 hop->flags = 0; 1205 return -EMULTIHOP; 1206 } 1207 ret = nouveau_bo_move_m2mf(bo, evict, ctx, 1208 new_reg); 1209 } else 1210 ret = -ENODEV; 1211 1212 if (ret) { 1213 /* Fallback to software copy. */ 1214 ret = ttm_bo_move_memcpy(bo, ctx, new_reg); 1215 } 1216 1217 out: 1218 if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA) { 1219 if (ret) 1220 nouveau_bo_vm_cleanup(bo, NULL, &new_tile); 1221 else 1222 nouveau_bo_vm_cleanup(bo, new_tile, &nvbo->tile); 1223 } 1224 out_ntfy: 1225 if (ret) { 1226 nouveau_bo_move_ntfy(bo, bo->resource); 1227 drm_gpuvm_bo_gem_evict(obj, !evict); 1228 } 1229 return ret; 1230 } 1231 1232 static void 1233 nouveau_ttm_io_mem_free_locked(struct nouveau_drm *drm, 1234 struct ttm_resource *reg) 1235 { 1236 struct nouveau_mem *mem = nouveau_mem(reg); 1237 1238 if (drm->client.mem->oclass >= NVIF_CLASS_MEM_NV50) { 1239 switch (reg->mem_type) { 1240 case TTM_PL_TT: 1241 if (mem->kind) 1242 nvif_object_unmap_handle(&mem->mem.object); 1243 break; 1244 case TTM_PL_VRAM: 1245 nvif_object_unmap_handle(&mem->mem.object); 1246 break; 1247 default: 1248 break; 1249 } 1250 } 1251 } 1252 1253 static int 1254 nouveau_ttm_io_mem_reserve(struct ttm_device *bdev, struct ttm_resource *reg) 1255 { 1256 struct nouveau_drm *drm = nouveau_bdev(bdev); 1257 struct nvkm_device *device = nvxx_device(drm); 1258 struct nouveau_mem *mem = nouveau_mem(reg); 1259 struct nvif_mmu *mmu = &drm->client.mmu; 1260 int ret; 1261 1262 mutex_lock(&drm->ttm.io_reserve_mutex); 1263 retry: 1264 switch (reg->mem_type) { 1265 case TTM_PL_SYSTEM: 1266 /* System memory */ 1267 ret = 0; 1268 goto out; 1269 case TTM_PL_TT: 1270 #if IS_ENABLED(CONFIG_AGP) 1271 if (drm->agp.bridge) { 1272 reg->bus.offset = (reg->start << PAGE_SHIFT) + 1273 drm->agp.base; 1274 reg->bus.is_iomem = !drm->agp.cma; 1275 reg->bus.caching = ttm_write_combined; 1276 } 1277 #endif 1278 if (drm->client.mem->oclass < NVIF_CLASS_MEM_NV50 || 1279 !mem->kind) { 1280 /* untiled */ 1281 ret = 0; 1282 break; 1283 } 1284 fallthrough; /* tiled memory */ 1285 case TTM_PL_VRAM: 1286 reg->bus.offset = (reg->start << PAGE_SHIFT) + 1287 device->func->resource_addr(device, NVKM_BAR1_FB); 1288 reg->bus.is_iomem = true; 1289 1290 /* Some BARs do not support being ioremapped WC */ 1291 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA && 1292 mmu->type[drm->ttm.type_vram].type & NVIF_MEM_UNCACHED) 1293 reg->bus.caching = ttm_uncached; 1294 else 1295 reg->bus.caching = ttm_write_combined; 1296 1297 if (drm->client.mem->oclass >= NVIF_CLASS_MEM_NV50) { 1298 union { 1299 struct nv50_mem_map_v0 nv50; 1300 struct gf100_mem_map_v0 gf100; 1301 } args; 1302 u64 handle, length; 1303 u32 argc = 0; 1304 1305 switch (mem->mem.object.oclass) { 1306 case NVIF_CLASS_MEM_NV50: 1307 args.nv50.version = 0; 1308 args.nv50.ro = 0; 1309 args.nv50.kind = mem->kind; 1310 args.nv50.comp = mem->comp; 1311 argc = sizeof(args.nv50); 1312 break; 1313 case NVIF_CLASS_MEM_GF100: 1314 args.gf100.version = 0; 1315 args.gf100.ro = 0; 1316 args.gf100.kind = mem->kind; 1317 argc = sizeof(args.gf100); 1318 break; 1319 default: 1320 WARN_ON(1); 1321 break; 1322 } 1323 1324 ret = nvif_object_map_handle(&mem->mem.object, 1325 &args, argc, 1326 &handle, &length); 1327 if (ret != 1) { 1328 if (WARN_ON(ret == 0)) 1329 ret = -EINVAL; 1330 goto out; 1331 } 1332 1333 reg->bus.offset = handle; 1334 } 1335 ret = 0; 1336 break; 1337 default: 1338 ret = -EINVAL; 1339 } 1340 1341 out: 1342 if (ret == -ENOSPC) { 1343 struct nouveau_bo *nvbo; 1344 1345 nvbo = list_first_entry_or_null(&drm->ttm.io_reserve_lru, 1346 typeof(*nvbo), 1347 io_reserve_lru); 1348 if (nvbo) { 1349 list_del_init(&nvbo->io_reserve_lru); 1350 drm_vma_node_unmap(&nvbo->bo.base.vma_node, 1351 bdev->dev_mapping); 1352 nouveau_ttm_io_mem_free_locked(drm, nvbo->bo.resource); 1353 nvbo->bo.resource->bus.offset = 0; 1354 nvbo->bo.resource->bus.addr = NULL; 1355 goto retry; 1356 } 1357 1358 } 1359 mutex_unlock(&drm->ttm.io_reserve_mutex); 1360 return ret; 1361 } 1362 1363 static void 1364 nouveau_ttm_io_mem_free(struct ttm_device *bdev, struct ttm_resource *reg) 1365 { 1366 struct nouveau_drm *drm = nouveau_bdev(bdev); 1367 1368 mutex_lock(&drm->ttm.io_reserve_mutex); 1369 nouveau_ttm_io_mem_free_locked(drm, reg); 1370 mutex_unlock(&drm->ttm.io_reserve_mutex); 1371 } 1372 1373 vm_fault_t nouveau_ttm_fault_reserve_notify(struct ttm_buffer_object *bo) 1374 { 1375 struct nouveau_drm *drm = nouveau_bdev(bo->bdev); 1376 struct nouveau_bo *nvbo = nouveau_bo(bo); 1377 struct nvkm_device *device = nvxx_device(drm); 1378 u32 mappable = device->func->resource_size(device, NVKM_BAR1_FB) >> PAGE_SHIFT; 1379 int i, ret; 1380 1381 /* as long as the bo isn't in vram, and isn't tiled, we've got 1382 * nothing to do here. 1383 */ 1384 if (bo->resource->mem_type != TTM_PL_VRAM) { 1385 if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA || 1386 !nvbo->kind) 1387 return 0; 1388 1389 if (bo->resource->mem_type != TTM_PL_SYSTEM) 1390 return 0; 1391 1392 nouveau_bo_placement_set(nvbo, NOUVEAU_GEM_DOMAIN_GART, 0); 1393 1394 } else { 1395 /* make sure bo is in mappable vram */ 1396 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA || 1397 bo->resource->start + PFN_UP(bo->resource->size) < mappable) 1398 return 0; 1399 1400 for (i = 0; i < nvbo->placement.num_placement; ++i) { 1401 nvbo->placements[i].fpfn = 0; 1402 nvbo->placements[i].lpfn = mappable; 1403 } 1404 1405 nouveau_bo_placement_set(nvbo, NOUVEAU_GEM_DOMAIN_VRAM, 0); 1406 } 1407 1408 ret = nouveau_bo_validate(nvbo, false, false); 1409 if (unlikely(ret == -EBUSY || ret == -ERESTARTSYS)) 1410 return VM_FAULT_NOPAGE; 1411 else if (unlikely(ret)) 1412 return VM_FAULT_SIGBUS; 1413 1414 ttm_bo_move_to_lru_tail_unlocked(bo); 1415 return 0; 1416 } 1417 1418 static int 1419 nouveau_ttm_tt_populate(struct ttm_device *bdev, 1420 struct ttm_tt *ttm, struct ttm_operation_ctx *ctx) 1421 { 1422 struct ttm_tt *ttm_dma = (void *)ttm; 1423 struct nouveau_drm *drm; 1424 bool slave = !!(ttm->page_flags & TTM_TT_FLAG_EXTERNAL); 1425 1426 if (ttm_tt_is_populated(ttm)) 1427 return 0; 1428 1429 if (slave && ttm->sg) { 1430 drm_prime_sg_to_dma_addr_array(ttm->sg, ttm_dma->dma_address, 1431 ttm->num_pages); 1432 return 0; 1433 } 1434 1435 drm = nouveau_bdev(bdev); 1436 1437 return ttm_pool_alloc(&drm->ttm.bdev.pool, ttm, ctx); 1438 } 1439 1440 static void 1441 nouveau_ttm_tt_unpopulate(struct ttm_device *bdev, 1442 struct ttm_tt *ttm) 1443 { 1444 struct nouveau_drm *drm; 1445 bool slave = !!(ttm->page_flags & TTM_TT_FLAG_EXTERNAL); 1446 1447 if (slave) 1448 return; 1449 1450 nouveau_ttm_tt_unbind(bdev, ttm); 1451 1452 drm = nouveau_bdev(bdev); 1453 1454 return ttm_pool_free(&drm->ttm.bdev.pool, ttm); 1455 } 1456 1457 static void 1458 nouveau_ttm_tt_destroy(struct ttm_device *bdev, 1459 struct ttm_tt *ttm) 1460 { 1461 #if IS_ENABLED(CONFIG_AGP) 1462 struct nouveau_drm *drm = nouveau_bdev(bdev); 1463 if (drm->agp.bridge) { 1464 ttm_agp_destroy(ttm); 1465 return; 1466 } 1467 #endif 1468 nouveau_sgdma_destroy(bdev, ttm); 1469 } 1470 1471 void 1472 nouveau_bo_fence(struct nouveau_bo *nvbo, struct nouveau_fence *fence, bool exclusive) 1473 { 1474 struct dma_resv *resv = nvbo->bo.base.resv; 1475 1476 if (!fence) 1477 return; 1478 1479 dma_resv_add_fence(resv, &fence->base, exclusive ? 1480 DMA_RESV_USAGE_WRITE : DMA_RESV_USAGE_READ); 1481 } 1482 1483 static void 1484 nouveau_bo_delete_mem_notify(struct ttm_buffer_object *bo) 1485 { 1486 nouveau_bo_move_ntfy(bo, NULL); 1487 } 1488 1489 struct ttm_device_funcs nouveau_bo_driver = { 1490 .ttm_tt_create = &nouveau_ttm_tt_create, 1491 .ttm_tt_populate = &nouveau_ttm_tt_populate, 1492 .ttm_tt_unpopulate = &nouveau_ttm_tt_unpopulate, 1493 .ttm_tt_destroy = &nouveau_ttm_tt_destroy, 1494 .eviction_valuable = ttm_bo_eviction_valuable, 1495 .evict_flags = nouveau_bo_evict_flags, 1496 .delete_mem_notify = nouveau_bo_delete_mem_notify, 1497 .move = nouveau_bo_move, 1498 .io_mem_reserve = &nouveau_ttm_io_mem_reserve, 1499 .io_mem_free = &nouveau_ttm_io_mem_free, 1500 }; 1501