1 /* 2 * Copyright 2007 Dave Airlied 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice (including the next 13 * paragraph) shall be included in all copies or substantial portions of the 14 * Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 * OTHER DEALINGS IN THE SOFTWARE. 23 */ 24 /* 25 * Authors: Dave Airlied <airlied@linux.ie> 26 * Ben Skeggs <darktama@iinet.net.au> 27 * Jeremy Kolb <jkolb@brandeis.edu> 28 */ 29 30 #include "drmP.h" 31 #include "ttm/ttm_page_alloc.h" 32 33 #include "nouveau_drm.h" 34 #include "nouveau_drv.h" 35 #include "nouveau_dma.h" 36 #include "nouveau_mm.h" 37 #include "nouveau_vm.h" 38 39 #include <linux/log2.h> 40 #include <linux/slab.h> 41 42 static void 43 nouveau_bo_del_ttm(struct ttm_buffer_object *bo) 44 { 45 struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev); 46 struct drm_device *dev = dev_priv->dev; 47 struct nouveau_bo *nvbo = nouveau_bo(bo); 48 49 if (unlikely(nvbo->gem)) 50 DRM_ERROR("bo %p still attached to GEM object\n", bo); 51 52 nv10_mem_put_tile_region(dev, nvbo->tile, NULL); 53 kfree(nvbo); 54 } 55 56 static void 57 nouveau_bo_fixup_align(struct nouveau_bo *nvbo, u32 flags, 58 int *align, int *size) 59 { 60 struct drm_nouveau_private *dev_priv = nouveau_bdev(nvbo->bo.bdev); 61 62 if (dev_priv->card_type < NV_50) { 63 if (nvbo->tile_mode) { 64 if (dev_priv->chipset >= 0x40) { 65 *align = 65536; 66 *size = roundup(*size, 64 * nvbo->tile_mode); 67 68 } else if (dev_priv->chipset >= 0x30) { 69 *align = 32768; 70 *size = roundup(*size, 64 * nvbo->tile_mode); 71 72 } else if (dev_priv->chipset >= 0x20) { 73 *align = 16384; 74 *size = roundup(*size, 64 * nvbo->tile_mode); 75 76 } else if (dev_priv->chipset >= 0x10) { 77 *align = 16384; 78 *size = roundup(*size, 32 * nvbo->tile_mode); 79 } 80 } 81 } else { 82 *size = roundup(*size, (1 << nvbo->page_shift)); 83 *align = max((1 << nvbo->page_shift), *align); 84 } 85 86 *size = roundup(*size, PAGE_SIZE); 87 } 88 89 int 90 nouveau_bo_new(struct drm_device *dev, int size, int align, 91 uint32_t flags, uint32_t tile_mode, uint32_t tile_flags, 92 struct nouveau_bo **pnvbo) 93 { 94 struct drm_nouveau_private *dev_priv = dev->dev_private; 95 struct nouveau_bo *nvbo; 96 size_t acc_size; 97 int ret; 98 99 nvbo = kzalloc(sizeof(struct nouveau_bo), GFP_KERNEL); 100 if (!nvbo) 101 return -ENOMEM; 102 INIT_LIST_HEAD(&nvbo->head); 103 INIT_LIST_HEAD(&nvbo->entry); 104 INIT_LIST_HEAD(&nvbo->vma_list); 105 nvbo->tile_mode = tile_mode; 106 nvbo->tile_flags = tile_flags; 107 nvbo->bo.bdev = &dev_priv->ttm.bdev; 108 109 nvbo->page_shift = 12; 110 if (dev_priv->bar1_vm) { 111 if (!(flags & TTM_PL_FLAG_TT) && size > 256 * 1024) 112 nvbo->page_shift = dev_priv->bar1_vm->lpg_shift; 113 } 114 115 nouveau_bo_fixup_align(nvbo, flags, &align, &size); 116 nvbo->bo.mem.num_pages = size >> PAGE_SHIFT; 117 nouveau_bo_placement_set(nvbo, flags, 0); 118 119 acc_size = ttm_bo_dma_acc_size(&dev_priv->ttm.bdev, size, 120 sizeof(struct nouveau_bo)); 121 122 ret = ttm_bo_init(&dev_priv->ttm.bdev, &nvbo->bo, size, 123 ttm_bo_type_device, &nvbo->placement, 124 align >> PAGE_SHIFT, 0, false, NULL, acc_size, 125 nouveau_bo_del_ttm); 126 if (ret) { 127 /* ttm will call nouveau_bo_del_ttm if it fails.. */ 128 return ret; 129 } 130 131 *pnvbo = nvbo; 132 return 0; 133 } 134 135 static void 136 set_placement_list(uint32_t *pl, unsigned *n, uint32_t type, uint32_t flags) 137 { 138 *n = 0; 139 140 if (type & TTM_PL_FLAG_VRAM) 141 pl[(*n)++] = TTM_PL_FLAG_VRAM | flags; 142 if (type & TTM_PL_FLAG_TT) 143 pl[(*n)++] = TTM_PL_FLAG_TT | flags; 144 if (type & TTM_PL_FLAG_SYSTEM) 145 pl[(*n)++] = TTM_PL_FLAG_SYSTEM | flags; 146 } 147 148 static void 149 set_placement_range(struct nouveau_bo *nvbo, uint32_t type) 150 { 151 struct drm_nouveau_private *dev_priv = nouveau_bdev(nvbo->bo.bdev); 152 int vram_pages = dev_priv->vram_size >> PAGE_SHIFT; 153 154 if (dev_priv->card_type == NV_10 && 155 nvbo->tile_mode && (type & TTM_PL_FLAG_VRAM) && 156 nvbo->bo.mem.num_pages < vram_pages / 4) { 157 /* 158 * Make sure that the color and depth buffers are handled 159 * by independent memory controller units. Up to a 9x 160 * speed up when alpha-blending and depth-test are enabled 161 * at the same time. 162 */ 163 if (nvbo->tile_flags & NOUVEAU_GEM_TILE_ZETA) { 164 nvbo->placement.fpfn = vram_pages / 2; 165 nvbo->placement.lpfn = ~0; 166 } else { 167 nvbo->placement.fpfn = 0; 168 nvbo->placement.lpfn = vram_pages / 2; 169 } 170 } 171 } 172 173 void 174 nouveau_bo_placement_set(struct nouveau_bo *nvbo, uint32_t type, uint32_t busy) 175 { 176 struct ttm_placement *pl = &nvbo->placement; 177 uint32_t flags = TTM_PL_MASK_CACHING | 178 (nvbo->pin_refcnt ? TTM_PL_FLAG_NO_EVICT : 0); 179 180 pl->placement = nvbo->placements; 181 set_placement_list(nvbo->placements, &pl->num_placement, 182 type, flags); 183 184 pl->busy_placement = nvbo->busy_placements; 185 set_placement_list(nvbo->busy_placements, &pl->num_busy_placement, 186 type | busy, flags); 187 188 set_placement_range(nvbo, type); 189 } 190 191 int 192 nouveau_bo_pin(struct nouveau_bo *nvbo, uint32_t memtype) 193 { 194 struct drm_nouveau_private *dev_priv = nouveau_bdev(nvbo->bo.bdev); 195 struct ttm_buffer_object *bo = &nvbo->bo; 196 int ret; 197 198 if (nvbo->pin_refcnt && !(memtype & (1 << bo->mem.mem_type))) { 199 NV_ERROR(nouveau_bdev(bo->bdev)->dev, 200 "bo %p pinned elsewhere: 0x%08x vs 0x%08x\n", bo, 201 1 << bo->mem.mem_type, memtype); 202 return -EINVAL; 203 } 204 205 if (nvbo->pin_refcnt++) 206 return 0; 207 208 ret = ttm_bo_reserve(bo, false, false, false, 0); 209 if (ret) 210 goto out; 211 212 nouveau_bo_placement_set(nvbo, memtype, 0); 213 214 ret = nouveau_bo_validate(nvbo, false, false, false); 215 if (ret == 0) { 216 switch (bo->mem.mem_type) { 217 case TTM_PL_VRAM: 218 dev_priv->fb_aper_free -= bo->mem.size; 219 break; 220 case TTM_PL_TT: 221 dev_priv->gart_info.aper_free -= bo->mem.size; 222 break; 223 default: 224 break; 225 } 226 } 227 ttm_bo_unreserve(bo); 228 out: 229 if (unlikely(ret)) 230 nvbo->pin_refcnt--; 231 return ret; 232 } 233 234 int 235 nouveau_bo_unpin(struct nouveau_bo *nvbo) 236 { 237 struct drm_nouveau_private *dev_priv = nouveau_bdev(nvbo->bo.bdev); 238 struct ttm_buffer_object *bo = &nvbo->bo; 239 int ret; 240 241 if (--nvbo->pin_refcnt) 242 return 0; 243 244 ret = ttm_bo_reserve(bo, false, false, false, 0); 245 if (ret) 246 return ret; 247 248 nouveau_bo_placement_set(nvbo, bo->mem.placement, 0); 249 250 ret = nouveau_bo_validate(nvbo, false, false, false); 251 if (ret == 0) { 252 switch (bo->mem.mem_type) { 253 case TTM_PL_VRAM: 254 dev_priv->fb_aper_free += bo->mem.size; 255 break; 256 case TTM_PL_TT: 257 dev_priv->gart_info.aper_free += bo->mem.size; 258 break; 259 default: 260 break; 261 } 262 } 263 264 ttm_bo_unreserve(bo); 265 return ret; 266 } 267 268 int 269 nouveau_bo_map(struct nouveau_bo *nvbo) 270 { 271 int ret; 272 273 ret = ttm_bo_reserve(&nvbo->bo, false, false, false, 0); 274 if (ret) 275 return ret; 276 277 ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.mem.num_pages, &nvbo->kmap); 278 ttm_bo_unreserve(&nvbo->bo); 279 return ret; 280 } 281 282 void 283 nouveau_bo_unmap(struct nouveau_bo *nvbo) 284 { 285 if (nvbo) 286 ttm_bo_kunmap(&nvbo->kmap); 287 } 288 289 int 290 nouveau_bo_validate(struct nouveau_bo *nvbo, bool interruptible, 291 bool no_wait_reserve, bool no_wait_gpu) 292 { 293 int ret; 294 295 ret = ttm_bo_validate(&nvbo->bo, &nvbo->placement, interruptible, 296 no_wait_reserve, no_wait_gpu); 297 if (ret) 298 return ret; 299 300 return 0; 301 } 302 303 u16 304 nouveau_bo_rd16(struct nouveau_bo *nvbo, unsigned index) 305 { 306 bool is_iomem; 307 u16 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem); 308 mem = &mem[index]; 309 if (is_iomem) 310 return ioread16_native((void __force __iomem *)mem); 311 else 312 return *mem; 313 } 314 315 void 316 nouveau_bo_wr16(struct nouveau_bo *nvbo, unsigned index, u16 val) 317 { 318 bool is_iomem; 319 u16 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem); 320 mem = &mem[index]; 321 if (is_iomem) 322 iowrite16_native(val, (void __force __iomem *)mem); 323 else 324 *mem = val; 325 } 326 327 u32 328 nouveau_bo_rd32(struct nouveau_bo *nvbo, unsigned index) 329 { 330 bool is_iomem; 331 u32 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem); 332 mem = &mem[index]; 333 if (is_iomem) 334 return ioread32_native((void __force __iomem *)mem); 335 else 336 return *mem; 337 } 338 339 void 340 nouveau_bo_wr32(struct nouveau_bo *nvbo, unsigned index, u32 val) 341 { 342 bool is_iomem; 343 u32 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem); 344 mem = &mem[index]; 345 if (is_iomem) 346 iowrite32_native(val, (void __force __iomem *)mem); 347 else 348 *mem = val; 349 } 350 351 static struct ttm_tt * 352 nouveau_ttm_tt_create(struct ttm_bo_device *bdev, 353 unsigned long size, uint32_t page_flags, 354 struct page *dummy_read_page) 355 { 356 struct drm_nouveau_private *dev_priv = nouveau_bdev(bdev); 357 struct drm_device *dev = dev_priv->dev; 358 359 switch (dev_priv->gart_info.type) { 360 #if __OS_HAS_AGP 361 case NOUVEAU_GART_AGP: 362 return ttm_agp_tt_create(bdev, dev->agp->bridge, 363 size, page_flags, dummy_read_page); 364 #endif 365 case NOUVEAU_GART_PDMA: 366 case NOUVEAU_GART_HW: 367 return nouveau_sgdma_create_ttm(bdev, size, page_flags, 368 dummy_read_page); 369 default: 370 NV_ERROR(dev, "Unknown GART type %d\n", 371 dev_priv->gart_info.type); 372 break; 373 } 374 375 return NULL; 376 } 377 378 static int 379 nouveau_bo_invalidate_caches(struct ttm_bo_device *bdev, uint32_t flags) 380 { 381 /* We'll do this from user space. */ 382 return 0; 383 } 384 385 static int 386 nouveau_bo_init_mem_type(struct ttm_bo_device *bdev, uint32_t type, 387 struct ttm_mem_type_manager *man) 388 { 389 struct drm_nouveau_private *dev_priv = nouveau_bdev(bdev); 390 struct drm_device *dev = dev_priv->dev; 391 392 switch (type) { 393 case TTM_PL_SYSTEM: 394 man->flags = TTM_MEMTYPE_FLAG_MAPPABLE; 395 man->available_caching = TTM_PL_MASK_CACHING; 396 man->default_caching = TTM_PL_FLAG_CACHED; 397 break; 398 case TTM_PL_VRAM: 399 if (dev_priv->card_type >= NV_50) { 400 man->func = &nouveau_vram_manager; 401 man->io_reserve_fastpath = false; 402 man->use_io_reserve_lru = true; 403 } else { 404 man->func = &ttm_bo_manager_func; 405 } 406 man->flags = TTM_MEMTYPE_FLAG_FIXED | 407 TTM_MEMTYPE_FLAG_MAPPABLE; 408 man->available_caching = TTM_PL_FLAG_UNCACHED | 409 TTM_PL_FLAG_WC; 410 man->default_caching = TTM_PL_FLAG_WC; 411 break; 412 case TTM_PL_TT: 413 if (dev_priv->card_type >= NV_50) 414 man->func = &nouveau_gart_manager; 415 else 416 man->func = &ttm_bo_manager_func; 417 switch (dev_priv->gart_info.type) { 418 case NOUVEAU_GART_AGP: 419 man->flags = TTM_MEMTYPE_FLAG_MAPPABLE; 420 man->available_caching = TTM_PL_FLAG_UNCACHED | 421 TTM_PL_FLAG_WC; 422 man->default_caching = TTM_PL_FLAG_WC; 423 break; 424 case NOUVEAU_GART_PDMA: 425 case NOUVEAU_GART_HW: 426 man->flags = TTM_MEMTYPE_FLAG_MAPPABLE | 427 TTM_MEMTYPE_FLAG_CMA; 428 man->available_caching = TTM_PL_MASK_CACHING; 429 man->default_caching = TTM_PL_FLAG_CACHED; 430 break; 431 default: 432 NV_ERROR(dev, "Unknown GART type: %d\n", 433 dev_priv->gart_info.type); 434 return -EINVAL; 435 } 436 break; 437 default: 438 NV_ERROR(dev, "Unsupported memory type %u\n", (unsigned)type); 439 return -EINVAL; 440 } 441 return 0; 442 } 443 444 static void 445 nouveau_bo_evict_flags(struct ttm_buffer_object *bo, struct ttm_placement *pl) 446 { 447 struct nouveau_bo *nvbo = nouveau_bo(bo); 448 449 switch (bo->mem.mem_type) { 450 case TTM_PL_VRAM: 451 nouveau_bo_placement_set(nvbo, TTM_PL_FLAG_TT, 452 TTM_PL_FLAG_SYSTEM); 453 break; 454 default: 455 nouveau_bo_placement_set(nvbo, TTM_PL_FLAG_SYSTEM, 0); 456 break; 457 } 458 459 *pl = nvbo->placement; 460 } 461 462 463 /* GPU-assisted copy using NV_MEMORY_TO_MEMORY_FORMAT, can access 464 * TTM_PL_{VRAM,TT} directly. 465 */ 466 467 static int 468 nouveau_bo_move_accel_cleanup(struct nouveau_channel *chan, 469 struct nouveau_bo *nvbo, bool evict, 470 bool no_wait_reserve, bool no_wait_gpu, 471 struct ttm_mem_reg *new_mem) 472 { 473 struct nouveau_fence *fence = NULL; 474 int ret; 475 476 ret = nouveau_fence_new(chan, &fence, true); 477 if (ret) 478 return ret; 479 480 ret = ttm_bo_move_accel_cleanup(&nvbo->bo, fence, NULL, evict, 481 no_wait_reserve, no_wait_gpu, new_mem); 482 nouveau_fence_unref(&fence); 483 return ret; 484 } 485 486 static int 487 nvc0_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo, 488 struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem) 489 { 490 struct nouveau_mem *node = old_mem->mm_node; 491 u64 src_offset = node->vma[0].offset; 492 u64 dst_offset = node->vma[1].offset; 493 u32 page_count = new_mem->num_pages; 494 int ret; 495 496 page_count = new_mem->num_pages; 497 while (page_count) { 498 int line_count = (page_count > 2047) ? 2047 : page_count; 499 500 ret = RING_SPACE(chan, 12); 501 if (ret) 502 return ret; 503 504 BEGIN_NVC0(chan, 2, NvSubM2MF, 0x0238, 2); 505 OUT_RING (chan, upper_32_bits(dst_offset)); 506 OUT_RING (chan, lower_32_bits(dst_offset)); 507 BEGIN_NVC0(chan, 2, NvSubM2MF, 0x030c, 6); 508 OUT_RING (chan, upper_32_bits(src_offset)); 509 OUT_RING (chan, lower_32_bits(src_offset)); 510 OUT_RING (chan, PAGE_SIZE); /* src_pitch */ 511 OUT_RING (chan, PAGE_SIZE); /* dst_pitch */ 512 OUT_RING (chan, PAGE_SIZE); /* line_length */ 513 OUT_RING (chan, line_count); 514 BEGIN_NVC0(chan, 2, NvSubM2MF, 0x0300, 1); 515 OUT_RING (chan, 0x00100110); 516 517 page_count -= line_count; 518 src_offset += (PAGE_SIZE * line_count); 519 dst_offset += (PAGE_SIZE * line_count); 520 } 521 522 return 0; 523 } 524 525 static int 526 nv50_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo, 527 struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem) 528 { 529 struct nouveau_mem *node = old_mem->mm_node; 530 struct nouveau_bo *nvbo = nouveau_bo(bo); 531 u64 length = (new_mem->num_pages << PAGE_SHIFT); 532 u64 src_offset = node->vma[0].offset; 533 u64 dst_offset = node->vma[1].offset; 534 int ret; 535 536 while (length) { 537 u32 amount, stride, height; 538 539 amount = min(length, (u64)(4 * 1024 * 1024)); 540 stride = 16 * 4; 541 height = amount / stride; 542 543 if (new_mem->mem_type == TTM_PL_VRAM && 544 nouveau_bo_tile_layout(nvbo)) { 545 ret = RING_SPACE(chan, 8); 546 if (ret) 547 return ret; 548 549 BEGIN_RING(chan, NvSubM2MF, 0x0200, 7); 550 OUT_RING (chan, 0); 551 OUT_RING (chan, 0); 552 OUT_RING (chan, stride); 553 OUT_RING (chan, height); 554 OUT_RING (chan, 1); 555 OUT_RING (chan, 0); 556 OUT_RING (chan, 0); 557 } else { 558 ret = RING_SPACE(chan, 2); 559 if (ret) 560 return ret; 561 562 BEGIN_RING(chan, NvSubM2MF, 0x0200, 1); 563 OUT_RING (chan, 1); 564 } 565 if (old_mem->mem_type == TTM_PL_VRAM && 566 nouveau_bo_tile_layout(nvbo)) { 567 ret = RING_SPACE(chan, 8); 568 if (ret) 569 return ret; 570 571 BEGIN_RING(chan, NvSubM2MF, 0x021c, 7); 572 OUT_RING (chan, 0); 573 OUT_RING (chan, 0); 574 OUT_RING (chan, stride); 575 OUT_RING (chan, height); 576 OUT_RING (chan, 1); 577 OUT_RING (chan, 0); 578 OUT_RING (chan, 0); 579 } else { 580 ret = RING_SPACE(chan, 2); 581 if (ret) 582 return ret; 583 584 BEGIN_RING(chan, NvSubM2MF, 0x021c, 1); 585 OUT_RING (chan, 1); 586 } 587 588 ret = RING_SPACE(chan, 14); 589 if (ret) 590 return ret; 591 592 BEGIN_RING(chan, NvSubM2MF, 0x0238, 2); 593 OUT_RING (chan, upper_32_bits(src_offset)); 594 OUT_RING (chan, upper_32_bits(dst_offset)); 595 BEGIN_RING(chan, NvSubM2MF, 0x030c, 8); 596 OUT_RING (chan, lower_32_bits(src_offset)); 597 OUT_RING (chan, lower_32_bits(dst_offset)); 598 OUT_RING (chan, stride); 599 OUT_RING (chan, stride); 600 OUT_RING (chan, stride); 601 OUT_RING (chan, height); 602 OUT_RING (chan, 0x00000101); 603 OUT_RING (chan, 0x00000000); 604 BEGIN_RING(chan, NvSubM2MF, NV_MEMORY_TO_MEMORY_FORMAT_NOP, 1); 605 OUT_RING (chan, 0); 606 607 length -= amount; 608 src_offset += amount; 609 dst_offset += amount; 610 } 611 612 return 0; 613 } 614 615 static inline uint32_t 616 nouveau_bo_mem_ctxdma(struct ttm_buffer_object *bo, 617 struct nouveau_channel *chan, struct ttm_mem_reg *mem) 618 { 619 if (mem->mem_type == TTM_PL_TT) 620 return chan->gart_handle; 621 return chan->vram_handle; 622 } 623 624 static int 625 nv04_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo, 626 struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem) 627 { 628 u32 src_offset = old_mem->start << PAGE_SHIFT; 629 u32 dst_offset = new_mem->start << PAGE_SHIFT; 630 u32 page_count = new_mem->num_pages; 631 int ret; 632 633 ret = RING_SPACE(chan, 3); 634 if (ret) 635 return ret; 636 637 BEGIN_RING(chan, NvSubM2MF, NV_MEMORY_TO_MEMORY_FORMAT_DMA_SOURCE, 2); 638 OUT_RING (chan, nouveau_bo_mem_ctxdma(bo, chan, old_mem)); 639 OUT_RING (chan, nouveau_bo_mem_ctxdma(bo, chan, new_mem)); 640 641 page_count = new_mem->num_pages; 642 while (page_count) { 643 int line_count = (page_count > 2047) ? 2047 : page_count; 644 645 ret = RING_SPACE(chan, 11); 646 if (ret) 647 return ret; 648 649 BEGIN_RING(chan, NvSubM2MF, 650 NV_MEMORY_TO_MEMORY_FORMAT_OFFSET_IN, 8); 651 OUT_RING (chan, src_offset); 652 OUT_RING (chan, dst_offset); 653 OUT_RING (chan, PAGE_SIZE); /* src_pitch */ 654 OUT_RING (chan, PAGE_SIZE); /* dst_pitch */ 655 OUT_RING (chan, PAGE_SIZE); /* line_length */ 656 OUT_RING (chan, line_count); 657 OUT_RING (chan, 0x00000101); 658 OUT_RING (chan, 0x00000000); 659 BEGIN_RING(chan, NvSubM2MF, NV_MEMORY_TO_MEMORY_FORMAT_NOP, 1); 660 OUT_RING (chan, 0); 661 662 page_count -= line_count; 663 src_offset += (PAGE_SIZE * line_count); 664 dst_offset += (PAGE_SIZE * line_count); 665 } 666 667 return 0; 668 } 669 670 static int 671 nouveau_vma_getmap(struct nouveau_channel *chan, struct nouveau_bo *nvbo, 672 struct ttm_mem_reg *mem, struct nouveau_vma *vma) 673 { 674 struct nouveau_mem *node = mem->mm_node; 675 int ret; 676 677 ret = nouveau_vm_get(chan->vm, mem->num_pages << PAGE_SHIFT, 678 node->page_shift, NV_MEM_ACCESS_RO, vma); 679 if (ret) 680 return ret; 681 682 if (mem->mem_type == TTM_PL_VRAM) 683 nouveau_vm_map(vma, node); 684 else 685 nouveau_vm_map_sg(vma, 0, mem->num_pages << PAGE_SHIFT, node); 686 687 return 0; 688 } 689 690 static int 691 nouveau_bo_move_m2mf(struct ttm_buffer_object *bo, int evict, bool intr, 692 bool no_wait_reserve, bool no_wait_gpu, 693 struct ttm_mem_reg *new_mem) 694 { 695 struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev); 696 struct nouveau_bo *nvbo = nouveau_bo(bo); 697 struct ttm_mem_reg *old_mem = &bo->mem; 698 struct nouveau_channel *chan; 699 int ret; 700 701 chan = nvbo->channel; 702 if (!chan) { 703 chan = dev_priv->channel; 704 mutex_lock_nested(&chan->mutex, NOUVEAU_KCHANNEL_MUTEX); 705 } 706 707 /* create temporary vmas for the transfer and attach them to the 708 * old nouveau_mem node, these will get cleaned up after ttm has 709 * destroyed the ttm_mem_reg 710 */ 711 if (dev_priv->card_type >= NV_50) { 712 struct nouveau_mem *node = old_mem->mm_node; 713 714 ret = nouveau_vma_getmap(chan, nvbo, old_mem, &node->vma[0]); 715 if (ret) 716 goto out; 717 718 ret = nouveau_vma_getmap(chan, nvbo, new_mem, &node->vma[1]); 719 if (ret) 720 goto out; 721 } 722 723 if (dev_priv->card_type < NV_50) 724 ret = nv04_bo_move_m2mf(chan, bo, &bo->mem, new_mem); 725 else 726 if (dev_priv->card_type < NV_C0) 727 ret = nv50_bo_move_m2mf(chan, bo, &bo->mem, new_mem); 728 else 729 ret = nvc0_bo_move_m2mf(chan, bo, &bo->mem, new_mem); 730 if (ret == 0) { 731 ret = nouveau_bo_move_accel_cleanup(chan, nvbo, evict, 732 no_wait_reserve, 733 no_wait_gpu, new_mem); 734 } 735 736 out: 737 if (chan == dev_priv->channel) 738 mutex_unlock(&chan->mutex); 739 return ret; 740 } 741 742 static int 743 nouveau_bo_move_flipd(struct ttm_buffer_object *bo, bool evict, bool intr, 744 bool no_wait_reserve, bool no_wait_gpu, 745 struct ttm_mem_reg *new_mem) 746 { 747 u32 placement_memtype = TTM_PL_FLAG_TT | TTM_PL_MASK_CACHING; 748 struct ttm_placement placement; 749 struct ttm_mem_reg tmp_mem; 750 int ret; 751 752 placement.fpfn = placement.lpfn = 0; 753 placement.num_placement = placement.num_busy_placement = 1; 754 placement.placement = placement.busy_placement = &placement_memtype; 755 756 tmp_mem = *new_mem; 757 tmp_mem.mm_node = NULL; 758 ret = ttm_bo_mem_space(bo, &placement, &tmp_mem, intr, no_wait_reserve, no_wait_gpu); 759 if (ret) 760 return ret; 761 762 ret = ttm_tt_bind(bo->ttm, &tmp_mem); 763 if (ret) 764 goto out; 765 766 ret = nouveau_bo_move_m2mf(bo, true, intr, no_wait_reserve, no_wait_gpu, &tmp_mem); 767 if (ret) 768 goto out; 769 770 ret = ttm_bo_move_ttm(bo, true, no_wait_reserve, no_wait_gpu, new_mem); 771 out: 772 ttm_bo_mem_put(bo, &tmp_mem); 773 return ret; 774 } 775 776 static int 777 nouveau_bo_move_flips(struct ttm_buffer_object *bo, bool evict, bool intr, 778 bool no_wait_reserve, bool no_wait_gpu, 779 struct ttm_mem_reg *new_mem) 780 { 781 u32 placement_memtype = TTM_PL_FLAG_TT | TTM_PL_MASK_CACHING; 782 struct ttm_placement placement; 783 struct ttm_mem_reg tmp_mem; 784 int ret; 785 786 placement.fpfn = placement.lpfn = 0; 787 placement.num_placement = placement.num_busy_placement = 1; 788 placement.placement = placement.busy_placement = &placement_memtype; 789 790 tmp_mem = *new_mem; 791 tmp_mem.mm_node = NULL; 792 ret = ttm_bo_mem_space(bo, &placement, &tmp_mem, intr, no_wait_reserve, no_wait_gpu); 793 if (ret) 794 return ret; 795 796 ret = ttm_bo_move_ttm(bo, true, no_wait_reserve, no_wait_gpu, &tmp_mem); 797 if (ret) 798 goto out; 799 800 ret = nouveau_bo_move_m2mf(bo, true, intr, no_wait_reserve, no_wait_gpu, new_mem); 801 if (ret) 802 goto out; 803 804 out: 805 ttm_bo_mem_put(bo, &tmp_mem); 806 return ret; 807 } 808 809 static void 810 nouveau_bo_move_ntfy(struct ttm_buffer_object *bo, struct ttm_mem_reg *new_mem) 811 { 812 struct nouveau_bo *nvbo = nouveau_bo(bo); 813 struct nouveau_vma *vma; 814 815 /* ttm can now (stupidly) pass the driver bos it didn't create... */ 816 if (bo->destroy != nouveau_bo_del_ttm) 817 return; 818 819 list_for_each_entry(vma, &nvbo->vma_list, head) { 820 if (new_mem && new_mem->mem_type == TTM_PL_VRAM) { 821 nouveau_vm_map(vma, new_mem->mm_node); 822 } else 823 if (new_mem && new_mem->mem_type == TTM_PL_TT && 824 nvbo->page_shift == vma->vm->spg_shift) { 825 nouveau_vm_map_sg(vma, 0, new_mem-> 826 num_pages << PAGE_SHIFT, 827 new_mem->mm_node); 828 } else { 829 nouveau_vm_unmap(vma); 830 } 831 } 832 } 833 834 static int 835 nouveau_bo_vm_bind(struct ttm_buffer_object *bo, struct ttm_mem_reg *new_mem, 836 struct nouveau_tile_reg **new_tile) 837 { 838 struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev); 839 struct drm_device *dev = dev_priv->dev; 840 struct nouveau_bo *nvbo = nouveau_bo(bo); 841 u64 offset = new_mem->start << PAGE_SHIFT; 842 843 *new_tile = NULL; 844 if (new_mem->mem_type != TTM_PL_VRAM) 845 return 0; 846 847 if (dev_priv->card_type >= NV_10) { 848 *new_tile = nv10_mem_set_tiling(dev, offset, new_mem->size, 849 nvbo->tile_mode, 850 nvbo->tile_flags); 851 } 852 853 return 0; 854 } 855 856 static void 857 nouveau_bo_vm_cleanup(struct ttm_buffer_object *bo, 858 struct nouveau_tile_reg *new_tile, 859 struct nouveau_tile_reg **old_tile) 860 { 861 struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev); 862 struct drm_device *dev = dev_priv->dev; 863 864 nv10_mem_put_tile_region(dev, *old_tile, bo->sync_obj); 865 *old_tile = new_tile; 866 } 867 868 static int 869 nouveau_bo_move(struct ttm_buffer_object *bo, bool evict, bool intr, 870 bool no_wait_reserve, bool no_wait_gpu, 871 struct ttm_mem_reg *new_mem) 872 { 873 struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev); 874 struct nouveau_bo *nvbo = nouveau_bo(bo); 875 struct ttm_mem_reg *old_mem = &bo->mem; 876 struct nouveau_tile_reg *new_tile = NULL; 877 int ret = 0; 878 879 if (dev_priv->card_type < NV_50) { 880 ret = nouveau_bo_vm_bind(bo, new_mem, &new_tile); 881 if (ret) 882 return ret; 883 } 884 885 /* Fake bo copy. */ 886 if (old_mem->mem_type == TTM_PL_SYSTEM && !bo->ttm) { 887 BUG_ON(bo->mem.mm_node != NULL); 888 bo->mem = *new_mem; 889 new_mem->mm_node = NULL; 890 goto out; 891 } 892 893 /* Software copy if the card isn't up and running yet. */ 894 if (!dev_priv->channel) { 895 ret = ttm_bo_move_memcpy(bo, evict, no_wait_reserve, no_wait_gpu, new_mem); 896 goto out; 897 } 898 899 /* Hardware assisted copy. */ 900 if (new_mem->mem_type == TTM_PL_SYSTEM) 901 ret = nouveau_bo_move_flipd(bo, evict, intr, no_wait_reserve, no_wait_gpu, new_mem); 902 else if (old_mem->mem_type == TTM_PL_SYSTEM) 903 ret = nouveau_bo_move_flips(bo, evict, intr, no_wait_reserve, no_wait_gpu, new_mem); 904 else 905 ret = nouveau_bo_move_m2mf(bo, evict, intr, no_wait_reserve, no_wait_gpu, new_mem); 906 907 if (!ret) 908 goto out; 909 910 /* Fallback to software copy. */ 911 ret = ttm_bo_move_memcpy(bo, evict, no_wait_reserve, no_wait_gpu, new_mem); 912 913 out: 914 if (dev_priv->card_type < NV_50) { 915 if (ret) 916 nouveau_bo_vm_cleanup(bo, NULL, &new_tile); 917 else 918 nouveau_bo_vm_cleanup(bo, new_tile, &nvbo->tile); 919 } 920 921 return ret; 922 } 923 924 static int 925 nouveau_bo_verify_access(struct ttm_buffer_object *bo, struct file *filp) 926 { 927 return 0; 928 } 929 930 static int 931 nouveau_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) 932 { 933 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; 934 struct drm_nouveau_private *dev_priv = nouveau_bdev(bdev); 935 struct drm_device *dev = dev_priv->dev; 936 int ret; 937 938 mem->bus.addr = NULL; 939 mem->bus.offset = 0; 940 mem->bus.size = mem->num_pages << PAGE_SHIFT; 941 mem->bus.base = 0; 942 mem->bus.is_iomem = false; 943 if (!(man->flags & TTM_MEMTYPE_FLAG_MAPPABLE)) 944 return -EINVAL; 945 switch (mem->mem_type) { 946 case TTM_PL_SYSTEM: 947 /* System memory */ 948 return 0; 949 case TTM_PL_TT: 950 #if __OS_HAS_AGP 951 if (dev_priv->gart_info.type == NOUVEAU_GART_AGP) { 952 mem->bus.offset = mem->start << PAGE_SHIFT; 953 mem->bus.base = dev_priv->gart_info.aper_base; 954 mem->bus.is_iomem = true; 955 } 956 #endif 957 break; 958 case TTM_PL_VRAM: 959 { 960 struct nouveau_mem *node = mem->mm_node; 961 u8 page_shift; 962 963 if (!dev_priv->bar1_vm) { 964 mem->bus.offset = mem->start << PAGE_SHIFT; 965 mem->bus.base = pci_resource_start(dev->pdev, 1); 966 mem->bus.is_iomem = true; 967 break; 968 } 969 970 if (dev_priv->card_type >= NV_C0) 971 page_shift = node->page_shift; 972 else 973 page_shift = 12; 974 975 ret = nouveau_vm_get(dev_priv->bar1_vm, mem->bus.size, 976 page_shift, NV_MEM_ACCESS_RW, 977 &node->bar_vma); 978 if (ret) 979 return ret; 980 981 nouveau_vm_map(&node->bar_vma, node); 982 if (ret) { 983 nouveau_vm_put(&node->bar_vma); 984 return ret; 985 } 986 987 mem->bus.offset = node->bar_vma.offset; 988 if (dev_priv->card_type == NV_50) /*XXX*/ 989 mem->bus.offset -= 0x0020000000ULL; 990 mem->bus.base = pci_resource_start(dev->pdev, 1); 991 mem->bus.is_iomem = true; 992 } 993 break; 994 default: 995 return -EINVAL; 996 } 997 return 0; 998 } 999 1000 static void 1001 nouveau_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) 1002 { 1003 struct drm_nouveau_private *dev_priv = nouveau_bdev(bdev); 1004 struct nouveau_mem *node = mem->mm_node; 1005 1006 if (!dev_priv->bar1_vm || mem->mem_type != TTM_PL_VRAM) 1007 return; 1008 1009 if (!node->bar_vma.node) 1010 return; 1011 1012 nouveau_vm_unmap(&node->bar_vma); 1013 nouveau_vm_put(&node->bar_vma); 1014 } 1015 1016 static int 1017 nouveau_ttm_fault_reserve_notify(struct ttm_buffer_object *bo) 1018 { 1019 struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev); 1020 struct nouveau_bo *nvbo = nouveau_bo(bo); 1021 1022 /* as long as the bo isn't in vram, and isn't tiled, we've got 1023 * nothing to do here. 1024 */ 1025 if (bo->mem.mem_type != TTM_PL_VRAM) { 1026 if (dev_priv->card_type < NV_50 || 1027 !nouveau_bo_tile_layout(nvbo)) 1028 return 0; 1029 } 1030 1031 /* make sure bo is in mappable vram */ 1032 if (bo->mem.start + bo->mem.num_pages < dev_priv->fb_mappable_pages) 1033 return 0; 1034 1035 1036 nvbo->placement.fpfn = 0; 1037 nvbo->placement.lpfn = dev_priv->fb_mappable_pages; 1038 nouveau_bo_placement_set(nvbo, TTM_PL_VRAM, 0); 1039 return nouveau_bo_validate(nvbo, false, true, false); 1040 } 1041 1042 void 1043 nouveau_bo_fence(struct nouveau_bo *nvbo, struct nouveau_fence *fence) 1044 { 1045 struct nouveau_fence *old_fence; 1046 1047 if (likely(fence)) 1048 nouveau_fence_ref(fence); 1049 1050 spin_lock(&nvbo->bo.bdev->fence_lock); 1051 old_fence = nvbo->bo.sync_obj; 1052 nvbo->bo.sync_obj = fence; 1053 spin_unlock(&nvbo->bo.bdev->fence_lock); 1054 1055 nouveau_fence_unref(&old_fence); 1056 } 1057 1058 static int 1059 nouveau_ttm_tt_populate(struct ttm_tt *ttm) 1060 { 1061 struct ttm_dma_tt *ttm_dma = (void *)ttm; 1062 struct drm_nouveau_private *dev_priv; 1063 struct drm_device *dev; 1064 unsigned i; 1065 int r; 1066 1067 if (ttm->state != tt_unpopulated) 1068 return 0; 1069 1070 dev_priv = nouveau_bdev(ttm->bdev); 1071 dev = dev_priv->dev; 1072 1073 #if __OS_HAS_AGP 1074 if (dev_priv->gart_info.type == NOUVEAU_GART_AGP) { 1075 return ttm_agp_tt_populate(ttm); 1076 } 1077 #endif 1078 1079 #ifdef CONFIG_SWIOTLB 1080 if (swiotlb_nr_tbl()) { 1081 return ttm_dma_populate((void *)ttm, dev->dev); 1082 } 1083 #endif 1084 1085 r = ttm_pool_populate(ttm); 1086 if (r) { 1087 return r; 1088 } 1089 1090 for (i = 0; i < ttm->num_pages; i++) { 1091 ttm_dma->dma_address[i] = pci_map_page(dev->pdev, ttm->pages[i], 1092 0, PAGE_SIZE, 1093 PCI_DMA_BIDIRECTIONAL); 1094 if (pci_dma_mapping_error(dev->pdev, ttm_dma->dma_address[i])) { 1095 while (--i) { 1096 pci_unmap_page(dev->pdev, ttm_dma->dma_address[i], 1097 PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); 1098 ttm_dma->dma_address[i] = 0; 1099 } 1100 ttm_pool_unpopulate(ttm); 1101 return -EFAULT; 1102 } 1103 } 1104 return 0; 1105 } 1106 1107 static void 1108 nouveau_ttm_tt_unpopulate(struct ttm_tt *ttm) 1109 { 1110 struct ttm_dma_tt *ttm_dma = (void *)ttm; 1111 struct drm_nouveau_private *dev_priv; 1112 struct drm_device *dev; 1113 unsigned i; 1114 1115 dev_priv = nouveau_bdev(ttm->bdev); 1116 dev = dev_priv->dev; 1117 1118 #if __OS_HAS_AGP 1119 if (dev_priv->gart_info.type == NOUVEAU_GART_AGP) { 1120 ttm_agp_tt_unpopulate(ttm); 1121 return; 1122 } 1123 #endif 1124 1125 #ifdef CONFIG_SWIOTLB 1126 if (swiotlb_nr_tbl()) { 1127 ttm_dma_unpopulate((void *)ttm, dev->dev); 1128 return; 1129 } 1130 #endif 1131 1132 for (i = 0; i < ttm->num_pages; i++) { 1133 if (ttm_dma->dma_address[i]) { 1134 pci_unmap_page(dev->pdev, ttm_dma->dma_address[i], 1135 PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); 1136 } 1137 } 1138 1139 ttm_pool_unpopulate(ttm); 1140 } 1141 1142 struct ttm_bo_driver nouveau_bo_driver = { 1143 .ttm_tt_create = &nouveau_ttm_tt_create, 1144 .ttm_tt_populate = &nouveau_ttm_tt_populate, 1145 .ttm_tt_unpopulate = &nouveau_ttm_tt_unpopulate, 1146 .invalidate_caches = nouveau_bo_invalidate_caches, 1147 .init_mem_type = nouveau_bo_init_mem_type, 1148 .evict_flags = nouveau_bo_evict_flags, 1149 .move_notify = nouveau_bo_move_ntfy, 1150 .move = nouveau_bo_move, 1151 .verify_access = nouveau_bo_verify_access, 1152 .sync_obj_signaled = __nouveau_fence_signalled, 1153 .sync_obj_wait = __nouveau_fence_wait, 1154 .sync_obj_flush = __nouveau_fence_flush, 1155 .sync_obj_unref = __nouveau_fence_unref, 1156 .sync_obj_ref = __nouveau_fence_ref, 1157 .fault_reserve_notify = &nouveau_ttm_fault_reserve_notify, 1158 .io_mem_reserve = &nouveau_ttm_io_mem_reserve, 1159 .io_mem_free = &nouveau_ttm_io_mem_free, 1160 }; 1161 1162 struct nouveau_vma * 1163 nouveau_bo_vma_find(struct nouveau_bo *nvbo, struct nouveau_vm *vm) 1164 { 1165 struct nouveau_vma *vma; 1166 list_for_each_entry(vma, &nvbo->vma_list, head) { 1167 if (vma->vm == vm) 1168 return vma; 1169 } 1170 1171 return NULL; 1172 } 1173 1174 int 1175 nouveau_bo_vma_add(struct nouveau_bo *nvbo, struct nouveau_vm *vm, 1176 struct nouveau_vma *vma) 1177 { 1178 const u32 size = nvbo->bo.mem.num_pages << PAGE_SHIFT; 1179 struct nouveau_mem *node = nvbo->bo.mem.mm_node; 1180 int ret; 1181 1182 ret = nouveau_vm_get(vm, size, nvbo->page_shift, 1183 NV_MEM_ACCESS_RW, vma); 1184 if (ret) 1185 return ret; 1186 1187 if (nvbo->bo.mem.mem_type == TTM_PL_VRAM) 1188 nouveau_vm_map(vma, nvbo->bo.mem.mm_node); 1189 else 1190 if (nvbo->bo.mem.mem_type == TTM_PL_TT) 1191 nouveau_vm_map_sg(vma, 0, size, node); 1192 1193 list_add_tail(&vma->head, &nvbo->vma_list); 1194 vma->refcount = 1; 1195 return 0; 1196 } 1197 1198 void 1199 nouveau_bo_vma_del(struct nouveau_bo *nvbo, struct nouveau_vma *vma) 1200 { 1201 if (vma->node) { 1202 if (nvbo->bo.mem.mem_type != TTM_PL_SYSTEM) { 1203 spin_lock(&nvbo->bo.bdev->fence_lock); 1204 ttm_bo_wait(&nvbo->bo, false, false, false); 1205 spin_unlock(&nvbo->bo.bdev->fence_lock); 1206 nouveau_vm_unmap(vma); 1207 } 1208 1209 nouveau_vm_put(vma); 1210 list_del(&vma->head); 1211 } 1212 } 1213