1 /************************************************************************** 2 * 3 * Copyright (c) 2007-2009 VMware, Inc., Palo Alto, CA., USA 4 * All Rights Reserved. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the 8 * "Software"), to deal in the Software without restriction, including 9 * without limitation the rights to use, copy, modify, merge, publish, 10 * distribute, sub license, and/or sell copies of the Software, and to 11 * permit persons to whom the Software is furnished to do so, subject to 12 * the following conditions: 13 * 14 * The above copyright notice and this permission notice (including the 15 * next paragraph) shall be included in all copies or substantial portions 16 * of the Software. 17 * 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 24 * USE OR OTHER DEALINGS IN THE SOFTWARE. 25 * 26 **************************************************************************/ 27 /* 28 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> 29 */ 30 31 #include <sys/cdefs.h> 32 #include <dev/drm2/drmP.h> 33 #include <dev/drm2/ttm/ttm_bo_driver.h> 34 #include <dev/drm2/ttm/ttm_placement.h> 35 #include <sys/sf_buf.h> 36 37 void ttm_bo_free_old_node(struct ttm_buffer_object *bo) 38 { 39 ttm_bo_mem_put(bo, &bo->mem); 40 } 41 42 int ttm_bo_move_ttm(struct ttm_buffer_object *bo, 43 bool evict, 44 bool no_wait_gpu, struct ttm_mem_reg *new_mem) 45 { 46 struct ttm_tt *ttm = bo->ttm; 47 struct ttm_mem_reg *old_mem = &bo->mem; 48 int ret; 49 50 if (old_mem->mem_type != TTM_PL_SYSTEM) { 51 ttm_tt_unbind(ttm); 52 ttm_bo_free_old_node(bo); 53 ttm_flag_masked(&old_mem->placement, TTM_PL_FLAG_SYSTEM, 54 TTM_PL_MASK_MEM); 55 old_mem->mem_type = TTM_PL_SYSTEM; 56 } 57 58 ret = ttm_tt_set_placement_caching(ttm, new_mem->placement); 59 if (unlikely(ret != 0)) 60 return ret; 61 62 if (new_mem->mem_type != TTM_PL_SYSTEM) { 63 ret = ttm_tt_bind(ttm, new_mem); 64 if (unlikely(ret != 0)) 65 return ret; 66 } 67 68 *old_mem = *new_mem; 69 new_mem->mm_node = NULL; 70 71 return 0; 72 } 73 74 int ttm_mem_io_lock(struct ttm_mem_type_manager *man, bool interruptible) 75 { 76 if (likely(man->io_reserve_fastpath)) 77 return 0; 78 79 if (interruptible) { 80 if (sx_xlock_sig(&man->io_reserve_mutex)) 81 return (-EINTR); 82 else 83 return (0); 84 } 85 86 sx_xlock(&man->io_reserve_mutex); 87 return 0; 88 } 89 90 void ttm_mem_io_unlock(struct ttm_mem_type_manager *man) 91 { 92 if (likely(man->io_reserve_fastpath)) 93 return; 94 95 sx_xunlock(&man->io_reserve_mutex); 96 } 97 98 static int ttm_mem_io_evict(struct ttm_mem_type_manager *man) 99 { 100 struct ttm_buffer_object *bo; 101 102 if (!man->use_io_reserve_lru || list_empty(&man->io_reserve_lru)) 103 return -EAGAIN; 104 105 bo = list_first_entry(&man->io_reserve_lru, 106 struct ttm_buffer_object, 107 io_reserve_lru); 108 list_del_init(&bo->io_reserve_lru); 109 ttm_bo_unmap_virtual_locked(bo); 110 111 return 0; 112 } 113 114 static int ttm_mem_io_reserve(struct ttm_bo_device *bdev, 115 struct ttm_mem_reg *mem) 116 { 117 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; 118 int ret = 0; 119 120 if (!bdev->driver->io_mem_reserve) 121 return 0; 122 if (likely(man->io_reserve_fastpath)) 123 return bdev->driver->io_mem_reserve(bdev, mem); 124 125 if (bdev->driver->io_mem_reserve && 126 mem->bus.io_reserved_count++ == 0) { 127 retry: 128 ret = bdev->driver->io_mem_reserve(bdev, mem); 129 if (ret == -EAGAIN) { 130 ret = ttm_mem_io_evict(man); 131 if (ret == 0) 132 goto retry; 133 } 134 } 135 return ret; 136 } 137 138 static void ttm_mem_io_free(struct ttm_bo_device *bdev, 139 struct ttm_mem_reg *mem) 140 { 141 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; 142 143 if (likely(man->io_reserve_fastpath)) 144 return; 145 146 if (bdev->driver->io_mem_reserve && 147 --mem->bus.io_reserved_count == 0 && 148 bdev->driver->io_mem_free) 149 bdev->driver->io_mem_free(bdev, mem); 150 151 } 152 153 int ttm_mem_io_reserve_vm(struct ttm_buffer_object *bo) 154 { 155 struct ttm_mem_reg *mem = &bo->mem; 156 int ret; 157 158 if (!mem->bus.io_reserved_vm) { 159 struct ttm_mem_type_manager *man = 160 &bo->bdev->man[mem->mem_type]; 161 162 ret = ttm_mem_io_reserve(bo->bdev, mem); 163 if (unlikely(ret != 0)) 164 return ret; 165 mem->bus.io_reserved_vm = true; 166 if (man->use_io_reserve_lru) 167 list_add_tail(&bo->io_reserve_lru, 168 &man->io_reserve_lru); 169 } 170 return 0; 171 } 172 173 void ttm_mem_io_free_vm(struct ttm_buffer_object *bo) 174 { 175 struct ttm_mem_reg *mem = &bo->mem; 176 177 if (mem->bus.io_reserved_vm) { 178 mem->bus.io_reserved_vm = false; 179 list_del_init(&bo->io_reserve_lru); 180 ttm_mem_io_free(bo->bdev, mem); 181 } 182 } 183 184 static 185 int ttm_mem_reg_ioremap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem, 186 void **virtual) 187 { 188 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; 189 int ret; 190 void *addr; 191 192 *virtual = NULL; 193 (void) ttm_mem_io_lock(man, false); 194 ret = ttm_mem_io_reserve(bdev, mem); 195 ttm_mem_io_unlock(man); 196 if (ret || !mem->bus.is_iomem) 197 return ret; 198 199 if (mem->bus.addr) { 200 addr = mem->bus.addr; 201 } else { 202 addr = pmap_mapdev_attr(mem->bus.base + mem->bus.offset, 203 mem->bus.size, (mem->placement & TTM_PL_FLAG_WC) ? 204 VM_MEMATTR_WRITE_COMBINING : VM_MEMATTR_UNCACHEABLE); 205 if (!addr) { 206 (void) ttm_mem_io_lock(man, false); 207 ttm_mem_io_free(bdev, mem); 208 ttm_mem_io_unlock(man); 209 return -ENOMEM; 210 } 211 } 212 *virtual = addr; 213 return 0; 214 } 215 216 static 217 void ttm_mem_reg_iounmap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem, 218 void *virtual) 219 { 220 struct ttm_mem_type_manager *man; 221 222 man = &bdev->man[mem->mem_type]; 223 224 if (virtual && mem->bus.addr == NULL) 225 pmap_unmapdev(virtual, mem->bus.size); 226 (void) ttm_mem_io_lock(man, false); 227 ttm_mem_io_free(bdev, mem); 228 ttm_mem_io_unlock(man); 229 } 230 231 static int ttm_copy_io_page(void *dst, void *src, unsigned long page) 232 { 233 uint32_t *dstP = 234 (uint32_t *) ((unsigned long)dst + (page << PAGE_SHIFT)); 235 uint32_t *srcP = 236 (uint32_t *) ((unsigned long)src + (page << PAGE_SHIFT)); 237 238 int i; 239 for (i = 0; i < PAGE_SIZE / sizeof(uint32_t); ++i) 240 /* iowrite32(ioread32(srcP++), dstP++); */ 241 *dstP++ = *srcP++; 242 return 0; 243 } 244 245 static int ttm_copy_io_ttm_page(struct ttm_tt *ttm, void *src, 246 unsigned long page, 247 vm_memattr_t prot) 248 { 249 vm_page_t d = ttm->pages[page]; 250 void *dst; 251 252 if (!d) 253 return -ENOMEM; 254 255 src = (void *)((unsigned long)src + (page << PAGE_SHIFT)); 256 257 /* XXXKIB can't sleep ? */ 258 dst = pmap_mapdev_attr(VM_PAGE_TO_PHYS(d), PAGE_SIZE, prot); 259 if (!dst) 260 return -ENOMEM; 261 262 memcpy(dst, src, PAGE_SIZE); 263 264 pmap_unmapdev(dst, PAGE_SIZE); 265 266 return 0; 267 } 268 269 static int ttm_copy_ttm_io_page(struct ttm_tt *ttm, void *dst, 270 unsigned long page, 271 vm_memattr_t prot) 272 { 273 vm_page_t s = ttm->pages[page]; 274 void *src; 275 276 if (!s) 277 return -ENOMEM; 278 279 dst = (void *)((unsigned long)dst + (page << PAGE_SHIFT)); 280 src = pmap_mapdev_attr(VM_PAGE_TO_PHYS(s), PAGE_SIZE, prot); 281 if (!src) 282 return -ENOMEM; 283 284 memcpy(dst, src, PAGE_SIZE); 285 286 pmap_unmapdev(src, PAGE_SIZE); 287 288 return 0; 289 } 290 291 int ttm_bo_move_memcpy(struct ttm_buffer_object *bo, 292 bool evict, bool no_wait_gpu, 293 struct ttm_mem_reg *new_mem) 294 { 295 struct ttm_bo_device *bdev = bo->bdev; 296 struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type]; 297 struct ttm_tt *ttm = bo->ttm; 298 struct ttm_mem_reg *old_mem = &bo->mem; 299 struct ttm_mem_reg old_copy = *old_mem; 300 void *old_iomap; 301 void *new_iomap; 302 int ret; 303 unsigned long i; 304 unsigned long page; 305 unsigned long add = 0; 306 int dir; 307 308 ret = ttm_mem_reg_ioremap(bdev, old_mem, &old_iomap); 309 if (ret) 310 return ret; 311 ret = ttm_mem_reg_ioremap(bdev, new_mem, &new_iomap); 312 if (ret) 313 goto out; 314 315 if (old_iomap == NULL && new_iomap == NULL) 316 goto out2; 317 if (old_iomap == NULL && ttm == NULL) 318 goto out2; 319 320 if (ttm->state == tt_unpopulated) { 321 ret = ttm->bdev->driver->ttm_tt_populate(ttm); 322 if (ret) { 323 /* if we fail here don't nuke the mm node 324 * as the bo still owns it */ 325 old_copy.mm_node = NULL; 326 goto out1; 327 } 328 } 329 330 add = 0; 331 dir = 1; 332 333 if ((old_mem->mem_type == new_mem->mem_type) && 334 (new_mem->start < old_mem->start + old_mem->size)) { 335 dir = -1; 336 add = new_mem->num_pages - 1; 337 } 338 339 for (i = 0; i < new_mem->num_pages; ++i) { 340 page = i * dir + add; 341 if (old_iomap == NULL) { 342 vm_memattr_t prot = ttm_io_prot(old_mem->placement); 343 ret = ttm_copy_ttm_io_page(ttm, new_iomap, page, 344 prot); 345 } else if (new_iomap == NULL) { 346 vm_memattr_t prot = ttm_io_prot(new_mem->placement); 347 ret = ttm_copy_io_ttm_page(ttm, old_iomap, page, 348 prot); 349 } else 350 ret = ttm_copy_io_page(new_iomap, old_iomap, page); 351 if (ret) { 352 /* failing here, means keep old copy as-is */ 353 old_copy.mm_node = NULL; 354 goto out1; 355 } 356 } 357 mb(); 358 out2: 359 old_copy = *old_mem; 360 *old_mem = *new_mem; 361 new_mem->mm_node = NULL; 362 363 if ((man->flags & TTM_MEMTYPE_FLAG_FIXED) && (ttm != NULL)) { 364 ttm_tt_unbind(ttm); 365 ttm_tt_destroy(ttm); 366 bo->ttm = NULL; 367 } 368 369 out1: 370 ttm_mem_reg_iounmap(bdev, old_mem, new_iomap); 371 out: 372 ttm_mem_reg_iounmap(bdev, &old_copy, old_iomap); 373 ttm_bo_mem_put(bo, &old_copy); 374 return ret; 375 } 376 377 MALLOC_DEFINE(M_TTM_TRANSF_OBJ, "ttm_transf_obj", "TTM Transfer Objects"); 378 379 static void ttm_transfered_destroy(struct ttm_buffer_object *bo) 380 { 381 free(bo, M_TTM_TRANSF_OBJ); 382 } 383 384 /** 385 * ttm_buffer_object_transfer 386 * 387 * @bo: A pointer to a struct ttm_buffer_object. 388 * @new_obj: A pointer to a pointer to a newly created ttm_buffer_object, 389 * holding the data of @bo with the old placement. 390 * 391 * This is a utility function that may be called after an accelerated move 392 * has been scheduled. A new buffer object is created as a placeholder for 393 * the old data while it's being copied. When that buffer object is idle, 394 * it can be destroyed, releasing the space of the old placement. 395 * Returns: 396 * !0: Failure. 397 */ 398 399 static int 400 ttm_buffer_object_transfer(struct ttm_buffer_object *bo, 401 struct ttm_buffer_object **new_obj) 402 { 403 struct ttm_buffer_object *fbo; 404 struct ttm_bo_device *bdev = bo->bdev; 405 struct ttm_bo_driver *driver = bdev->driver; 406 407 fbo = malloc(sizeof(*fbo), M_TTM_TRANSF_OBJ, M_WAITOK); 408 *fbo = *bo; 409 410 /** 411 * Fix up members that we shouldn't copy directly: 412 * TODO: Explicit member copy would probably be better here. 413 */ 414 415 INIT_LIST_HEAD(&fbo->ddestroy); 416 INIT_LIST_HEAD(&fbo->lru); 417 INIT_LIST_HEAD(&fbo->swap); 418 INIT_LIST_HEAD(&fbo->io_reserve_lru); 419 fbo->vm_node = NULL; 420 atomic_set(&fbo->cpu_writers, 0); 421 422 mtx_lock(&bdev->fence_lock); 423 if (bo->sync_obj) 424 fbo->sync_obj = driver->sync_obj_ref(bo->sync_obj); 425 else 426 fbo->sync_obj = NULL; 427 mtx_unlock(&bdev->fence_lock); 428 refcount_init(&fbo->list_kref, 1); 429 refcount_init(&fbo->kref, 1); 430 fbo->destroy = &ttm_transfered_destroy; 431 fbo->acc_size = 0; 432 433 *new_obj = fbo; 434 return 0; 435 } 436 437 vm_memattr_t 438 ttm_io_prot(uint32_t caching_flags) 439 { 440 #if defined(__i386__) || defined(__amd64__) || defined(__powerpc__) || \ 441 defined(__arm__) 442 if (caching_flags & TTM_PL_FLAG_WC) 443 return (VM_MEMATTR_WRITE_COMBINING); 444 else 445 /* 446 * We do not support i386, look at the linux source 447 * for the reason of the comment. 448 */ 449 return (VM_MEMATTR_UNCACHEABLE); 450 #else 451 #error Port me 452 #endif 453 } 454 455 static int ttm_bo_ioremap(struct ttm_buffer_object *bo, 456 unsigned long offset, 457 unsigned long size, 458 struct ttm_bo_kmap_obj *map) 459 { 460 struct ttm_mem_reg *mem = &bo->mem; 461 462 if (bo->mem.bus.addr) { 463 map->bo_kmap_type = ttm_bo_map_premapped; 464 map->virtual = (void *)(((u8 *)bo->mem.bus.addr) + offset); 465 } else { 466 map->bo_kmap_type = ttm_bo_map_iomap; 467 map->virtual = pmap_mapdev_attr(bo->mem.bus.base + 468 bo->mem.bus.offset + offset, size, 469 (mem->placement & TTM_PL_FLAG_WC) ? 470 VM_MEMATTR_WRITE_COMBINING : VM_MEMATTR_UNCACHEABLE); 471 map->size = size; 472 } 473 return (!map->virtual) ? -ENOMEM : 0; 474 } 475 476 static int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo, 477 unsigned long start_page, 478 unsigned long num_pages, 479 struct ttm_bo_kmap_obj *map) 480 { 481 struct ttm_mem_reg *mem = &bo->mem; 482 vm_memattr_t prot; 483 struct ttm_tt *ttm = bo->ttm; 484 int i, ret; 485 486 MPASS(ttm != NULL); 487 488 if (ttm->state == tt_unpopulated) { 489 ret = ttm->bdev->driver->ttm_tt_populate(ttm); 490 if (ret) 491 return ret; 492 } 493 494 if (num_pages == 1 && (mem->placement & TTM_PL_FLAG_CACHED)) { 495 /* 496 * We're mapping a single page, and the desired 497 * page protection is consistent with the bo. 498 */ 499 500 map->bo_kmap_type = ttm_bo_map_kmap; 501 map->page = ttm->pages[start_page]; 502 map->sf = sf_buf_alloc(map->page, 0); 503 map->virtual = (void *)sf_buf_kva(map->sf); 504 } else { 505 /* 506 * We need to use vmap to get the desired page protection 507 * or to make the buffer object look contiguous. 508 */ 509 prot = (mem->placement & TTM_PL_FLAG_CACHED) ? 510 VM_MEMATTR_DEFAULT : ttm_io_prot(mem->placement); 511 map->bo_kmap_type = ttm_bo_map_vmap; 512 map->num_pages = num_pages; 513 map->virtual = (void *)kva_alloc(num_pages * PAGE_SIZE); 514 if (map->virtual != NULL) { 515 for (i = 0; i < num_pages; i++) { 516 /* XXXKIB hack */ 517 pmap_page_set_memattr(ttm->pages[start_page + 518 i], prot); 519 } 520 pmap_qenter((vm_offset_t)map->virtual, 521 &ttm->pages[start_page], num_pages); 522 } 523 } 524 return (!map->virtual) ? -ENOMEM : 0; 525 } 526 527 int ttm_bo_kmap(struct ttm_buffer_object *bo, 528 unsigned long start_page, unsigned long num_pages, 529 struct ttm_bo_kmap_obj *map) 530 { 531 struct ttm_mem_type_manager *man = 532 &bo->bdev->man[bo->mem.mem_type]; 533 unsigned long offset, size; 534 int ret; 535 536 MPASS(list_empty(&bo->swap)); 537 map->virtual = NULL; 538 map->bo = bo; 539 if (num_pages > bo->num_pages) 540 return -EINVAL; 541 if (start_page > bo->num_pages) 542 return -EINVAL; 543 #if 0 544 if (num_pages > 1 && !DRM_SUSER(DRM_CURPROC)) 545 return -EPERM; 546 #endif 547 (void) ttm_mem_io_lock(man, false); 548 ret = ttm_mem_io_reserve(bo->bdev, &bo->mem); 549 ttm_mem_io_unlock(man); 550 if (ret) 551 return ret; 552 if (!bo->mem.bus.is_iomem) { 553 return ttm_bo_kmap_ttm(bo, start_page, num_pages, map); 554 } else { 555 offset = start_page << PAGE_SHIFT; 556 size = num_pages << PAGE_SHIFT; 557 return ttm_bo_ioremap(bo, offset, size, map); 558 } 559 } 560 561 void ttm_bo_kunmap(struct ttm_bo_kmap_obj *map) 562 { 563 struct ttm_buffer_object *bo = map->bo; 564 struct ttm_mem_type_manager *man = 565 &bo->bdev->man[bo->mem.mem_type]; 566 567 if (!map->virtual) 568 return; 569 switch (map->bo_kmap_type) { 570 case ttm_bo_map_iomap: 571 pmap_unmapdev(map->virtual, map->size); 572 break; 573 case ttm_bo_map_vmap: 574 pmap_qremove((vm_offset_t)(map->virtual), map->num_pages); 575 kva_free((vm_offset_t)map->virtual, 576 map->num_pages * PAGE_SIZE); 577 break; 578 case ttm_bo_map_kmap: 579 sf_buf_free(map->sf); 580 break; 581 case ttm_bo_map_premapped: 582 break; 583 default: 584 MPASS(0); 585 } 586 (void) ttm_mem_io_lock(man, false); 587 ttm_mem_io_free(map->bo->bdev, &map->bo->mem); 588 ttm_mem_io_unlock(man); 589 map->virtual = NULL; 590 map->page = NULL; 591 map->sf = NULL; 592 } 593 594 int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo, 595 void *sync_obj, 596 bool evict, 597 bool no_wait_gpu, 598 struct ttm_mem_reg *new_mem) 599 { 600 struct ttm_bo_device *bdev = bo->bdev; 601 struct ttm_bo_driver *driver = bdev->driver; 602 struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type]; 603 struct ttm_mem_reg *old_mem = &bo->mem; 604 int ret; 605 struct ttm_buffer_object *ghost_obj; 606 void *tmp_obj = NULL; 607 608 mtx_lock(&bdev->fence_lock); 609 if (bo->sync_obj) { 610 tmp_obj = bo->sync_obj; 611 bo->sync_obj = NULL; 612 } 613 bo->sync_obj = driver->sync_obj_ref(sync_obj); 614 if (evict) { 615 ret = ttm_bo_wait(bo, false, false, false); 616 mtx_unlock(&bdev->fence_lock); 617 if (tmp_obj) 618 driver->sync_obj_unref(&tmp_obj); 619 if (ret) 620 return ret; 621 622 if ((man->flags & TTM_MEMTYPE_FLAG_FIXED) && 623 (bo->ttm != NULL)) { 624 ttm_tt_unbind(bo->ttm); 625 ttm_tt_destroy(bo->ttm); 626 bo->ttm = NULL; 627 } 628 ttm_bo_free_old_node(bo); 629 } else { 630 /** 631 * This should help pipeline ordinary buffer moves. 632 * 633 * Hang old buffer memory on a new buffer object, 634 * and leave it to be released when the GPU 635 * operation has completed. 636 */ 637 638 set_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags); 639 mtx_unlock(&bdev->fence_lock); 640 if (tmp_obj) 641 driver->sync_obj_unref(&tmp_obj); 642 643 ret = ttm_buffer_object_transfer(bo, &ghost_obj); 644 if (ret) 645 return ret; 646 647 /** 648 * If we're not moving to fixed memory, the TTM object 649 * needs to stay alive. Otherwhise hang it on the ghost 650 * bo to be unbound and destroyed. 651 */ 652 653 if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED)) 654 ghost_obj->ttm = NULL; 655 else 656 bo->ttm = NULL; 657 658 ttm_bo_unreserve(ghost_obj); 659 ttm_bo_unref(&ghost_obj); 660 } 661 662 *old_mem = *new_mem; 663 new_mem->mm_node = NULL; 664 665 return 0; 666 } 667