1 /************************************************************************** 2 * 3 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA 4 * All Rights Reserved. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the 8 * "Software"), to deal in the Software without restriction, including 9 * without limitation the rights to use, copy, modify, merge, publish, 10 * distribute, sub license, and/or sell copies of the Software, and to 11 * permit persons to whom the Software is furnished to do so, subject to 12 * the following conditions: 13 * 14 * The above copyright notice and this permission notice (including the 15 * next paragraph) shall be included in all copies or substantial portions 16 * of the Software. 17 * 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 24 * USE OR OTHER DEALINGS IN THE SOFTWARE. 25 * 26 **************************************************************************/ 27 /* 28 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> 29 */ 30 /* Notes: 31 * 32 * We store bo pointer in drm_mm_node struct so we know which bo own a 33 * specific node. There is no protection on the pointer, thus to make 34 * sure things don't go berserk you have to access this pointer while 35 * holding the global lru lock and make sure anytime you free a node you 36 * reset the pointer to NULL. 37 */ 38 39 #include "ttm/ttm_module.h" 40 #include "ttm/ttm_bo_driver.h" 41 #include "ttm/ttm_placement.h" 42 #include <linux/jiffies.h> 43 #include <linux/slab.h> 44 #include <linux/sched.h> 45 #include <linux/mm.h> 46 #include <linux/file.h> 47 #include <linux/module.h> 48 49 #define TTM_ASSERT_LOCKED(param) 50 #define TTM_DEBUG(fmt, arg...) 51 #define TTM_BO_HASH_ORDER 13 52 53 static int ttm_bo_setup_vm(struct ttm_buffer_object *bo); 54 static int ttm_bo_swapout(struct ttm_mem_shrink *shrink); 55 static void ttm_bo_global_kobj_release(struct kobject *kobj); 56 57 static struct attribute ttm_bo_count = { 58 .name = "bo_count", 59 .mode = S_IRUGO 60 }; 61 62 static inline int ttm_mem_type_from_flags(uint32_t flags, uint32_t *mem_type) 63 { 64 int i; 65 66 for (i = 0; i <= TTM_PL_PRIV5; i++) 67 if (flags & (1 << i)) { 68 *mem_type = i; 69 return 0; 70 } 71 return -EINVAL; 72 } 73 74 static void ttm_mem_type_debug(struct ttm_bo_device *bdev, int mem_type) 75 { 76 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 77 78 printk(KERN_ERR TTM_PFX " has_type: %d\n", man->has_type); 79 printk(KERN_ERR TTM_PFX " use_type: %d\n", man->use_type); 80 printk(KERN_ERR TTM_PFX " flags: 0x%08X\n", man->flags); 81 printk(KERN_ERR TTM_PFX " gpu_offset: 0x%08lX\n", man->gpu_offset); 82 printk(KERN_ERR TTM_PFX " size: %llu\n", man->size); 83 printk(KERN_ERR TTM_PFX " available_caching: 0x%08X\n", 84 man->available_caching); 85 printk(KERN_ERR TTM_PFX " default_caching: 0x%08X\n", 86 man->default_caching); 87 if (mem_type != TTM_PL_SYSTEM) { 88 spin_lock(&bdev->glob->lru_lock); 89 drm_mm_debug_table(&man->manager, TTM_PFX); 90 spin_unlock(&bdev->glob->lru_lock); 91 } 92 } 93 94 static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo, 95 struct ttm_placement *placement) 96 { 97 int i, ret, mem_type; 98 99 printk(KERN_ERR TTM_PFX "No space for %p (%lu pages, %luK, %luM)\n", 100 bo, bo->mem.num_pages, bo->mem.size >> 10, 101 bo->mem.size >> 20); 102 for (i = 0; i < placement->num_placement; i++) { 103 ret = ttm_mem_type_from_flags(placement->placement[i], 104 &mem_type); 105 if (ret) 106 return; 107 printk(KERN_ERR TTM_PFX " placement[%d]=0x%08X (%d)\n", 108 i, placement->placement[i], mem_type); 109 ttm_mem_type_debug(bo->bdev, mem_type); 110 } 111 } 112 113 static ssize_t ttm_bo_global_show(struct kobject *kobj, 114 struct attribute *attr, 115 char *buffer) 116 { 117 struct ttm_bo_global *glob = 118 container_of(kobj, struct ttm_bo_global, kobj); 119 120 return snprintf(buffer, PAGE_SIZE, "%lu\n", 121 (unsigned long) atomic_read(&glob->bo_count)); 122 } 123 124 static struct attribute *ttm_bo_global_attrs[] = { 125 &ttm_bo_count, 126 NULL 127 }; 128 129 static const struct sysfs_ops ttm_bo_global_ops = { 130 .show = &ttm_bo_global_show 131 }; 132 133 static struct kobj_type ttm_bo_glob_kobj_type = { 134 .release = &ttm_bo_global_kobj_release, 135 .sysfs_ops = &ttm_bo_global_ops, 136 .default_attrs = ttm_bo_global_attrs 137 }; 138 139 140 static inline uint32_t ttm_bo_type_flags(unsigned type) 141 { 142 return 1 << (type); 143 } 144 145 static void ttm_bo_release_list(struct kref *list_kref) 146 { 147 struct ttm_buffer_object *bo = 148 container_of(list_kref, struct ttm_buffer_object, list_kref); 149 struct ttm_bo_device *bdev = bo->bdev; 150 151 BUG_ON(atomic_read(&bo->list_kref.refcount)); 152 BUG_ON(atomic_read(&bo->kref.refcount)); 153 BUG_ON(atomic_read(&bo->cpu_writers)); 154 BUG_ON(bo->sync_obj != NULL); 155 BUG_ON(bo->mem.mm_node != NULL); 156 BUG_ON(!list_empty(&bo->lru)); 157 BUG_ON(!list_empty(&bo->ddestroy)); 158 159 if (bo->ttm) 160 ttm_tt_destroy(bo->ttm); 161 atomic_dec(&bo->glob->bo_count); 162 if (bo->destroy) 163 bo->destroy(bo); 164 else { 165 ttm_mem_global_free(bdev->glob->mem_glob, bo->acc_size); 166 kfree(bo); 167 } 168 } 169 170 int ttm_bo_wait_unreserved(struct ttm_buffer_object *bo, bool interruptible) 171 { 172 173 if (interruptible) { 174 int ret = 0; 175 176 ret = wait_event_interruptible(bo->event_queue, 177 atomic_read(&bo->reserved) == 0); 178 if (unlikely(ret != 0)) 179 return ret; 180 } else { 181 wait_event(bo->event_queue, atomic_read(&bo->reserved) == 0); 182 } 183 return 0; 184 } 185 EXPORT_SYMBOL(ttm_bo_wait_unreserved); 186 187 static void ttm_bo_add_to_lru(struct ttm_buffer_object *bo) 188 { 189 struct ttm_bo_device *bdev = bo->bdev; 190 struct ttm_mem_type_manager *man; 191 192 BUG_ON(!atomic_read(&bo->reserved)); 193 194 if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) { 195 196 BUG_ON(!list_empty(&bo->lru)); 197 198 man = &bdev->man[bo->mem.mem_type]; 199 list_add_tail(&bo->lru, &man->lru); 200 kref_get(&bo->list_kref); 201 202 if (bo->ttm != NULL) { 203 list_add_tail(&bo->swap, &bo->glob->swap_lru); 204 kref_get(&bo->list_kref); 205 } 206 } 207 } 208 209 /** 210 * Call with the lru_lock held. 211 */ 212 213 static int ttm_bo_del_from_lru(struct ttm_buffer_object *bo) 214 { 215 int put_count = 0; 216 217 if (!list_empty(&bo->swap)) { 218 list_del_init(&bo->swap); 219 ++put_count; 220 } 221 if (!list_empty(&bo->lru)) { 222 list_del_init(&bo->lru); 223 ++put_count; 224 } 225 226 /* 227 * TODO: Add a driver hook to delete from 228 * driver-specific LRU's here. 229 */ 230 231 return put_count; 232 } 233 234 int ttm_bo_reserve_locked(struct ttm_buffer_object *bo, 235 bool interruptible, 236 bool no_wait, bool use_sequence, uint32_t sequence) 237 { 238 struct ttm_bo_global *glob = bo->glob; 239 int ret; 240 241 while (unlikely(atomic_cmpxchg(&bo->reserved, 0, 1) != 0)) { 242 if (use_sequence && bo->seq_valid && 243 (sequence - bo->val_seq < (1 << 31))) { 244 return -EAGAIN; 245 } 246 247 if (no_wait) 248 return -EBUSY; 249 250 spin_unlock(&glob->lru_lock); 251 ret = ttm_bo_wait_unreserved(bo, interruptible); 252 spin_lock(&glob->lru_lock); 253 254 if (unlikely(ret)) 255 return ret; 256 } 257 258 if (use_sequence) { 259 bo->val_seq = sequence; 260 bo->seq_valid = true; 261 } else { 262 bo->seq_valid = false; 263 } 264 265 return 0; 266 } 267 EXPORT_SYMBOL(ttm_bo_reserve); 268 269 static void ttm_bo_ref_bug(struct kref *list_kref) 270 { 271 BUG(); 272 } 273 274 int ttm_bo_reserve(struct ttm_buffer_object *bo, 275 bool interruptible, 276 bool no_wait, bool use_sequence, uint32_t sequence) 277 { 278 struct ttm_bo_global *glob = bo->glob; 279 int put_count = 0; 280 int ret; 281 282 spin_lock(&glob->lru_lock); 283 ret = ttm_bo_reserve_locked(bo, interruptible, no_wait, use_sequence, 284 sequence); 285 if (likely(ret == 0)) 286 put_count = ttm_bo_del_from_lru(bo); 287 spin_unlock(&glob->lru_lock); 288 289 while (put_count--) 290 kref_put(&bo->list_kref, ttm_bo_ref_bug); 291 292 return ret; 293 } 294 295 void ttm_bo_unreserve(struct ttm_buffer_object *bo) 296 { 297 struct ttm_bo_global *glob = bo->glob; 298 299 spin_lock(&glob->lru_lock); 300 ttm_bo_add_to_lru(bo); 301 atomic_set(&bo->reserved, 0); 302 wake_up_all(&bo->event_queue); 303 spin_unlock(&glob->lru_lock); 304 } 305 EXPORT_SYMBOL(ttm_bo_unreserve); 306 307 /* 308 * Call bo->mutex locked. 309 */ 310 static int ttm_bo_add_ttm(struct ttm_buffer_object *bo, bool zero_alloc) 311 { 312 struct ttm_bo_device *bdev = bo->bdev; 313 struct ttm_bo_global *glob = bo->glob; 314 int ret = 0; 315 uint32_t page_flags = 0; 316 317 TTM_ASSERT_LOCKED(&bo->mutex); 318 bo->ttm = NULL; 319 320 if (bdev->need_dma32) 321 page_flags |= TTM_PAGE_FLAG_DMA32; 322 323 switch (bo->type) { 324 case ttm_bo_type_device: 325 if (zero_alloc) 326 page_flags |= TTM_PAGE_FLAG_ZERO_ALLOC; 327 case ttm_bo_type_kernel: 328 bo->ttm = ttm_tt_create(bdev, bo->num_pages << PAGE_SHIFT, 329 page_flags, glob->dummy_read_page); 330 if (unlikely(bo->ttm == NULL)) 331 ret = -ENOMEM; 332 break; 333 case ttm_bo_type_user: 334 bo->ttm = ttm_tt_create(bdev, bo->num_pages << PAGE_SHIFT, 335 page_flags | TTM_PAGE_FLAG_USER, 336 glob->dummy_read_page); 337 if (unlikely(bo->ttm == NULL)) { 338 ret = -ENOMEM; 339 break; 340 } 341 342 ret = ttm_tt_set_user(bo->ttm, current, 343 bo->buffer_start, bo->num_pages); 344 if (unlikely(ret != 0)) 345 ttm_tt_destroy(bo->ttm); 346 break; 347 default: 348 printk(KERN_ERR TTM_PFX "Illegal buffer object type\n"); 349 ret = -EINVAL; 350 break; 351 } 352 353 return ret; 354 } 355 356 static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, 357 struct ttm_mem_reg *mem, 358 bool evict, bool interruptible, 359 bool no_wait_reserve, bool no_wait_gpu) 360 { 361 struct ttm_bo_device *bdev = bo->bdev; 362 bool old_is_pci = ttm_mem_reg_is_pci(bdev, &bo->mem); 363 bool new_is_pci = ttm_mem_reg_is_pci(bdev, mem); 364 struct ttm_mem_type_manager *old_man = &bdev->man[bo->mem.mem_type]; 365 struct ttm_mem_type_manager *new_man = &bdev->man[mem->mem_type]; 366 int ret = 0; 367 368 if (old_is_pci || new_is_pci || 369 ((mem->placement & bo->mem.placement & TTM_PL_MASK_CACHING) == 0)) 370 ttm_bo_unmap_virtual(bo); 371 372 /* 373 * Create and bind a ttm if required. 374 */ 375 376 if (!(new_man->flags & TTM_MEMTYPE_FLAG_FIXED) && (bo->ttm == NULL)) { 377 ret = ttm_bo_add_ttm(bo, false); 378 if (ret) 379 goto out_err; 380 381 ret = ttm_tt_set_placement_caching(bo->ttm, mem->placement); 382 if (ret) 383 goto out_err; 384 385 if (mem->mem_type != TTM_PL_SYSTEM) { 386 ret = ttm_tt_bind(bo->ttm, mem); 387 if (ret) 388 goto out_err; 389 } 390 391 if (bo->mem.mem_type == TTM_PL_SYSTEM) { 392 bo->mem = *mem; 393 mem->mm_node = NULL; 394 goto moved; 395 } 396 397 } 398 399 if (bdev->driver->move_notify) 400 bdev->driver->move_notify(bo, mem); 401 402 if (!(old_man->flags & TTM_MEMTYPE_FLAG_FIXED) && 403 !(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) 404 ret = ttm_bo_move_ttm(bo, evict, no_wait_reserve, no_wait_gpu, mem); 405 else if (bdev->driver->move) 406 ret = bdev->driver->move(bo, evict, interruptible, 407 no_wait_reserve, no_wait_gpu, mem); 408 else 409 ret = ttm_bo_move_memcpy(bo, evict, no_wait_reserve, no_wait_gpu, mem); 410 411 if (ret) 412 goto out_err; 413 414 moved: 415 if (bo->evicted) { 416 ret = bdev->driver->invalidate_caches(bdev, bo->mem.placement); 417 if (ret) 418 printk(KERN_ERR TTM_PFX "Can not flush read caches\n"); 419 bo->evicted = false; 420 } 421 422 if (bo->mem.mm_node) { 423 spin_lock(&bo->lock); 424 bo->offset = (bo->mem.mm_node->start << PAGE_SHIFT) + 425 bdev->man[bo->mem.mem_type].gpu_offset; 426 bo->cur_placement = bo->mem.placement; 427 spin_unlock(&bo->lock); 428 } else 429 bo->offset = 0; 430 431 return 0; 432 433 out_err: 434 new_man = &bdev->man[bo->mem.mem_type]; 435 if ((new_man->flags & TTM_MEMTYPE_FLAG_FIXED) && bo->ttm) { 436 ttm_tt_unbind(bo->ttm); 437 ttm_tt_destroy(bo->ttm); 438 bo->ttm = NULL; 439 } 440 441 return ret; 442 } 443 444 /** 445 * If bo idle, remove from delayed- and lru lists, and unref. 446 * If not idle, and already on delayed list, do nothing. 447 * If not idle, and not on delayed list, put on delayed list, 448 * up the list_kref and schedule a delayed list check. 449 */ 450 451 static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo, bool remove_all) 452 { 453 struct ttm_bo_device *bdev = bo->bdev; 454 struct ttm_bo_global *glob = bo->glob; 455 struct ttm_bo_driver *driver = bdev->driver; 456 int ret; 457 458 spin_lock(&bo->lock); 459 (void) ttm_bo_wait(bo, false, false, !remove_all); 460 461 if (!bo->sync_obj) { 462 int put_count; 463 464 spin_unlock(&bo->lock); 465 466 spin_lock(&glob->lru_lock); 467 put_count = ttm_bo_del_from_lru(bo); 468 469 ret = ttm_bo_reserve_locked(bo, false, false, false, 0); 470 BUG_ON(ret); 471 if (bo->ttm) 472 ttm_tt_unbind(bo->ttm); 473 474 if (!list_empty(&bo->ddestroy)) { 475 list_del_init(&bo->ddestroy); 476 ++put_count; 477 } 478 if (bo->mem.mm_node) { 479 bo->mem.mm_node->private = NULL; 480 drm_mm_put_block(bo->mem.mm_node); 481 bo->mem.mm_node = NULL; 482 } 483 spin_unlock(&glob->lru_lock); 484 485 atomic_set(&bo->reserved, 0); 486 487 while (put_count--) 488 kref_put(&bo->list_kref, ttm_bo_ref_bug); 489 490 return 0; 491 } 492 493 spin_lock(&glob->lru_lock); 494 if (list_empty(&bo->ddestroy)) { 495 void *sync_obj = bo->sync_obj; 496 void *sync_obj_arg = bo->sync_obj_arg; 497 498 kref_get(&bo->list_kref); 499 list_add_tail(&bo->ddestroy, &bdev->ddestroy); 500 spin_unlock(&glob->lru_lock); 501 spin_unlock(&bo->lock); 502 503 if (sync_obj) 504 driver->sync_obj_flush(sync_obj, sync_obj_arg); 505 schedule_delayed_work(&bdev->wq, 506 ((HZ / 100) < 1) ? 1 : HZ / 100); 507 ret = 0; 508 509 } else { 510 spin_unlock(&glob->lru_lock); 511 spin_unlock(&bo->lock); 512 ret = -EBUSY; 513 } 514 515 return ret; 516 } 517 518 /** 519 * Traverse the delayed list, and call ttm_bo_cleanup_refs on all 520 * encountered buffers. 521 */ 522 523 static int ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all) 524 { 525 struct ttm_bo_global *glob = bdev->glob; 526 struct ttm_buffer_object *entry = NULL; 527 int ret = 0; 528 529 spin_lock(&glob->lru_lock); 530 if (list_empty(&bdev->ddestroy)) 531 goto out_unlock; 532 533 entry = list_first_entry(&bdev->ddestroy, 534 struct ttm_buffer_object, ddestroy); 535 kref_get(&entry->list_kref); 536 537 for (;;) { 538 struct ttm_buffer_object *nentry = NULL; 539 540 if (entry->ddestroy.next != &bdev->ddestroy) { 541 nentry = list_first_entry(&entry->ddestroy, 542 struct ttm_buffer_object, ddestroy); 543 kref_get(&nentry->list_kref); 544 } 545 546 spin_unlock(&glob->lru_lock); 547 ret = ttm_bo_cleanup_refs(entry, remove_all); 548 kref_put(&entry->list_kref, ttm_bo_release_list); 549 entry = nentry; 550 551 if (ret || !entry) 552 goto out; 553 554 spin_lock(&glob->lru_lock); 555 if (list_empty(&entry->ddestroy)) 556 break; 557 } 558 559 out_unlock: 560 spin_unlock(&glob->lru_lock); 561 out: 562 if (entry) 563 kref_put(&entry->list_kref, ttm_bo_release_list); 564 return ret; 565 } 566 567 static void ttm_bo_delayed_workqueue(struct work_struct *work) 568 { 569 struct ttm_bo_device *bdev = 570 container_of(work, struct ttm_bo_device, wq.work); 571 572 if (ttm_bo_delayed_delete(bdev, false)) { 573 schedule_delayed_work(&bdev->wq, 574 ((HZ / 100) < 1) ? 1 : HZ / 100); 575 } 576 } 577 578 static void ttm_bo_release(struct kref *kref) 579 { 580 struct ttm_buffer_object *bo = 581 container_of(kref, struct ttm_buffer_object, kref); 582 struct ttm_bo_device *bdev = bo->bdev; 583 584 if (likely(bo->vm_node != NULL)) { 585 rb_erase(&bo->vm_rb, &bdev->addr_space_rb); 586 drm_mm_put_block(bo->vm_node); 587 bo->vm_node = NULL; 588 } 589 write_unlock(&bdev->vm_lock); 590 ttm_bo_cleanup_refs(bo, false); 591 kref_put(&bo->list_kref, ttm_bo_release_list); 592 write_lock(&bdev->vm_lock); 593 } 594 595 void ttm_bo_unref(struct ttm_buffer_object **p_bo) 596 { 597 struct ttm_buffer_object *bo = *p_bo; 598 struct ttm_bo_device *bdev = bo->bdev; 599 600 *p_bo = NULL; 601 write_lock(&bdev->vm_lock); 602 kref_put(&bo->kref, ttm_bo_release); 603 write_unlock(&bdev->vm_lock); 604 } 605 EXPORT_SYMBOL(ttm_bo_unref); 606 607 int ttm_bo_lock_delayed_workqueue(struct ttm_bo_device *bdev) 608 { 609 return cancel_delayed_work_sync(&bdev->wq); 610 } 611 EXPORT_SYMBOL(ttm_bo_lock_delayed_workqueue); 612 613 void ttm_bo_unlock_delayed_workqueue(struct ttm_bo_device *bdev, int resched) 614 { 615 if (resched) 616 schedule_delayed_work(&bdev->wq, 617 ((HZ / 100) < 1) ? 1 : HZ / 100); 618 } 619 EXPORT_SYMBOL(ttm_bo_unlock_delayed_workqueue); 620 621 static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible, 622 bool no_wait_reserve, bool no_wait_gpu) 623 { 624 struct ttm_bo_device *bdev = bo->bdev; 625 struct ttm_bo_global *glob = bo->glob; 626 struct ttm_mem_reg evict_mem; 627 struct ttm_placement placement; 628 int ret = 0; 629 630 spin_lock(&bo->lock); 631 ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu); 632 spin_unlock(&bo->lock); 633 634 if (unlikely(ret != 0)) { 635 if (ret != -ERESTARTSYS) { 636 printk(KERN_ERR TTM_PFX 637 "Failed to expire sync object before " 638 "buffer eviction.\n"); 639 } 640 goto out; 641 } 642 643 BUG_ON(!atomic_read(&bo->reserved)); 644 645 evict_mem = bo->mem; 646 evict_mem.mm_node = NULL; 647 evict_mem.bus.io_reserved = false; 648 649 placement.fpfn = 0; 650 placement.lpfn = 0; 651 placement.num_placement = 0; 652 placement.num_busy_placement = 0; 653 bdev->driver->evict_flags(bo, &placement); 654 ret = ttm_bo_mem_space(bo, &placement, &evict_mem, interruptible, 655 no_wait_reserve, no_wait_gpu); 656 if (ret) { 657 if (ret != -ERESTARTSYS) { 658 printk(KERN_ERR TTM_PFX 659 "Failed to find memory space for " 660 "buffer 0x%p eviction.\n", bo); 661 ttm_bo_mem_space_debug(bo, &placement); 662 } 663 goto out; 664 } 665 666 ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, interruptible, 667 no_wait_reserve, no_wait_gpu); 668 if (ret) { 669 if (ret != -ERESTARTSYS) 670 printk(KERN_ERR TTM_PFX "Buffer eviction failed\n"); 671 spin_lock(&glob->lru_lock); 672 if (evict_mem.mm_node) { 673 evict_mem.mm_node->private = NULL; 674 drm_mm_put_block(evict_mem.mm_node); 675 evict_mem.mm_node = NULL; 676 } 677 spin_unlock(&glob->lru_lock); 678 goto out; 679 } 680 bo->evicted = true; 681 out: 682 return ret; 683 } 684 685 static int ttm_mem_evict_first(struct ttm_bo_device *bdev, 686 uint32_t mem_type, 687 bool interruptible, bool no_wait_reserve, 688 bool no_wait_gpu) 689 { 690 struct ttm_bo_global *glob = bdev->glob; 691 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 692 struct ttm_buffer_object *bo; 693 int ret, put_count = 0; 694 695 retry: 696 spin_lock(&glob->lru_lock); 697 if (list_empty(&man->lru)) { 698 spin_unlock(&glob->lru_lock); 699 return -EBUSY; 700 } 701 702 bo = list_first_entry(&man->lru, struct ttm_buffer_object, lru); 703 kref_get(&bo->list_kref); 704 705 ret = ttm_bo_reserve_locked(bo, false, no_wait_reserve, false, 0); 706 707 if (unlikely(ret == -EBUSY)) { 708 spin_unlock(&glob->lru_lock); 709 if (likely(!no_wait_gpu)) 710 ret = ttm_bo_wait_unreserved(bo, interruptible); 711 712 kref_put(&bo->list_kref, ttm_bo_release_list); 713 714 /** 715 * We *need* to retry after releasing the lru lock. 716 */ 717 718 if (unlikely(ret != 0)) 719 return ret; 720 goto retry; 721 } 722 723 put_count = ttm_bo_del_from_lru(bo); 724 spin_unlock(&glob->lru_lock); 725 726 BUG_ON(ret != 0); 727 728 while (put_count--) 729 kref_put(&bo->list_kref, ttm_bo_ref_bug); 730 731 ret = ttm_bo_evict(bo, interruptible, no_wait_reserve, no_wait_gpu); 732 ttm_bo_unreserve(bo); 733 734 kref_put(&bo->list_kref, ttm_bo_release_list); 735 return ret; 736 } 737 738 static int ttm_bo_man_get_node(struct ttm_buffer_object *bo, 739 struct ttm_mem_type_manager *man, 740 struct ttm_placement *placement, 741 struct ttm_mem_reg *mem, 742 struct drm_mm_node **node) 743 { 744 struct ttm_bo_global *glob = bo->glob; 745 unsigned long lpfn; 746 int ret; 747 748 lpfn = placement->lpfn; 749 if (!lpfn) 750 lpfn = man->size; 751 *node = NULL; 752 do { 753 ret = drm_mm_pre_get(&man->manager); 754 if (unlikely(ret)) 755 return ret; 756 757 spin_lock(&glob->lru_lock); 758 *node = drm_mm_search_free_in_range(&man->manager, 759 mem->num_pages, mem->page_alignment, 760 placement->fpfn, lpfn, 1); 761 if (unlikely(*node == NULL)) { 762 spin_unlock(&glob->lru_lock); 763 return 0; 764 } 765 *node = drm_mm_get_block_atomic_range(*node, mem->num_pages, 766 mem->page_alignment, 767 placement->fpfn, 768 lpfn); 769 spin_unlock(&glob->lru_lock); 770 } while (*node == NULL); 771 return 0; 772 } 773 774 /** 775 * Repeatedly evict memory from the LRU for @mem_type until we create enough 776 * space, or we've evicted everything and there isn't enough space. 777 */ 778 static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo, 779 uint32_t mem_type, 780 struct ttm_placement *placement, 781 struct ttm_mem_reg *mem, 782 bool interruptible, 783 bool no_wait_reserve, 784 bool no_wait_gpu) 785 { 786 struct ttm_bo_device *bdev = bo->bdev; 787 struct ttm_bo_global *glob = bdev->glob; 788 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 789 struct drm_mm_node *node; 790 int ret; 791 792 do { 793 ret = ttm_bo_man_get_node(bo, man, placement, mem, &node); 794 if (unlikely(ret != 0)) 795 return ret; 796 if (node) 797 break; 798 spin_lock(&glob->lru_lock); 799 if (list_empty(&man->lru)) { 800 spin_unlock(&glob->lru_lock); 801 break; 802 } 803 spin_unlock(&glob->lru_lock); 804 ret = ttm_mem_evict_first(bdev, mem_type, interruptible, 805 no_wait_reserve, no_wait_gpu); 806 if (unlikely(ret != 0)) 807 return ret; 808 } while (1); 809 if (node == NULL) 810 return -ENOMEM; 811 mem->mm_node = node; 812 mem->mem_type = mem_type; 813 return 0; 814 } 815 816 static uint32_t ttm_bo_select_caching(struct ttm_mem_type_manager *man, 817 uint32_t cur_placement, 818 uint32_t proposed_placement) 819 { 820 uint32_t caching = proposed_placement & TTM_PL_MASK_CACHING; 821 uint32_t result = proposed_placement & ~TTM_PL_MASK_CACHING; 822 823 /** 824 * Keep current caching if possible. 825 */ 826 827 if ((cur_placement & caching) != 0) 828 result |= (cur_placement & caching); 829 else if ((man->default_caching & caching) != 0) 830 result |= man->default_caching; 831 else if ((TTM_PL_FLAG_CACHED & caching) != 0) 832 result |= TTM_PL_FLAG_CACHED; 833 else if ((TTM_PL_FLAG_WC & caching) != 0) 834 result |= TTM_PL_FLAG_WC; 835 else if ((TTM_PL_FLAG_UNCACHED & caching) != 0) 836 result |= TTM_PL_FLAG_UNCACHED; 837 838 return result; 839 } 840 841 static bool ttm_bo_mt_compatible(struct ttm_mem_type_manager *man, 842 bool disallow_fixed, 843 uint32_t mem_type, 844 uint32_t proposed_placement, 845 uint32_t *masked_placement) 846 { 847 uint32_t cur_flags = ttm_bo_type_flags(mem_type); 848 849 if ((man->flags & TTM_MEMTYPE_FLAG_FIXED) && disallow_fixed) 850 return false; 851 852 if ((cur_flags & proposed_placement & TTM_PL_MASK_MEM) == 0) 853 return false; 854 855 if ((proposed_placement & man->available_caching) == 0) 856 return false; 857 858 cur_flags |= (proposed_placement & man->available_caching); 859 860 *masked_placement = cur_flags; 861 return true; 862 } 863 864 /** 865 * Creates space for memory region @mem according to its type. 866 * 867 * This function first searches for free space in compatible memory types in 868 * the priority order defined by the driver. If free space isn't found, then 869 * ttm_bo_mem_force_space is attempted in priority order to evict and find 870 * space. 871 */ 872 int ttm_bo_mem_space(struct ttm_buffer_object *bo, 873 struct ttm_placement *placement, 874 struct ttm_mem_reg *mem, 875 bool interruptible, bool no_wait_reserve, 876 bool no_wait_gpu) 877 { 878 struct ttm_bo_device *bdev = bo->bdev; 879 struct ttm_mem_type_manager *man; 880 uint32_t mem_type = TTM_PL_SYSTEM; 881 uint32_t cur_flags = 0; 882 bool type_found = false; 883 bool type_ok = false; 884 bool has_erestartsys = false; 885 struct drm_mm_node *node = NULL; 886 int i, ret; 887 888 mem->mm_node = NULL; 889 for (i = 0; i < placement->num_placement; ++i) { 890 ret = ttm_mem_type_from_flags(placement->placement[i], 891 &mem_type); 892 if (ret) 893 return ret; 894 man = &bdev->man[mem_type]; 895 896 type_ok = ttm_bo_mt_compatible(man, 897 bo->type == ttm_bo_type_user, 898 mem_type, 899 placement->placement[i], 900 &cur_flags); 901 902 if (!type_ok) 903 continue; 904 905 cur_flags = ttm_bo_select_caching(man, bo->mem.placement, 906 cur_flags); 907 /* 908 * Use the access and other non-mapping-related flag bits from 909 * the memory placement flags to the current flags 910 */ 911 ttm_flag_masked(&cur_flags, placement->placement[i], 912 ~TTM_PL_MASK_MEMTYPE); 913 914 if (mem_type == TTM_PL_SYSTEM) 915 break; 916 917 if (man->has_type && man->use_type) { 918 type_found = true; 919 ret = ttm_bo_man_get_node(bo, man, placement, mem, 920 &node); 921 if (unlikely(ret)) 922 return ret; 923 } 924 if (node) 925 break; 926 } 927 928 if ((type_ok && (mem_type == TTM_PL_SYSTEM)) || node) { 929 mem->mm_node = node; 930 mem->mem_type = mem_type; 931 mem->placement = cur_flags; 932 if (node) 933 node->private = bo; 934 return 0; 935 } 936 937 if (!type_found) 938 return -EINVAL; 939 940 for (i = 0; i < placement->num_busy_placement; ++i) { 941 ret = ttm_mem_type_from_flags(placement->busy_placement[i], 942 &mem_type); 943 if (ret) 944 return ret; 945 man = &bdev->man[mem_type]; 946 if (!man->has_type) 947 continue; 948 if (!ttm_bo_mt_compatible(man, 949 bo->type == ttm_bo_type_user, 950 mem_type, 951 placement->busy_placement[i], 952 &cur_flags)) 953 continue; 954 955 cur_flags = ttm_bo_select_caching(man, bo->mem.placement, 956 cur_flags); 957 /* 958 * Use the access and other non-mapping-related flag bits from 959 * the memory placement flags to the current flags 960 */ 961 ttm_flag_masked(&cur_flags, placement->busy_placement[i], 962 ~TTM_PL_MASK_MEMTYPE); 963 964 965 if (mem_type == TTM_PL_SYSTEM) { 966 mem->mem_type = mem_type; 967 mem->placement = cur_flags; 968 mem->mm_node = NULL; 969 return 0; 970 } 971 972 ret = ttm_bo_mem_force_space(bo, mem_type, placement, mem, 973 interruptible, no_wait_reserve, no_wait_gpu); 974 if (ret == 0 && mem->mm_node) { 975 mem->placement = cur_flags; 976 mem->mm_node->private = bo; 977 return 0; 978 } 979 if (ret == -ERESTARTSYS) 980 has_erestartsys = true; 981 } 982 ret = (has_erestartsys) ? -ERESTARTSYS : -ENOMEM; 983 return ret; 984 } 985 EXPORT_SYMBOL(ttm_bo_mem_space); 986 987 int ttm_bo_wait_cpu(struct ttm_buffer_object *bo, bool no_wait) 988 { 989 if ((atomic_read(&bo->cpu_writers) > 0) && no_wait) 990 return -EBUSY; 991 992 return wait_event_interruptible(bo->event_queue, 993 atomic_read(&bo->cpu_writers) == 0); 994 } 995 EXPORT_SYMBOL(ttm_bo_wait_cpu); 996 997 int ttm_bo_move_buffer(struct ttm_buffer_object *bo, 998 struct ttm_placement *placement, 999 bool interruptible, bool no_wait_reserve, 1000 bool no_wait_gpu) 1001 { 1002 struct ttm_bo_global *glob = bo->glob; 1003 int ret = 0; 1004 struct ttm_mem_reg mem; 1005 1006 BUG_ON(!atomic_read(&bo->reserved)); 1007 1008 /* 1009 * FIXME: It's possible to pipeline buffer moves. 1010 * Have the driver move function wait for idle when necessary, 1011 * instead of doing it here. 1012 */ 1013 spin_lock(&bo->lock); 1014 ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu); 1015 spin_unlock(&bo->lock); 1016 if (ret) 1017 return ret; 1018 mem.num_pages = bo->num_pages; 1019 mem.size = mem.num_pages << PAGE_SHIFT; 1020 mem.page_alignment = bo->mem.page_alignment; 1021 mem.bus.io_reserved = false; 1022 /* 1023 * Determine where to move the buffer. 1024 */ 1025 ret = ttm_bo_mem_space(bo, placement, &mem, interruptible, no_wait_reserve, no_wait_gpu); 1026 if (ret) 1027 goto out_unlock; 1028 ret = ttm_bo_handle_move_mem(bo, &mem, false, interruptible, no_wait_reserve, no_wait_gpu); 1029 out_unlock: 1030 if (ret && mem.mm_node) { 1031 spin_lock(&glob->lru_lock); 1032 mem.mm_node->private = NULL; 1033 drm_mm_put_block(mem.mm_node); 1034 spin_unlock(&glob->lru_lock); 1035 } 1036 return ret; 1037 } 1038 1039 static int ttm_bo_mem_compat(struct ttm_placement *placement, 1040 struct ttm_mem_reg *mem) 1041 { 1042 int i; 1043 struct drm_mm_node *node = mem->mm_node; 1044 1045 if (node && placement->lpfn != 0 && 1046 (node->start < placement->fpfn || 1047 node->start + node->size > placement->lpfn)) 1048 return -1; 1049 1050 for (i = 0; i < placement->num_placement; i++) { 1051 if ((placement->placement[i] & mem->placement & 1052 TTM_PL_MASK_CACHING) && 1053 (placement->placement[i] & mem->placement & 1054 TTM_PL_MASK_MEM)) 1055 return i; 1056 } 1057 return -1; 1058 } 1059 1060 int ttm_bo_validate(struct ttm_buffer_object *bo, 1061 struct ttm_placement *placement, 1062 bool interruptible, bool no_wait_reserve, 1063 bool no_wait_gpu) 1064 { 1065 int ret; 1066 1067 BUG_ON(!atomic_read(&bo->reserved)); 1068 /* Check that range is valid */ 1069 if (placement->lpfn || placement->fpfn) 1070 if (placement->fpfn > placement->lpfn || 1071 (placement->lpfn - placement->fpfn) < bo->num_pages) 1072 return -EINVAL; 1073 /* 1074 * Check whether we need to move buffer. 1075 */ 1076 ret = ttm_bo_mem_compat(placement, &bo->mem); 1077 if (ret < 0) { 1078 ret = ttm_bo_move_buffer(bo, placement, interruptible, no_wait_reserve, no_wait_gpu); 1079 if (ret) 1080 return ret; 1081 } else { 1082 /* 1083 * Use the access and other non-mapping-related flag bits from 1084 * the compatible memory placement flags to the active flags 1085 */ 1086 ttm_flag_masked(&bo->mem.placement, placement->placement[ret], 1087 ~TTM_PL_MASK_MEMTYPE); 1088 } 1089 /* 1090 * We might need to add a TTM. 1091 */ 1092 if (bo->mem.mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) { 1093 ret = ttm_bo_add_ttm(bo, true); 1094 if (ret) 1095 return ret; 1096 } 1097 return 0; 1098 } 1099 EXPORT_SYMBOL(ttm_bo_validate); 1100 1101 int ttm_bo_check_placement(struct ttm_buffer_object *bo, 1102 struct ttm_placement *placement) 1103 { 1104 int i; 1105 1106 if (placement->fpfn || placement->lpfn) { 1107 if (bo->mem.num_pages > (placement->lpfn - placement->fpfn)) { 1108 printk(KERN_ERR TTM_PFX "Page number range to small " 1109 "Need %lu pages, range is [%u, %u]\n", 1110 bo->mem.num_pages, placement->fpfn, 1111 placement->lpfn); 1112 return -EINVAL; 1113 } 1114 } 1115 for (i = 0; i < placement->num_placement; i++) { 1116 if (!capable(CAP_SYS_ADMIN)) { 1117 if (placement->placement[i] & TTM_PL_FLAG_NO_EVICT) { 1118 printk(KERN_ERR TTM_PFX "Need to be root to " 1119 "modify NO_EVICT status.\n"); 1120 return -EINVAL; 1121 } 1122 } 1123 } 1124 for (i = 0; i < placement->num_busy_placement; i++) { 1125 if (!capable(CAP_SYS_ADMIN)) { 1126 if (placement->busy_placement[i] & TTM_PL_FLAG_NO_EVICT) { 1127 printk(KERN_ERR TTM_PFX "Need to be root to " 1128 "modify NO_EVICT status.\n"); 1129 return -EINVAL; 1130 } 1131 } 1132 } 1133 return 0; 1134 } 1135 1136 int ttm_bo_init(struct ttm_bo_device *bdev, 1137 struct ttm_buffer_object *bo, 1138 unsigned long size, 1139 enum ttm_bo_type type, 1140 struct ttm_placement *placement, 1141 uint32_t page_alignment, 1142 unsigned long buffer_start, 1143 bool interruptible, 1144 struct file *persistant_swap_storage, 1145 size_t acc_size, 1146 void (*destroy) (struct ttm_buffer_object *)) 1147 { 1148 int ret = 0; 1149 unsigned long num_pages; 1150 1151 size += buffer_start & ~PAGE_MASK; 1152 num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; 1153 if (num_pages == 0) { 1154 printk(KERN_ERR TTM_PFX "Illegal buffer object size.\n"); 1155 return -EINVAL; 1156 } 1157 bo->destroy = destroy; 1158 1159 spin_lock_init(&bo->lock); 1160 kref_init(&bo->kref); 1161 kref_init(&bo->list_kref); 1162 atomic_set(&bo->cpu_writers, 0); 1163 atomic_set(&bo->reserved, 1); 1164 init_waitqueue_head(&bo->event_queue); 1165 INIT_LIST_HEAD(&bo->lru); 1166 INIT_LIST_HEAD(&bo->ddestroy); 1167 INIT_LIST_HEAD(&bo->swap); 1168 bo->bdev = bdev; 1169 bo->glob = bdev->glob; 1170 bo->type = type; 1171 bo->num_pages = num_pages; 1172 bo->mem.size = num_pages << PAGE_SHIFT; 1173 bo->mem.mem_type = TTM_PL_SYSTEM; 1174 bo->mem.num_pages = bo->num_pages; 1175 bo->mem.mm_node = NULL; 1176 bo->mem.page_alignment = page_alignment; 1177 bo->mem.bus.io_reserved = false; 1178 bo->buffer_start = buffer_start & PAGE_MASK; 1179 bo->priv_flags = 0; 1180 bo->mem.placement = (TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED); 1181 bo->seq_valid = false; 1182 bo->persistant_swap_storage = persistant_swap_storage; 1183 bo->acc_size = acc_size; 1184 atomic_inc(&bo->glob->bo_count); 1185 1186 ret = ttm_bo_check_placement(bo, placement); 1187 if (unlikely(ret != 0)) 1188 goto out_err; 1189 1190 /* 1191 * For ttm_bo_type_device buffers, allocate 1192 * address space from the device. 1193 */ 1194 if (bo->type == ttm_bo_type_device) { 1195 ret = ttm_bo_setup_vm(bo); 1196 if (ret) 1197 goto out_err; 1198 } 1199 1200 ret = ttm_bo_validate(bo, placement, interruptible, false, false); 1201 if (ret) 1202 goto out_err; 1203 1204 ttm_bo_unreserve(bo); 1205 return 0; 1206 1207 out_err: 1208 ttm_bo_unreserve(bo); 1209 ttm_bo_unref(&bo); 1210 1211 return ret; 1212 } 1213 EXPORT_SYMBOL(ttm_bo_init); 1214 1215 static inline size_t ttm_bo_size(struct ttm_bo_global *glob, 1216 unsigned long num_pages) 1217 { 1218 size_t page_array_size = (num_pages * sizeof(void *) + PAGE_SIZE - 1) & 1219 PAGE_MASK; 1220 1221 return glob->ttm_bo_size + 2 * page_array_size; 1222 } 1223 1224 int ttm_bo_create(struct ttm_bo_device *bdev, 1225 unsigned long size, 1226 enum ttm_bo_type type, 1227 struct ttm_placement *placement, 1228 uint32_t page_alignment, 1229 unsigned long buffer_start, 1230 bool interruptible, 1231 struct file *persistant_swap_storage, 1232 struct ttm_buffer_object **p_bo) 1233 { 1234 struct ttm_buffer_object *bo; 1235 struct ttm_mem_global *mem_glob = bdev->glob->mem_glob; 1236 int ret; 1237 1238 size_t acc_size = 1239 ttm_bo_size(bdev->glob, (size + PAGE_SIZE - 1) >> PAGE_SHIFT); 1240 ret = ttm_mem_global_alloc(mem_glob, acc_size, false, false); 1241 if (unlikely(ret != 0)) 1242 return ret; 1243 1244 bo = kzalloc(sizeof(*bo), GFP_KERNEL); 1245 1246 if (unlikely(bo == NULL)) { 1247 ttm_mem_global_free(mem_glob, acc_size); 1248 return -ENOMEM; 1249 } 1250 1251 ret = ttm_bo_init(bdev, bo, size, type, placement, page_alignment, 1252 buffer_start, interruptible, 1253 persistant_swap_storage, acc_size, NULL); 1254 if (likely(ret == 0)) 1255 *p_bo = bo; 1256 1257 return ret; 1258 } 1259 1260 static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev, 1261 unsigned mem_type, bool allow_errors) 1262 { 1263 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 1264 struct ttm_bo_global *glob = bdev->glob; 1265 int ret; 1266 1267 /* 1268 * Can't use standard list traversal since we're unlocking. 1269 */ 1270 1271 spin_lock(&glob->lru_lock); 1272 while (!list_empty(&man->lru)) { 1273 spin_unlock(&glob->lru_lock); 1274 ret = ttm_mem_evict_first(bdev, mem_type, false, false, false); 1275 if (ret) { 1276 if (allow_errors) { 1277 return ret; 1278 } else { 1279 printk(KERN_ERR TTM_PFX 1280 "Cleanup eviction failed\n"); 1281 } 1282 } 1283 spin_lock(&glob->lru_lock); 1284 } 1285 spin_unlock(&glob->lru_lock); 1286 return 0; 1287 } 1288 1289 int ttm_bo_clean_mm(struct ttm_bo_device *bdev, unsigned mem_type) 1290 { 1291 struct ttm_bo_global *glob = bdev->glob; 1292 struct ttm_mem_type_manager *man; 1293 int ret = -EINVAL; 1294 1295 if (mem_type >= TTM_NUM_MEM_TYPES) { 1296 printk(KERN_ERR TTM_PFX "Illegal memory type %d\n", mem_type); 1297 return ret; 1298 } 1299 man = &bdev->man[mem_type]; 1300 1301 if (!man->has_type) { 1302 printk(KERN_ERR TTM_PFX "Trying to take down uninitialized " 1303 "memory manager type %u\n", mem_type); 1304 return ret; 1305 } 1306 1307 man->use_type = false; 1308 man->has_type = false; 1309 1310 ret = 0; 1311 if (mem_type > 0) { 1312 ttm_bo_force_list_clean(bdev, mem_type, false); 1313 1314 spin_lock(&glob->lru_lock); 1315 if (drm_mm_clean(&man->manager)) 1316 drm_mm_takedown(&man->manager); 1317 else 1318 ret = -EBUSY; 1319 1320 spin_unlock(&glob->lru_lock); 1321 } 1322 1323 return ret; 1324 } 1325 EXPORT_SYMBOL(ttm_bo_clean_mm); 1326 1327 int ttm_bo_evict_mm(struct ttm_bo_device *bdev, unsigned mem_type) 1328 { 1329 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 1330 1331 if (mem_type == 0 || mem_type >= TTM_NUM_MEM_TYPES) { 1332 printk(KERN_ERR TTM_PFX 1333 "Illegal memory manager memory type %u.\n", 1334 mem_type); 1335 return -EINVAL; 1336 } 1337 1338 if (!man->has_type) { 1339 printk(KERN_ERR TTM_PFX 1340 "Memory type %u has not been initialized.\n", 1341 mem_type); 1342 return 0; 1343 } 1344 1345 return ttm_bo_force_list_clean(bdev, mem_type, true); 1346 } 1347 EXPORT_SYMBOL(ttm_bo_evict_mm); 1348 1349 int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type, 1350 unsigned long p_size) 1351 { 1352 int ret = -EINVAL; 1353 struct ttm_mem_type_manager *man; 1354 1355 if (type >= TTM_NUM_MEM_TYPES) { 1356 printk(KERN_ERR TTM_PFX "Illegal memory type %d\n", type); 1357 return ret; 1358 } 1359 1360 man = &bdev->man[type]; 1361 if (man->has_type) { 1362 printk(KERN_ERR TTM_PFX 1363 "Memory manager already initialized for type %d\n", 1364 type); 1365 return ret; 1366 } 1367 1368 ret = bdev->driver->init_mem_type(bdev, type, man); 1369 if (ret) 1370 return ret; 1371 1372 ret = 0; 1373 if (type != TTM_PL_SYSTEM) { 1374 if (!p_size) { 1375 printk(KERN_ERR TTM_PFX 1376 "Zero size memory manager type %d\n", 1377 type); 1378 return ret; 1379 } 1380 ret = drm_mm_init(&man->manager, 0, p_size); 1381 if (ret) 1382 return ret; 1383 } 1384 man->has_type = true; 1385 man->use_type = true; 1386 man->size = p_size; 1387 1388 INIT_LIST_HEAD(&man->lru); 1389 1390 return 0; 1391 } 1392 EXPORT_SYMBOL(ttm_bo_init_mm); 1393 1394 static void ttm_bo_global_kobj_release(struct kobject *kobj) 1395 { 1396 struct ttm_bo_global *glob = 1397 container_of(kobj, struct ttm_bo_global, kobj); 1398 1399 ttm_mem_unregister_shrink(glob->mem_glob, &glob->shrink); 1400 __free_page(glob->dummy_read_page); 1401 kfree(glob); 1402 } 1403 1404 void ttm_bo_global_release(struct ttm_global_reference *ref) 1405 { 1406 struct ttm_bo_global *glob = ref->object; 1407 1408 kobject_del(&glob->kobj); 1409 kobject_put(&glob->kobj); 1410 } 1411 EXPORT_SYMBOL(ttm_bo_global_release); 1412 1413 int ttm_bo_global_init(struct ttm_global_reference *ref) 1414 { 1415 struct ttm_bo_global_ref *bo_ref = 1416 container_of(ref, struct ttm_bo_global_ref, ref); 1417 struct ttm_bo_global *glob = ref->object; 1418 int ret; 1419 1420 mutex_init(&glob->device_list_mutex); 1421 spin_lock_init(&glob->lru_lock); 1422 glob->mem_glob = bo_ref->mem_glob; 1423 glob->dummy_read_page = alloc_page(__GFP_ZERO | GFP_DMA32); 1424 1425 if (unlikely(glob->dummy_read_page == NULL)) { 1426 ret = -ENOMEM; 1427 goto out_no_drp; 1428 } 1429 1430 INIT_LIST_HEAD(&glob->swap_lru); 1431 INIT_LIST_HEAD(&glob->device_list); 1432 1433 ttm_mem_init_shrink(&glob->shrink, ttm_bo_swapout); 1434 ret = ttm_mem_register_shrink(glob->mem_glob, &glob->shrink); 1435 if (unlikely(ret != 0)) { 1436 printk(KERN_ERR TTM_PFX 1437 "Could not register buffer object swapout.\n"); 1438 goto out_no_shrink; 1439 } 1440 1441 glob->ttm_bo_extra_size = 1442 ttm_round_pot(sizeof(struct ttm_tt)) + 1443 ttm_round_pot(sizeof(struct ttm_backend)); 1444 1445 glob->ttm_bo_size = glob->ttm_bo_extra_size + 1446 ttm_round_pot(sizeof(struct ttm_buffer_object)); 1447 1448 atomic_set(&glob->bo_count, 0); 1449 1450 ret = kobject_init_and_add( 1451 &glob->kobj, &ttm_bo_glob_kobj_type, ttm_get_kobj(), "buffer_objects"); 1452 if (unlikely(ret != 0)) 1453 kobject_put(&glob->kobj); 1454 return ret; 1455 out_no_shrink: 1456 __free_page(glob->dummy_read_page); 1457 out_no_drp: 1458 kfree(glob); 1459 return ret; 1460 } 1461 EXPORT_SYMBOL(ttm_bo_global_init); 1462 1463 1464 int ttm_bo_device_release(struct ttm_bo_device *bdev) 1465 { 1466 int ret = 0; 1467 unsigned i = TTM_NUM_MEM_TYPES; 1468 struct ttm_mem_type_manager *man; 1469 struct ttm_bo_global *glob = bdev->glob; 1470 1471 while (i--) { 1472 man = &bdev->man[i]; 1473 if (man->has_type) { 1474 man->use_type = false; 1475 if ((i != TTM_PL_SYSTEM) && ttm_bo_clean_mm(bdev, i)) { 1476 ret = -EBUSY; 1477 printk(KERN_ERR TTM_PFX 1478 "DRM memory manager type %d " 1479 "is not clean.\n", i); 1480 } 1481 man->has_type = false; 1482 } 1483 } 1484 1485 mutex_lock(&glob->device_list_mutex); 1486 list_del(&bdev->device_list); 1487 mutex_unlock(&glob->device_list_mutex); 1488 1489 if (!cancel_delayed_work(&bdev->wq)) 1490 flush_scheduled_work(); 1491 1492 while (ttm_bo_delayed_delete(bdev, true)) 1493 ; 1494 1495 spin_lock(&glob->lru_lock); 1496 if (list_empty(&bdev->ddestroy)) 1497 TTM_DEBUG("Delayed destroy list was clean\n"); 1498 1499 if (list_empty(&bdev->man[0].lru)) 1500 TTM_DEBUG("Swap list was clean\n"); 1501 spin_unlock(&glob->lru_lock); 1502 1503 BUG_ON(!drm_mm_clean(&bdev->addr_space_mm)); 1504 write_lock(&bdev->vm_lock); 1505 drm_mm_takedown(&bdev->addr_space_mm); 1506 write_unlock(&bdev->vm_lock); 1507 1508 return ret; 1509 } 1510 EXPORT_SYMBOL(ttm_bo_device_release); 1511 1512 int ttm_bo_device_init(struct ttm_bo_device *bdev, 1513 struct ttm_bo_global *glob, 1514 struct ttm_bo_driver *driver, 1515 uint64_t file_page_offset, 1516 bool need_dma32) 1517 { 1518 int ret = -EINVAL; 1519 1520 rwlock_init(&bdev->vm_lock); 1521 bdev->driver = driver; 1522 1523 memset(bdev->man, 0, sizeof(bdev->man)); 1524 1525 /* 1526 * Initialize the system memory buffer type. 1527 * Other types need to be driver / IOCTL initialized. 1528 */ 1529 ret = ttm_bo_init_mm(bdev, TTM_PL_SYSTEM, 0); 1530 if (unlikely(ret != 0)) 1531 goto out_no_sys; 1532 1533 bdev->addr_space_rb = RB_ROOT; 1534 ret = drm_mm_init(&bdev->addr_space_mm, file_page_offset, 0x10000000); 1535 if (unlikely(ret != 0)) 1536 goto out_no_addr_mm; 1537 1538 INIT_DELAYED_WORK(&bdev->wq, ttm_bo_delayed_workqueue); 1539 bdev->nice_mode = true; 1540 INIT_LIST_HEAD(&bdev->ddestroy); 1541 bdev->dev_mapping = NULL; 1542 bdev->glob = glob; 1543 bdev->need_dma32 = need_dma32; 1544 1545 mutex_lock(&glob->device_list_mutex); 1546 list_add_tail(&bdev->device_list, &glob->device_list); 1547 mutex_unlock(&glob->device_list_mutex); 1548 1549 return 0; 1550 out_no_addr_mm: 1551 ttm_bo_clean_mm(bdev, 0); 1552 out_no_sys: 1553 return ret; 1554 } 1555 EXPORT_SYMBOL(ttm_bo_device_init); 1556 1557 /* 1558 * buffer object vm functions. 1559 */ 1560 1561 bool ttm_mem_reg_is_pci(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) 1562 { 1563 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; 1564 1565 if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED)) { 1566 if (mem->mem_type == TTM_PL_SYSTEM) 1567 return false; 1568 1569 if (man->flags & TTM_MEMTYPE_FLAG_CMA) 1570 return false; 1571 1572 if (mem->placement & TTM_PL_FLAG_CACHED) 1573 return false; 1574 } 1575 return true; 1576 } 1577 1578 void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo) 1579 { 1580 struct ttm_bo_device *bdev = bo->bdev; 1581 loff_t offset = (loff_t) bo->addr_space_offset; 1582 loff_t holelen = ((loff_t) bo->mem.num_pages) << PAGE_SHIFT; 1583 1584 if (!bdev->dev_mapping) 1585 return; 1586 unmap_mapping_range(bdev->dev_mapping, offset, holelen, 1); 1587 ttm_mem_io_free(bdev, &bo->mem); 1588 } 1589 EXPORT_SYMBOL(ttm_bo_unmap_virtual); 1590 1591 static void ttm_bo_vm_insert_rb(struct ttm_buffer_object *bo) 1592 { 1593 struct ttm_bo_device *bdev = bo->bdev; 1594 struct rb_node **cur = &bdev->addr_space_rb.rb_node; 1595 struct rb_node *parent = NULL; 1596 struct ttm_buffer_object *cur_bo; 1597 unsigned long offset = bo->vm_node->start; 1598 unsigned long cur_offset; 1599 1600 while (*cur) { 1601 parent = *cur; 1602 cur_bo = rb_entry(parent, struct ttm_buffer_object, vm_rb); 1603 cur_offset = cur_bo->vm_node->start; 1604 if (offset < cur_offset) 1605 cur = &parent->rb_left; 1606 else if (offset > cur_offset) 1607 cur = &parent->rb_right; 1608 else 1609 BUG(); 1610 } 1611 1612 rb_link_node(&bo->vm_rb, parent, cur); 1613 rb_insert_color(&bo->vm_rb, &bdev->addr_space_rb); 1614 } 1615 1616 /** 1617 * ttm_bo_setup_vm: 1618 * 1619 * @bo: the buffer to allocate address space for 1620 * 1621 * Allocate address space in the drm device so that applications 1622 * can mmap the buffer and access the contents. This only 1623 * applies to ttm_bo_type_device objects as others are not 1624 * placed in the drm device address space. 1625 */ 1626 1627 static int ttm_bo_setup_vm(struct ttm_buffer_object *bo) 1628 { 1629 struct ttm_bo_device *bdev = bo->bdev; 1630 int ret; 1631 1632 retry_pre_get: 1633 ret = drm_mm_pre_get(&bdev->addr_space_mm); 1634 if (unlikely(ret != 0)) 1635 return ret; 1636 1637 write_lock(&bdev->vm_lock); 1638 bo->vm_node = drm_mm_search_free(&bdev->addr_space_mm, 1639 bo->mem.num_pages, 0, 0); 1640 1641 if (unlikely(bo->vm_node == NULL)) { 1642 ret = -ENOMEM; 1643 goto out_unlock; 1644 } 1645 1646 bo->vm_node = drm_mm_get_block_atomic(bo->vm_node, 1647 bo->mem.num_pages, 0); 1648 1649 if (unlikely(bo->vm_node == NULL)) { 1650 write_unlock(&bdev->vm_lock); 1651 goto retry_pre_get; 1652 } 1653 1654 ttm_bo_vm_insert_rb(bo); 1655 write_unlock(&bdev->vm_lock); 1656 bo->addr_space_offset = ((uint64_t) bo->vm_node->start) << PAGE_SHIFT; 1657 1658 return 0; 1659 out_unlock: 1660 write_unlock(&bdev->vm_lock); 1661 return ret; 1662 } 1663 1664 int ttm_bo_wait(struct ttm_buffer_object *bo, 1665 bool lazy, bool interruptible, bool no_wait) 1666 { 1667 struct ttm_bo_driver *driver = bo->bdev->driver; 1668 void *sync_obj; 1669 void *sync_obj_arg; 1670 int ret = 0; 1671 1672 if (likely(bo->sync_obj == NULL)) 1673 return 0; 1674 1675 while (bo->sync_obj) { 1676 1677 if (driver->sync_obj_signaled(bo->sync_obj, bo->sync_obj_arg)) { 1678 void *tmp_obj = bo->sync_obj; 1679 bo->sync_obj = NULL; 1680 clear_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags); 1681 spin_unlock(&bo->lock); 1682 driver->sync_obj_unref(&tmp_obj); 1683 spin_lock(&bo->lock); 1684 continue; 1685 } 1686 1687 if (no_wait) 1688 return -EBUSY; 1689 1690 sync_obj = driver->sync_obj_ref(bo->sync_obj); 1691 sync_obj_arg = bo->sync_obj_arg; 1692 spin_unlock(&bo->lock); 1693 ret = driver->sync_obj_wait(sync_obj, sync_obj_arg, 1694 lazy, interruptible); 1695 if (unlikely(ret != 0)) { 1696 driver->sync_obj_unref(&sync_obj); 1697 spin_lock(&bo->lock); 1698 return ret; 1699 } 1700 spin_lock(&bo->lock); 1701 if (likely(bo->sync_obj == sync_obj && 1702 bo->sync_obj_arg == sync_obj_arg)) { 1703 void *tmp_obj = bo->sync_obj; 1704 bo->sync_obj = NULL; 1705 clear_bit(TTM_BO_PRIV_FLAG_MOVING, 1706 &bo->priv_flags); 1707 spin_unlock(&bo->lock); 1708 driver->sync_obj_unref(&sync_obj); 1709 driver->sync_obj_unref(&tmp_obj); 1710 spin_lock(&bo->lock); 1711 } else { 1712 spin_unlock(&bo->lock); 1713 driver->sync_obj_unref(&sync_obj); 1714 spin_lock(&bo->lock); 1715 } 1716 } 1717 return 0; 1718 } 1719 EXPORT_SYMBOL(ttm_bo_wait); 1720 1721 int ttm_bo_synccpu_write_grab(struct ttm_buffer_object *bo, bool no_wait) 1722 { 1723 int ret = 0; 1724 1725 /* 1726 * Using ttm_bo_reserve makes sure the lru lists are updated. 1727 */ 1728 1729 ret = ttm_bo_reserve(bo, true, no_wait, false, 0); 1730 if (unlikely(ret != 0)) 1731 return ret; 1732 spin_lock(&bo->lock); 1733 ret = ttm_bo_wait(bo, false, true, no_wait); 1734 spin_unlock(&bo->lock); 1735 if (likely(ret == 0)) 1736 atomic_inc(&bo->cpu_writers); 1737 ttm_bo_unreserve(bo); 1738 return ret; 1739 } 1740 EXPORT_SYMBOL(ttm_bo_synccpu_write_grab); 1741 1742 void ttm_bo_synccpu_write_release(struct ttm_buffer_object *bo) 1743 { 1744 if (atomic_dec_and_test(&bo->cpu_writers)) 1745 wake_up_all(&bo->event_queue); 1746 } 1747 EXPORT_SYMBOL(ttm_bo_synccpu_write_release); 1748 1749 /** 1750 * A buffer object shrink method that tries to swap out the first 1751 * buffer object on the bo_global::swap_lru list. 1752 */ 1753 1754 static int ttm_bo_swapout(struct ttm_mem_shrink *shrink) 1755 { 1756 struct ttm_bo_global *glob = 1757 container_of(shrink, struct ttm_bo_global, shrink); 1758 struct ttm_buffer_object *bo; 1759 int ret = -EBUSY; 1760 int put_count; 1761 uint32_t swap_placement = (TTM_PL_FLAG_CACHED | TTM_PL_FLAG_SYSTEM); 1762 1763 spin_lock(&glob->lru_lock); 1764 while (ret == -EBUSY) { 1765 if (unlikely(list_empty(&glob->swap_lru))) { 1766 spin_unlock(&glob->lru_lock); 1767 return -EBUSY; 1768 } 1769 1770 bo = list_first_entry(&glob->swap_lru, 1771 struct ttm_buffer_object, swap); 1772 kref_get(&bo->list_kref); 1773 1774 /** 1775 * Reserve buffer. Since we unlock while sleeping, we need 1776 * to re-check that nobody removed us from the swap-list while 1777 * we slept. 1778 */ 1779 1780 ret = ttm_bo_reserve_locked(bo, false, true, false, 0); 1781 if (unlikely(ret == -EBUSY)) { 1782 spin_unlock(&glob->lru_lock); 1783 ttm_bo_wait_unreserved(bo, false); 1784 kref_put(&bo->list_kref, ttm_bo_release_list); 1785 spin_lock(&glob->lru_lock); 1786 } 1787 } 1788 1789 BUG_ON(ret != 0); 1790 put_count = ttm_bo_del_from_lru(bo); 1791 spin_unlock(&glob->lru_lock); 1792 1793 while (put_count--) 1794 kref_put(&bo->list_kref, ttm_bo_ref_bug); 1795 1796 /** 1797 * Wait for GPU, then move to system cached. 1798 */ 1799 1800 spin_lock(&bo->lock); 1801 ret = ttm_bo_wait(bo, false, false, false); 1802 spin_unlock(&bo->lock); 1803 1804 if (unlikely(ret != 0)) 1805 goto out; 1806 1807 if ((bo->mem.placement & swap_placement) != swap_placement) { 1808 struct ttm_mem_reg evict_mem; 1809 1810 evict_mem = bo->mem; 1811 evict_mem.mm_node = NULL; 1812 evict_mem.placement = TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED; 1813 evict_mem.mem_type = TTM_PL_SYSTEM; 1814 1815 ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, 1816 false, false, false); 1817 if (unlikely(ret != 0)) 1818 goto out; 1819 } 1820 1821 ttm_bo_unmap_virtual(bo); 1822 1823 /** 1824 * Swap out. Buffer will be swapped in again as soon as 1825 * anyone tries to access a ttm page. 1826 */ 1827 1828 if (bo->bdev->driver->swap_notify) 1829 bo->bdev->driver->swap_notify(bo); 1830 1831 ret = ttm_tt_swapout(bo->ttm, bo->persistant_swap_storage); 1832 out: 1833 1834 /** 1835 * 1836 * Unreserve without putting on LRU to avoid swapping out an 1837 * already swapped buffer. 1838 */ 1839 1840 atomic_set(&bo->reserved, 0); 1841 wake_up_all(&bo->event_queue); 1842 kref_put(&bo->list_kref, ttm_bo_release_list); 1843 return ret; 1844 } 1845 1846 void ttm_bo_swapout_all(struct ttm_bo_device *bdev) 1847 { 1848 while (ttm_bo_swapout(&bdev->glob->shrink) == 0) 1849 ; 1850 } 1851 EXPORT_SYMBOL(ttm_bo_swapout_all); 1852