xref: /linux/drivers/gpu/drm/ttm/ttm_bo.c (revision d91517839e5d95adc0cf4b28caa7af62a71de526)
1 /**************************************************************************
2  *
3  * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
4  * All Rights Reserved.
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a
7  * copy of this software and associated documentation files (the
8  * "Software"), to deal in the Software without restriction, including
9  * without limitation the rights to use, copy, modify, merge, publish,
10  * distribute, sub license, and/or sell copies of the Software, and to
11  * permit persons to whom the Software is furnished to do so, subject to
12  * the following conditions:
13  *
14  * The above copyright notice and this permission notice (including the
15  * next paragraph) shall be included in all copies or substantial portions
16  * of the Software.
17  *
18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
21  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
22  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
23  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
24  * USE OR OTHER DEALINGS IN THE SOFTWARE.
25  *
26  **************************************************************************/
27 /*
28  * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
29  */
30 
31 #define pr_fmt(fmt) "[TTM] " fmt
32 
33 #include <drm/ttm/ttm_module.h>
34 #include <drm/ttm/ttm_bo_driver.h>
35 #include <drm/ttm/ttm_placement.h>
36 #include <linux/jiffies.h>
37 #include <linux/slab.h>
38 #include <linux/sched.h>
39 #include <linux/mm.h>
40 #include <linux/file.h>
41 #include <linux/module.h>
42 #include <linux/atomic.h>
43 
44 #define TTM_ASSERT_LOCKED(param)
45 #define TTM_DEBUG(fmt, arg...)
46 #define TTM_BO_HASH_ORDER 13
47 
48 static int ttm_bo_swapout(struct ttm_mem_shrink *shrink);
49 static void ttm_bo_global_kobj_release(struct kobject *kobj);
50 
51 static struct attribute ttm_bo_count = {
52 	.name = "bo_count",
53 	.mode = S_IRUGO
54 };
55 
56 static inline int ttm_mem_type_from_flags(uint32_t flags, uint32_t *mem_type)
57 {
58 	int i;
59 
60 	for (i = 0; i <= TTM_PL_PRIV5; i++)
61 		if (flags & (1 << i)) {
62 			*mem_type = i;
63 			return 0;
64 		}
65 	return -EINVAL;
66 }
67 
68 static void ttm_mem_type_debug(struct ttm_bo_device *bdev, int mem_type)
69 {
70 	struct ttm_mem_type_manager *man = &bdev->man[mem_type];
71 
72 	pr_err("    has_type: %d\n", man->has_type);
73 	pr_err("    use_type: %d\n", man->use_type);
74 	pr_err("    flags: 0x%08X\n", man->flags);
75 	pr_err("    gpu_offset: 0x%08lX\n", man->gpu_offset);
76 	pr_err("    size: %llu\n", man->size);
77 	pr_err("    available_caching: 0x%08X\n", man->available_caching);
78 	pr_err("    default_caching: 0x%08X\n", man->default_caching);
79 	if (mem_type != TTM_PL_SYSTEM)
80 		(*man->func->debug)(man, TTM_PFX);
81 }
82 
83 static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo,
84 					struct ttm_placement *placement)
85 {
86 	int i, ret, mem_type;
87 
88 	pr_err("No space for %p (%lu pages, %luK, %luM)\n",
89 	       bo, bo->mem.num_pages, bo->mem.size >> 10,
90 	       bo->mem.size >> 20);
91 	for (i = 0; i < placement->num_placement; i++) {
92 		ret = ttm_mem_type_from_flags(placement->placement[i],
93 						&mem_type);
94 		if (ret)
95 			return;
96 		pr_err("  placement[%d]=0x%08X (%d)\n",
97 		       i, placement->placement[i], mem_type);
98 		ttm_mem_type_debug(bo->bdev, mem_type);
99 	}
100 }
101 
102 static ssize_t ttm_bo_global_show(struct kobject *kobj,
103 				  struct attribute *attr,
104 				  char *buffer)
105 {
106 	struct ttm_bo_global *glob =
107 		container_of(kobj, struct ttm_bo_global, kobj);
108 
109 	return snprintf(buffer, PAGE_SIZE, "%lu\n",
110 			(unsigned long) atomic_read(&glob->bo_count));
111 }
112 
113 static struct attribute *ttm_bo_global_attrs[] = {
114 	&ttm_bo_count,
115 	NULL
116 };
117 
118 static const struct sysfs_ops ttm_bo_global_ops = {
119 	.show = &ttm_bo_global_show
120 };
121 
122 static struct kobj_type ttm_bo_glob_kobj_type  = {
123 	.release = &ttm_bo_global_kobj_release,
124 	.sysfs_ops = &ttm_bo_global_ops,
125 	.default_attrs = ttm_bo_global_attrs
126 };
127 
128 
129 static inline uint32_t ttm_bo_type_flags(unsigned type)
130 {
131 	return 1 << (type);
132 }
133 
134 static void ttm_bo_release_list(struct kref *list_kref)
135 {
136 	struct ttm_buffer_object *bo =
137 	    container_of(list_kref, struct ttm_buffer_object, list_kref);
138 	struct ttm_bo_device *bdev = bo->bdev;
139 	size_t acc_size = bo->acc_size;
140 
141 	BUG_ON(atomic_read(&bo->list_kref.refcount));
142 	BUG_ON(atomic_read(&bo->kref.refcount));
143 	BUG_ON(atomic_read(&bo->cpu_writers));
144 	BUG_ON(bo->sync_obj != NULL);
145 	BUG_ON(bo->mem.mm_node != NULL);
146 	BUG_ON(!list_empty(&bo->lru));
147 	BUG_ON(!list_empty(&bo->ddestroy));
148 
149 	if (bo->ttm)
150 		ttm_tt_destroy(bo->ttm);
151 	atomic_dec(&bo->glob->bo_count);
152 	if (bo->resv == &bo->ttm_resv)
153 		reservation_object_fini(&bo->ttm_resv);
154 	mutex_destroy(&bo->wu_mutex);
155 	if (bo->destroy)
156 		bo->destroy(bo);
157 	else {
158 		kfree(bo);
159 	}
160 	ttm_mem_global_free(bdev->glob->mem_glob, acc_size);
161 }
162 
163 void ttm_bo_add_to_lru(struct ttm_buffer_object *bo)
164 {
165 	struct ttm_bo_device *bdev = bo->bdev;
166 	struct ttm_mem_type_manager *man;
167 
168 	lockdep_assert_held(&bo->resv->lock.base);
169 
170 	if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) {
171 
172 		BUG_ON(!list_empty(&bo->lru));
173 
174 		man = &bdev->man[bo->mem.mem_type];
175 		list_add_tail(&bo->lru, &man->lru);
176 		kref_get(&bo->list_kref);
177 
178 		if (bo->ttm != NULL) {
179 			list_add_tail(&bo->swap, &bo->glob->swap_lru);
180 			kref_get(&bo->list_kref);
181 		}
182 	}
183 }
184 EXPORT_SYMBOL(ttm_bo_add_to_lru);
185 
186 int ttm_bo_del_from_lru(struct ttm_buffer_object *bo)
187 {
188 	int put_count = 0;
189 
190 	if (!list_empty(&bo->swap)) {
191 		list_del_init(&bo->swap);
192 		++put_count;
193 	}
194 	if (!list_empty(&bo->lru)) {
195 		list_del_init(&bo->lru);
196 		++put_count;
197 	}
198 
199 	/*
200 	 * TODO: Add a driver hook to delete from
201 	 * driver-specific LRU's here.
202 	 */
203 
204 	return put_count;
205 }
206 
207 static void ttm_bo_ref_bug(struct kref *list_kref)
208 {
209 	BUG();
210 }
211 
212 void ttm_bo_list_ref_sub(struct ttm_buffer_object *bo, int count,
213 			 bool never_free)
214 {
215 	kref_sub(&bo->list_kref, count,
216 		 (never_free) ? ttm_bo_ref_bug : ttm_bo_release_list);
217 }
218 
219 void ttm_bo_del_sub_from_lru(struct ttm_buffer_object *bo)
220 {
221 	int put_count;
222 
223 	spin_lock(&bo->glob->lru_lock);
224 	put_count = ttm_bo_del_from_lru(bo);
225 	spin_unlock(&bo->glob->lru_lock);
226 	ttm_bo_list_ref_sub(bo, put_count, true);
227 }
228 EXPORT_SYMBOL(ttm_bo_del_sub_from_lru);
229 
230 /*
231  * Call bo->mutex locked.
232  */
233 static int ttm_bo_add_ttm(struct ttm_buffer_object *bo, bool zero_alloc)
234 {
235 	struct ttm_bo_device *bdev = bo->bdev;
236 	struct ttm_bo_global *glob = bo->glob;
237 	int ret = 0;
238 	uint32_t page_flags = 0;
239 
240 	TTM_ASSERT_LOCKED(&bo->mutex);
241 	bo->ttm = NULL;
242 
243 	if (bdev->need_dma32)
244 		page_flags |= TTM_PAGE_FLAG_DMA32;
245 
246 	switch (bo->type) {
247 	case ttm_bo_type_device:
248 		if (zero_alloc)
249 			page_flags |= TTM_PAGE_FLAG_ZERO_ALLOC;
250 	case ttm_bo_type_kernel:
251 		bo->ttm = bdev->driver->ttm_tt_create(bdev, bo->num_pages << PAGE_SHIFT,
252 						      page_flags, glob->dummy_read_page);
253 		if (unlikely(bo->ttm == NULL))
254 			ret = -ENOMEM;
255 		break;
256 	case ttm_bo_type_sg:
257 		bo->ttm = bdev->driver->ttm_tt_create(bdev, bo->num_pages << PAGE_SHIFT,
258 						      page_flags | TTM_PAGE_FLAG_SG,
259 						      glob->dummy_read_page);
260 		if (unlikely(bo->ttm == NULL)) {
261 			ret = -ENOMEM;
262 			break;
263 		}
264 		bo->ttm->sg = bo->sg;
265 		break;
266 	default:
267 		pr_err("Illegal buffer object type\n");
268 		ret = -EINVAL;
269 		break;
270 	}
271 
272 	return ret;
273 }
274 
275 static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo,
276 				  struct ttm_mem_reg *mem,
277 				  bool evict, bool interruptible,
278 				  bool no_wait_gpu)
279 {
280 	struct ttm_bo_device *bdev = bo->bdev;
281 	bool old_is_pci = ttm_mem_reg_is_pci(bdev, &bo->mem);
282 	bool new_is_pci = ttm_mem_reg_is_pci(bdev, mem);
283 	struct ttm_mem_type_manager *old_man = &bdev->man[bo->mem.mem_type];
284 	struct ttm_mem_type_manager *new_man = &bdev->man[mem->mem_type];
285 	int ret = 0;
286 
287 	if (old_is_pci || new_is_pci ||
288 	    ((mem->placement & bo->mem.placement & TTM_PL_MASK_CACHING) == 0)) {
289 		ret = ttm_mem_io_lock(old_man, true);
290 		if (unlikely(ret != 0))
291 			goto out_err;
292 		ttm_bo_unmap_virtual_locked(bo);
293 		ttm_mem_io_unlock(old_man);
294 	}
295 
296 	/*
297 	 * Create and bind a ttm if required.
298 	 */
299 
300 	if (!(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) {
301 		if (bo->ttm == NULL) {
302 			bool zero = !(old_man->flags & TTM_MEMTYPE_FLAG_FIXED);
303 			ret = ttm_bo_add_ttm(bo, zero);
304 			if (ret)
305 				goto out_err;
306 		}
307 
308 		ret = ttm_tt_set_placement_caching(bo->ttm, mem->placement);
309 		if (ret)
310 			goto out_err;
311 
312 		if (mem->mem_type != TTM_PL_SYSTEM) {
313 			ret = ttm_tt_bind(bo->ttm, mem);
314 			if (ret)
315 				goto out_err;
316 		}
317 
318 		if (bo->mem.mem_type == TTM_PL_SYSTEM) {
319 			if (bdev->driver->move_notify)
320 				bdev->driver->move_notify(bo, mem);
321 			bo->mem = *mem;
322 			mem->mm_node = NULL;
323 			goto moved;
324 		}
325 	}
326 
327 	if (bdev->driver->move_notify)
328 		bdev->driver->move_notify(bo, mem);
329 
330 	if (!(old_man->flags & TTM_MEMTYPE_FLAG_FIXED) &&
331 	    !(new_man->flags & TTM_MEMTYPE_FLAG_FIXED))
332 		ret = ttm_bo_move_ttm(bo, evict, no_wait_gpu, mem);
333 	else if (bdev->driver->move)
334 		ret = bdev->driver->move(bo, evict, interruptible,
335 					 no_wait_gpu, mem);
336 	else
337 		ret = ttm_bo_move_memcpy(bo, evict, no_wait_gpu, mem);
338 
339 	if (ret) {
340 		if (bdev->driver->move_notify) {
341 			struct ttm_mem_reg tmp_mem = *mem;
342 			*mem = bo->mem;
343 			bo->mem = tmp_mem;
344 			bdev->driver->move_notify(bo, mem);
345 			bo->mem = *mem;
346 			*mem = tmp_mem;
347 		}
348 
349 		goto out_err;
350 	}
351 
352 moved:
353 	if (bo->evicted) {
354 		ret = bdev->driver->invalidate_caches(bdev, bo->mem.placement);
355 		if (ret)
356 			pr_err("Can not flush read caches\n");
357 		bo->evicted = false;
358 	}
359 
360 	if (bo->mem.mm_node) {
361 		bo->offset = (bo->mem.start << PAGE_SHIFT) +
362 		    bdev->man[bo->mem.mem_type].gpu_offset;
363 		bo->cur_placement = bo->mem.placement;
364 	} else
365 		bo->offset = 0;
366 
367 	return 0;
368 
369 out_err:
370 	new_man = &bdev->man[bo->mem.mem_type];
371 	if ((new_man->flags & TTM_MEMTYPE_FLAG_FIXED) && bo->ttm) {
372 		ttm_tt_unbind(bo->ttm);
373 		ttm_tt_destroy(bo->ttm);
374 		bo->ttm = NULL;
375 	}
376 
377 	return ret;
378 }
379 
380 /**
381  * Call bo::reserved.
382  * Will release GPU memory type usage on destruction.
383  * This is the place to put in driver specific hooks to release
384  * driver private resources.
385  * Will release the bo::reserved lock.
386  */
387 
388 static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo)
389 {
390 	if (bo->bdev->driver->move_notify)
391 		bo->bdev->driver->move_notify(bo, NULL);
392 
393 	if (bo->ttm) {
394 		ttm_tt_unbind(bo->ttm);
395 		ttm_tt_destroy(bo->ttm);
396 		bo->ttm = NULL;
397 	}
398 	ttm_bo_mem_put(bo, &bo->mem);
399 
400 	ww_mutex_unlock (&bo->resv->lock);
401 }
402 
403 static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo)
404 {
405 	struct ttm_bo_device *bdev = bo->bdev;
406 	struct ttm_bo_global *glob = bo->glob;
407 	struct ttm_bo_driver *driver = bdev->driver;
408 	void *sync_obj = NULL;
409 	int put_count;
410 	int ret;
411 
412 	spin_lock(&glob->lru_lock);
413 	ret = ttm_bo_reserve_nolru(bo, false, true, false, 0);
414 
415 	spin_lock(&bdev->fence_lock);
416 	(void) ttm_bo_wait(bo, false, false, true);
417 	if (!ret && !bo->sync_obj) {
418 		spin_unlock(&bdev->fence_lock);
419 		put_count = ttm_bo_del_from_lru(bo);
420 
421 		spin_unlock(&glob->lru_lock);
422 		ttm_bo_cleanup_memtype_use(bo);
423 
424 		ttm_bo_list_ref_sub(bo, put_count, true);
425 
426 		return;
427 	}
428 	if (bo->sync_obj)
429 		sync_obj = driver->sync_obj_ref(bo->sync_obj);
430 	spin_unlock(&bdev->fence_lock);
431 
432 	if (!ret) {
433 
434 		/*
435 		 * Make NO_EVICT bos immediately available to
436 		 * shrinkers, now that they are queued for
437 		 * destruction.
438 		 */
439 		if (bo->mem.placement & TTM_PL_FLAG_NO_EVICT) {
440 			bo->mem.placement &= ~TTM_PL_FLAG_NO_EVICT;
441 			ttm_bo_add_to_lru(bo);
442 		}
443 
444 		ww_mutex_unlock(&bo->resv->lock);
445 	}
446 
447 	kref_get(&bo->list_kref);
448 	list_add_tail(&bo->ddestroy, &bdev->ddestroy);
449 	spin_unlock(&glob->lru_lock);
450 
451 	if (sync_obj) {
452 		driver->sync_obj_flush(sync_obj);
453 		driver->sync_obj_unref(&sync_obj);
454 	}
455 	schedule_delayed_work(&bdev->wq,
456 			      ((HZ / 100) < 1) ? 1 : HZ / 100);
457 }
458 
459 /**
460  * function ttm_bo_cleanup_refs_and_unlock
461  * If bo idle, remove from delayed- and lru lists, and unref.
462  * If not idle, do nothing.
463  *
464  * Must be called with lru_lock and reservation held, this function
465  * will drop both before returning.
466  *
467  * @interruptible         Any sleeps should occur interruptibly.
468  * @no_wait_gpu           Never wait for gpu. Return -EBUSY instead.
469  */
470 
471 static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo,
472 					  bool interruptible,
473 					  bool no_wait_gpu)
474 {
475 	struct ttm_bo_device *bdev = bo->bdev;
476 	struct ttm_bo_driver *driver = bdev->driver;
477 	struct ttm_bo_global *glob = bo->glob;
478 	int put_count;
479 	int ret;
480 
481 	spin_lock(&bdev->fence_lock);
482 	ret = ttm_bo_wait(bo, false, false, true);
483 
484 	if (ret && !no_wait_gpu) {
485 		void *sync_obj;
486 
487 		/*
488 		 * Take a reference to the fence and unreserve,
489 		 * at this point the buffer should be dead, so
490 		 * no new sync objects can be attached.
491 		 */
492 		sync_obj = driver->sync_obj_ref(bo->sync_obj);
493 		spin_unlock(&bdev->fence_lock);
494 
495 		ww_mutex_unlock(&bo->resv->lock);
496 		spin_unlock(&glob->lru_lock);
497 
498 		ret = driver->sync_obj_wait(sync_obj, false, interruptible);
499 		driver->sync_obj_unref(&sync_obj);
500 		if (ret)
501 			return ret;
502 
503 		/*
504 		 * remove sync_obj with ttm_bo_wait, the wait should be
505 		 * finished, and no new wait object should have been added.
506 		 */
507 		spin_lock(&bdev->fence_lock);
508 		ret = ttm_bo_wait(bo, false, false, true);
509 		WARN_ON(ret);
510 		spin_unlock(&bdev->fence_lock);
511 		if (ret)
512 			return ret;
513 
514 		spin_lock(&glob->lru_lock);
515 		ret = ttm_bo_reserve_nolru(bo, false, true, false, 0);
516 
517 		/*
518 		 * We raced, and lost, someone else holds the reservation now,
519 		 * and is probably busy in ttm_bo_cleanup_memtype_use.
520 		 *
521 		 * Even if it's not the case, because we finished waiting any
522 		 * delayed destruction would succeed, so just return success
523 		 * here.
524 		 */
525 		if (ret) {
526 			spin_unlock(&glob->lru_lock);
527 			return 0;
528 		}
529 	} else
530 		spin_unlock(&bdev->fence_lock);
531 
532 	if (ret || unlikely(list_empty(&bo->ddestroy))) {
533 		ww_mutex_unlock(&bo->resv->lock);
534 		spin_unlock(&glob->lru_lock);
535 		return ret;
536 	}
537 
538 	put_count = ttm_bo_del_from_lru(bo);
539 	list_del_init(&bo->ddestroy);
540 	++put_count;
541 
542 	spin_unlock(&glob->lru_lock);
543 	ttm_bo_cleanup_memtype_use(bo);
544 
545 	ttm_bo_list_ref_sub(bo, put_count, true);
546 
547 	return 0;
548 }
549 
550 /**
551  * Traverse the delayed list, and call ttm_bo_cleanup_refs on all
552  * encountered buffers.
553  */
554 
555 static int ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all)
556 {
557 	struct ttm_bo_global *glob = bdev->glob;
558 	struct ttm_buffer_object *entry = NULL;
559 	int ret = 0;
560 
561 	spin_lock(&glob->lru_lock);
562 	if (list_empty(&bdev->ddestroy))
563 		goto out_unlock;
564 
565 	entry = list_first_entry(&bdev->ddestroy,
566 		struct ttm_buffer_object, ddestroy);
567 	kref_get(&entry->list_kref);
568 
569 	for (;;) {
570 		struct ttm_buffer_object *nentry = NULL;
571 
572 		if (entry->ddestroy.next != &bdev->ddestroy) {
573 			nentry = list_first_entry(&entry->ddestroy,
574 				struct ttm_buffer_object, ddestroy);
575 			kref_get(&nentry->list_kref);
576 		}
577 
578 		ret = ttm_bo_reserve_nolru(entry, false, true, false, 0);
579 		if (remove_all && ret) {
580 			spin_unlock(&glob->lru_lock);
581 			ret = ttm_bo_reserve_nolru(entry, false, false,
582 						   false, 0);
583 			spin_lock(&glob->lru_lock);
584 		}
585 
586 		if (!ret)
587 			ret = ttm_bo_cleanup_refs_and_unlock(entry, false,
588 							     !remove_all);
589 		else
590 			spin_unlock(&glob->lru_lock);
591 
592 		kref_put(&entry->list_kref, ttm_bo_release_list);
593 		entry = nentry;
594 
595 		if (ret || !entry)
596 			goto out;
597 
598 		spin_lock(&glob->lru_lock);
599 		if (list_empty(&entry->ddestroy))
600 			break;
601 	}
602 
603 out_unlock:
604 	spin_unlock(&glob->lru_lock);
605 out:
606 	if (entry)
607 		kref_put(&entry->list_kref, ttm_bo_release_list);
608 	return ret;
609 }
610 
611 static void ttm_bo_delayed_workqueue(struct work_struct *work)
612 {
613 	struct ttm_bo_device *bdev =
614 	    container_of(work, struct ttm_bo_device, wq.work);
615 
616 	if (ttm_bo_delayed_delete(bdev, false)) {
617 		schedule_delayed_work(&bdev->wq,
618 				      ((HZ / 100) < 1) ? 1 : HZ / 100);
619 	}
620 }
621 
622 static void ttm_bo_release(struct kref *kref)
623 {
624 	struct ttm_buffer_object *bo =
625 	    container_of(kref, struct ttm_buffer_object, kref);
626 	struct ttm_bo_device *bdev = bo->bdev;
627 	struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type];
628 
629 	drm_vma_offset_remove(&bdev->vma_manager, &bo->vma_node);
630 	ttm_mem_io_lock(man, false);
631 	ttm_mem_io_free_vm(bo);
632 	ttm_mem_io_unlock(man);
633 	ttm_bo_cleanup_refs_or_queue(bo);
634 	kref_put(&bo->list_kref, ttm_bo_release_list);
635 }
636 
637 void ttm_bo_unref(struct ttm_buffer_object **p_bo)
638 {
639 	struct ttm_buffer_object *bo = *p_bo;
640 
641 	*p_bo = NULL;
642 	kref_put(&bo->kref, ttm_bo_release);
643 }
644 EXPORT_SYMBOL(ttm_bo_unref);
645 
646 int ttm_bo_lock_delayed_workqueue(struct ttm_bo_device *bdev)
647 {
648 	return cancel_delayed_work_sync(&bdev->wq);
649 }
650 EXPORT_SYMBOL(ttm_bo_lock_delayed_workqueue);
651 
652 void ttm_bo_unlock_delayed_workqueue(struct ttm_bo_device *bdev, int resched)
653 {
654 	if (resched)
655 		schedule_delayed_work(&bdev->wq,
656 				      ((HZ / 100) < 1) ? 1 : HZ / 100);
657 }
658 EXPORT_SYMBOL(ttm_bo_unlock_delayed_workqueue);
659 
660 static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible,
661 			bool no_wait_gpu)
662 {
663 	struct ttm_bo_device *bdev = bo->bdev;
664 	struct ttm_mem_reg evict_mem;
665 	struct ttm_placement placement;
666 	int ret = 0;
667 
668 	spin_lock(&bdev->fence_lock);
669 	ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu);
670 	spin_unlock(&bdev->fence_lock);
671 
672 	if (unlikely(ret != 0)) {
673 		if (ret != -ERESTARTSYS) {
674 			pr_err("Failed to expire sync object before buffer eviction\n");
675 		}
676 		goto out;
677 	}
678 
679 	lockdep_assert_held(&bo->resv->lock.base);
680 
681 	evict_mem = bo->mem;
682 	evict_mem.mm_node = NULL;
683 	evict_mem.bus.io_reserved_vm = false;
684 	evict_mem.bus.io_reserved_count = 0;
685 
686 	placement.fpfn = 0;
687 	placement.lpfn = 0;
688 	placement.num_placement = 0;
689 	placement.num_busy_placement = 0;
690 	bdev->driver->evict_flags(bo, &placement);
691 	ret = ttm_bo_mem_space(bo, &placement, &evict_mem, interruptible,
692 				no_wait_gpu);
693 	if (ret) {
694 		if (ret != -ERESTARTSYS) {
695 			pr_err("Failed to find memory space for buffer 0x%p eviction\n",
696 			       bo);
697 			ttm_bo_mem_space_debug(bo, &placement);
698 		}
699 		goto out;
700 	}
701 
702 	ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, interruptible,
703 				     no_wait_gpu);
704 	if (ret) {
705 		if (ret != -ERESTARTSYS)
706 			pr_err("Buffer eviction failed\n");
707 		ttm_bo_mem_put(bo, &evict_mem);
708 		goto out;
709 	}
710 	bo->evicted = true;
711 out:
712 	return ret;
713 }
714 
715 static int ttm_mem_evict_first(struct ttm_bo_device *bdev,
716 				uint32_t mem_type,
717 				bool interruptible,
718 				bool no_wait_gpu)
719 {
720 	struct ttm_bo_global *glob = bdev->glob;
721 	struct ttm_mem_type_manager *man = &bdev->man[mem_type];
722 	struct ttm_buffer_object *bo;
723 	int ret = -EBUSY, put_count;
724 
725 	spin_lock(&glob->lru_lock);
726 	list_for_each_entry(bo, &man->lru, lru) {
727 		ret = ttm_bo_reserve_nolru(bo, false, true, false, 0);
728 		if (!ret)
729 			break;
730 	}
731 
732 	if (ret) {
733 		spin_unlock(&glob->lru_lock);
734 		return ret;
735 	}
736 
737 	kref_get(&bo->list_kref);
738 
739 	if (!list_empty(&bo->ddestroy)) {
740 		ret = ttm_bo_cleanup_refs_and_unlock(bo, interruptible,
741 						     no_wait_gpu);
742 		kref_put(&bo->list_kref, ttm_bo_release_list);
743 		return ret;
744 	}
745 
746 	put_count = ttm_bo_del_from_lru(bo);
747 	spin_unlock(&glob->lru_lock);
748 
749 	BUG_ON(ret != 0);
750 
751 	ttm_bo_list_ref_sub(bo, put_count, true);
752 
753 	ret = ttm_bo_evict(bo, interruptible, no_wait_gpu);
754 	ttm_bo_unreserve(bo);
755 
756 	kref_put(&bo->list_kref, ttm_bo_release_list);
757 	return ret;
758 }
759 
760 void ttm_bo_mem_put(struct ttm_buffer_object *bo, struct ttm_mem_reg *mem)
761 {
762 	struct ttm_mem_type_manager *man = &bo->bdev->man[mem->mem_type];
763 
764 	if (mem->mm_node)
765 		(*man->func->put_node)(man, mem);
766 }
767 EXPORT_SYMBOL(ttm_bo_mem_put);
768 
769 /**
770  * Repeatedly evict memory from the LRU for @mem_type until we create enough
771  * space, or we've evicted everything and there isn't enough space.
772  */
773 static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo,
774 					uint32_t mem_type,
775 					struct ttm_placement *placement,
776 					struct ttm_mem_reg *mem,
777 					bool interruptible,
778 					bool no_wait_gpu)
779 {
780 	struct ttm_bo_device *bdev = bo->bdev;
781 	struct ttm_mem_type_manager *man = &bdev->man[mem_type];
782 	int ret;
783 
784 	do {
785 		ret = (*man->func->get_node)(man, bo, placement, mem);
786 		if (unlikely(ret != 0))
787 			return ret;
788 		if (mem->mm_node)
789 			break;
790 		ret = ttm_mem_evict_first(bdev, mem_type,
791 					  interruptible, no_wait_gpu);
792 		if (unlikely(ret != 0))
793 			return ret;
794 	} while (1);
795 	if (mem->mm_node == NULL)
796 		return -ENOMEM;
797 	mem->mem_type = mem_type;
798 	return 0;
799 }
800 
801 static uint32_t ttm_bo_select_caching(struct ttm_mem_type_manager *man,
802 				      uint32_t cur_placement,
803 				      uint32_t proposed_placement)
804 {
805 	uint32_t caching = proposed_placement & TTM_PL_MASK_CACHING;
806 	uint32_t result = proposed_placement & ~TTM_PL_MASK_CACHING;
807 
808 	/**
809 	 * Keep current caching if possible.
810 	 */
811 
812 	if ((cur_placement & caching) != 0)
813 		result |= (cur_placement & caching);
814 	else if ((man->default_caching & caching) != 0)
815 		result |= man->default_caching;
816 	else if ((TTM_PL_FLAG_CACHED & caching) != 0)
817 		result |= TTM_PL_FLAG_CACHED;
818 	else if ((TTM_PL_FLAG_WC & caching) != 0)
819 		result |= TTM_PL_FLAG_WC;
820 	else if ((TTM_PL_FLAG_UNCACHED & caching) != 0)
821 		result |= TTM_PL_FLAG_UNCACHED;
822 
823 	return result;
824 }
825 
826 static bool ttm_bo_mt_compatible(struct ttm_mem_type_manager *man,
827 				 uint32_t mem_type,
828 				 uint32_t proposed_placement,
829 				 uint32_t *masked_placement)
830 {
831 	uint32_t cur_flags = ttm_bo_type_flags(mem_type);
832 
833 	if ((cur_flags & proposed_placement & TTM_PL_MASK_MEM) == 0)
834 		return false;
835 
836 	if ((proposed_placement & man->available_caching) == 0)
837 		return false;
838 
839 	cur_flags |= (proposed_placement & man->available_caching);
840 
841 	*masked_placement = cur_flags;
842 	return true;
843 }
844 
845 /**
846  * Creates space for memory region @mem according to its type.
847  *
848  * This function first searches for free space in compatible memory types in
849  * the priority order defined by the driver.  If free space isn't found, then
850  * ttm_bo_mem_force_space is attempted in priority order to evict and find
851  * space.
852  */
853 int ttm_bo_mem_space(struct ttm_buffer_object *bo,
854 			struct ttm_placement *placement,
855 			struct ttm_mem_reg *mem,
856 			bool interruptible,
857 			bool no_wait_gpu)
858 {
859 	struct ttm_bo_device *bdev = bo->bdev;
860 	struct ttm_mem_type_manager *man;
861 	uint32_t mem_type = TTM_PL_SYSTEM;
862 	uint32_t cur_flags = 0;
863 	bool type_found = false;
864 	bool type_ok = false;
865 	bool has_erestartsys = false;
866 	int i, ret;
867 
868 	mem->mm_node = NULL;
869 	for (i = 0; i < placement->num_placement; ++i) {
870 		ret = ttm_mem_type_from_flags(placement->placement[i],
871 						&mem_type);
872 		if (ret)
873 			return ret;
874 		man = &bdev->man[mem_type];
875 
876 		type_ok = ttm_bo_mt_compatible(man,
877 						mem_type,
878 						placement->placement[i],
879 						&cur_flags);
880 
881 		if (!type_ok)
882 			continue;
883 
884 		cur_flags = ttm_bo_select_caching(man, bo->mem.placement,
885 						  cur_flags);
886 		/*
887 		 * Use the access and other non-mapping-related flag bits from
888 		 * the memory placement flags to the current flags
889 		 */
890 		ttm_flag_masked(&cur_flags, placement->placement[i],
891 				~TTM_PL_MASK_MEMTYPE);
892 
893 		if (mem_type == TTM_PL_SYSTEM)
894 			break;
895 
896 		if (man->has_type && man->use_type) {
897 			type_found = true;
898 			ret = (*man->func->get_node)(man, bo, placement, mem);
899 			if (unlikely(ret))
900 				return ret;
901 		}
902 		if (mem->mm_node)
903 			break;
904 	}
905 
906 	if ((type_ok && (mem_type == TTM_PL_SYSTEM)) || mem->mm_node) {
907 		mem->mem_type = mem_type;
908 		mem->placement = cur_flags;
909 		return 0;
910 	}
911 
912 	if (!type_found)
913 		return -EINVAL;
914 
915 	for (i = 0; i < placement->num_busy_placement; ++i) {
916 		ret = ttm_mem_type_from_flags(placement->busy_placement[i],
917 						&mem_type);
918 		if (ret)
919 			return ret;
920 		man = &bdev->man[mem_type];
921 		if (!man->has_type)
922 			continue;
923 		if (!ttm_bo_mt_compatible(man,
924 						mem_type,
925 						placement->busy_placement[i],
926 						&cur_flags))
927 			continue;
928 
929 		cur_flags = ttm_bo_select_caching(man, bo->mem.placement,
930 						  cur_flags);
931 		/*
932 		 * Use the access and other non-mapping-related flag bits from
933 		 * the memory placement flags to the current flags
934 		 */
935 		ttm_flag_masked(&cur_flags, placement->busy_placement[i],
936 				~TTM_PL_MASK_MEMTYPE);
937 
938 
939 		if (mem_type == TTM_PL_SYSTEM) {
940 			mem->mem_type = mem_type;
941 			mem->placement = cur_flags;
942 			mem->mm_node = NULL;
943 			return 0;
944 		}
945 
946 		ret = ttm_bo_mem_force_space(bo, mem_type, placement, mem,
947 						interruptible, no_wait_gpu);
948 		if (ret == 0 && mem->mm_node) {
949 			mem->placement = cur_flags;
950 			return 0;
951 		}
952 		if (ret == -ERESTARTSYS)
953 			has_erestartsys = true;
954 	}
955 	ret = (has_erestartsys) ? -ERESTARTSYS : -ENOMEM;
956 	return ret;
957 }
958 EXPORT_SYMBOL(ttm_bo_mem_space);
959 
960 static int ttm_bo_move_buffer(struct ttm_buffer_object *bo,
961 			struct ttm_placement *placement,
962 			bool interruptible,
963 			bool no_wait_gpu)
964 {
965 	int ret = 0;
966 	struct ttm_mem_reg mem;
967 	struct ttm_bo_device *bdev = bo->bdev;
968 
969 	lockdep_assert_held(&bo->resv->lock.base);
970 
971 	/*
972 	 * FIXME: It's possible to pipeline buffer moves.
973 	 * Have the driver move function wait for idle when necessary,
974 	 * instead of doing it here.
975 	 */
976 	spin_lock(&bdev->fence_lock);
977 	ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu);
978 	spin_unlock(&bdev->fence_lock);
979 	if (ret)
980 		return ret;
981 	mem.num_pages = bo->num_pages;
982 	mem.size = mem.num_pages << PAGE_SHIFT;
983 	mem.page_alignment = bo->mem.page_alignment;
984 	mem.bus.io_reserved_vm = false;
985 	mem.bus.io_reserved_count = 0;
986 	/*
987 	 * Determine where to move the buffer.
988 	 */
989 	ret = ttm_bo_mem_space(bo, placement, &mem,
990 			       interruptible, no_wait_gpu);
991 	if (ret)
992 		goto out_unlock;
993 	ret = ttm_bo_handle_move_mem(bo, &mem, false,
994 				     interruptible, no_wait_gpu);
995 out_unlock:
996 	if (ret && mem.mm_node)
997 		ttm_bo_mem_put(bo, &mem);
998 	return ret;
999 }
1000 
1001 static bool ttm_bo_mem_compat(struct ttm_placement *placement,
1002 			      struct ttm_mem_reg *mem,
1003 			      uint32_t *new_flags)
1004 {
1005 	int i;
1006 
1007 	if (mem->mm_node && placement->lpfn != 0 &&
1008 	    (mem->start < placement->fpfn ||
1009 	     mem->start + mem->num_pages > placement->lpfn))
1010 		return false;
1011 
1012 	for (i = 0; i < placement->num_placement; i++) {
1013 		*new_flags = placement->placement[i];
1014 		if ((*new_flags & mem->placement & TTM_PL_MASK_CACHING) &&
1015 		    (*new_flags & mem->placement & TTM_PL_MASK_MEM))
1016 			return true;
1017 	}
1018 
1019 	for (i = 0; i < placement->num_busy_placement; i++) {
1020 		*new_flags = placement->busy_placement[i];
1021 		if ((*new_flags & mem->placement & TTM_PL_MASK_CACHING) &&
1022 		    (*new_flags & mem->placement & TTM_PL_MASK_MEM))
1023 			return true;
1024 	}
1025 
1026 	return false;
1027 }
1028 
1029 int ttm_bo_validate(struct ttm_buffer_object *bo,
1030 			struct ttm_placement *placement,
1031 			bool interruptible,
1032 			bool no_wait_gpu)
1033 {
1034 	int ret;
1035 	uint32_t new_flags;
1036 
1037 	lockdep_assert_held(&bo->resv->lock.base);
1038 	/* Check that range is valid */
1039 	if (placement->lpfn || placement->fpfn)
1040 		if (placement->fpfn > placement->lpfn ||
1041 			(placement->lpfn - placement->fpfn) < bo->num_pages)
1042 			return -EINVAL;
1043 	/*
1044 	 * Check whether we need to move buffer.
1045 	 */
1046 	if (!ttm_bo_mem_compat(placement, &bo->mem, &new_flags)) {
1047 		ret = ttm_bo_move_buffer(bo, placement, interruptible,
1048 					 no_wait_gpu);
1049 		if (ret)
1050 			return ret;
1051 	} else {
1052 		/*
1053 		 * Use the access and other non-mapping-related flag bits from
1054 		 * the compatible memory placement flags to the active flags
1055 		 */
1056 		ttm_flag_masked(&bo->mem.placement, new_flags,
1057 				~TTM_PL_MASK_MEMTYPE);
1058 	}
1059 	/*
1060 	 * We might need to add a TTM.
1061 	 */
1062 	if (bo->mem.mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) {
1063 		ret = ttm_bo_add_ttm(bo, true);
1064 		if (ret)
1065 			return ret;
1066 	}
1067 	return 0;
1068 }
1069 EXPORT_SYMBOL(ttm_bo_validate);
1070 
1071 int ttm_bo_check_placement(struct ttm_buffer_object *bo,
1072 				struct ttm_placement *placement)
1073 {
1074 	BUG_ON((placement->fpfn || placement->lpfn) &&
1075 	       (bo->mem.num_pages > (placement->lpfn - placement->fpfn)));
1076 
1077 	return 0;
1078 }
1079 
1080 int ttm_bo_init(struct ttm_bo_device *bdev,
1081 		struct ttm_buffer_object *bo,
1082 		unsigned long size,
1083 		enum ttm_bo_type type,
1084 		struct ttm_placement *placement,
1085 		uint32_t page_alignment,
1086 		bool interruptible,
1087 		struct file *persistent_swap_storage,
1088 		size_t acc_size,
1089 		struct sg_table *sg,
1090 		void (*destroy) (struct ttm_buffer_object *))
1091 {
1092 	int ret = 0;
1093 	unsigned long num_pages;
1094 	struct ttm_mem_global *mem_glob = bdev->glob->mem_glob;
1095 	bool locked;
1096 
1097 	ret = ttm_mem_global_alloc(mem_glob, acc_size, false, false);
1098 	if (ret) {
1099 		pr_err("Out of kernel memory\n");
1100 		if (destroy)
1101 			(*destroy)(bo);
1102 		else
1103 			kfree(bo);
1104 		return -ENOMEM;
1105 	}
1106 
1107 	num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
1108 	if (num_pages == 0) {
1109 		pr_err("Illegal buffer object size\n");
1110 		if (destroy)
1111 			(*destroy)(bo);
1112 		else
1113 			kfree(bo);
1114 		ttm_mem_global_free(mem_glob, acc_size);
1115 		return -EINVAL;
1116 	}
1117 	bo->destroy = destroy;
1118 
1119 	kref_init(&bo->kref);
1120 	kref_init(&bo->list_kref);
1121 	atomic_set(&bo->cpu_writers, 0);
1122 	INIT_LIST_HEAD(&bo->lru);
1123 	INIT_LIST_HEAD(&bo->ddestroy);
1124 	INIT_LIST_HEAD(&bo->swap);
1125 	INIT_LIST_HEAD(&bo->io_reserve_lru);
1126 	mutex_init(&bo->wu_mutex);
1127 	bo->bdev = bdev;
1128 	bo->glob = bdev->glob;
1129 	bo->type = type;
1130 	bo->num_pages = num_pages;
1131 	bo->mem.size = num_pages << PAGE_SHIFT;
1132 	bo->mem.mem_type = TTM_PL_SYSTEM;
1133 	bo->mem.num_pages = bo->num_pages;
1134 	bo->mem.mm_node = NULL;
1135 	bo->mem.page_alignment = page_alignment;
1136 	bo->mem.bus.io_reserved_vm = false;
1137 	bo->mem.bus.io_reserved_count = 0;
1138 	bo->priv_flags = 0;
1139 	bo->mem.placement = (TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED);
1140 	bo->persistent_swap_storage = persistent_swap_storage;
1141 	bo->acc_size = acc_size;
1142 	bo->sg = sg;
1143 	bo->resv = &bo->ttm_resv;
1144 	reservation_object_init(bo->resv);
1145 	atomic_inc(&bo->glob->bo_count);
1146 	drm_vma_node_reset(&bo->vma_node);
1147 
1148 	ret = ttm_bo_check_placement(bo, placement);
1149 
1150 	/*
1151 	 * For ttm_bo_type_device buffers, allocate
1152 	 * address space from the device.
1153 	 */
1154 	if (likely(!ret) &&
1155 	    (bo->type == ttm_bo_type_device ||
1156 	     bo->type == ttm_bo_type_sg))
1157 		ret = drm_vma_offset_add(&bdev->vma_manager, &bo->vma_node,
1158 					 bo->mem.num_pages);
1159 
1160 	locked = ww_mutex_trylock(&bo->resv->lock);
1161 	WARN_ON(!locked);
1162 
1163 	if (likely(!ret))
1164 		ret = ttm_bo_validate(bo, placement, interruptible, false);
1165 
1166 	ttm_bo_unreserve(bo);
1167 
1168 	if (unlikely(ret))
1169 		ttm_bo_unref(&bo);
1170 
1171 	return ret;
1172 }
1173 EXPORT_SYMBOL(ttm_bo_init);
1174 
1175 size_t ttm_bo_acc_size(struct ttm_bo_device *bdev,
1176 		       unsigned long bo_size,
1177 		       unsigned struct_size)
1178 {
1179 	unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT;
1180 	size_t size = 0;
1181 
1182 	size += ttm_round_pot(struct_size);
1183 	size += PAGE_ALIGN(npages * sizeof(void *));
1184 	size += ttm_round_pot(sizeof(struct ttm_tt));
1185 	return size;
1186 }
1187 EXPORT_SYMBOL(ttm_bo_acc_size);
1188 
1189 size_t ttm_bo_dma_acc_size(struct ttm_bo_device *bdev,
1190 			   unsigned long bo_size,
1191 			   unsigned struct_size)
1192 {
1193 	unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT;
1194 	size_t size = 0;
1195 
1196 	size += ttm_round_pot(struct_size);
1197 	size += PAGE_ALIGN(npages * sizeof(void *));
1198 	size += PAGE_ALIGN(npages * sizeof(dma_addr_t));
1199 	size += ttm_round_pot(sizeof(struct ttm_dma_tt));
1200 	return size;
1201 }
1202 EXPORT_SYMBOL(ttm_bo_dma_acc_size);
1203 
1204 int ttm_bo_create(struct ttm_bo_device *bdev,
1205 			unsigned long size,
1206 			enum ttm_bo_type type,
1207 			struct ttm_placement *placement,
1208 			uint32_t page_alignment,
1209 			bool interruptible,
1210 			struct file *persistent_swap_storage,
1211 			struct ttm_buffer_object **p_bo)
1212 {
1213 	struct ttm_buffer_object *bo;
1214 	size_t acc_size;
1215 	int ret;
1216 
1217 	bo = kzalloc(sizeof(*bo), GFP_KERNEL);
1218 	if (unlikely(bo == NULL))
1219 		return -ENOMEM;
1220 
1221 	acc_size = ttm_bo_acc_size(bdev, size, sizeof(struct ttm_buffer_object));
1222 	ret = ttm_bo_init(bdev, bo, size, type, placement, page_alignment,
1223 			  interruptible, persistent_swap_storage, acc_size,
1224 			  NULL, NULL);
1225 	if (likely(ret == 0))
1226 		*p_bo = bo;
1227 
1228 	return ret;
1229 }
1230 EXPORT_SYMBOL(ttm_bo_create);
1231 
1232 static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev,
1233 					unsigned mem_type, bool allow_errors)
1234 {
1235 	struct ttm_mem_type_manager *man = &bdev->man[mem_type];
1236 	struct ttm_bo_global *glob = bdev->glob;
1237 	int ret;
1238 
1239 	/*
1240 	 * Can't use standard list traversal since we're unlocking.
1241 	 */
1242 
1243 	spin_lock(&glob->lru_lock);
1244 	while (!list_empty(&man->lru)) {
1245 		spin_unlock(&glob->lru_lock);
1246 		ret = ttm_mem_evict_first(bdev, mem_type, false, false);
1247 		if (ret) {
1248 			if (allow_errors) {
1249 				return ret;
1250 			} else {
1251 				pr_err("Cleanup eviction failed\n");
1252 			}
1253 		}
1254 		spin_lock(&glob->lru_lock);
1255 	}
1256 	spin_unlock(&glob->lru_lock);
1257 	return 0;
1258 }
1259 
1260 int ttm_bo_clean_mm(struct ttm_bo_device *bdev, unsigned mem_type)
1261 {
1262 	struct ttm_mem_type_manager *man;
1263 	int ret = -EINVAL;
1264 
1265 	if (mem_type >= TTM_NUM_MEM_TYPES) {
1266 		pr_err("Illegal memory type %d\n", mem_type);
1267 		return ret;
1268 	}
1269 	man = &bdev->man[mem_type];
1270 
1271 	if (!man->has_type) {
1272 		pr_err("Trying to take down uninitialized memory manager type %u\n",
1273 		       mem_type);
1274 		return ret;
1275 	}
1276 
1277 	man->use_type = false;
1278 	man->has_type = false;
1279 
1280 	ret = 0;
1281 	if (mem_type > 0) {
1282 		ttm_bo_force_list_clean(bdev, mem_type, false);
1283 
1284 		ret = (*man->func->takedown)(man);
1285 	}
1286 
1287 	return ret;
1288 }
1289 EXPORT_SYMBOL(ttm_bo_clean_mm);
1290 
1291 int ttm_bo_evict_mm(struct ttm_bo_device *bdev, unsigned mem_type)
1292 {
1293 	struct ttm_mem_type_manager *man = &bdev->man[mem_type];
1294 
1295 	if (mem_type == 0 || mem_type >= TTM_NUM_MEM_TYPES) {
1296 		pr_err("Illegal memory manager memory type %u\n", mem_type);
1297 		return -EINVAL;
1298 	}
1299 
1300 	if (!man->has_type) {
1301 		pr_err("Memory type %u has not been initialized\n", mem_type);
1302 		return 0;
1303 	}
1304 
1305 	return ttm_bo_force_list_clean(bdev, mem_type, true);
1306 }
1307 EXPORT_SYMBOL(ttm_bo_evict_mm);
1308 
1309 int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type,
1310 			unsigned long p_size)
1311 {
1312 	int ret = -EINVAL;
1313 	struct ttm_mem_type_manager *man;
1314 
1315 	BUG_ON(type >= TTM_NUM_MEM_TYPES);
1316 	man = &bdev->man[type];
1317 	BUG_ON(man->has_type);
1318 	man->io_reserve_fastpath = true;
1319 	man->use_io_reserve_lru = false;
1320 	mutex_init(&man->io_reserve_mutex);
1321 	INIT_LIST_HEAD(&man->io_reserve_lru);
1322 
1323 	ret = bdev->driver->init_mem_type(bdev, type, man);
1324 	if (ret)
1325 		return ret;
1326 	man->bdev = bdev;
1327 
1328 	ret = 0;
1329 	if (type != TTM_PL_SYSTEM) {
1330 		ret = (*man->func->init)(man, p_size);
1331 		if (ret)
1332 			return ret;
1333 	}
1334 	man->has_type = true;
1335 	man->use_type = true;
1336 	man->size = p_size;
1337 
1338 	INIT_LIST_HEAD(&man->lru);
1339 
1340 	return 0;
1341 }
1342 EXPORT_SYMBOL(ttm_bo_init_mm);
1343 
1344 static void ttm_bo_global_kobj_release(struct kobject *kobj)
1345 {
1346 	struct ttm_bo_global *glob =
1347 		container_of(kobj, struct ttm_bo_global, kobj);
1348 
1349 	ttm_mem_unregister_shrink(glob->mem_glob, &glob->shrink);
1350 	__free_page(glob->dummy_read_page);
1351 	kfree(glob);
1352 }
1353 
1354 void ttm_bo_global_release(struct drm_global_reference *ref)
1355 {
1356 	struct ttm_bo_global *glob = ref->object;
1357 
1358 	kobject_del(&glob->kobj);
1359 	kobject_put(&glob->kobj);
1360 }
1361 EXPORT_SYMBOL(ttm_bo_global_release);
1362 
1363 int ttm_bo_global_init(struct drm_global_reference *ref)
1364 {
1365 	struct ttm_bo_global_ref *bo_ref =
1366 		container_of(ref, struct ttm_bo_global_ref, ref);
1367 	struct ttm_bo_global *glob = ref->object;
1368 	int ret;
1369 
1370 	mutex_init(&glob->device_list_mutex);
1371 	spin_lock_init(&glob->lru_lock);
1372 	glob->mem_glob = bo_ref->mem_glob;
1373 	glob->dummy_read_page = alloc_page(__GFP_ZERO | GFP_DMA32);
1374 
1375 	if (unlikely(glob->dummy_read_page == NULL)) {
1376 		ret = -ENOMEM;
1377 		goto out_no_drp;
1378 	}
1379 
1380 	INIT_LIST_HEAD(&glob->swap_lru);
1381 	INIT_LIST_HEAD(&glob->device_list);
1382 
1383 	ttm_mem_init_shrink(&glob->shrink, ttm_bo_swapout);
1384 	ret = ttm_mem_register_shrink(glob->mem_glob, &glob->shrink);
1385 	if (unlikely(ret != 0)) {
1386 		pr_err("Could not register buffer object swapout\n");
1387 		goto out_no_shrink;
1388 	}
1389 
1390 	atomic_set(&glob->bo_count, 0);
1391 
1392 	ret = kobject_init_and_add(
1393 		&glob->kobj, &ttm_bo_glob_kobj_type, ttm_get_kobj(), "buffer_objects");
1394 	if (unlikely(ret != 0))
1395 		kobject_put(&glob->kobj);
1396 	return ret;
1397 out_no_shrink:
1398 	__free_page(glob->dummy_read_page);
1399 out_no_drp:
1400 	kfree(glob);
1401 	return ret;
1402 }
1403 EXPORT_SYMBOL(ttm_bo_global_init);
1404 
1405 
1406 int ttm_bo_device_release(struct ttm_bo_device *bdev)
1407 {
1408 	int ret = 0;
1409 	unsigned i = TTM_NUM_MEM_TYPES;
1410 	struct ttm_mem_type_manager *man;
1411 	struct ttm_bo_global *glob = bdev->glob;
1412 
1413 	while (i--) {
1414 		man = &bdev->man[i];
1415 		if (man->has_type) {
1416 			man->use_type = false;
1417 			if ((i != TTM_PL_SYSTEM) && ttm_bo_clean_mm(bdev, i)) {
1418 				ret = -EBUSY;
1419 				pr_err("DRM memory manager type %d is not clean\n",
1420 				       i);
1421 			}
1422 			man->has_type = false;
1423 		}
1424 	}
1425 
1426 	mutex_lock(&glob->device_list_mutex);
1427 	list_del(&bdev->device_list);
1428 	mutex_unlock(&glob->device_list_mutex);
1429 
1430 	cancel_delayed_work_sync(&bdev->wq);
1431 
1432 	while (ttm_bo_delayed_delete(bdev, true))
1433 		;
1434 
1435 	spin_lock(&glob->lru_lock);
1436 	if (list_empty(&bdev->ddestroy))
1437 		TTM_DEBUG("Delayed destroy list was clean\n");
1438 
1439 	if (list_empty(&bdev->man[0].lru))
1440 		TTM_DEBUG("Swap list was clean\n");
1441 	spin_unlock(&glob->lru_lock);
1442 
1443 	drm_vma_offset_manager_destroy(&bdev->vma_manager);
1444 
1445 	return ret;
1446 }
1447 EXPORT_SYMBOL(ttm_bo_device_release);
1448 
1449 int ttm_bo_device_init(struct ttm_bo_device *bdev,
1450 		       struct ttm_bo_global *glob,
1451 		       struct ttm_bo_driver *driver,
1452 		       uint64_t file_page_offset,
1453 		       bool need_dma32)
1454 {
1455 	int ret = -EINVAL;
1456 
1457 	bdev->driver = driver;
1458 
1459 	memset(bdev->man, 0, sizeof(bdev->man));
1460 
1461 	/*
1462 	 * Initialize the system memory buffer type.
1463 	 * Other types need to be driver / IOCTL initialized.
1464 	 */
1465 	ret = ttm_bo_init_mm(bdev, TTM_PL_SYSTEM, 0);
1466 	if (unlikely(ret != 0))
1467 		goto out_no_sys;
1468 
1469 	drm_vma_offset_manager_init(&bdev->vma_manager, file_page_offset,
1470 				    0x10000000);
1471 	INIT_DELAYED_WORK(&bdev->wq, ttm_bo_delayed_workqueue);
1472 	INIT_LIST_HEAD(&bdev->ddestroy);
1473 	bdev->dev_mapping = NULL;
1474 	bdev->glob = glob;
1475 	bdev->need_dma32 = need_dma32;
1476 	bdev->val_seq = 0;
1477 	spin_lock_init(&bdev->fence_lock);
1478 	mutex_lock(&glob->device_list_mutex);
1479 	list_add_tail(&bdev->device_list, &glob->device_list);
1480 	mutex_unlock(&glob->device_list_mutex);
1481 
1482 	return 0;
1483 out_no_sys:
1484 	return ret;
1485 }
1486 EXPORT_SYMBOL(ttm_bo_device_init);
1487 
1488 /*
1489  * buffer object vm functions.
1490  */
1491 
1492 bool ttm_mem_reg_is_pci(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
1493 {
1494 	struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
1495 
1496 	if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED)) {
1497 		if (mem->mem_type == TTM_PL_SYSTEM)
1498 			return false;
1499 
1500 		if (man->flags & TTM_MEMTYPE_FLAG_CMA)
1501 			return false;
1502 
1503 		if (mem->placement & TTM_PL_FLAG_CACHED)
1504 			return false;
1505 	}
1506 	return true;
1507 }
1508 
1509 void ttm_bo_unmap_virtual_locked(struct ttm_buffer_object *bo)
1510 {
1511 	struct ttm_bo_device *bdev = bo->bdev;
1512 
1513 	drm_vma_node_unmap(&bo->vma_node, bdev->dev_mapping);
1514 	ttm_mem_io_free_vm(bo);
1515 }
1516 
1517 void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo)
1518 {
1519 	struct ttm_bo_device *bdev = bo->bdev;
1520 	struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type];
1521 
1522 	ttm_mem_io_lock(man, false);
1523 	ttm_bo_unmap_virtual_locked(bo);
1524 	ttm_mem_io_unlock(man);
1525 }
1526 
1527 
1528 EXPORT_SYMBOL(ttm_bo_unmap_virtual);
1529 
1530 
1531 int ttm_bo_wait(struct ttm_buffer_object *bo,
1532 		bool lazy, bool interruptible, bool no_wait)
1533 {
1534 	struct ttm_bo_driver *driver = bo->bdev->driver;
1535 	struct ttm_bo_device *bdev = bo->bdev;
1536 	void *sync_obj;
1537 	int ret = 0;
1538 
1539 	if (likely(bo->sync_obj == NULL))
1540 		return 0;
1541 
1542 	while (bo->sync_obj) {
1543 
1544 		if (driver->sync_obj_signaled(bo->sync_obj)) {
1545 			void *tmp_obj = bo->sync_obj;
1546 			bo->sync_obj = NULL;
1547 			clear_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags);
1548 			spin_unlock(&bdev->fence_lock);
1549 			driver->sync_obj_unref(&tmp_obj);
1550 			spin_lock(&bdev->fence_lock);
1551 			continue;
1552 		}
1553 
1554 		if (no_wait)
1555 			return -EBUSY;
1556 
1557 		sync_obj = driver->sync_obj_ref(bo->sync_obj);
1558 		spin_unlock(&bdev->fence_lock);
1559 		ret = driver->sync_obj_wait(sync_obj,
1560 					    lazy, interruptible);
1561 		if (unlikely(ret != 0)) {
1562 			driver->sync_obj_unref(&sync_obj);
1563 			spin_lock(&bdev->fence_lock);
1564 			return ret;
1565 		}
1566 		spin_lock(&bdev->fence_lock);
1567 		if (likely(bo->sync_obj == sync_obj)) {
1568 			void *tmp_obj = bo->sync_obj;
1569 			bo->sync_obj = NULL;
1570 			clear_bit(TTM_BO_PRIV_FLAG_MOVING,
1571 				  &bo->priv_flags);
1572 			spin_unlock(&bdev->fence_lock);
1573 			driver->sync_obj_unref(&sync_obj);
1574 			driver->sync_obj_unref(&tmp_obj);
1575 			spin_lock(&bdev->fence_lock);
1576 		} else {
1577 			spin_unlock(&bdev->fence_lock);
1578 			driver->sync_obj_unref(&sync_obj);
1579 			spin_lock(&bdev->fence_lock);
1580 		}
1581 	}
1582 	return 0;
1583 }
1584 EXPORT_SYMBOL(ttm_bo_wait);
1585 
1586 int ttm_bo_synccpu_write_grab(struct ttm_buffer_object *bo, bool no_wait)
1587 {
1588 	struct ttm_bo_device *bdev = bo->bdev;
1589 	int ret = 0;
1590 
1591 	/*
1592 	 * Using ttm_bo_reserve makes sure the lru lists are updated.
1593 	 */
1594 
1595 	ret = ttm_bo_reserve(bo, true, no_wait, false, 0);
1596 	if (unlikely(ret != 0))
1597 		return ret;
1598 	spin_lock(&bdev->fence_lock);
1599 	ret = ttm_bo_wait(bo, false, true, no_wait);
1600 	spin_unlock(&bdev->fence_lock);
1601 	if (likely(ret == 0))
1602 		atomic_inc(&bo->cpu_writers);
1603 	ttm_bo_unreserve(bo);
1604 	return ret;
1605 }
1606 EXPORT_SYMBOL(ttm_bo_synccpu_write_grab);
1607 
1608 void ttm_bo_synccpu_write_release(struct ttm_buffer_object *bo)
1609 {
1610 	atomic_dec(&bo->cpu_writers);
1611 }
1612 EXPORT_SYMBOL(ttm_bo_synccpu_write_release);
1613 
1614 /**
1615  * A buffer object shrink method that tries to swap out the first
1616  * buffer object on the bo_global::swap_lru list.
1617  */
1618 
1619 static int ttm_bo_swapout(struct ttm_mem_shrink *shrink)
1620 {
1621 	struct ttm_bo_global *glob =
1622 	    container_of(shrink, struct ttm_bo_global, shrink);
1623 	struct ttm_buffer_object *bo;
1624 	int ret = -EBUSY;
1625 	int put_count;
1626 	uint32_t swap_placement = (TTM_PL_FLAG_CACHED | TTM_PL_FLAG_SYSTEM);
1627 
1628 	spin_lock(&glob->lru_lock);
1629 	list_for_each_entry(bo, &glob->swap_lru, swap) {
1630 		ret = ttm_bo_reserve_nolru(bo, false, true, false, 0);
1631 		if (!ret)
1632 			break;
1633 	}
1634 
1635 	if (ret) {
1636 		spin_unlock(&glob->lru_lock);
1637 		return ret;
1638 	}
1639 
1640 	kref_get(&bo->list_kref);
1641 
1642 	if (!list_empty(&bo->ddestroy)) {
1643 		ret = ttm_bo_cleanup_refs_and_unlock(bo, false, false);
1644 		kref_put(&bo->list_kref, ttm_bo_release_list);
1645 		return ret;
1646 	}
1647 
1648 	put_count = ttm_bo_del_from_lru(bo);
1649 	spin_unlock(&glob->lru_lock);
1650 
1651 	ttm_bo_list_ref_sub(bo, put_count, true);
1652 
1653 	/**
1654 	 * Wait for GPU, then move to system cached.
1655 	 */
1656 
1657 	spin_lock(&bo->bdev->fence_lock);
1658 	ret = ttm_bo_wait(bo, false, false, false);
1659 	spin_unlock(&bo->bdev->fence_lock);
1660 
1661 	if (unlikely(ret != 0))
1662 		goto out;
1663 
1664 	if ((bo->mem.placement & swap_placement) != swap_placement) {
1665 		struct ttm_mem_reg evict_mem;
1666 
1667 		evict_mem = bo->mem;
1668 		evict_mem.mm_node = NULL;
1669 		evict_mem.placement = TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED;
1670 		evict_mem.mem_type = TTM_PL_SYSTEM;
1671 
1672 		ret = ttm_bo_handle_move_mem(bo, &evict_mem, true,
1673 					     false, false);
1674 		if (unlikely(ret != 0))
1675 			goto out;
1676 	}
1677 
1678 	ttm_bo_unmap_virtual(bo);
1679 
1680 	/**
1681 	 * Swap out. Buffer will be swapped in again as soon as
1682 	 * anyone tries to access a ttm page.
1683 	 */
1684 
1685 	if (bo->bdev->driver->swap_notify)
1686 		bo->bdev->driver->swap_notify(bo);
1687 
1688 	ret = ttm_tt_swapout(bo->ttm, bo->persistent_swap_storage);
1689 out:
1690 
1691 	/**
1692 	 *
1693 	 * Unreserve without putting on LRU to avoid swapping out an
1694 	 * already swapped buffer.
1695 	 */
1696 
1697 	ww_mutex_unlock(&bo->resv->lock);
1698 	kref_put(&bo->list_kref, ttm_bo_release_list);
1699 	return ret;
1700 }
1701 
1702 void ttm_bo_swapout_all(struct ttm_bo_device *bdev)
1703 {
1704 	while (ttm_bo_swapout(&bdev->glob->shrink) == 0)
1705 		;
1706 }
1707 EXPORT_SYMBOL(ttm_bo_swapout_all);
1708 
1709 /**
1710  * ttm_bo_wait_unreserved - interruptible wait for a buffer object to become
1711  * unreserved
1712  *
1713  * @bo: Pointer to buffer
1714  */
1715 int ttm_bo_wait_unreserved(struct ttm_buffer_object *bo)
1716 {
1717 	int ret;
1718 
1719 	/*
1720 	 * In the absense of a wait_unlocked API,
1721 	 * Use the bo::wu_mutex to avoid triggering livelocks due to
1722 	 * concurrent use of this function. Note that this use of
1723 	 * bo::wu_mutex can go away if we change locking order to
1724 	 * mmap_sem -> bo::reserve.
1725 	 */
1726 	ret = mutex_lock_interruptible(&bo->wu_mutex);
1727 	if (unlikely(ret != 0))
1728 		return -ERESTARTSYS;
1729 	if (!ww_mutex_is_locked(&bo->resv->lock))
1730 		goto out_unlock;
1731 	ret = ttm_bo_reserve_nolru(bo, true, false, false, NULL);
1732 	if (unlikely(ret != 0))
1733 		goto out_unlock;
1734 	ww_mutex_unlock(&bo->resv->lock);
1735 
1736 out_unlock:
1737 	mutex_unlock(&bo->wu_mutex);
1738 	return ret;
1739 }
1740