184 int drm_gem_object_init(struct drm_device *dev, struct drm_gem_object *obj,  in drm_gem_object_init()  argument
191 	drm_gem_private_object_init(dev, obj, size);  in drm_gem_object_init()
203 	obj->filp = filp;  in drm_gem_object_init()
220 				 struct drm_gem_object *obj, size_t size)  in drm_gem_private_object_init()  argument
224 	obj->dev = dev;  in drm_gem_private_object_init()
225 	obj->filp = NULL;  in drm_gem_private_object_init()
227 	kref_init(&obj->refcount);  in drm_gem_private_object_init()
228 	obj->handle_count = 0;  in drm_gem_private_object_init()
229 	obj->size = size;  in drm_gem_private_object_init()
230 	mutex_init(&obj->gpuva.lock);  in drm_gem_private_object_init()
231 	dma_resv_init(&obj->_resv);  in drm_gem_private_object_init()
232 	if (!obj->resv)  in drm_gem_private_object_init()
233 		obj->resv = &obj->_resv;  in drm_gem_private_object_init()
236 		drm_gem_gpuva_init(obj);  in drm_gem_private_object_init()
238 	drm_vma_node_reset(&obj->vma_node);  in drm_gem_private_object_init()
239 	INIT_LIST_HEAD(&obj->lru_node);  in drm_gem_private_object_init()
249 void drm_gem_private_object_fini(struct drm_gem_object *obj)  in drm_gem_private_object_fini()  argument
251 	WARN_ON(obj->dma_buf);  in drm_gem_private_object_fini()
253 	dma_resv_fini(&obj->_resv);  in drm_gem_private_object_fini()
254 	mutex_destroy(&obj->gpuva.lock);  in drm_gem_private_object_fini()
258 static void drm_gem_object_handle_get(struct drm_gem_object *obj)  in drm_gem_object_handle_get()  argument
260 	struct drm_device *dev = obj->dev;  in drm_gem_object_handle_get()
264 	if (obj->handle_count++ == 0)  in drm_gem_object_handle_get()
265 		drm_gem_object_get(obj);  in drm_gem_object_handle_get()
279 bool drm_gem_object_handle_get_if_exists_unlocked(struct drm_gem_object *obj)  in drm_gem_object_handle_get_if_exists_unlocked()  argument
281 	struct drm_device *dev = obj->dev;  in drm_gem_object_handle_get_if_exists_unlocked()
290 	if (!obj->handle_count)  in drm_gem_object_handle_get_if_exists_unlocked()
293 	drm_gem_object_handle_get(obj);  in drm_gem_object_handle_get_if_exists_unlocked()
308 static void drm_gem_object_handle_free(struct drm_gem_object *obj)  in drm_gem_object_handle_free()  argument
310 	struct drm_device *dev = obj->dev;  in drm_gem_object_handle_free()
313 	if (obj->name) {  in drm_gem_object_handle_free()
314 		idr_remove(&dev->object_name_idr, obj->name);  in drm_gem_object_handle_free()
315 		obj->name = 0;  in drm_gem_object_handle_free()
319 static void drm_gem_object_exported_dma_buf_free(struct drm_gem_object *obj)  in drm_gem_object_exported_dma_buf_free()  argument
322 	if (obj->dma_buf) {  in drm_gem_object_exported_dma_buf_free()
323 		dma_buf_put(obj->dma_buf);  in drm_gem_object_exported_dma_buf_free()
324 		obj->dma_buf = NULL;  in drm_gem_object_exported_dma_buf_free()
335 void drm_gem_object_handle_put_unlocked(struct drm_gem_object *obj)  in drm_gem_object_handle_put_unlocked()  argument
337 	struct drm_device *dev = obj->dev;  in drm_gem_object_handle_put_unlocked()
340 	if (drm_WARN_ON(dev, READ_ONCE(obj->handle_count) == 0))  in drm_gem_object_handle_put_unlocked()
350 	if (--obj->handle_count == 0) {  in drm_gem_object_handle_put_unlocked()
351 		drm_gem_object_handle_free(obj);  in drm_gem_object_handle_put_unlocked()
352 		drm_gem_object_exported_dma_buf_free(obj);  in drm_gem_object_handle_put_unlocked()
358 		drm_gem_object_put(obj);  in drm_gem_object_handle_put_unlocked()
369 	struct drm_gem_object *obj = ptr;  in drm_gem_object_release_handle()  local
371 	if (drm_WARN_ON(obj->dev, !data))  in drm_gem_object_release_handle()
374 	if (obj->funcs->close)  in drm_gem_object_release_handle()
375 		obj->funcs->close(obj, file_priv);  in drm_gem_object_release_handle()
383 	drm_vma_node_revoke(&obj->vma_node, file_priv);  in drm_gem_object_release_handle()
385 	drm_gem_object_handle_put_unlocked(obj);  in drm_gem_object_release_handle()
402 	struct drm_gem_object *obj;  in drm_gem_handle_delete()  local
407 	obj = idr_replace(&filp->object_idr, NULL, handle);  in drm_gem_handle_delete()
409 	if (IS_ERR_OR_NULL(obj))  in drm_gem_handle_delete()
413 	drm_gem_object_release_handle(handle, obj, filp);  in drm_gem_handle_delete()
440 	struct drm_gem_object *obj;  in drm_gem_dumb_map_offset()  local
443 	obj = drm_gem_object_lookup(file, handle);  in drm_gem_dumb_map_offset()
444 	if (!obj)  in drm_gem_dumb_map_offset()
448 	if (drm_gem_is_imported(obj)) {  in drm_gem_dumb_map_offset()
453 	ret = drm_gem_create_mmap_offset(obj);  in drm_gem_dumb_map_offset()
457 	*offset = drm_vma_node_offset_addr(&obj->vma_node);  in drm_gem_dumb_map_offset()
459 	drm_gem_object_put(obj);  in drm_gem_dumb_map_offset()
481 			   struct drm_gem_object *obj,  in drm_gem_handle_create_tail()  argument
484 	struct drm_device *dev = obj->dev;  in drm_gem_handle_create_tail()
490 	drm_gem_object_handle_get(obj);  in drm_gem_handle_create_tail()
510 	ret = drm_vma_node_allow(&obj->vma_node, file_priv);  in drm_gem_handle_create_tail()
514 	if (obj->funcs->open) {  in drm_gem_handle_create_tail()
515 		ret = obj->funcs->open(obj, file_priv);  in drm_gem_handle_create_tail()
522 	obj = idr_replace(&file_priv->object_idr, obj, handle);  in drm_gem_handle_create_tail()
523 	WARN_ON(obj != NULL);  in drm_gem_handle_create_tail()
529 	drm_vma_node_revoke(&obj->vma_node, file_priv);  in drm_gem_handle_create_tail()
535 	drm_gem_object_handle_put_unlocked(obj);  in drm_gem_handle_create_tail()
553 			  struct drm_gem_object *obj,  in drm_gem_handle_create()  argument
556 	mutex_lock(&obj->dev->object_name_lock);  in drm_gem_handle_create()
558 	return drm_gem_handle_create_tail(file_priv, obj, handlep);  in drm_gem_handle_create()
574 drm_gem_free_mmap_offset(struct drm_gem_object *obj)  in drm_gem_free_mmap_offset()  argument
576 	struct drm_device *dev = obj->dev;  in drm_gem_free_mmap_offset()
578 	drm_vma_offset_remove(dev->vma_offset_manager, &obj->vma_node);  in drm_gem_free_mmap_offset()
600 drm_gem_create_mmap_offset_size(struct drm_gem_object *obj, size_t size)  in drm_gem_create_mmap_offset_size()  argument
602 	struct drm_device *dev = obj->dev;  in drm_gem_create_mmap_offset_size()
604 	return drm_vma_offset_add(dev->vma_offset_manager, &obj->vma_node,  in drm_gem_create_mmap_offset_size()
623 int drm_gem_create_mmap_offset(struct drm_gem_object *obj)  in drm_gem_create_mmap_offset()  argument
625 	return drm_gem_create_mmap_offset_size(obj, obj->size);  in drm_gem_create_mmap_offset()
665 struct page **drm_gem_get_pages(struct drm_gem_object *obj)  in drm_gem_get_pages()  argument
673 	if (WARN_ON(!obj->filp))  in drm_gem_get_pages()
677 	mapping = obj->filp->f_mapping;  in drm_gem_get_pages()
683 	WARN_ON((obj->size & (PAGE_SIZE - 1)) != 0);  in drm_gem_get_pages()
685 	npages = obj->size >> PAGE_SHIFT;  in drm_gem_get_pages()
740 void drm_gem_put_pages(struct drm_gem_object *obj, struct page **pages,  in drm_gem_put_pages()  argument
747 	mapping = file_inode(obj->filp)->i_mapping;  in drm_gem_put_pages()
754 	WARN_ON((obj->size & (PAGE_SIZE - 1)) != 0);  in drm_gem_put_pages()
756 	npages = obj->size >> PAGE_SHIFT;  in drm_gem_put_pages()
788 	struct drm_gem_object *obj;  in objects_lookup()  local
794 		obj = idr_find(&filp->object_idr, handle[i]);  in objects_lookup()
795 		if (!obj) {  in objects_lookup()
799 		drm_gem_object_get(obj);  in objects_lookup()
800 		objs[i] = obj;  in objects_lookup()
867 	struct drm_gem_object *obj = NULL;  in drm_gem_object_lookup()  local
869 	objects_lookup(filp, &handle, 1, &obj);  in drm_gem_object_lookup()
870 	return obj;  in drm_gem_object_lookup()
890 	struct drm_gem_object *obj;  in drm_gem_dma_resv_wait()  local
893 	obj = drm_gem_object_lookup(filep, handle);  in drm_gem_dma_resv_wait()
894 	if (!obj) {  in drm_gem_dma_resv_wait()
899 	ret = dma_resv_wait_timeout(obj->resv, dma_resv_usage_rw(wait_all),  in drm_gem_dma_resv_wait()
906 	drm_gem_object_put(obj);  in drm_gem_dma_resv_wait()
932 	struct drm_gem_object *obj;  in drm_gem_flink_ioctl()  local
938 	obj = drm_gem_object_lookup(file_priv, args->handle);  in drm_gem_flink_ioctl()
939 	if (obj == NULL)  in drm_gem_flink_ioctl()
944 	if (obj->handle_count == 0) {  in drm_gem_flink_ioctl()
949 	if (!obj->name) {  in drm_gem_flink_ioctl()
950 		ret = idr_alloc(&dev->object_name_idr, obj, 1, 0, GFP_KERNEL);  in drm_gem_flink_ioctl()
954 		obj->name = ret;  in drm_gem_flink_ioctl()
957 	args->name = (uint64_t) obj->name;  in drm_gem_flink_ioctl()
962 	drm_gem_object_put(obj);  in drm_gem_flink_ioctl()
971 	struct drm_gem_object *obj;  in drm_gem_open_ioctl()  local
979 	obj = idr_find(&dev->object_name_idr, (int) args->name);  in drm_gem_open_ioctl()
980 	if (obj) {  in drm_gem_open_ioctl()
981 		drm_gem_object_get(obj);  in drm_gem_open_ioctl()
988 	ret = drm_gem_handle_create_tail(file_priv, obj, &handle);  in drm_gem_open_ioctl()
993 	args->size = obj->size;  in drm_gem_open_ioctl()
996 	drm_gem_object_put(obj);  in drm_gem_open_ioctl()
1004 	struct drm_gem_object *obj;  in drm_gem_change_handle_ioctl()  local
1015 	obj = drm_gem_object_lookup(file_priv, args->handle);  in drm_gem_change_handle_ioctl()
1016 	if (!obj)  in drm_gem_change_handle_ioctl()
1027 	ret = idr_alloc(&file_priv->object_idr, obj, handle, handle + 1,  in drm_gem_change_handle_ioctl()
1034 	if (obj->dma_buf) {  in drm_gem_change_handle_ioctl()
1035 		ret = drm_prime_add_buf_handle(&file_priv->prime, obj->dma_buf,  in drm_gem_change_handle_ioctl()
1056 	drm_gem_object_put(obj);  in drm_gem_change_handle_ioctl()
1101 drm_gem_object_release(struct drm_gem_object *obj)  in drm_gem_object_release()  argument
1103 	if (obj->filp)  in drm_gem_object_release()
1104 		fput(obj->filp);  in drm_gem_object_release()
1106 	drm_gem_private_object_fini(obj);  in drm_gem_object_release()
1108 	drm_gem_free_mmap_offset(obj);  in drm_gem_object_release()
1109 	drm_gem_lru_remove(obj);  in drm_gem_object_release()
1124 	struct drm_gem_object *obj =  in drm_gem_object_free()  local
1127 	if (WARN_ON(!obj->funcs->free))  in drm_gem_object_free()
1130 	obj->funcs->free(obj);  in drm_gem_object_free()
1143 	struct drm_gem_object *obj = vma->vm_private_data;  in drm_gem_vm_open()  local
1145 	drm_gem_object_get(obj);  in drm_gem_vm_open()
1158 	struct drm_gem_object *obj = vma->vm_private_data;  in drm_gem_vm_close()  local
1160 	drm_gem_object_put(obj);  in drm_gem_vm_close()
1188 int drm_gem_mmap_obj(struct drm_gem_object *obj, unsigned long obj_size,  in drm_gem_mmap_obj()  argument
1203 	drm_gem_object_get(obj);  in drm_gem_mmap_obj()
1205 	vma->vm_private_data = obj;  in drm_gem_mmap_obj()
1206 	vma->vm_ops = obj->funcs->vm_ops;  in drm_gem_mmap_obj()
1208 	if (obj->funcs->mmap) {  in drm_gem_mmap_obj()
1209 		ret = obj->funcs->mmap(obj, vma);  in drm_gem_mmap_obj()
1227 	drm_gem_object_put(obj);  in drm_gem_mmap_obj()
1244 	struct drm_gem_object *obj = NULL;  in drm_gem_object_lookup_at_offset()  local
1254 		obj = container_of(node, struct drm_gem_object, vma_node);  in drm_gem_object_lookup_at_offset()
1265 		if (!kref_get_unless_zero(&obj->refcount))  in drm_gem_object_lookup_at_offset()
1266 			obj = NULL;  in drm_gem_object_lookup_at_offset()
1270 	if (!obj)  in drm_gem_object_lookup_at_offset()
1274 		drm_gem_object_put(obj);  in drm_gem_object_lookup_at_offset()
1278 	return obj;  in drm_gem_object_lookup_at_offset()
1308 	struct drm_gem_object *obj;  in drm_gem_get_unmapped_area()  local
1311 	obj = drm_gem_object_lookup_at_offset(filp, pgoff, len >> PAGE_SHIFT);  in drm_gem_get_unmapped_area()
1312 	if (IS_ERR(obj))  in drm_gem_get_unmapped_area()
1313 		obj = NULL;  in drm_gem_get_unmapped_area()
1315 	if (!obj || !obj->filp || !obj->filp->f_op->get_unmapped_area)  in drm_gem_get_unmapped_area()
1318 		ret = obj->filp->f_op->get_unmapped_area(obj->filp, uaddr, len, 0, flags);  in drm_gem_get_unmapped_area()
1320 	drm_gem_object_put(obj);  in drm_gem_get_unmapped_area()
1344 	struct drm_gem_object *obj;  in drm_gem_mmap()  local
1347 	obj = drm_gem_object_lookup_at_offset(filp, vma->vm_pgoff,  in drm_gem_mmap()
1349 	if (IS_ERR(obj))  in drm_gem_mmap()
1350 		return PTR_ERR(obj);  in drm_gem_mmap()
1352 	ret = drm_gem_mmap_obj(obj,  in drm_gem_mmap()
1353 			       drm_vma_node_size(&obj->vma_node) << PAGE_SHIFT,  in drm_gem_mmap()
1356 	drm_gem_object_put(obj);  in drm_gem_mmap()
1363 			const struct drm_gem_object *obj)  in drm_gem_print_info()  argument
1365 	drm_printf_indent(p, indent, "name=%d\n", obj->name);  in drm_gem_print_info()
1367 			  kref_read(&obj->refcount));  in drm_gem_print_info()
1369 			  drm_vma_node_start(&obj->vma_node));  in drm_gem_print_info()
1370 	drm_printf_indent(p, indent, "size=%zu\n", obj->size);  in drm_gem_print_info()
1372 			  str_yes_no(drm_gem_is_imported(obj)));  in drm_gem_print_info()
1374 	if (obj->funcs->print_info)  in drm_gem_print_info()
1375 		obj->funcs->print_info(p, indent, obj);  in drm_gem_print_info()
1378 int drm_gem_vmap_locked(struct drm_gem_object *obj, struct iosys_map *map)  in drm_gem_vmap_locked()  argument
1382 	dma_resv_assert_held(obj->resv);  in drm_gem_vmap_locked()
1384 	if (!obj->funcs->vmap)  in drm_gem_vmap_locked()
1387 	ret = obj->funcs->vmap(obj, map);  in drm_gem_vmap_locked()
1397 void drm_gem_vunmap_locked(struct drm_gem_object *obj, struct iosys_map *map)  in drm_gem_vunmap_locked()  argument
1399 	dma_resv_assert_held(obj->resv);  in drm_gem_vunmap_locked()
1404 	if (obj->funcs->vunmap)  in drm_gem_vunmap_locked()
1405 		obj->funcs->vunmap(obj, map);  in drm_gem_vunmap_locked()
1412 void drm_gem_lock(struct drm_gem_object *obj)  in drm_gem_lock()  argument
1414 	dma_resv_lock(obj->resv, NULL);  in drm_gem_lock()
1418 void drm_gem_unlock(struct drm_gem_object *obj)  in drm_gem_unlock()  argument
1420 	dma_resv_unlock(obj->resv);  in drm_gem_unlock()
1424 int drm_gem_vmap(struct drm_gem_object *obj, struct iosys_map *map)  in drm_gem_vmap()  argument
1428 	dma_resv_lock(obj->resv, NULL);  in drm_gem_vmap()
1429 	ret = drm_gem_vmap_locked(obj, map);  in drm_gem_vmap()
1430 	dma_resv_unlock(obj->resv);  in drm_gem_vmap()
1436 void drm_gem_vunmap(struct drm_gem_object *obj, struct iosys_map *map)  in drm_gem_vunmap()  argument
1438 	dma_resv_lock(obj->resv, NULL);  in drm_gem_vunmap()
1439 	drm_gem_vunmap_locked(obj, map);  in drm_gem_vunmap()
1440 	dma_resv_unlock(obj->resv);  in drm_gem_vunmap()
1468 		struct drm_gem_object *obj = objs[contended];  in drm_gem_lock_reservations()  local
1470 		ret = dma_resv_lock_slow_interruptible(obj->resv,  in drm_gem_lock_reservations()
1538 drm_gem_lru_remove_locked(struct drm_gem_object *obj)  in drm_gem_lru_remove_locked()  argument
1540 	obj->lru->count -= obj->size >> PAGE_SHIFT;  in drm_gem_lru_remove_locked()
1541 	WARN_ON(obj->lru->count < 0);  in drm_gem_lru_remove_locked()
1542 	list_del(&obj->lru_node);  in drm_gem_lru_remove_locked()
1543 	obj->lru = NULL;  in drm_gem_lru_remove_locked()
1554 drm_gem_lru_remove(struct drm_gem_object *obj)  in drm_gem_lru_remove()  argument
1556 	struct drm_gem_lru *lru = obj->lru;  in drm_gem_lru_remove()
1562 	drm_gem_lru_remove_locked(obj);  in drm_gem_lru_remove()
1576 drm_gem_lru_move_tail_locked(struct drm_gem_lru *lru, struct drm_gem_object *obj)  in drm_gem_lru_move_tail_locked()  argument
1580 	if (obj->lru)  in drm_gem_lru_move_tail_locked()
1581 		drm_gem_lru_remove_locked(obj);  in drm_gem_lru_move_tail_locked()
1583 	lru->count += obj->size >> PAGE_SHIFT;  in drm_gem_lru_move_tail_locked()
1584 	list_add_tail(&obj->lru_node, &lru->list);  in drm_gem_lru_move_tail_locked()
1585 	obj->lru = lru;  in drm_gem_lru_move_tail_locked()
1600 drm_gem_lru_move_tail(struct drm_gem_lru *lru, struct drm_gem_object *obj)  in drm_gem_lru_move_tail()  argument
1603 	drm_gem_lru_move_tail_locked(lru, obj);  in drm_gem_lru_move_tail()
1628 		 bool (*shrink)(struct drm_gem_object *obj, struct ww_acquire_ctx *ticket),  in drm_gem_lru_scan()  argument
1632 	struct drm_gem_object *obj;  in drm_gem_lru_scan()  local
1640 		obj = list_first_entry_or_null(&lru->list, typeof(*obj), lru_node);  in drm_gem_lru_scan()
1642 		if (!obj)  in drm_gem_lru_scan()
1645 		drm_gem_lru_move_tail_locked(&still_in_lru, obj);  in drm_gem_lru_scan()
1652 		if (!kref_get_unless_zero(&obj->refcount))  in drm_gem_lru_scan()
1670 		if (!ww_mutex_trylock(&obj->resv->lock, ticket)) {  in drm_gem_lru_scan()
1671 			*remaining += obj->size >> PAGE_SHIFT;  in drm_gem_lru_scan()
1675 		if (shrink(obj, ticket)) {  in drm_gem_lru_scan()
1676 			freed += obj->size >> PAGE_SHIFT;  in drm_gem_lru_scan()
1683 			WARN_ON(obj->lru == &still_in_lru);  in drm_gem_lru_scan()
1684 			WARN_ON(obj->lru == lru);  in drm_gem_lru_scan()
1687 		dma_resv_unlock(obj->resv);  in drm_gem_lru_scan()
1693 		drm_gem_object_put(obj);  in drm_gem_lru_scan()
1701 	list_for_each_entry (obj, &still_in_lru.list, lru_node)  in drm_gem_lru_scan()
1702 		obj->lru = lru;  in drm_gem_lru_scan()
1716 int drm_gem_evict_locked(struct drm_gem_object *obj)  in drm_gem_evict_locked()  argument
1718 	dma_resv_assert_held(obj->resv);  in drm_gem_evict_locked()
1720 	if (!dma_resv_test_signaled(obj->resv, DMA_RESV_USAGE_READ))  in drm_gem_evict_locked()
1723 	if (obj->funcs->evict)  in drm_gem_evict_locked()
1724 		return obj->funcs->evict(obj);  in drm_gem_evict_locked()