27 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in physaddr()  local
29 	return (((dma_addr_t)msm_obj->vram_node->start) << PAGE_SHIFT) +  in physaddr()
35 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in use_pages()  local
36 	return !msm_obj->vram_node;  in use_pages()
81 static void sync_for_device(struct msm_gem_object *msm_obj)  in sync_for_device()  argument
83 	struct device *dev = msm_obj->base.dev->dev;  in sync_for_device()
85 	dma_map_sgtable(dev, msm_obj->sgt, DMA_BIDIRECTIONAL, 0);  in sync_for_device()
88 static void sync_for_cpu(struct msm_gem_object *msm_obj)  in sync_for_cpu()  argument
90 	struct device *dev = msm_obj->base.dev->dev;  in sync_for_cpu()
92 	dma_unmap_sgtable(dev, msm_obj->sgt, DMA_BIDIRECTIONAL, 0);  in sync_for_cpu()
98 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in update_lru_active()  local
100 	GEM_WARN_ON(!msm_obj->pages);  in update_lru_active()
102 	if (msm_obj->pin_count) {  in update_lru_active()
104 	} else if (msm_obj->madv == MSM_MADV_WILLNEED) {  in update_lru_active()
107 		GEM_WARN_ON(msm_obj->madv != MSM_MADV_DONTNEED);  in update_lru_active()
116 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in update_lru_locked()  local
118 	msm_gem_assert_locked(&msm_obj->base);  in update_lru_locked()
120 	if (!msm_obj->pages) {  in update_lru_locked()
121 		GEM_WARN_ON(msm_obj->pin_count);  in update_lru_locked()
141 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in get_pages_vram()  local
152 	ret = drm_mm_insert_node(&priv->vram.mm, msm_obj->vram_node, npages);  in get_pages_vram()
170 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in get_pages()  local
174 	if (!msm_obj->pages) {  in get_pages()
192 		msm_obj->pages = p;  in get_pages()
194 		msm_obj->sgt = drm_prime_pages_to_sg(obj->dev, p, npages);  in get_pages()
195 		if (IS_ERR(msm_obj->sgt)) {  in get_pages()
196 			void *ptr = ERR_CAST(msm_obj->sgt);  in get_pages()
199 			msm_obj->sgt = NULL;  in get_pages()
206 		if (msm_obj->flags & MSM_BO_WC)  in get_pages()
207 			sync_for_device(msm_obj);  in get_pages()
212 	return msm_obj->pages;  in get_pages()
217 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in put_pages_vram()  local
221 	drm_mm_remove_node(msm_obj->vram_node);  in put_pages_vram()
224 	kvfree(msm_obj->pages);  in put_pages_vram()
229 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in put_pages()  local
231 	if (msm_obj->pages) {  in put_pages()
232 		if (msm_obj->sgt) {  in put_pages()
237 			if (msm_obj->flags & MSM_BO_WC)  in put_pages()
238 				sync_for_cpu(msm_obj);  in put_pages()
240 			sg_free_table(msm_obj->sgt);  in put_pages()
241 			kfree(msm_obj->sgt);  in put_pages()
242 			msm_obj->sgt = NULL;  in put_pages()
248 			drm_gem_put_pages(obj, msm_obj->pages, true, false);  in put_pages()
252 		msm_obj->pages = NULL;  in put_pages()
260 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in msm_gem_get_pages_locked()  local
264 	if (msm_obj->madv > madv) {  in msm_gem_get_pages_locked()
266 				     msm_obj->madv, madv);  in msm_gem_get_pages_locked()
315 static pgprot_t msm_gem_pgprot(struct msm_gem_object *msm_obj, pgprot_t prot)  in msm_gem_pgprot()  argument
317 	if (msm_obj->flags & MSM_BO_WC)  in msm_gem_pgprot()
326 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in msm_gem_fault()  local
343 	if (GEM_WARN_ON(msm_obj->madv != MSM_MADV_WILLNEED)) {  in msm_gem_fault()
403 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in add_vma()  local
412 	list_add_tail(&vma->list, &msm_obj->vmas);  in add_vma()
420 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in lookup_vma()  local
425 	list_for_each_entry(vma, &msm_obj->vmas, list) {  in lookup_vma()
451 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in put_iova_spaces()  local
456 	list_for_each_entry(vma, &msm_obj->vmas, list) {  in put_iova_spaces()
469 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in put_iova_vmas()  local
474 	list_for_each_entry_safe(vma, tmp, &msm_obj->vmas, list) {  in put_iova_vmas()
512 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in msm_gem_pin_vma_locked()  local
516 	if (!(msm_obj->flags & MSM_BO_GPU_READONLY))  in msm_gem_pin_vma_locked()
519 	if (msm_obj->flags & MSM_BO_MAP_PRIV)  in msm_gem_pin_vma_locked()
522 	if (msm_obj->flags & MSM_BO_CACHED_COHERENT)  in msm_gem_pin_vma_locked()
531 	return msm_gem_vma_map(vma, prot, msm_obj->sgt, obj->size);  in msm_gem_pin_vma_locked()
537 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in msm_gem_unpin_locked()  local
542 	msm_obj->pin_count--;  in msm_gem_unpin_locked()
543 	GEM_WARN_ON(msm_obj->pin_count < 0);  in msm_gem_unpin_locked()
556 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in msm_gem_unpin_active()  local
558 	msm_obj->pin_count--;  in msm_gem_unpin_active()
559 	GEM_WARN_ON(msm_obj->pin_count < 0);  in msm_gem_unpin_active()
732 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in get_vaddr()  local
753 	msm_obj->vmap_count++;  in get_vaddr()
755 	if (!msm_obj->vaddr) {  in get_vaddr()
756 		msm_obj->vaddr = vmap(pages, obj->size >> PAGE_SHIFT,  in get_vaddr()
757 				VM_MAP, msm_gem_pgprot(msm_obj, PAGE_KERNEL));  in get_vaddr()
758 		if (msm_obj->vaddr == NULL) {  in get_vaddr()
764 	return msm_obj->vaddr;  in get_vaddr()
767 	msm_obj->vmap_count--;  in get_vaddr()
801 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in msm_gem_put_vaddr_locked()  local
804 	GEM_WARN_ON(msm_obj->vmap_count < 1);  in msm_gem_put_vaddr_locked()
806 	msm_obj->vmap_count--;  in msm_gem_put_vaddr_locked()
823 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in msm_gem_madvise()  local
829 	if (msm_obj->madv != __MSM_MADV_PURGED)  in msm_gem_madvise()
830 		msm_obj->madv = madv;  in msm_gem_madvise()
832 	madv = msm_obj->madv;  in msm_gem_madvise()
850 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in msm_gem_purge()  local
853 	GEM_WARN_ON(!is_purgeable(msm_obj));  in msm_gem_purge()
868 	msm_obj->madv = __MSM_MADV_PURGED;  in msm_gem_purge()
890 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in msm_gem_evict()  local
893 	GEM_WARN_ON(is_unevictable(msm_obj));  in msm_gem_evict()
905 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in msm_gem_vunmap()  local
909 	if (!msm_obj->vaddr || GEM_WARN_ON(!is_vunmapable(msm_obj)))  in msm_gem_vunmap()
912 	vunmap(msm_obj->vaddr);  in msm_gem_vunmap()
913 	msm_obj->vaddr = NULL;  in msm_gem_vunmap()
960 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in msm_gem_describe()  local
976 	if (msm_obj->pages) {  in msm_gem_describe()
981 	switch (msm_obj->madv) {  in msm_gem_describe()
999 			msm_obj->flags, msm_gem_active(obj) ? 'A' : 'I',  in msm_gem_describe()
1001 			off, msm_obj->vaddr);  in msm_gem_describe()
1003 	seq_printf(m, " %08zu %9s %-32s\n", obj->size, madv, msm_obj->name);  in msm_gem_describe()
1005 	if (!list_empty(&msm_obj->vmas)) {  in msm_gem_describe()
1009 		list_for_each_entry(vma, &msm_obj->vmas, list) {  in msm_gem_describe()
1042 	struct msm_gem_object *msm_obj;  in msm_gem_describe_objects()  local
1045 	list_for_each_entry(msm_obj, list, node) {  in msm_gem_describe_objects()
1046 		struct drm_gem_object *obj = &msm_obj->base;  in msm_gem_describe_objects()
1067 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in msm_gem_free_object()  local
1072 	list_del(&msm_obj->node);  in msm_gem_free_object()
1078 		GEM_WARN_ON(msm_obj->vaddr);  in msm_gem_free_object()
1083 		kvfree(msm_obj->pages);  in msm_gem_free_object()
1087 		drm_prime_gem_destroy(obj, msm_obj->sgt);  in msm_gem_free_object()
1096 	kfree(msm_obj->metadata);  in msm_gem_free_object()
1097 	kfree(msm_obj);  in msm_gem_free_object()
1102 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in msm_gem_object_mmap()  local
1105 	vma->vm_page_prot = msm_gem_pgprot(msm_obj, vm_get_page_prot(vma->vm_flags));  in msm_gem_object_mmap()
1136 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in msm_gem_status()  local
1139 	if (msm_obj->pages)  in msm_gem_status()
1142 	if (msm_obj->madv == MSM_MADV_DONTNEED)  in msm_gem_status()
1173 	struct msm_gem_object *msm_obj;  in msm_gem_new_impl()  local
1189 	msm_obj = kzalloc(sizeof(*msm_obj), GFP_KERNEL);  in msm_gem_new_impl()
1190 	if (!msm_obj)  in msm_gem_new_impl()
1193 	msm_obj->flags = flags;  in msm_gem_new_impl()
1194 	msm_obj->madv = MSM_MADV_WILLNEED;  in msm_gem_new_impl()
1196 	INIT_LIST_HEAD(&msm_obj->node);  in msm_gem_new_impl()
1197 	INIT_LIST_HEAD(&msm_obj->vmas);  in msm_gem_new_impl()
1199 	*obj = &msm_obj->base;  in msm_gem_new_impl()
1208 	struct msm_gem_object *msm_obj;  in msm_gem_new()  local
1233 	msm_obj = to_msm_bo(obj);  in msm_gem_new()
1277 	list_add_tail(&msm_obj->node, &priv->objects);  in msm_gem_new()
1295 	struct msm_gem_object *msm_obj;  in msm_gem_import()  local
1316 	msm_obj = to_msm_bo(obj);  in msm_gem_import()
1318 	msm_obj->sgt = sgt;  in msm_gem_import()
1319 	msm_obj->pages = kvmalloc_array(npages, sizeof(struct page *), GFP_KERNEL);  in msm_gem_import()
1320 	if (!msm_obj->pages) {  in msm_gem_import()
1326 	ret = drm_prime_sg_to_page_array(sgt, msm_obj->pages, npages);  in msm_gem_import()
1337 	list_add_tail(&msm_obj->node, &priv->objects);  in msm_gem_import()
1399 	struct msm_gem_object *msm_obj = to_msm_bo(bo);  in msm_gem_object_set_name()  local
1406 	vsnprintf(msm_obj->name, sizeof(msm_obj->name), fmt, ap);  in msm_gem_object_set_name()