793 static struct sg_table *__map_dma_buf(struct dma_buf_attachment *attach,  in __map_dma_buf()  argument
799 	sg_table = attach->dmabuf->ops->map_dma_buf(attach, direction);  in __map_dma_buf()
803 	if (!dma_buf_attachment_is_dynamic(attach)) {  in __map_dma_buf()
804 		ret = dma_resv_wait_timeout(attach->dmabuf->resv,  in __map_dma_buf()
808 			attach->dmabuf->ops->unmap_dma_buf(attach, sg_table,  in __map_dma_buf()
909 	struct dma_buf_attachment *attach;  in dma_buf_dynamic_attach()  local
918 	attach = kzalloc(sizeof(*attach), GFP_KERNEL);  in dma_buf_dynamic_attach()
919 	if (!attach)  in dma_buf_dynamic_attach()
922 	attach->dev = dev;  in dma_buf_dynamic_attach()
923 	attach->dmabuf = dmabuf;  in dma_buf_dynamic_attach()
925 		attach->peer2peer = importer_ops->allow_peer2peer;  in dma_buf_dynamic_attach()
926 	attach->importer_ops = importer_ops;  in dma_buf_dynamic_attach()
927 	attach->importer_priv = importer_priv;  in dma_buf_dynamic_attach()
929 	if (dmabuf->ops->attach) {  in dma_buf_dynamic_attach()
930 		ret = dmabuf->ops->attach(dmabuf, attach);  in dma_buf_dynamic_attach()
935 	list_add(&attach->node, &dmabuf->attachments);  in dma_buf_dynamic_attach()
942 	if (dma_buf_attachment_is_dynamic(attach) !=  in dma_buf_dynamic_attach()
946 		dma_resv_lock(attach->dmabuf->resv, NULL);  in dma_buf_dynamic_attach()
947 		if (dma_buf_is_dynamic(attach->dmabuf)) {  in dma_buf_dynamic_attach()
948 			ret = dmabuf->ops->pin(attach);  in dma_buf_dynamic_attach()
953 		sgt = __map_dma_buf(attach, DMA_BIDIRECTIONAL);  in dma_buf_dynamic_attach()
960 		dma_resv_unlock(attach->dmabuf->resv);  in dma_buf_dynamic_attach()
961 		attach->sgt = sgt;  in dma_buf_dynamic_attach()
962 		attach->dir = DMA_BIDIRECTIONAL;  in dma_buf_dynamic_attach()
965 	return attach;  in dma_buf_dynamic_attach()
968 	kfree(attach);  in dma_buf_dynamic_attach()
972 	if (dma_buf_is_dynamic(attach->dmabuf))  in dma_buf_dynamic_attach()
973 		dmabuf->ops->unpin(attach);  in dma_buf_dynamic_attach()
976 	dma_resv_unlock(attach->dmabuf->resv);  in dma_buf_dynamic_attach()
978 	dma_buf_detach(dmabuf, attach);  in dma_buf_dynamic_attach()
998 static void __unmap_dma_buf(struct dma_buf_attachment *attach,  in __unmap_dma_buf()  argument
1005 	attach->dmabuf->ops->unmap_dma_buf(attach, sg_table, direction);  in __unmap_dma_buf()
1017 void dma_buf_detach(struct dma_buf *dmabuf, struct dma_buf_attachment *attach)  in dma_buf_detach()  argument
1019 	if (WARN_ON(!dmabuf || !attach || dmabuf != attach->dmabuf))  in dma_buf_detach()
1024 	if (attach->sgt) {  in dma_buf_detach()
1026 		__unmap_dma_buf(attach, attach->sgt, attach->dir);  in dma_buf_detach()
1028 		if (dma_buf_is_dynamic(attach->dmabuf))  in dma_buf_detach()
1029 			dmabuf->ops->unpin(attach);  in dma_buf_detach()
1031 	list_del(&attach->node);  in dma_buf_detach()
1036 		dmabuf->ops->detach(dmabuf, attach);  in dma_buf_detach()
1038 	kfree(attach);  in dma_buf_detach()
1056 int dma_buf_pin(struct dma_buf_attachment *attach)  in dma_buf_pin()  argument
1058 	struct dma_buf *dmabuf = attach->dmabuf;  in dma_buf_pin()
1061 	WARN_ON(!dma_buf_attachment_is_dynamic(attach));  in dma_buf_pin()
1066 		ret = dmabuf->ops->pin(attach);  in dma_buf_pin()
1080 void dma_buf_unpin(struct dma_buf_attachment *attach)  in dma_buf_unpin()  argument
1082 	struct dma_buf *dmabuf = attach->dmabuf;  in dma_buf_unpin()
1084 	WARN_ON(!dma_buf_attachment_is_dynamic(attach));  in dma_buf_unpin()
1089 		dmabuf->ops->unpin(attach);  in dma_buf_unpin()
1114 struct sg_table *dma_buf_map_attachment(struct dma_buf_attachment *attach,  in dma_buf_map_attachment()  argument
1122 	if (WARN_ON(!attach || !attach->dmabuf))  in dma_buf_map_attachment()
1125 	dma_resv_assert_held(attach->dmabuf->resv);  in dma_buf_map_attachment()
1127 	if (attach->sgt) {  in dma_buf_map_attachment()
1132 		if (attach->dir != direction &&  in dma_buf_map_attachment()
1133 		    attach->dir != DMA_BIDIRECTIONAL)  in dma_buf_map_attachment()
1136 		return attach->sgt;  in dma_buf_map_attachment()
1139 	if (dma_buf_is_dynamic(attach->dmabuf)) {  in dma_buf_map_attachment()
1141 			r = attach->dmabuf->ops->pin(attach);  in dma_buf_map_attachment()
1147 	sg_table = __map_dma_buf(attach, direction);  in dma_buf_map_attachment()
1151 	if (IS_ERR(sg_table) && dma_buf_is_dynamic(attach->dmabuf) &&  in dma_buf_map_attachment()
1153 		attach->dmabuf->ops->unpin(attach);  in dma_buf_map_attachment()
1155 	if (!IS_ERR(sg_table) && attach->dmabuf->ops->cache_sgt_mapping) {  in dma_buf_map_attachment()
1156 		attach->sgt = sg_table;  in dma_buf_map_attachment()
1157 		attach->dir = direction;  in dma_buf_map_attachment()
1191 dma_buf_map_attachment_unlocked(struct dma_buf_attachment *attach,  in dma_buf_map_attachment_unlocked()  argument
1198 	if (WARN_ON(!attach || !attach->dmabuf))  in dma_buf_map_attachment_unlocked()
1201 	dma_resv_lock(attach->dmabuf->resv, NULL);  in dma_buf_map_attachment_unlocked()
1202 	sg_table = dma_buf_map_attachment(attach, direction);  in dma_buf_map_attachment_unlocked()
1203 	dma_resv_unlock(attach->dmabuf->resv);  in dma_buf_map_attachment_unlocked()
1219 void dma_buf_unmap_attachment(struct dma_buf_attachment *attach,  in dma_buf_unmap_attachment()  argument
1225 	if (WARN_ON(!attach || !attach->dmabuf || !sg_table))  in dma_buf_unmap_attachment()
1228 	dma_resv_assert_held(attach->dmabuf->resv);  in dma_buf_unmap_attachment()
1230 	if (attach->sgt == sg_table)  in dma_buf_unmap_attachment()
1233 	__unmap_dma_buf(attach, sg_table, direction);  in dma_buf_unmap_attachment()
1235 	if (dma_buf_is_dynamic(attach->dmabuf) &&  in dma_buf_unmap_attachment()
1237 		dma_buf_unpin(attach);  in dma_buf_unmap_attachment()
1251 void dma_buf_unmap_attachment_unlocked(struct dma_buf_attachment *attach,  in dma_buf_unmap_attachment_unlocked()  argument
1257 	if (WARN_ON(!attach || !attach->dmabuf || !sg_table))  in dma_buf_unmap_attachment_unlocked()
1260 	dma_resv_lock(attach->dmabuf->resv, NULL);  in dma_buf_unmap_attachment_unlocked()
1261 	dma_buf_unmap_attachment(attach, sg_table, direction);  in dma_buf_unmap_attachment_unlocked()
1262 	dma_resv_unlock(attach->dmabuf->resv);  in dma_buf_unmap_attachment_unlocked()
1276 	struct dma_buf_attachment *attach;  in dma_buf_move_notify()  local
1280 	list_for_each_entry(attach, &dmabuf->attachments, node)  in dma_buf_move_notify()
1281 		if (attach->importer_ops)  in dma_buf_move_notify()
1282 			attach->importer_ops->move_notify(attach);  in dma_buf_move_notify()