13 #include <linux/iommu.h>
18 #include "iommu.h"
24 			     struct intel_iommu *iommu, struct device *dev,  in cache_tage_match()  argument
34 		return tag->iommu == iommu;  in cache_tage_match()
48 	struct intel_iommu *iommu = info->iommu;  in cache_tag_assign()  local
57 	tag->iommu = iommu;  in cache_tag_assign()
65 		tag->dev = iommu->iommu.dev;  in cache_tag_assign()
69 		if (cache_tage_match(temp, did, iommu, dev, pasid, type)) {  in cache_tag_assign()
90 	struct intel_iommu *iommu = info->iommu;  in cache_tag_unassign()  local
96 		if (cache_tage_match(tag, did, iommu, dev, pasid, type)) {  in cache_tag_unassign()
167 	struct intel_iommu *iommu = info->iommu;  in domain_get_id_for_dev()  local
176 	return domain_id_iommu(domain, iommu);  in domain_get_id_for_dev()
265 static void qi_batch_flush_descs(struct intel_iommu *iommu, struct qi_batch *batch)  in qi_batch_flush_descs()  argument
267 	if (!iommu || !batch->index)  in qi_batch_flush_descs()
270 	qi_submit_sync(iommu, batch->descs, batch->index, 0);  in qi_batch_flush_descs()
276 static void qi_batch_increment_index(struct intel_iommu *iommu, struct qi_batch *batch)  in qi_batch_increment_index()  argument
279 		qi_batch_flush_descs(iommu, batch);  in qi_batch_increment_index()
282 static void qi_batch_add_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,  in qi_batch_add_iotlb()  argument
286 	qi_desc_iotlb(iommu, did, addr, size_order, type, &batch->descs[batch->index]);  in qi_batch_add_iotlb()
287 	qi_batch_increment_index(iommu, batch);  in qi_batch_add_iotlb()
290 static void qi_batch_add_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid,  in qi_batch_add_dev_iotlb()  argument
298 	if (!(iommu->gcmd & DMA_GCMD_TE))  in qi_batch_add_dev_iotlb()
302 	qi_batch_increment_index(iommu, batch);  in qi_batch_add_dev_iotlb()
305 static void qi_batch_add_piotlb(struct intel_iommu *iommu, u16 did, u32 pasid,  in qi_batch_add_piotlb()  argument
318 	qi_batch_increment_index(iommu, batch);  in qi_batch_add_piotlb()
321 static void qi_batch_add_pasid_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid,  in qi_batch_add_pasid_dev_iotlb()  argument
330 	if (!(iommu->gcmd & DMA_GCMD_TE))  in qi_batch_add_pasid_dev_iotlb()
335 	qi_batch_increment_index(iommu, batch);  in qi_batch_add_pasid_dev_iotlb()
342 	struct intel_iommu *iommu = tag->iommu;  in cache_tag_flush_iotlb()  local
346 		qi_batch_add_piotlb(iommu, tag->domain_id, tag->pasid, addr,  in cache_tag_flush_iotlb()
355 	if (!cap_pgsel_inv(iommu->cap) ||  in cache_tag_flush_iotlb()
356 	    mask > cap_max_amask_val(iommu->cap) || pages == -1) {  in cache_tag_flush_iotlb()
363 	if (ecap_qis(iommu->ecap))  in cache_tag_flush_iotlb()
364 		qi_batch_add_iotlb(iommu, tag->domain_id, addr | ih, mask, type,  in cache_tag_flush_iotlb()
367 		__iommu_flush_iotlb(iommu, tag->domain_id, addr | ih, mask, type);  in cache_tag_flush_iotlb()
373 	struct intel_iommu *iommu = tag->iommu;  in cache_tag_flush_devtlb_psi()  local
381 		qi_batch_add_dev_iotlb(iommu, sid, info->pfsid, info->ats_qdep,  in cache_tag_flush_devtlb_psi()
384 			qi_batch_add_dev_iotlb(iommu, sid, info->pfsid, info->ats_qdep,  in cache_tag_flush_devtlb_psi()
389 	qi_batch_add_pasid_dev_iotlb(iommu, sid, info->pfsid, tag->pasid,  in cache_tag_flush_devtlb_psi()
392 		qi_batch_add_pasid_dev_iotlb(iommu, sid, info->pfsid, tag->pasid,  in cache_tag_flush_devtlb_psi()
399 	struct intel_iommu *iommu = tag->iommu;  in cache_tag_flush_devtlb_all()  local
406 	qi_batch_add_dev_iotlb(iommu, sid, info->pfsid, info->ats_qdep, 0,  in cache_tag_flush_devtlb_all()
409 		qi_batch_add_dev_iotlb(iommu, sid, info->pfsid, info->ats_qdep, 0,  in cache_tag_flush_devtlb_all()
420 	struct intel_iommu *iommu = NULL;  in cache_tag_flush_range()  local
429 		if (iommu && iommu != tag->iommu)  in cache_tag_flush_range()
430 			qi_batch_flush_descs(iommu, domain->qi_batch);  in cache_tag_flush_range()
431 		iommu = tag->iommu;  in cache_tag_flush_range()
456 	qi_batch_flush_descs(iommu, domain->qi_batch);  in cache_tag_flush_range()
466 	struct intel_iommu *iommu = NULL;  in cache_tag_flush_all()  local
472 		if (iommu && iommu != tag->iommu)  in cache_tag_flush_all()
473 			qi_batch_flush_descs(iommu, domain->qi_batch);  in cache_tag_flush_all()
474 		iommu = tag->iommu;  in cache_tag_flush_all()
489 	qi_batch_flush_descs(iommu, domain->qi_batch);  in cache_tag_flush_all()
507 	struct intel_iommu *iommu = NULL;  in cache_tag_flush_range_np()  local
516 		if (iommu && iommu != tag->iommu)  in cache_tag_flush_range_np()
517 			qi_batch_flush_descs(iommu, domain->qi_batch);  in cache_tag_flush_range_np()
518 		iommu = tag->iommu;  in cache_tag_flush_range_np()
520 		if (!cap_caching_mode(iommu->cap) || domain->use_first_level) {  in cache_tag_flush_range_np()
521 			iommu_flush_write_buffer(iommu);  in cache_tag_flush_range_np()
531 	qi_batch_flush_descs(iommu, domain->qi_batch);  in cache_tag_flush_range_np()