54 	struct gk20a_instmem *imem;  member
145 	struct gk20a_instmem *imem = obj->base.imem;  in gk20a_instobj_iommu_recycle_vaddr()  local
151 	imem->vaddr_use -= nvkm_memory_size(&obj->base.base.memory);  in gk20a_instobj_iommu_recycle_vaddr()
152 	nvkm_debug(&imem->base.subdev, "vaddr used: %x/%x\n", imem->vaddr_use,  in gk20a_instobj_iommu_recycle_vaddr()
153 		   imem->vaddr_max);  in gk20a_instobj_iommu_recycle_vaddr()
160 gk20a_instmem_vaddr_gc(struct gk20a_instmem *imem, const u64 size)  in gk20a_instmem_vaddr_gc()  argument
162 	while (imem->vaddr_use + size > imem->vaddr_max) {  in gk20a_instmem_vaddr_gc()
164 		if (list_empty(&imem->vaddr_lru))  in gk20a_instmem_vaddr_gc()
168 				list_first_entry(&imem->vaddr_lru,  in gk20a_instmem_vaddr_gc()
177 	struct gk20a_instmem *imem = node->imem;  in gk20a_instobj_acquire_dma()  local
178 	struct nvkm_ltc *ltc = imem->base.subdev.device->ltc;  in gk20a_instobj_acquire_dma()
189 	struct gk20a_instmem *imem = node->base.imem;  in gk20a_instobj_acquire_iommu()  local
190 	struct nvkm_ltc *ltc = imem->base.subdev.device->ltc;  in gk20a_instobj_acquire_iommu()
195 	mutex_lock(&imem->lock);  in gk20a_instobj_acquire_iommu()
206 	gk20a_instmem_vaddr_gc(imem, size);  in gk20a_instobj_acquire_iommu()
212 		nvkm_error(&imem->base.subdev, "cannot map instobj - "  in gk20a_instobj_acquire_iommu()
217 	imem->vaddr_use += size;  in gk20a_instobj_acquire_iommu()
218 	nvkm_debug(&imem->base.subdev, "vaddr used: %x/%x\n",  in gk20a_instobj_acquire_iommu()
219 		   imem->vaddr_use, imem->vaddr_max);  in gk20a_instobj_acquire_iommu()
223 	mutex_unlock(&imem->lock);  in gk20a_instobj_acquire_iommu()
232 	struct gk20a_instmem *imem = node->imem;  in gk20a_instobj_release_dma()  local
233 	struct nvkm_ltc *ltc = imem->base.subdev.device->ltc;  in gk20a_instobj_release_dma()
244 	struct gk20a_instmem *imem = node->base.imem;  in gk20a_instobj_release_iommu()  local
245 	struct nvkm_ltc *ltc = imem->base.subdev.device->ltc;  in gk20a_instobj_release_iommu()
247 	mutex_lock(&imem->lock);  in gk20a_instobj_release_iommu()
255 		list_add_tail(&node->vaddr_node, &imem->vaddr_lru);  in gk20a_instobj_release_iommu()
258 	mutex_unlock(&imem->lock);  in gk20a_instobj_release_iommu()
298 	struct gk20a_instmem *imem = node->base.imem;  in gk20a_instobj_dtor_dma()  local
299 	struct device *dev = imem->base.subdev.device->dev;  in gk20a_instobj_dtor_dma()
305 		       node->base.vaddr, node->handle, imem->attrs);  in gk20a_instobj_dtor_dma()
315 	struct gk20a_instmem *imem = node->base.imem;  in gk20a_instobj_dtor_iommu()  local
316 	struct device *dev = imem->base.subdev.device->dev;  in gk20a_instobj_dtor_iommu()
323 	mutex_lock(&imem->lock);  in gk20a_instobj_dtor_iommu()
329 	mutex_unlock(&imem->lock);  in gk20a_instobj_dtor_iommu()
332 	r->offset &= ~BIT(imem->iommu_bit - imem->iommu_pgshift);  in gk20a_instobj_dtor_iommu()
336 		iommu_unmap(imem->domain,  in gk20a_instobj_dtor_iommu()
337 			    (r->offset + i) << imem->iommu_pgshift, PAGE_SIZE);  in gk20a_instobj_dtor_iommu()
344 	mutex_lock(imem->mm_mutex);  in gk20a_instobj_dtor_iommu()
345 	nvkm_mm_free(imem->mm, &r);  in gk20a_instobj_dtor_iommu()
346 	mutex_unlock(imem->mm_mutex);  in gk20a_instobj_dtor_iommu()
383 gk20a_instobj_ctor_dma(struct gk20a_instmem *imem, u32 npages, u32 align,  in gk20a_instobj_ctor_dma()  argument
387 	struct nvkm_subdev *subdev = &imem->base.subdev;  in gk20a_instobj_ctor_dma()
399 					   imem->attrs);  in gk20a_instobj_ctor_dma()
421 gk20a_instobj_ctor_iommu(struct gk20a_instmem *imem, u32 npages, u32 align,  in gk20a_instobj_ctor_iommu()  argument
425 	struct nvkm_subdev *subdev = &imem->base.subdev;  in gk20a_instobj_ctor_iommu()
463 	mutex_lock(imem->mm_mutex);  in gk20a_instobj_ctor_iommu()
465 	ret = nvkm_mm_head(imem->mm, 0, 1, npages, npages,  in gk20a_instobj_ctor_iommu()
466 			   align >> imem->iommu_pgshift, &r);  in gk20a_instobj_ctor_iommu()
467 	mutex_unlock(imem->mm_mutex);  in gk20a_instobj_ctor_iommu()
475 		u32 offset = (r->offset + i) << imem->iommu_pgshift;  in gk20a_instobj_ctor_iommu()
477 		ret = iommu_map(imem->domain, offset, node->dma_addrs[i],  in gk20a_instobj_ctor_iommu()
485 				iommu_unmap(imem->domain, offset, PAGE_SIZE);  in gk20a_instobj_ctor_iommu()
492 	r->offset |= BIT(imem->iommu_bit - imem->iommu_pgshift);  in gk20a_instobj_ctor_iommu()
498 	mutex_lock(imem->mm_mutex);  in gk20a_instobj_ctor_iommu()
499 	nvkm_mm_free(imem->mm, &r);  in gk20a_instobj_ctor_iommu()
500 	mutex_unlock(imem->mm_mutex);  in gk20a_instobj_ctor_iommu()
518 	struct gk20a_instmem *imem = gk20a_instmem(base);  in gk20a_instobj_new()  local
519 	struct nvkm_subdev *subdev = &imem->base.subdev;  in gk20a_instobj_new()
524 		   imem->domain ? "IOMMU" : "DMA", size, align);  in gk20a_instobj_new()
530 	if (imem->domain)  in gk20a_instobj_new()
531 		ret = gk20a_instobj_ctor_iommu(imem, size >> PAGE_SHIFT,  in gk20a_instobj_new()
534 		ret = gk20a_instobj_ctor_dma(imem, size >> PAGE_SHIFT,  in gk20a_instobj_new()
540 	node->imem = imem;  in gk20a_instobj_new()
551 	struct gk20a_instmem *imem = gk20a_instmem(base);  in gk20a_instmem_dtor()  local
554 	if (!list_empty(&imem->vaddr_lru))  in gk20a_instmem_dtor()
557 	if (imem->vaddr_use != 0)  in gk20a_instmem_dtor()
559 			  "0x%x bytes still mapped\n", imem->vaddr_use);  in gk20a_instmem_dtor()
561 	return imem;  in gk20a_instmem_dtor()
578 	struct gk20a_instmem *imem;  in gk20a_instmem_new()  local
580 	if (!(imem = kzalloc(sizeof(*imem), GFP_KERNEL)))  in gk20a_instmem_new()
582 	nvkm_instmem_ctor(&gk20a_instmem, device, type, inst, &imem->base);  in gk20a_instmem_new()
583 	mutex_init(&imem->lock);  in gk20a_instmem_new()
584 	*pimem = &imem->base;  in gk20a_instmem_new()
587 	imem->vaddr_use = 0;  in gk20a_instmem_new()
588 	imem->vaddr_max = 0x100000;  in gk20a_instmem_new()
589 	INIT_LIST_HEAD(&imem->vaddr_lru);  in gk20a_instmem_new()
592 		imem->mm_mutex = &tdev->iommu.mutex;  in gk20a_instmem_new()
593 		imem->mm = &tdev->iommu.mm;  in gk20a_instmem_new()
594 		imem->domain = tdev->iommu.domain;  in gk20a_instmem_new()
595 		imem->iommu_pgshift = tdev->iommu.pgshift;  in gk20a_instmem_new()
596 		imem->iommu_bit = tdev->func->iommu_bit;  in gk20a_instmem_new()
598 		nvkm_info(&imem->base.subdev, "using IOMMU\n");  in gk20a_instmem_new()
600 		imem->attrs = DMA_ATTR_WEAK_ORDERING |  in gk20a_instmem_new()
603 		nvkm_info(&imem->base.subdev, "using DMA API\n");  in gk20a_instmem_new()