52  * struct panthor_mmu - MMU related data
55 	/** @irq: The MMU irq. */
147 	/** @rsvd_page_tables: Pages reserved for the MMU page table update. */
156 		 * @rsvd_page_tables.pages: Array of pages to be used for an MMU page table update.
212  * It embeds the MMU page table for this address space, a tree containing
230 	 * and only implement hooks to update the MMU page table.
401  * @pt_cache: Cache used to allocate MMU page tables.
498 	/* Wait for the MMU status to indicate there is no active command, in  in wait_ready()
517 	/* write AS_COMMAND when MMU is ready to accept another command */  in write_cmd()
566 	lockdep_assert_held(&ptdev->mmu->as.slots_lock);  in mmu_hw_do_operation_locked()
616 	mutex_lock(&ptdev->mmu->as.slots_lock);  in mmu_hw_do_operation()
618 	mutex_unlock(&ptdev->mmu->as.slots_lock);  in mmu_hw_do_operation()
691 	lockdep_assert_held(&ptdev->mmu->as.slots_lock);  in panthor_vm_release_as_locked()
696 	ptdev->mmu->as.slots[vm->as.id].vm = NULL;  in panthor_vm_release_as_locked()
697 	clear_bit(vm->as.id, &ptdev->mmu->as.alloc_mask);  in panthor_vm_release_as_locked()
725 	mutex_lock(&ptdev->mmu->as.slots_lock);  in panthor_vm_active()
732 		/* Unhandled pagefault on this AS, the MMU was disabled. We need to  in panthor_vm_active()
733 		 * re-enable the MMU after clearing+unmasking the AS interrupts.  in panthor_vm_active()
735 		if (ptdev->mmu->as.faulty_mask & panthor_mmu_as_fault_mask(ptdev, as))  in panthor_vm_active()
743 		drm_WARN_ON(&ptdev->base, ptdev->mmu->as.alloc_mask & BIT(0));  in panthor_vm_active()
746 		as = ffz(ptdev->mmu->as.alloc_mask | BIT(0));  in panthor_vm_active()
752 		lru_vm = list_first_entry_or_null(&ptdev->mmu->as.lru_list,  in panthor_vm_active()
767 	set_bit(as, &ptdev->mmu->as.alloc_mask);  in panthor_vm_active()
768 	ptdev->mmu->as.slots[as].vm = vm;  in panthor_vm_active()
785 	if (ptdev->mmu->as.faulty_mask & panthor_mmu_as_fault_mask(ptdev, as)) {  in panthor_vm_active()
787 		ptdev->mmu->as.faulty_mask &= ~panthor_mmu_as_fault_mask(ptdev, as);  in panthor_vm_active()
788 		ptdev->mmu->irq.mask |= panthor_mmu_as_fault_mask(ptdev, as);  in panthor_vm_active()
789 		gpu_write(ptdev, MMU_INT_MASK, ~ptdev->mmu->as.faulty_mask);  in panthor_vm_active()
801 	mutex_unlock(&ptdev->mmu->as.slots_lock);  in panthor_vm_active()
818  * out of free slot, limiting the number of MMU operations (TLB flush
825 	if (!refcount_dec_and_mutex_lock(&vm->as.active_cnt, &ptdev->mmu->as.slots_lock))  in panthor_vm_idle()
829 		list_add_tail(&vm->as.lru_node, &ptdev->mmu->as.lru_list);  in panthor_vm_idle()
832 	mutex_unlock(&ptdev->mmu->as.slots_lock);  in panthor_vm_idle()
1667 		mutex_lock(&ptdev->mmu->as.slots_lock);  in panthor_mmu_irq_handler()
1669 		ptdev->mmu->as.faulty_mask |= mask;  in panthor_mmu_irq_handler()
1671 			panthor_mmu_fault_mask(ptdev, ~ptdev->mmu->as.faulty_mask);  in panthor_mmu_irq_handler()
1695 		/* Ignore MMU interrupts on this AS until it's been  in panthor_mmu_irq_handler()
1698 		ptdev->mmu->irq.mask = new_int_mask;  in panthor_mmu_irq_handler()
1700 		if (ptdev->mmu->as.slots[as].vm)  in panthor_mmu_irq_handler()
1701 			ptdev->mmu->as.slots[as].vm->unhandled_fault = true;  in panthor_mmu_irq_handler()
1703 		/* Disable the MMU to kill jobs on this AS. */  in panthor_mmu_irq_handler()
1705 		mutex_unlock(&ptdev->mmu->as.slots_lock);  in panthor_mmu_irq_handler()
1714 PANTHOR_IRQ_HANDLER(mmu, MMU, panthor_mmu_irq_handler);
1717  * panthor_mmu_suspend() - Suspend the MMU logic
1724  * We also suspend the MMU IRQ.
1728 	mutex_lock(&ptdev->mmu->as.slots_lock);  in panthor_mmu_suspend()
1729 	for (u32 i = 0; i < ARRAY_SIZE(ptdev->mmu->as.slots); i++) {  in panthor_mmu_suspend()
1730 		struct panthor_vm *vm = ptdev->mmu->as.slots[i].vm;  in panthor_mmu_suspend()
1737 	mutex_unlock(&ptdev->mmu->as.slots_lock);  in panthor_mmu_suspend()
1739 	panthor_mmu_irq_suspend(&ptdev->mmu->irq);  in panthor_mmu_suspend()
1743  * panthor_mmu_resume() - Resume the MMU logic
1753 	mutex_lock(&ptdev->mmu->as.slots_lock);  in panthor_mmu_resume()
1754 	ptdev->mmu->as.alloc_mask = 0;  in panthor_mmu_resume()
1755 	ptdev->mmu->as.faulty_mask = 0;  in panthor_mmu_resume()
1756 	mutex_unlock(&ptdev->mmu->as.slots_lock);  in panthor_mmu_resume()
1758 	panthor_mmu_irq_resume(&ptdev->mmu->irq, panthor_mmu_fault_mask(ptdev, ~0));  in panthor_mmu_resume()
1775 	panthor_mmu_irq_suspend(&ptdev->mmu->irq);  in panthor_mmu_pre_reset()
1777 	mutex_lock(&ptdev->mmu->vm.lock);  in panthor_mmu_pre_reset()
1778 	ptdev->mmu->vm.reset_in_progress = true;  in panthor_mmu_pre_reset()
1779 	list_for_each_entry(vm, &ptdev->mmu->vm.list, node)  in panthor_mmu_pre_reset()
1781 	mutex_unlock(&ptdev->mmu->vm.lock);  in panthor_mmu_pre_reset()
1788  * Put the MMU logic back in action after a reset. That implies resuming the
1795 	mutex_lock(&ptdev->mmu->as.slots_lock);  in panthor_mmu_post_reset()
1800 	ptdev->mmu->as.alloc_mask = 0;  in panthor_mmu_post_reset()
1801 	ptdev->mmu->as.faulty_mask = 0;  in panthor_mmu_post_reset()
1803 	for (u32 i = 0; i < ARRAY_SIZE(ptdev->mmu->as.slots); i++) {  in panthor_mmu_post_reset()
1804 		struct panthor_vm *vm = ptdev->mmu->as.slots[i].vm;  in panthor_mmu_post_reset()
1810 	mutex_unlock(&ptdev->mmu->as.slots_lock);  in panthor_mmu_post_reset()
1812 	panthor_mmu_irq_resume(&ptdev->mmu->irq, panthor_mmu_fault_mask(ptdev, ~0));  in panthor_mmu_post_reset()
1815 	mutex_lock(&ptdev->mmu->vm.lock);  in panthor_mmu_post_reset()
1816 	list_for_each_entry(vm, &ptdev->mmu->vm.list, node) {  in panthor_mmu_post_reset()
1819 	ptdev->mmu->vm.reset_in_progress = false;  in panthor_mmu_post_reset()
1820 	mutex_unlock(&ptdev->mmu->vm.lock);  in panthor_mmu_post_reset()
1834 	mutex_lock(&ptdev->mmu->vm.lock);  in panthor_vm_free()
1841 	if (ptdev->mmu->vm.reset_in_progress)  in panthor_vm_free()
1843 	mutex_unlock(&ptdev->mmu->vm.lock);  in panthor_vm_free()
1848 	mutex_lock(&ptdev->mmu->as.slots_lock);  in panthor_vm_free()
1857 		ptdev->mmu->as.slots[vm->as.id].vm = NULL;  in panthor_vm_free()
1858 		clear_bit(vm->as.id, &ptdev->mmu->as.alloc_mask);  in panthor_vm_free()
1861 	mutex_unlock(&ptdev->mmu->as.slots_lock);  in panthor_vm_free()
2287 		.submit_wq = ptdev->mmu->vm.wq,  in panthor_vm_create()
2364 	mutex_lock(&ptdev->mmu->vm.lock);  in panthor_vm_create()
2365 	list_add_tail(&vm->node, &ptdev->mmu->vm.list);  in panthor_vm_create()
2368 	if (ptdev->mmu->vm.reset_in_progress)  in panthor_vm_create()
2370 	mutex_unlock(&ptdev->mmu->vm.lock);  in panthor_vm_create()
2681  * panthor_mmu_unplug() - Unplug the MMU logic
2684  * No access to the MMU regs should be done after this function is called.
2690 		panthor_mmu_irq_suspend(&ptdev->mmu->irq);  in panthor_mmu_unplug()
2692 	mutex_lock(&ptdev->mmu->as.slots_lock);  in panthor_mmu_unplug()
2693 	for (u32 i = 0; i < ARRAY_SIZE(ptdev->mmu->as.slots); i++) {  in panthor_mmu_unplug()
2694 		struct panthor_vm *vm = ptdev->mmu->as.slots[i].vm;  in panthor_mmu_unplug()
2701 	mutex_unlock(&ptdev->mmu->as.slots_lock);  in panthor_mmu_unplug()
2710  * panthor_mmu_init() - Initialize the MMU logic.
2718 	struct panthor_mmu *mmu;  in panthor_mmu_init()  local
2721 	mmu = drmm_kzalloc(&ptdev->base, sizeof(*mmu), GFP_KERNEL);  in panthor_mmu_init()
2722 	if (!mmu)  in panthor_mmu_init()
2725 	INIT_LIST_HEAD(&mmu->as.lru_list);  in panthor_mmu_init()
2727 	ret = drmm_mutex_init(&ptdev->base, &mmu->as.slots_lock);  in panthor_mmu_init()
2731 	INIT_LIST_HEAD(&mmu->vm.list);  in panthor_mmu_init()
2732 	ret = drmm_mutex_init(&ptdev->base, &mmu->vm.lock);  in panthor_mmu_init()
2736 	ptdev->mmu = mmu;  in panthor_mmu_init()
2738 	irq = platform_get_irq_byname(to_platform_device(ptdev->base.dev), "mmu");  in panthor_mmu_init()
2742 	ret = panthor_request_mmu_irq(ptdev, &mmu->irq, irq,  in panthor_mmu_init()
2747 	mmu->vm.wq = alloc_workqueue("panthor-vm-bind", WQ_UNBOUND, 0);  in panthor_mmu_init()
2748 	if (!mmu->vm.wq)  in panthor_mmu_init()
2760 	return drmm_add_action_or_reset(&ptdev->base, panthor_mmu_release_wq, mmu->vm.wq);  in panthor_mmu_init()
2784 	mutex_lock(&ptdev->mmu->vm.lock);  in show_each_vm()
2785 	list_for_each_entry(vm, &ptdev->mmu->vm.list, node) {  in show_each_vm()
2792 	mutex_unlock(&ptdev->mmu->vm.lock);  in show_each_vm()
2802  * panthor_mmu_debugfs_init() - Initialize MMU debugfs entries
2820 	pt_cache = kmem_cache_create("panthor-mmu-pt", SZ_4K, SZ_4K, 0, NULL);  in panthor_mmu_pt_cache_init()