Lines Matching refs:vm
45 /** @vm: VM bound to this slot. NULL is no VM is bound. */
46 struct panthor_vm *vm;
89 /** @vm: VMs management fields */
91 /** @vm.lock: Lock protecting access to list. */
94 /** @vm.list: List containing all VMs. */
97 /** @vm.reset_in_progress: True if a reset is in progress. */
100 /** @vm.wq: Workqueue used for the VM_BIND queues. */
102 } vm;
350 /** @node: Used to insert the VM in the panthor_mmu::vm::list. */
405 /** @vm: VM targeted by the VM operation. */
406 struct panthor_vm *vm;
438 struct panthor_vm *vm = cookie;
442 if (unlikely(!vm->root_page_table)) {
445 drm_WARN_ON(&vm->ptdev->base, vm->op_ctx);
446 p = alloc_pages_node(dev_to_node(vm->ptdev->base.dev),
449 vm->root_page_table = page;
456 if (drm_WARN_ON(&vm->ptdev->base, size != SZ_4K))
462 if (drm_WARN_ON(&vm->ptdev->base, !vm->op_ctx) ||
463 drm_WARN_ON(&vm->ptdev->base,
464 vm->op_ctx->rsvd_page_tables.ptr >= vm->op_ctx->rsvd_page_tables.count))
467 page = vm->op_ctx->rsvd_page_tables.pages[vm->op_ctx->rsvd_page_tables.ptr++];
490 struct panthor_vm *vm = cookie;
492 if (unlikely(vm->root_page_table == data)) {
494 vm->root_page_table = NULL;
498 if (drm_WARN_ON(&vm->ptdev->base, size != SZ_4K))
596 static int mmu_hw_do_operation(struct panthor_vm *vm,
599 struct panthor_device *ptdev = vm->ptdev;
603 ret = mmu_hw_do_operation_locked(ptdev, vm->as.id, iova, size, op);
663 * @vm: VM to check.
667 bool panthor_vm_has_unhandled_faults(struct panthor_vm *vm)
669 return vm->unhandled_fault;
674 * @vm: VM to check.
678 bool panthor_vm_is_unusable(struct panthor_vm *vm)
680 return vm->unusable;
683 static void panthor_vm_release_as_locked(struct panthor_vm *vm)
685 struct panthor_device *ptdev = vm->ptdev;
689 if (drm_WARN_ON(&ptdev->base, vm->as.id < 0))
692 ptdev->mmu->as.slots[vm->as.id].vm = NULL;
693 clear_bit(vm->as.id, &ptdev->mmu->as.alloc_mask);
694 refcount_set(&vm->as.active_cnt, 0);
695 list_del_init(&vm->as.lru_node);
696 vm->as.id = -1;
701 * @vm: VM to flag as active.
707 int panthor_vm_active(struct panthor_vm *vm)
709 struct panthor_device *ptdev = vm->ptdev;
711 struct io_pgtable_cfg *cfg = &io_pgtable_ops_to_pgtable(vm->pgtbl_ops)->cfg;
718 if (refcount_inc_not_zero(&vm->as.active_cnt))
723 if (refcount_inc_not_zero(&vm->as.active_cnt))
726 as = vm->as.id;
738 if (vm->for_mcu) {
762 vm->as.id = as;
764 ptdev->mmu->as.slots[as].vm = vm;
776 vm->unhandled_fault = false;
787 ret = panthor_mmu_as_enable(vm->ptdev, vm->as.id, transtab, transcfg, vm->memattr);
791 refcount_set(&vm->as.active_cnt, 1);
792 list_del_init(&vm->as.lru_node);
805 * @vm: VM to flag as idle.
816 void panthor_vm_idle(struct panthor_vm *vm)
818 struct panthor_device *ptdev = vm->ptdev;
820 if (!refcount_dec_and_mutex_lock(&vm->as.active_cnt, &ptdev->mmu->as.slots_lock))
823 if (!drm_WARN_ON(&ptdev->base, vm->as.id == -1 || !list_empty(&vm->as.lru_node)))
824 list_add_tail(&vm->as.lru_node, &ptdev->mmu->as.lru_list);
826 refcount_set(&vm->as.active_cnt, 0);
830 u32 panthor_vm_page_size(struct panthor_vm *vm)
832 const struct io_pgtable *pgt = io_pgtable_ops_to_pgtable(vm->pgtbl_ops);
838 static void panthor_vm_stop(struct panthor_vm *vm)
840 drm_sched_stop(&vm->sched, NULL);
843 static void panthor_vm_start(struct panthor_vm *vm)
845 drm_sched_start(&vm->sched, 0);
850 * @vm: VM to get the AS slot of.
854 int panthor_vm_as(struct panthor_vm *vm)
856 return vm->as.id;
880 static int panthor_vm_flush_range(struct panthor_vm *vm, u64 iova, u64 size)
882 struct panthor_device *ptdev = vm->ptdev;
885 if (vm->as.id < 0)
892 ret = mmu_hw_do_operation(vm, iova, size, AS_COMMAND_FLUSH_PT);
900 * @vm: VM whose cache to flush
904 int panthor_vm_flush_all(struct panthor_vm *vm)
906 return panthor_vm_flush_range(vm, vm->base.mm_start, vm->base.mm_range);
909 static int panthor_vm_unmap_pages(struct panthor_vm *vm, u64 iova, u64 size)
911 struct panthor_device *ptdev = vm->ptdev;
912 struct io_pgtable_ops *ops = vm->pgtbl_ops;
915 drm_dbg(&ptdev->base, "unmap: as=%d, iova=%llx, len=%llx", vm->as.id, iova, size);
928 panthor_vm_flush_range(vm, iova, offset + unmapped_sz);
934 return panthor_vm_flush_range(vm, iova, size);
938 panthor_vm_map_pages(struct panthor_vm *vm, u64 iova, int prot,
941 struct panthor_device *ptdev = vm->ptdev;
944 struct io_pgtable_ops *ops = vm->pgtbl_ops;
966 vm->as.id, iova, &paddr, len);
986 panthor_vm_unmap_pages(vm, start_iova,
998 return panthor_vm_flush_range(vm, start_iova, iova - start_iova);
1021 * @vm: VM to allocate a region on.
1036 panthor_vm_alloc_va(struct panthor_vm *vm, u64 va, u64 size,
1039 ssize_t vm_pgsz = panthor_vm_page_size(vm);
1048 mutex_lock(&vm->mm_lock);
1052 ret = drm_mm_reserve_node(&vm->mm, va_node);
1054 ret = drm_mm_insert_node_in_range(&vm->mm, va_node, size,
1056 0, vm->kernel_auto_va.start,
1057 vm->kernel_auto_va.end,
1060 mutex_unlock(&vm->mm_lock);
1067 * @vm: VM to free the region on.
1070 void panthor_vm_free_va(struct panthor_vm *vm, struct drm_mm_node *va_node)
1072 mutex_lock(&vm->mm_lock);
1074 mutex_unlock(&vm->mm_lock);
1080 struct drm_gpuvm *vm = vm_bo->vm;
1088 drm_gpuvm_get(vm);
1097 dma_resv_lock(drm_gpuvm_resv(vm), NULL);
1101 dma_resv_unlock(drm_gpuvm_resv(vm));
1109 drm_gpuvm_put(vm);
1114 struct panthor_vm *vm)
1200 struct panthor_vm *vm,
1224 bo->exclusive_vm_root_gem != panthor_vm_root_gem(vm))
1257 preallocated_vm_bo = drm_gpuvm_bo_create(&vm->base, &bo->base.base);
1272 dma_resv_lock(panthor_vm_resv(vm), NULL);
1276 dma_resv_unlock(panthor_vm_resv(vm));
1316 dma_resv_lock(panthor_vm_resv(vm), NULL);
1318 dma_resv_unlock(panthor_vm_resv(vm));
1323 panthor_vm_cleanup_op_ctx(op_ctx, vm);
1328 struct panthor_vm *vm,
1375 panthor_vm_cleanup_op_ctx(op_ctx, vm);
1380 struct panthor_vm *vm)
1389 * @vm: VM to look into.
1402 panthor_vm_get_bo_for_va(struct panthor_vm *vm, u64 va, u64 *bo_offset)
1409 mutex_lock(&vm->op_lock);
1410 gpuva = drm_gpuva_find_first(&vm->base, va, 1);
1417 mutex_unlock(&vm->op_lock);
1507 struct panthor_vm *vm;
1515 vm = panthor_vm_create(ptdev, false, kernel_va_start, kernel_va_range,
1517 if (IS_ERR(vm))
1518 return PTR_ERR(vm);
1520 ret = xa_alloc(&pool->xa, &id, vm,
1524 panthor_vm_put(vm);
1532 static void panthor_vm_destroy(struct panthor_vm *vm)
1534 if (!vm)
1537 vm->destroyed = true;
1539 mutex_lock(&vm->heaps.lock);
1540 panthor_heap_pool_destroy(vm->heaps.pool);
1541 vm->heaps.pool = NULL;
1542 mutex_unlock(&vm->heaps.lock);
1544 drm_WARN_ON(&vm->ptdev->base,
1545 panthor_vm_unmap_range(vm, vm->base.mm_start, vm->base.mm_range));
1546 panthor_vm_put(vm);
1567 struct panthor_vm *vm;
1569 vm = xa_erase(&pool->xa, handle);
1571 panthor_vm_destroy(vm);
1573 return vm ? 0 : -EINVAL;
1586 struct panthor_vm *vm;
1589 vm = panthor_vm_get(xa_load(&pool->xa, handle));
1592 return vm;
1606 struct panthor_vm *vm;
1612 xa_for_each(&pfile->vms->xa, i, vm)
1613 panthor_vm_destroy(vm);
1718 if (ptdev->mmu->as.slots[as].vm)
1719 ptdev->mmu->as.slots[as].vm->unhandled_fault = true;
1748 struct panthor_vm *vm = ptdev->mmu->as.slots[i].vm;
1750 if (vm) {
1752 panthor_vm_release_as_locked(vm);
1791 struct panthor_vm *vm;
1795 mutex_lock(&ptdev->mmu->vm.lock);
1796 ptdev->mmu->vm.reset_in_progress = true;
1797 list_for_each_entry(vm, &ptdev->mmu->vm.list, node)
1798 panthor_vm_stop(vm);
1799 mutex_unlock(&ptdev->mmu->vm.lock);
1811 struct panthor_vm *vm;
1822 struct panthor_vm *vm = ptdev->mmu->as.slots[i].vm;
1824 if (vm)
1825 panthor_vm_release_as_locked(vm);
1833 mutex_lock(&ptdev->mmu->vm.lock);
1834 list_for_each_entry(vm, &ptdev->mmu->vm.list, node) {
1835 panthor_vm_start(vm);
1837 ptdev->mmu->vm.reset_in_progress = false;
1838 mutex_unlock(&ptdev->mmu->vm.lock);
1843 struct panthor_vm *vm = container_of(gpuvm, struct panthor_vm, base);
1844 struct panthor_device *ptdev = vm->ptdev;
1846 mutex_lock(&vm->heaps.lock);
1847 if (drm_WARN_ON(&ptdev->base, vm->heaps.pool))
1848 panthor_heap_pool_destroy(vm->heaps.pool);
1849 mutex_unlock(&vm->heaps.lock);
1850 mutex_destroy(&vm->heaps.lock);
1852 mutex_lock(&ptdev->mmu->vm.lock);
1853 list_del(&vm->node);
1859 if (ptdev->mmu->vm.reset_in_progress)
1860 panthor_vm_start(vm);
1861 mutex_unlock(&ptdev->mmu->vm.lock);
1863 drm_sched_entity_destroy(&vm->entity);
1864 drm_sched_fini(&vm->sched);
1867 if (vm->as.id >= 0) {
1871 panthor_mmu_as_disable(ptdev, vm->as.id);
1875 ptdev->mmu->as.slots[vm->as.id].vm = NULL;
1876 clear_bit(vm->as.id, &ptdev->mmu->as.alloc_mask);
1877 list_del(&vm->as.lru_node);
1881 free_io_pgtable_ops(vm->pgtbl_ops);
1883 drm_mm_takedown(&vm->mm);
1884 kfree(vm);
1889 * @vm: VM to release the reference on. Can be NULL.
1891 void panthor_vm_put(struct panthor_vm *vm)
1893 drm_gpuvm_put(vm ? &vm->base : NULL);
1898 * @vm: VM to get the reference on. Can be NULL.
1900 * Return: @vm value.
1902 struct panthor_vm *panthor_vm_get(struct panthor_vm *vm)
1904 if (vm)
1905 drm_gpuvm_get(&vm->base);
1907 return vm;
1912 * @vm: VM to query the heap pool on.
1924 struct panthor_heap_pool *panthor_vm_get_heap_pool(struct panthor_vm *vm, bool create)
1928 mutex_lock(&vm->heaps.lock);
1929 if (!vm->heaps.pool && create) {
1930 if (vm->destroyed)
1933 pool = panthor_heap_pool_create(vm->ptdev, vm);
1936 vm->heaps.pool = panthor_heap_pool_get(pool);
1938 pool = panthor_heap_pool_get(vm->heaps.pool);
1942 mutex_unlock(&vm->heaps.lock);
1958 struct panthor_vm *vm;
1965 xa_for_each(&pfile->vms->xa, i, vm) {
1966 size_t size = panthor_heap_pool_size(vm->heaps.pool);
1968 if (vm->as.id >= 0)
2016 static void panthor_vma_link(struct panthor_vm *vm,
2024 drm_WARN_ON(&vm->ptdev->base, drm_gpuvm_bo_put(vm_bo));
2028 static void panthor_vma_unlink(struct panthor_vm *vm,
2043 list_add_tail(&vma->node, &vm->op_ctx->returned_vmas);
2059 struct panthor_vm *vm = priv;
2060 struct panthor_vm_op_ctx *op_ctx = vm->op_ctx;
2069 ret = panthor_vm_map_pages(vm, op->map.va.addr, flags_to_prot(vma->flags),
2078 drm_gpuva_map(&vm->base, &vma->base, &op->map);
2079 panthor_vma_link(vm, vma, op_ctx->map.vm_bo);
2088 struct panthor_vm *vm = priv;
2089 struct panthor_vm_op_ctx *op_ctx = vm->op_ctx;
2095 ret = panthor_vm_unmap_pages(vm, unmap_start, unmap_range);
2119 panthor_vma_link(vm, prev_vma,
2124 panthor_vma_link(vm, next_vma,
2128 panthor_vma_unlink(vm, unmap_vma);
2136 struct panthor_vm *vm = priv;
2139 ret = panthor_vm_unmap_pages(vm, unmap_vma->base.va.addr,
2141 if (drm_WARN_ON(&vm->ptdev->base, ret))
2145 panthor_vma_unlink(vm, unmap_vma);
2158 * @vm: VM to get the dma_resv of.
2162 struct dma_resv *panthor_vm_resv(struct panthor_vm *vm)
2164 return drm_gpuvm_resv(&vm->base);
2167 struct drm_gem_object *panthor_vm_root_gem(struct panthor_vm *vm)
2169 if (!vm)
2172 return vm->base.r_obj;
2176 panthor_vm_exec_op(struct panthor_vm *vm, struct panthor_vm_op_ctx *op,
2185 mutex_lock(&vm->op_lock);
2186 vm->op_ctx = op;
2189 if (vm->unusable) {
2194 ret = drm_gpuvm_sm_map(&vm->base, vm, op->va.addr, op->va.range,
2199 ret = drm_gpuvm_sm_unmap(&vm->base, vm, op->va.addr, op->va.range);
2208 vm->unusable = true;
2210 vm->op_ctx = NULL;
2211 mutex_unlock(&vm->op_lock);
2229 ret = panthor_vm_exec_op(job->vm, &job->ctx, true);
2242 panthor_vm_cleanup_op_ctx(&job->ctx, job->vm);
2243 panthor_vm_put(job->vm);
2310 .submit_wq = ptdev->mmu->vm.wq,
2315 .name = "panthor-vm-bind",
2320 struct panthor_vm *vm;
2323 vm = kzalloc(sizeof(*vm), GFP_KERNEL);
2324 if (!vm)
2334 mutex_init(&vm->heaps.lock);
2335 vm->for_mcu = for_mcu;
2336 vm->ptdev = ptdev;
2337 mutex_init(&vm->op_lock);
2348 mutex_init(&vm->mm_lock);
2349 drm_mm_init(&vm->mm, kernel_va_start, kernel_va_size);
2350 vm->kernel_auto_va.start = auto_kernel_va_start;
2351 vm->kernel_auto_va.end = vm->kernel_auto_va.start + auto_kernel_va_size - 1;
2353 INIT_LIST_HEAD(&vm->node);
2354 INIT_LIST_HEAD(&vm->as.lru_node);
2355 vm->as.id = -1;
2356 refcount_set(&vm->as.active_cnt, 0);
2369 vm->pgtbl_ops = alloc_io_pgtable_ops(ARM_64_LPAE_S1, &pgtbl_cfg, vm);
2370 if (!vm->pgtbl_ops) {
2375 ret = drm_sched_init(&vm->sched, &sched_args);
2379 sched = &vm->sched;
2380 ret = drm_sched_entity_init(&vm->entity, 0, &sched, 1, NULL);
2384 mair = io_pgtable_ops_to_pgtable(vm->pgtbl_ops)->cfg.arm_lpae_s1_cfg.mair;
2385 vm->memattr = mair_to_memattr(mair, ptdev->coherent);
2387 mutex_lock(&ptdev->mmu->vm.lock);
2388 list_add_tail(&vm->node, &ptdev->mmu->vm.list);
2391 if (ptdev->mmu->vm.reset_in_progress)
2392 panthor_vm_stop(vm);
2393 mutex_unlock(&ptdev->mmu->vm.lock);
2398 drm_gpuvm_init(&vm->base, for_mcu ? "panthor-MCU-VM" : "panthor-GPU-VM",
2402 return vm;
2405 drm_sched_fini(&vm->sched);
2408 free_io_pgtable_ops(vm->pgtbl_ops);
2411 drm_mm_takedown(&vm->mm);
2415 kfree(vm);
2421 struct panthor_vm *vm,
2425 ssize_t vm_pgsz = panthor_vm_page_size(vm);
2436 ret = panthor_vm_prepare_map_op_ctx(op_ctx, vm,
2452 return panthor_vm_prepare_unmap_op_ctx(op_ctx, vm, op->va, op->size);
2467 panthor_vm_prepare_sync_only_op_ctx(op_ctx, vm);
2486 * @vm: VM targeted by the VM_BIND job.
2493 struct panthor_vm *vm,
2499 if (!vm)
2502 if (vm->destroyed || vm->unusable)
2509 ret = panthor_vm_bind_prepare_op_ctx(file, vm, op, &job->ctx);
2517 job->vm = panthor_vm_get(vm);
2519 ret = drm_sched_job_init(&job->base, &vm->entity, 1, vm);
2548 ret = drm_gpuvm_prepare_vm(&job->vm->base, exec, 1);
2573 drm_gpuvm_resv_add_fence(&job->vm->base, exec,
2579 void panthor_vm_update_resvs(struct panthor_vm *vm, struct drm_exec *exec,
2584 drm_gpuvm_resv_add_fence(&vm->base, exec, fence, private_usage, extobj_usage);
2590 * @vm: VM targeted by the VM operation.
2596 struct panthor_vm *vm,
2609 ret = panthor_vm_bind_prepare_op_ctx(file, vm, op, &op_ctx);
2613 ret = panthor_vm_exec_op(vm, &op_ctx, false);
2614 panthor_vm_cleanup_op_ctx(&op_ctx, vm);
2621 * @vm: VM to map the GEM to.
2634 int panthor_vm_map_bo_range(struct panthor_vm *vm, struct panthor_gem_object *bo,
2640 ret = panthor_vm_prepare_map_op_ctx(&op_ctx, vm, bo, offset, size, va, flags);
2644 ret = panthor_vm_exec_op(vm, &op_ctx, false);
2645 panthor_vm_cleanup_op_ctx(&op_ctx, vm);
2652 * @vm: VM to unmap the region from.
2661 int panthor_vm_unmap_range(struct panthor_vm *vm, u64 va, u64 size)
2666 ret = panthor_vm_prepare_unmap_op_ctx(&op_ctx, vm, va, size);
2670 ret = panthor_vm_exec_op(vm, &op_ctx, false);
2671 panthor_vm_cleanup_op_ctx(&op_ctx, vm);
2679 * @vm: VM targeted by the GPU job.
2689 int panthor_vm_prepare_mapped_bos_resvs(struct drm_exec *exec, struct panthor_vm *vm,
2695 ret = drm_gpuvm_prepare_vm(&vm->base, exec, slot_count);
2699 return drm_gpuvm_prepare_objects(&vm->base, exec, slot_count);
2716 struct panthor_vm *vm = ptdev->mmu->as.slots[i].vm;
2718 if (vm) {
2720 panthor_vm_release_as_locked(vm);
2753 INIT_LIST_HEAD(&mmu->vm.list);
2754 ret = drmm_mutex_init(&ptdev->base, &mmu->vm.lock);
2769 mmu->vm.wq = alloc_workqueue("panthor-vm-bind", WQ_UNBOUND, 0);
2770 if (!mmu->vm.wq)
2782 return drmm_add_action_or_reset(&ptdev->base, panthor_mmu_release_wq, mmu->vm.wq);
2786 static int show_vm_gpuvas(struct panthor_vm *vm, struct seq_file *m)
2790 mutex_lock(&vm->op_lock);
2791 ret = drm_debugfs_gpuva_info(m, &vm->base);
2792 mutex_unlock(&vm->op_lock);
2803 struct panthor_vm *vm;
2806 mutex_lock(&ptdev->mmu->vm.lock);
2807 list_for_each_entry(vm, &ptdev->mmu->vm.list, node) {
2808 ret = show(vm, m);
2814 mutex_unlock(&ptdev->mmu->vm.lock);