Home
last modified time | relevance | path

Searched refs:vmf (Results 1 – 25 of 113) sorted by relevance

12345

/linux/mm/
H A Dmemory.c97 static vm_fault_t do_fault(struct vm_fault *vmf);
98 static vm_fault_t do_anonymous_page(struct vm_fault *vmf);
99 static bool vmf_pte_changed(struct vm_fault *vmf);
105 static __always_inline bool vmf_orig_pte_uffd_wp(struct vm_fault *vmf) in vmf_orig_pte_uffd_wp() argument
107 if (!userfaultfd_wp(vmf->vma)) in vmf_orig_pte_uffd_wp()
109 if (!(vmf->flags & FAULT_FLAG_ORIG_PTE_VALID)) in vmf_orig_pte_uffd_wp()
112 return pte_is_uffd_wp_marker(vmf->orig_pte); in vmf_orig_pte_uffd_wp()
2999 vm_fault_t vmf_insert_page_mkwrite(struct vm_fault *vmf, struct page *page, in vmf_insert_page_mkwrite() argument
3002 pgprot_t pgprot = vmf->vma->vm_page_prot; in vmf_insert_page_mkwrite()
3003 unsigned long addr = vmf->address; in vmf_insert_page_mkwrite()
[all …]
H A Dhuge_memory.c1424 static vm_fault_t __do_huge_pmd_anonymous_page(struct vm_fault *vmf) in __do_huge_pmd_anonymous_page() argument
1426 unsigned long haddr = vmf->address & HPAGE_PMD_MASK; in __do_huge_pmd_anonymous_page()
1427 struct vm_area_struct *vma = vmf->vma; in __do_huge_pmd_anonymous_page()
1432 folio = vma_alloc_anon_folio_pmd(vma, vmf->address); in __do_huge_pmd_anonymous_page()
1442 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); in __do_huge_pmd_anonymous_page()
1443 if (unlikely(!pmd_none(*vmf->pmd))) { in __do_huge_pmd_anonymous_page()
1452 spin_unlock(vmf->ptl); in __do_huge_pmd_anonymous_page()
1455 ret = handle_userfault(vmf, VM_UFFD_MISSING); in __do_huge_pmd_anonymous_page()
1459 pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, pgtable); in __do_huge_pmd_anonymous_page()
1460 map_anon_folio_pmd_pf(folio, vmf->pmd, vma, haddr); in __do_huge_pmd_anonymous_page()
[all …]
H A Dfilemap.c1759 vm_fault_t __folio_lock_or_retry(struct folio *folio, struct vm_fault *vmf) in __folio_lock_or_retry() argument
1761 unsigned int flags = vmf->flags; in __folio_lock_or_retry()
1771 release_fault_lock(vmf); in __folio_lock_or_retry()
1783 release_fault_lock(vmf); in __folio_lock_or_retry()
3288 static int lock_folio_maybe_drop_mmap(struct vm_fault *vmf, struct folio *folio, in lock_folio_maybe_drop_mmap() argument
3299 if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT) in lock_folio_maybe_drop_mmap()
3302 *fpin = maybe_unlock_mmap_for_io(vmf, *fpin); in lock_folio_maybe_drop_mmap()
3303 if (vmf->flags & FAULT_FLAG_KILLABLE) { in lock_folio_maybe_drop_mmap()
3313 release_fault_lock(vmf); in lock_folio_maybe_drop_mmap()
3329 static struct file *do_sync_mmap_readahead(struct vm_fault *vmf) in do_sync_mmap_readahead() argument
[all …]
H A Dhugetlb.c4802 static vm_fault_t hugetlb_vm_op_fault(struct vm_fault *vmf) in hugetlb_vm_op_fault() argument
5491 static vm_fault_t hugetlb_wp(struct vm_fault *vmf) in hugetlb_wp() argument
5493 struct vm_area_struct *vma = vmf->vma; in hugetlb_wp()
5495 const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE; in hugetlb_wp()
5496 pte_t pte = huge_ptep_get(mm, vmf->address, vmf->pte); in hugetlb_wp()
5517 set_huge_ptep_writable(vma, vmf->address, vmf->pte); in hugetlb_wp()
5543 set_huge_ptep_maybe_writable(vma, vmf->address, in hugetlb_wp()
5544 vmf->pte); in hugetlb_wp()
5572 spin_unlock(vmf->ptl); in hugetlb_wp()
5573 new_folio = alloc_hugetlb_folio(vma, vmf->address, cow_from_owner); in hugetlb_wp()
[all …]
H A Dswap_state.c411 unsigned int order, struct vm_fault *vmf, in __swap_cache_alloc() argument
420 struct vm_area_struct *vma = vmf ? vmf->vma : NULL; in __swap_cache_alloc()
439 if (mpol || !vmf) { in __swap_cache_alloc()
442 address = round_down(vmf->address, PAGE_SIZE << order); in __swap_cache_alloc()
443 folio = vma_alloc_folio(gfp, order, vmf->vma, address); in __swap_cache_alloc()
463 vmf ? vmf->vma->vm_mm : NULL, gfp)) { in __swap_cache_alloc()
515 unsigned long orders, struct vm_fault *vmf, in swap_cache_alloc_folio() argument
531 vmf, mpol, ilx); in swap_cache_alloc_folio()
684 struct vm_fault *vmf, struct mempolicy *mpol, pgoff_t ilx) in swapin_sync() argument
693 folio = swap_cache_alloc_folio(entry, gfp, orders, vmf, mpol, ilx); in swapin_sync()
[all …]
H A Dsecretmem.c110 static vm_fault_t secretmem_fault(struct vm_fault *vmf) in secretmem_fault() argument
112 struct address_space *mapping = vmf->vma->vm_file->f_mapping; in secretmem_fault()
113 struct inode *inode = file_inode(vmf->vma->vm_file); in secretmem_fault()
115 pgoff_t offset = vmf->pgoff; in secretmem_fault()
116 gfp_t gfp = vmf->gfp_mask; in secretmem_fault()
122 if (((loff_t)vmf->pgoff << PAGE_SHIFT) >= i_size_read(inode)) in secretmem_fault()
172 vmf->page = folio_file_page(folio, vmf->pgoff); in secretmem_fault()
H A Dswap.h316 unsigned long orders, struct vm_fault *vmf,
333 struct vm_fault *vmf);
335 struct vm_fault *vmf, struct mempolicy *mpol, pgoff_t ilx);
409 struct vm_fault *vmf) in swapin_readahead() argument
416 struct vm_fault *vmf, struct mempolicy *mpol, pgoff_t ilx) in swapin_sync() argument
H A Dinternal.h566 vm_fault_t __vmf_anon_prepare(struct vm_fault *vmf);
567 static inline vm_fault_t vmf_anon_prepare(struct vm_fault *vmf) in vmf_anon_prepare() argument
569 vm_fault_t ret = __vmf_anon_prepare(vmf); in vmf_anon_prepare()
572 vma_end_read(vmf->vma); in vmf_anon_prepare()
576 vm_fault_t do_swap_page(struct vm_fault *vmf);
1104 static inline struct file *maybe_unlock_mmap_for_io(struct vm_fault *vmf, in maybe_unlock_mmap_for_io() argument
1107 int flags = vmf->flags; in maybe_unlock_mmap_for_io()
1119 fpin = get_file(vmf->vma->vm_file); in maybe_unlock_mmap_for_io()
1120 release_fault_lock(vmf); in maybe_unlock_mmap_for_io()
1272 int numa_migrate_check(struct folio *folio, struct vm_fault *vmf,
/linux/fs/
H A Ddax.c1043 static int copy_cow_page_dax(struct vm_fault *vmf, const struct iomap_iter *iter) in copy_cow_page_dax() argument
1057 vto = kmap_atomic(vmf->cow_page); in copy_cow_page_dax()
1058 copy_user_page(vto, kaddr, vmf->address, vmf->cow_page); in copy_cow_page_dax()
1082 static void *dax_insert_entry(struct xa_state *xas, struct vm_fault *vmf, in dax_insert_entry() argument
1086 struct address_space *mapping = vmf->vma->vm_file->f_mapping; in dax_insert_entry()
1089 bool dirty = write && !dax_fault_is_synchronous(iter, vmf->vma); in dax_insert_entry()
1111 dax_associate_entry(new_entry, mapping, vmf->vma, in dax_insert_entry()
1112 vmf->address, shared); in dax_insert_entry()
1398 static vm_fault_t dax_load_hole(struct xa_state *xas, struct vm_fault *vmf, in dax_load_hole() argument
1402 unsigned long vaddr = vmf->address; in dax_load_hole()
[all …]
/linux/drivers/gpu/drm/ttm/
H A Dttm_bo_vm.c45 struct vm_fault *vmf) in ttm_bo_vm_fault_idle() argument
60 if (fault_flag_allow_retry_first(vmf->flags)) { in ttm_bo_vm_fault_idle()
61 if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT) in ttm_bo_vm_fault_idle()
65 mmap_read_unlock(vmf->vma->vm_mm); in ttm_bo_vm_fault_idle()
120 struct vm_fault *vmf) in ttm_bo_vm_reserve() argument
134 if (fault_flag_allow_retry_first(vmf->flags)) { in ttm_bo_vm_reserve()
135 if (!(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) { in ttm_bo_vm_reserve()
137 mmap_read_unlock(vmf->vma->vm_mm); in ttm_bo_vm_reserve()
184 vm_fault_t ttm_bo_vm_fault_reserved(struct vm_fault *vmf, in ttm_bo_vm_fault_reserved() argument
188 struct vm_area_struct *vma = vmf->vma; in ttm_bo_vm_fault_reserved()
[all …]
/linux/drivers/gpu/drm/xe/
H A Dxe_mmio_gem.c199 static vm_fault_t xe_mmio_gem_vm_fault_dummy_page(struct vm_fault *vmf) in xe_mmio_gem_vm_fault_dummy_page() argument
201 struct vm_area_struct *vma = vmf->vma; in xe_mmio_gem_vm_fault_dummy_page()
211 return vmf_insert_pfn_prot(vma, vmf->address, pfn, in xe_mmio_gem_vm_fault_dummy_page()
215 static vm_fault_t xe_mmio_gem_vm_fault_locked(struct vm_fault *vmf) in xe_mmio_gem_vm_fault_locked() argument
217 struct vm_area_struct *vma = vmf->vma; in xe_mmio_gem_vm_fault_locked()
236 return xe_mmio_gem_vm_fault_dummy_page(vmf); in xe_mmio_gem_vm_fault_locked()
252 static vm_fault_t xe_mmio_gem_vm_fault(struct vm_fault *vmf) in xe_mmio_gem_vm_fault() argument
254 struct vm_area_struct *vma = vmf->vma; in xe_mmio_gem_vm_fault()
259 ret = xe_mmio_gem_vm_fault_locked(vmf); in xe_mmio_gem_vm_fault()
/linux/drivers/gpu/drm/
H A Ddrm_gem_shmem_helper.c591 static void drm_gem_shmem_record_mkwrite(struct vm_fault *vmf) in drm_gem_shmem_record_mkwrite() argument
593 struct vm_area_struct *vma = vmf->vma; in drm_gem_shmem_record_mkwrite()
597 pgoff_t page_offset = vmf->pgoff - vma->vm_pgoff; /* page offset within VMA */ in drm_gem_shmem_record_mkwrite()
606 static vm_fault_t try_insert_pfn(struct vm_fault *vmf, unsigned int order, in try_insert_pfn() argument
610 return vmf_insert_pfn(vmf->vma, vmf->address, pfn); in try_insert_pfn()
614 struct vm_area_struct *vma = vmf->vma; in try_insert_pfn()
615 unsigned long start = ALIGN_DOWN(vmf->address, PMD_SIZE); in try_insert_pfn()
618 bool aligned = (vmf->address & ~PMD_MASK) == (paddr & ~PMD_MASK); in try_insert_pfn()
633 ret = vmf_insert_pfn_pmd(vmf, pfn, in try_insert_pfn()
634 vmf->flags & FAULT_FLAG_WRITE); in try_insert_pfn()
[all …]
/linux/fs/xfs/
H A Dxfs_file.c1946 struct vm_fault *vmf, in xfs_dax_fault_locked() argument
1957 ret = dax_iomap_fault(vmf, order, &pfn, NULL, in xfs_dax_fault_locked()
1958 (write_fault && !vmf->cow_page) ? in xfs_dax_fault_locked()
1962 ret = dax_finish_sync_fault(vmf, order, pfn); in xfs_dax_fault_locked()
1968 struct vm_fault *vmf, in xfs_dax_read_fault() argument
1971 struct xfs_inode *ip = XFS_I(file_inode(vmf->vma->vm_file)); in xfs_dax_read_fault()
1977 ret = xfs_dax_fault_locked(vmf, order, false); in xfs_dax_read_fault()
1995 struct vm_fault *vmf, in __xfs_write_fault() argument
1999 struct inode *inode = file_inode(vmf->vma->vm_file); in __xfs_write_fault()
2007 file_update_time(vmf->vma->vm_file); in __xfs_write_fault()
[all …]
/linux/drivers/xen/
H A Dprivcmd-buf.c118 static vm_fault_t privcmd_buf_vma_fault(struct vm_fault *vmf) in privcmd_buf_vma_fault() argument
121 vmf->vma, vmf->vma->vm_start, vmf->vma->vm_end, in privcmd_buf_vma_fault()
122 vmf->pgoff, (void *)vmf->address); in privcmd_buf_vma_fault()
/linux/drivers/hv/
H A Dmshv_vtl_main.c799 static vm_fault_t mshv_vtl_fault(struct vm_fault *vmf) in mshv_vtl_fault() argument
802 int cpu = vmf->pgoff & MSHV_PG_OFF_CPU_MASK; in mshv_vtl_fault()
803 int real_off = vmf->pgoff >> MSHV_REAL_OFF_SHIFT; in mshv_vtl_fault()
823 vmf->page = page; in mshv_vtl_fault()
1232 static bool can_fault(struct vm_fault *vmf, unsigned long size, unsigned long *pfn) in can_fault() argument
1235 unsigned long start = vmf->address & ~mask; in can_fault()
1239 is_valid = (vmf->address & mask) == ((vmf->pgoff << PAGE_SHIFT) & mask) && in can_fault()
1240 start >= vmf->vma->vm_start && in can_fault()
1241 end <= vmf->vma->vm_end; in can_fault()
1244 *pfn = vmf->pgoff & ~(mask >> PAGE_SHIFT); in can_fault()
[all …]
/linux/drivers/gpu/drm/panthor/
H A Dpanthor_gem.c797 static vm_fault_t insert_page(struct vm_fault *vmf, unsigned int order, struct page *page) in insert_page() argument
800 return vmf_insert_pfn(vmf->vma, vmf->address, page_to_pfn(page)); in insert_page()
805 struct vm_area_struct *vma = vmf->vma; in insert_page()
806 unsigned long start = ALIGN_DOWN(vmf->address, PMD_SIZE); in insert_page()
809 bool aligned = (vmf->address & ~PMD_MASK) == (paddr & ~PMD_MASK); in insert_page()
814 return vmf_insert_pfn_pmd(vmf, pfn, vmf->flags & FAULT_FLAG_WRITE); in insert_page()
822 static vm_fault_t nonblocking_page_setup(struct vm_fault *vmf, in nonblocking_page_setup() argument
826 struct vm_area_struct *vma = vmf->vma; in nonblocking_page_setup()
834 ret = insert_page(vmf, order, bo->backing.pages[page_offset]); in nonblocking_page_setup()
842 static vm_fault_t blocking_page_setup(struct vm_fault *vmf, unsigned int order, in blocking_page_setup() argument
[all …]
/linux/drivers/misc/ocxl/
H A Dsysfs.c109 static vm_fault_t global_mmio_fault(struct vm_fault *vmf) in global_mmio_fault() argument
111 struct vm_area_struct *vma = vmf->vma; in global_mmio_fault()
115 if (vmf->pgoff >= (afu->config.global_mmio_size >> PAGE_SHIFT)) in global_mmio_fault()
118 offset = vmf->pgoff; in global_mmio_fault()
120 return vmf_insert_pfn(vma, vmf->address, offset); in global_mmio_fault()
H A Dcontext.c139 static vm_fault_t ocxl_mmap_fault(struct vm_fault *vmf) in ocxl_mmap_fault() argument
141 struct vm_area_struct *vma = vmf->vma; in ocxl_mmap_fault()
146 offset = vmf->pgoff << PAGE_SHIFT; in ocxl_mmap_fault()
148 ctx->pasid, vmf->address, offset); in ocxl_mmap_fault()
151 ret = map_pp_mmio(vma, vmf->address, offset, ctx); in ocxl_mmap_fault()
153 ret = map_afu_irq(vma, vmf->address, offset, ctx); in ocxl_mmap_fault()
/linux/drivers/char/agp/
H A Dalpha-agp.c14 static vm_fault_t alpha_core_agp_vm_fault(struct vm_fault *vmf) in alpha_core_agp_vm_fault() argument
21 dma_addr = vmf->address - vmf->vma->vm_start + agp->aperture.bus_base; in alpha_core_agp_vm_fault()
32 vmf->page = page; in alpha_core_agp_vm_fault()
/linux/sound/usb/usx2y/
H A DusX2Yhwdep.c21 static vm_fault_t snd_us428ctls_vm_fault(struct vm_fault *vmf) in snd_us428ctls_vm_fault() argument
27 offset = vmf->pgoff << PAGE_SHIFT; in snd_us428ctls_vm_fault()
30 vaddr = (char *)((struct usx2ydev *)vmf->vma->vm_private_data)->us428ctls_sharedmem + offset; in snd_us428ctls_vm_fault()
33 vmf->page = page; in snd_us428ctls_vm_fault()
/linux/drivers/gpu/drm/nouveau/
H A Dnouveau_dmem.c183 static vm_fault_t nouveau_dmem_migrate_to_ram(struct vm_fault *vmf) in nouveau_dmem_migrate_to_ram() argument
185 struct nouveau_drm *drm = page_to_drm(vmf->page); in nouveau_dmem_migrate_to_ram()
193 .vma = vmf->vma, in nouveau_dmem_migrate_to_ram()
195 .fault_page = vmf->page, in nouveau_dmem_migrate_to_ram()
205 sfolio = page_folio(vmf->page); in nouveau_dmem_migrate_to_ram()
213 if (vmf->pte) { in nouveau_dmem_migrate_to_ram()
221 args.start = ALIGN_DOWN(vmf->address, (PAGE_SIZE << order)); in nouveau_dmem_migrate_to_ram()
222 args.vma = vmf->vma; in nouveau_dmem_migrate_to_ram()
243 order, vmf->vma, vmf->address), 0); in nouveau_dmem_migrate_to_ram()
245 dpage = alloc_page_vma(GFP_HIGHUSER | __GFP_ZERO, vmf->vma, in nouveau_dmem_migrate_to_ram()
[all …]
/linux/fs/ext4/
H A Dfile.c800 static vm_fault_t ext4_dax_huge_fault(struct vm_fault *vmf, unsigned int order) in ext4_dax_huge_fault() argument
806 struct inode *inode = file_inode(vmf->vma->vm_file); in ext4_dax_huge_fault()
820 bool write = (vmf->flags & FAULT_FLAG_WRITE) && in ext4_dax_huge_fault()
821 (vmf->vma->vm_flags & VM_SHARED); in ext4_dax_huge_fault()
822 struct address_space *mapping = vmf->vma->vm_file->f_mapping; in ext4_dax_huge_fault()
827 file_update_time(vmf->vma->vm_file); in ext4_dax_huge_fault()
840 result = dax_iomap_fault(vmf, order, &pfn, &error, &ext4_iomap_ops); in ext4_dax_huge_fault()
849 result = dax_finish_sync_fault(vmf, order, pfn); in ext4_dax_huge_fault()
859 static vm_fault_t ext4_dax_fault(struct vm_fault *vmf) in ext4_dax_fault() argument
861 return ext4_dax_huge_fault(vmf, 0); in ext4_dax_fault()
/linux/arch/powerpc/platforms/cell/spufs/
H A Dfile.c230 spufs_mem_mmap_fault(struct vm_fault *vmf) in spufs_mem_mmap_fault() argument
232 struct vm_area_struct *vma = vmf->vma; in spufs_mem_mmap_fault()
237 offset = vmf->pgoff << PAGE_SHIFT; in spufs_mem_mmap_fault()
242 vmf->address, offset); in spufs_mem_mmap_fault()
254 ret = vmf_insert_pfn(vma, vmf->address, pfn); in spufs_mem_mmap_fault()
312 static vm_fault_t spufs_ps_fault(struct vm_fault *vmf, in spufs_ps_fault() argument
316 struct spu_context *ctx = vmf->vma->vm_file->private_data; in spufs_ps_fault()
317 unsigned long area, offset = vmf->pgoff << PAGE_SHIFT; in spufs_ps_fault()
355 ret = vmf_insert_pfn(vmf->vma, vmf->address, in spufs_ps_fault()
369 static vm_fault_t spufs_cntl_mmap_fault(struct vm_fault *vmf) in spufs_cntl_mmap_fault() argument
[all …]
/linux/drivers/gpu/drm/armada/
H A Darmada_gem.c21 static vm_fault_t armada_gem_vm_fault(struct vm_fault *vmf) in armada_gem_vm_fault() argument
23 struct drm_gem_object *gobj = vmf->vma->vm_private_data; in armada_gem_vm_fault()
27 pfn += (vmf->address - vmf->vma->vm_start) >> PAGE_SHIFT; in armada_gem_vm_fault()
28 return vmf_insert_pfn(vmf->vma, vmf->address, pfn); in armada_gem_vm_fault()
/linux/drivers/infiniband/hw/hfi1/
H A Dfile_ops.c73 static vm_fault_t vma_fault(struct vm_fault *vmf);
307 static inline void mmap_cdbg(u16 ctxt, u8 subctxt, u8 type, u8 mapio, u8 vmf, in mmap_cdbg() argument
313 ctxt, subctxt, type, mapio, vmf, !!memdma, in mmap_cdbg()
328 u8 subctxt, mapio = 0, vmf = 0, type; in hfi1_file_mmap() local
451 mmap_cdbg(ctxt, subctxt, type, mapio, vmf, memaddr, in hfi1_file_mmap()
497 vmf = 1; in hfi1_file_mmap()
530 vmf = 1; in hfi1_file_mmap()
536 vmf = 1; in hfi1_file_mmap()
543 vmf = 1; in hfi1_file_mmap()
555 vmf = 1; in hfi1_file_mmap()
[all …]

12345