151 				 unsigned long dst_addr)  in mfill_file_over_size()  argument
160 	offset = linear_page_index(dst_vma, dst_addr);  in mfill_file_over_size()
173 			     unsigned long dst_addr, struct page *page,  in mfill_atomic_install_pte()  argument
195 	dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl);  in mfill_atomic_install_pte()
199 	if (mfill_file_over_size(dst_vma, dst_addr)) {  in mfill_atomic_install_pte()
219 		folio_add_new_anon_rmap(folio, dst_vma, dst_addr, RMAP_EXCLUSIVE);  in mfill_atomic_install_pte()
229 	set_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte);  in mfill_atomic_install_pte()
232 	update_mmu_cache(dst_vma, dst_addr, dst_pte);  in mfill_atomic_install_pte()
242 				 unsigned long dst_addr,  in mfill_atomic_pte_copy()  argument
254 					dst_addr, false);  in mfill_atomic_pte_copy()
305 	ret = mfill_atomic_install_pte(dst_pmd, dst_vma, dst_addr,  in mfill_atomic_pte_copy()
318 					 unsigned long dst_addr)  in mfill_atomic_pte_zeroed_folio()  argument
323 	folio = vma_alloc_zeroed_movable_folio(dst_vma, dst_addr);  in mfill_atomic_pte_zeroed_folio()
337 	ret = mfill_atomic_install_pte(dst_pmd, dst_vma, dst_addr,  in mfill_atomic_pte_zeroed_folio()
350 				     unsigned long dst_addr)  in mfill_atomic_pte_zeropage()  argument
357 		return mfill_atomic_pte_zeroed_folio(dst_pmd, dst_vma, dst_addr);  in mfill_atomic_pte_zeropage()
359 	_dst_pte = pte_mkspecial(pfn_pte(my_zero_pfn(dst_addr),  in mfill_atomic_pte_zeropage()
362 	dst_pte = pte_offset_map_lock(dst_vma->vm_mm, dst_pmd, dst_addr, &ptl);  in mfill_atomic_pte_zeropage()
365 	if (mfill_file_over_size(dst_vma, dst_addr)) {  in mfill_atomic_pte_zeropage()
372 	set_pte_at(dst_vma->vm_mm, dst_addr, dst_pte, _dst_pte);  in mfill_atomic_pte_zeropage()
374 	update_mmu_cache(dst_vma, dst_addr, dst_pte);  in mfill_atomic_pte_zeropage()
385 				     unsigned long dst_addr,  in mfill_atomic_pte_continue()  argument
389 	pgoff_t pgoff = linear_page_index(dst_vma, dst_addr);  in mfill_atomic_pte_continue()
411 	ret = mfill_atomic_install_pte(dst_pmd, dst_vma, dst_addr,  in mfill_atomic_pte_continue()
429 				   unsigned long dst_addr,  in mfill_atomic_pte_poison()  argument
439 	dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl);  in mfill_atomic_pte_poison()
443 	if (mfill_file_over_size(dst_vma, dst_addr)) {  in mfill_atomic_pte_poison()
453 	set_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte);  in mfill_atomic_pte_poison()
456 	update_mmu_cache(dst_vma, dst_addr, dst_pte);  in mfill_atomic_pte_poison()
502 	unsigned long src_addr, dst_addr;  in mfill_atomic_hugetlb()  local
523 	dst_addr = dst_start;  in mfill_atomic_hugetlb()
567 		BUG_ON(dst_addr >= dst_start + len);  in mfill_atomic_hugetlb()
575 		idx = linear_page_index(dst_vma, dst_addr);  in mfill_atomic_hugetlb()
582 		dst_pte = huge_pte_alloc(dst_mm, dst_vma, dst_addr, vma_hpagesize);  in mfill_atomic_hugetlb()
590 		    !huge_pte_none_mostly(huge_ptep_get(dst_mm, dst_addr, dst_pte))) {  in mfill_atomic_hugetlb()
597 		err = hugetlb_mfill_atomic_pte(dst_pte, dst_vma, dst_addr,  in mfill_atomic_hugetlb()
623 			dst_addr += vma_hpagesize;  in mfill_atomic_hugetlb()
658 						unsigned long dst_addr,  in mfill_atomic_pte()  argument
667 						 dst_addr, flags);  in mfill_atomic_pte()
670 					       dst_addr, flags);  in mfill_atomic_pte()
686 						    dst_addr, src_addr,  in mfill_atomic_pte()
690 						 dst_vma, dst_addr);  in mfill_atomic_pte()
693 					     dst_addr, src_addr,  in mfill_atomic_pte()
710 	unsigned long src_addr, dst_addr;  in mfill_atomic()  local
725 	dst_addr = dst_start;  in mfill_atomic()
781 		BUG_ON(dst_addr >= dst_start + len);  in mfill_atomic()
783 		dst_pmd = mm_alloc_pmd(dst_mm, dst_addr);  in mfill_atomic()
815 		err = mfill_atomic_pte(dst_pmd, dst_vma, dst_addr,  in mfill_atomic()
841 			dst_addr += PAGE_SIZE;  in mfill_atomic()
1027 			    unsigned long dst_addr, unsigned long src_addr,  in move_present_pte()  argument
1058 	src_folio->index = linear_page_index(dst_vma, dst_addr);  in move_present_pte()
1064 	set_pte_at(mm, dst_addr, dst_pte, orig_dst_pte);  in move_present_pte()
1071 			 unsigned long dst_addr, unsigned long src_addr,  in move_swap_pte()  argument
1088 	set_pte_at(mm, dst_addr, dst_pte, orig_src_pte);  in move_swap_pte()
1097 			     unsigned long dst_addr, unsigned long src_addr,  in move_zeropage_pte()  argument
1111 	zero_pte = pte_mkspecial(pfn_pte(my_zero_pfn(dst_addr),  in move_zeropage_pte()
1114 	set_pte_at(mm, dst_addr, dst_pte, zero_pte);  in move_zeropage_pte()
1129 			  unsigned long dst_addr, unsigned long src_addr,  in move_pages_pte()  argument
1149 	dst_pte = pte_offset_map_nolock(mm, dst_pmd, dst_addr, &dst_ptl);  in move_pages_pte()
1205 					       dst_addr, src_addr, dst_pte, src_pte,  in move_pages_pte()
1296 				       dst_addr, src_addr, dst_pte, src_pte,  in move_pages_pte()
1313 		err = move_swap_pte(mm, dst_addr, src_addr,  in move_pages_pte()
1338 static inline bool move_splits_huge_pmd(unsigned long dst_addr,  in move_splits_huge_pmd()  argument
1342 	return (src_addr & ~HPAGE_PMD_MASK) || (dst_addr & ~HPAGE_PMD_MASK) ||  in move_splits_huge_pmd()
1346 static inline bool move_splits_huge_pmd(unsigned long dst_addr,  in move_splits_huge_pmd()  argument
1597 	unsigned long src_addr, dst_addr;  in move_pages()  local
1642 	for (src_addr = src_start, dst_addr = dst_start;  in move_pages()
1665 		dst_pmd = mm_alloc_pmd(mm, dst_addr);  in move_pages()
1692 			if (move_splits_huge_pmd(dst_addr, src_addr, src_start + len) ||  in move_pages()
1711 						  dst_addr, src_addr);  in move_pages()
1732 					     dst_addr, src_addr, mode);  in move_pages()
1752 		dst_addr += step_size;  in move_pages()