173 static int move_ptes(struct pagetable_move_control *pmc,  in move_ptes()  argument
176 	struct vm_area_struct *vma = pmc->old;  in move_ptes()
183 	unsigned long old_addr = pmc->old_addr;  in move_ptes()
184 	unsigned long new_addr = pmc->new_addr;  in move_ptes()
207 	if (pmc->need_rmap_locks)  in move_ptes()
283 	if (pmc->need_rmap_locks)  in move_ptes()
298 static bool move_normal_pmd(struct pagetable_move_control *pmc,  in move_normal_pmd()  argument
302 	struct vm_area_struct *vma = pmc->old;  in move_normal_pmd()
365 	flush_tlb_range(vma, pmc->old_addr, pmc->old_addr + PMD_SIZE);  in move_normal_pmd()
374 static inline bool move_normal_pmd(struct pagetable_move_control *pmc,  in move_normal_pmd()  argument
382 static bool move_normal_pud(struct pagetable_move_control *pmc,  in move_normal_pud()  argument
386 	struct vm_area_struct *vma = pmc->old;  in move_normal_pud()
424 	flush_tlb_range(vma, pmc->old_addr, pmc->old_addr + PUD_SIZE);  in move_normal_pud()
432 static inline bool move_normal_pud(struct pagetable_move_control *pmc,  in move_normal_pud()  argument
440 static bool move_huge_pud(struct pagetable_move_control *pmc,  in move_huge_pud()  argument
444 	struct vm_area_struct *vma = pmc->old;  in move_huge_pud()
472 	set_pud_at(mm, pmc->new_addr, new_pud, pud);  in move_huge_pud()
473 	flush_pud_tlb_range(vma, pmc->old_addr, pmc->old_addr + HPAGE_PUD_SIZE);  in move_huge_pud()
481 static bool move_huge_pud(struct pagetable_move_control *pmc,  in move_huge_pud()  argument
504 						struct pagetable_move_control *pmc)  in get_extent()  argument
507 	unsigned long old_addr = pmc->old_addr;  in get_extent()
508 	unsigned long old_end = pmc->old_end;  in get_extent()
509 	unsigned long new_addr = pmc->new_addr;  in get_extent()
540  * the PMC, or overridden in the case of normal, larger page tables.
542 static bool should_take_rmap_locks(struct pagetable_move_control *pmc,  in should_take_rmap_locks()  argument
550 		return pmc->need_rmap_locks;  in should_take_rmap_locks()
558 static bool move_pgt_entry(struct pagetable_move_control *pmc,  in move_pgt_entry()  argument
562 	bool need_rmap_locks = should_take_rmap_locks(pmc, entry);  in move_pgt_entry()
566 		take_rmap_locks(pmc->old);  in move_pgt_entry()
570 		moved = move_normal_pmd(pmc, old_entry, new_entry);  in move_pgt_entry()
573 		moved = move_normal_pud(pmc, old_entry, new_entry);  in move_pgt_entry()
577 			move_huge_pmd(pmc->old, pmc->old_addr, pmc->new_addr, old_entry,  in move_pgt_entry()
582 			move_huge_pud(pmc, old_entry, new_entry);  in move_pgt_entry()
591 		drop_rmap_locks(pmc->old);  in move_pgt_entry()
602 static bool can_align_down(struct pagetable_move_control *pmc,  in can_align_down()  argument
613 	if (!pmc->for_stack && vma->vm_start != addr_to_align)  in can_align_down()
617 	if (pmc->for_stack && addr_masked >= vma->vm_start)  in can_align_down()
631 static bool can_realign_addr(struct pagetable_move_control *pmc,  in can_realign_addr()  argument
635 	unsigned long old_align = pmc->old_addr & align_mask;  in can_realign_addr()
636 	unsigned long new_align = pmc->new_addr & align_mask;  in can_realign_addr()
660 	if (pmc->len_in < old_align_next)  in can_realign_addr()
672 	if (!can_align_down(pmc, pmc->old, pmc->old_addr, pagetable_mask) ||  in can_realign_addr()
673 	    !can_align_down(pmc, pmc->new, pmc->new_addr, pagetable_mask))  in can_realign_addr()
690  *      .         pmc->old_addr         .      pmc->old_end
698  * The idea here is simply to align pmc->old_addr, pmc->new_addr down to the
706  * pmc->old_addr                        .      pmc->old_end
710 static void try_realign_addr(struct pagetable_move_control *pmc,  in try_realign_addr()  argument
714 	if (!can_realign_addr(pmc, pagetable_mask))  in try_realign_addr()
719 	 * pmc->old_end value, and since the move_page_tables() operation spans  in try_realign_addr()
723 	pmc->old_addr &= pagetable_mask;  in try_realign_addr()
724 	pmc->new_addr &= pagetable_mask;  in try_realign_addr()
728 static bool pmc_done(struct pagetable_move_control *pmc)  in pmc_done()  argument
730 	return pmc->old_addr >= pmc->old_end;  in pmc_done()
734 static void pmc_next(struct pagetable_move_control *pmc, unsigned long extent)  in pmc_next()  argument
736 	pmc->old_addr += extent;  in pmc_next()
737 	pmc->new_addr += extent;  in pmc_next()
744 static unsigned long pmc_progress(struct pagetable_move_control *pmc)  in pmc_progress()  argument
746 	unsigned long orig_old_addr = pmc->old_end - pmc->len_in;  in pmc_progress()
747 	unsigned long old_addr = pmc->old_addr;  in pmc_progress()
757 unsigned long move_page_tables(struct pagetable_move_control *pmc)  in move_page_tables()  argument
763 	struct mm_struct *mm = pmc->old->vm_mm;  in move_page_tables()
765 	if (!pmc->len_in)  in move_page_tables()
768 	if (is_vm_hugetlb_page(pmc->old))  in move_page_tables()
769 		return move_hugetlb_page_tables(pmc->old, pmc->new, pmc->old_addr,  in move_page_tables()
770 						pmc->new_addr, pmc->len_in);  in move_page_tables()
776 	try_realign_addr(pmc, PMD_MASK);  in move_page_tables()
778 	flush_cache_range(pmc->old, pmc->old_addr, pmc->old_end);  in move_page_tables()
780 				pmc->old_addr, pmc->old_end);  in move_page_tables()
783 	for (; !pmc_done(pmc); pmc_next(pmc, extent)) {  in move_page_tables()
789 		extent = get_extent(NORMAL_PUD, pmc);  in move_page_tables()
791 		old_pud = get_old_pud(mm, pmc->old_addr);  in move_page_tables()
794 		new_pud = alloc_new_pud(mm, pmc->new_addr);  in move_page_tables()
799 				move_pgt_entry(pmc, HPAGE_PUD, old_pud, new_pud);  in move_page_tables()
804 			if (move_pgt_entry(pmc, NORMAL_PUD, old_pud, new_pud))  in move_page_tables()
808 		extent = get_extent(NORMAL_PMD, pmc);  in move_page_tables()
809 		old_pmd = get_old_pmd(mm, pmc->old_addr);  in move_page_tables()
812 		new_pmd = alloc_new_pmd(mm, pmc->new_addr);  in move_page_tables()
819 			    move_pgt_entry(pmc, HPAGE_PMD, old_pmd, new_pmd))  in move_page_tables()
821 			split_huge_pmd(pmc->old, old_pmd, pmc->old_addr);  in move_page_tables()
828 			if (move_pgt_entry(pmc, NORMAL_PMD, old_pmd, new_pmd))  in move_page_tables()
833 		if (pte_alloc(pmc->new->vm_mm, new_pmd))  in move_page_tables()
835 		if (move_ptes(pmc, extent, old_pmd, new_pmd) < 0)  in move_page_tables()
841 	return pmc_progress(pmc);  in move_page_tables()
1155 	PAGETABLE_MOVE(pmc, NULL, NULL, vrm->addr, vrm->new_addr, vrm->old_len);  in copy_vma_and_data()
1158 			   &pmc.need_rmap_locks);  in copy_vma_and_data()
1165 	pmc.old = vma;  in copy_vma_and_data()
1166 	pmc.new = new_vma;  in copy_vma_and_data()
1168 	moved_len = move_page_tables(&pmc);  in copy_vma_and_data()