1 // SPDX-License-Identifier: GPL-2.0
6 pte_t huge_ptep_get(struct mm_struct *mm, unsigned long addr, pte_t *ptep)  in huge_ptep_get()  argument
17 	for (i = 0; i < pte_num; i++, ptep++) {  in huge_ptep_get()
30 pte_t *huge_pte_alloc(struct mm_struct *mm,  in huge_pte_alloc()  argument
42 	pgd = pgd_offset(mm, addr);  in huge_pte_alloc()
43 	p4d = p4d_alloc(mm, pgd, addr);  in huge_pte_alloc()
47 	pud = pud_alloc(mm, p4d, addr);  in huge_pte_alloc()
58 			pte = huge_pmd_share(mm, vma, addr, pud);  in huge_pte_alloc()
60 			pte = (pte_t *)pmd_alloc(mm, pud, addr);  in huge_pte_alloc()
64 	pmd = pmd_alloc(mm, pud, addr);  in huge_pte_alloc()
70 			pte = pte_alloc_huge(mm, pmd, addr & napot_cont_mask(order));  in huge_pte_alloc()
84 pte_t *huge_pte_offset(struct mm_struct *mm,  in huge_pte_offset()  argument
95 	pgd = pgd_offset(mm, addr);  in huge_pte_offset()
105 		/* must be pud huge, non-present or none */  in huge_pte_offset()
113 		/* must be pmd huge, non-present or none */  in huge_pte_offset()
135 		return P4D_SIZE - PUD_SIZE;  in hugetlb_mask_last_page()
138 		return PUD_SIZE - PMD_SIZE;  in hugetlb_mask_last_page()
140 		return PMD_SIZE - napot_cont_size(NAPOT_CONT64KB_ORDER);  in hugetlb_mask_last_page()
145 	return 0UL;  in hugetlb_mask_last_page()
148 static pte_t get_clear_contig(struct mm_struct *mm,  in get_clear_contig()  argument
156 	for (i = 0; i < pte_num; i++, addr += PAGE_SIZE, ptep++) {  in get_clear_contig()
157 		pte_t pte = ptep_get_and_clear(mm, addr, ptep);  in get_clear_contig()
169 static pte_t get_clear_contig_flush(struct mm_struct *mm,  in get_clear_contig_flush()  argument
174 	pte_t orig_pte = get_clear_contig(mm, addr, ptep, pte_num);  in get_clear_contig_flush()
175 	struct vm_area_struct vma = TLB_FLUSH_VMA(mm, 0);  in get_clear_contig_flush()
200 static void clear_flush(struct mm_struct *mm,  in clear_flush()  argument
206 	struct vm_area_struct vma = TLB_FLUSH_VMA(mm, 0);  in clear_flush()
209 	for (i = 0; i < ncontig; i++, addr += pgsize, ptep++)  in clear_flush()
210 		ptep_get_and_clear(mm, addr, ptep);  in clear_flush()
220  * 4.2.1.". That's the equivalent of the Break-Before-Make approach used by
223 void set_huge_pte_at(struct mm_struct *mm,  in set_huge_pte_at()  argument
247 		for (i = 0; i < pte_num; i++, ptep++, addr += pgsize)  in set_huge_pte_at()
248 			set_ptes(mm, addr, ptep, pte, 1);  in set_huge_pte_at()
253 		set_ptes(mm, addr, ptep, pte, 1);  in set_huge_pte_at()
257 	clear_flush(mm, addr, ptep, pgsize, pte_num);  in set_huge_pte_at()
259 	for (i = 0; i < pte_num; i++, ptep++, addr += pgsize)  in set_huge_pte_at()
260 		set_pte_at(mm, addr, ptep, pte);  in set_huge_pte_at()
269 	struct mm_struct *mm = vma->vm_mm;  in huge_ptep_set_access_flags()  local
279 	ptep = huge_pte_offset(mm, addr, napot_cont_size(order));  in huge_ptep_set_access_flags()
280 	orig_pte = get_clear_contig_flush(mm, addr, ptep, pte_num);  in huge_ptep_set_access_flags()
288 	for (i = 0; i < pte_num; i++, addr += PAGE_SIZE, ptep++)  in huge_ptep_set_access_flags()
289 		set_pte_at(mm, addr, ptep, pte);  in huge_ptep_set_access_flags()
294 pte_t huge_ptep_get_and_clear(struct mm_struct *mm,  in huge_ptep_get_and_clear()  argument
302 		return ptep_get_and_clear(mm, addr, ptep);  in huge_ptep_get_and_clear()
306 	return get_clear_contig(mm, addr, ptep, pte_num);  in huge_ptep_get_and_clear()
309 void huge_ptep_set_wrprotect(struct mm_struct *mm,  in huge_ptep_set_wrprotect()  argument
319 		ptep_set_wrprotect(mm, addr, ptep);  in huge_ptep_set_wrprotect()
325 	ptep = huge_pte_offset(mm, addr, napot_cont_size(order));  in huge_ptep_set_wrprotect()
326 	orig_pte = get_clear_contig_flush(mm, addr, ptep, pte_num);  in huge_ptep_set_wrprotect()
330 	for (i = 0; i < pte_num; i++, addr += PAGE_SIZE, ptep++)  in huge_ptep_set_wrprotect()
331 		set_pte_at(mm, addr, ptep, orig_pte);  in huge_ptep_set_wrprotect()
346 	return get_clear_contig_flush(vma->vm_mm, addr, ptep, pte_num);  in huge_ptep_clear_flush()
349 void huge_pte_clear(struct mm_struct *mm,  in huge_pte_clear()  argument
358 		pte_clear(mm, addr, ptep);  in huge_pte_clear()
363 	for (i = 0; i < pte_num; i++, addr += PAGE_SIZE, ptep++)  in huge_pte_clear()
364 		pte_clear(mm, addr, ptep);  in huge_pte_clear()
389 	return 0;  in napot_hugetlbpages_init()
431 		hugetlb_add_hstate(PUD_SHIFT - PAGE_SHIFT);  in gigantic_pages_init()
432 	return 0;  in gigantic_pages_init()