1 // SPDX-License-Identifier: GPL-2.0-or-later
20 #define pr_fmt(fmt) "hash-mmu: " fmt
41 #include <linux/elf-randomize.h>
61 #include <asm/text-patching.h>
67 #include <asm/pte-walk.h>
68 #include <asm/asm-prototypes.h>
94  * Note:  pte   --> Linux PTE
95  *        HPTE  --> PowerPC Hashed Page Table Entry
143 		.penc   = {[MMU_PAGE_4K] = 0, [1 ... MMU_PAGE_COUNT - 1] = -1},
158 		.penc   = {[MMU_PAGE_4K] = 0, [1 ... MMU_PAGE_COUNT - 1] = -1},
165 		.penc   = {[0 ... MMU_PAGE_16M - 1] = -1, [MMU_PAGE_16M] = 0,
166 			    [MMU_PAGE_16M + 1 ... MMU_PAGE_COUNT - 1] = -1 },
221 	 * Flush the partition table cache if this is HV mode.  in tlbiel_all_isa300()
233 	 *      * PRS=1, R=0, and RIC!=2 (The only process-scoped  in tlbiel_all_isa300()
239 	 * Then flush the sets of the TLB proper. Hash mode uses  in tlbiel_all_isa300()
241 	 * in !HV mode.  in tlbiel_all_isa300()
273 		WARN(1, "%s called on pre-POWER7 CPU\n", __func__);  in hash__tlbiel_all()
283 	unsigned long mode = htab_convert_pte_flags(pgprot_val(PAGE_KERNEL), HPTE_USE_KERNEL_KEY);  in kernel_map_linear_page()  local
295 	ret = hpte_insert_repeating(hash, vpn, __pa(vaddr), mode,  in kernel_map_linear_page()
417 	/* allocate linear map for kfence within RMA region */  in hash_kfence_alloc_pool()
424 		pr_err("%s: memblock for linear map (%lu) failed\n", __func__,  in hash_kfence_alloc_pool()
468 	unsigned long lmi = (vaddr - (unsigned long)__kfence_pool)  in hash_kfence_add_slot()
487 		lmi = (vaddr - (unsigned long)__kfence_pool) >> PAGE_SHIFT;  in hash_kfence_map_pages()
542  *  - Under pHyp or KVM, the updatepp path will not set C, thus it *will*
545  *  - The above is however not a problem, because we also don't do that
549  *    - Under bare metal,  we do have the race, so we need R and C set
550  *    - We make sure R is always set and never lost
551  *    - C is _PAGE_DIRTY, and *should* always be set for a writeable mapping
557 	/* _PAGE_EXEC -> NOEXEC */  in htab_convert_pte_flags()
565 	 * or PPP=0b011 for read-only (including writeable but clean pages).  in htab_convert_pte_flags()
577 		VM_WARN_ONCE(!(pteflags & _PAGE_RWX), "no-access mapping request");  in htab_convert_pte_flags()
586 		VM_WARN_ONCE(!(pteflags & _PAGE_RWX), "no-access mapping request");  in htab_convert_pte_flags()
631 	DBG("htab_bolt_mapping(%lx..%lx -> %lx (%lx,%d,%d)\n",  in htab_bolt_mapping()
650 			return -1;  in htab_bolt_mapping()
663 		 * mode the vector region need to be marked as executable.  in htab_bolt_mapping()
677 		if (ret == -1) {  in htab_bolt_mapping()
683 			if (ret != -1)  in htab_bolt_mapping()
687 			if (ret == -1 && !secondary_hash) {  in htab_bolt_mapping()
698 		/* add slot info in debug_pagealloc / kfence linear map */  in htab_bolt_mapping()
716 		return -ENODEV;  in htab_remove_mapping()
733 		if (rc == -ENOENT) {  in htab_remove_mapping()
734 			ret = -ENOENT;  in htab_remove_mapping()
765  * per-CPU array allocated if we enable stress_hpt.
799 	prop = of_get_flat_dt_prop(node, "ibm,processor-segment-sizes", &size);  in htab_dt_scan_seg_sizes()
802 	for (; size >= 4; size -= 4, ++prop) {  in htab_dt_scan_seg_sizes()
811 			cur_cpu_spec->mmu_features |= MMU_FTR_1T_SEGMENT;  in htab_dt_scan_seg_sizes()
815 	cur_cpu_spec->mmu_features &= ~MMU_FTR_NO_SLBIE_B;  in htab_dt_scan_seg_sizes()
821 	int idx = -1;  in get_idx_from_shift()
855 	prop = of_get_flat_dt_prop(node, "ibm,segment-page-sizes", &size);  in htab_dt_scan_page_sizes()
859 	pr_info("Page sizes from device-tree:\n");  in htab_dt_scan_page_sizes()
861 	cur_cpu_spec->mmu_features &= ~(MMU_FTR_16M_PAGE);  in htab_dt_scan_page_sizes()
869 		size -= 3; prop += 3;  in htab_dt_scan_page_sizes()
873 			prop += lpnum * 2; size -= lpnum * 2;  in htab_dt_scan_page_sizes()
878 			cur_cpu_spec->mmu_features |= MMU_FTR_16M_PAGE;  in htab_dt_scan_page_sizes()
880 		def->shift = base_shift;  in htab_dt_scan_page_sizes()
882 			def->avpnm = 0;  in htab_dt_scan_page_sizes()
884 			def->avpnm = (1 << (base_shift - 23)) - 1;  in htab_dt_scan_page_sizes()
885 		def->sllp = slbenc;  in htab_dt_scan_page_sizes()
891 			def->tlbiel = 1;  in htab_dt_scan_page_sizes()
893 			def->tlbiel = 0;  in htab_dt_scan_page_sizes()
899 			prop += 2; size -= 2;  in htab_dt_scan_page_sizes()
900 			lpnum--;  in htab_dt_scan_page_sizes()
906 			if (penc == -1)  in htab_dt_scan_page_sizes()
910 			def->penc[idx] = penc;  in htab_dt_scan_page_sizes()
913 				base_shift, shift, def->sllp,  in htab_dt_scan_page_sizes()
914 				def->avpnm, def->tlbiel, def->penc[idx]);  in htab_dt_scan_page_sizes()
971 			mmu_psize_defs[bpsize].penc[apsize] = -1;  in mmu_psize_set_default_penc()
997 	/* se the invalid penc to -1 */  in htab_scan_page_sizes()
1005 	 * Try to find the available page sizes in the device-tree  in htab_scan_page_sizes()
1010 		 * Nothing in the device-tree, but the CPU supports 16M pages,  in htab_scan_page_sizes()
1042  * The zzzz bits are implementation-specific but are chosen so that
1044  * low-order N bits as the encoding for the 2^(12+N) byte page size
1057 			if (penc == -1 || !mmu_psize_defs[ap].shift)  in init_hpte_page_sizes()
1059 			shift = mmu_psize_defs[ap].shift - LP_SHIFT;  in init_hpte_page_sizes()
1082 		 * Pick a size for the linear mapping. Currently, we only  in htab_init_page_sizes()
1088 				pr_warn("Kernel not 16M aligned, disabling 16M linear map alignment\n");  in htab_init_page_sizes()
1103 	 * (and firmware) support cache-inhibited large pages.  in htab_init_page_sizes()
1105 	 * hash_page knows to switch processes that use cache-inhibited  in htab_init_page_sizes()
1138 	printk(KERN_DEBUG "Page orders: linear mapping = %d, "  in htab_init_page_sizes()
1164 	prop = of_get_flat_dt_prop(node, "ibm,pft-size", NULL);  in htab_dt_scan_pftsize()
1184 	pteg_shift = memshift - (pshift + 1);  in htab_shift_for_mem_size()
1197 	 * retrieve it from the device-tree. If it's not there neither, we  in htab_get_table_size()
1227 	    target_hpt_shift < ppc64_pft_size - 1)  in resize_hpt_for_hotplug()
1240 		return -1;  in hash__create_section_mapping()
1252 		BUG_ON(rc2 && (rc2 != -ENOENT));  in hash__create_section_mapping()
1262 	if (resize_hpt_for_hotplug(memblock_phys_mem_size()) == -ENOSPC)  in hash__remove_section_mapping()
1278 	htab_size =  __ilog2(htab_size) - 18;  in hash_init_partition_table()
1308 	DBG(" -> htab_initialize()\n");  in htab_initialize()
1341 	htab_hash_mask = pteg_count - 1;  in htab_initialize()
1374 		_SDR1 = table + __ilog2(htab_size_bytes) - 18;  in htab_initialize()
1390 	/* create bolted the linear mapping in the hash table */  in htab_initialize()
1392 		size = end - base;  in htab_initialize()
1395 		DBG("creating mapping for region: %lx..%lx (prot: %lx)\n",  in htab_initialize()
1413 	 * tce_alloc_start/end are 16MB aligned so the mapping should work  in htab_initialize()
1429 	DBG(" <- htab_initialize()\n");  in htab_initialize()
1452 	 * Where the slot number is between 0-15, and values of 8-15 indicate  in hash__early_init_mmu()
1508 	 * Initialize the MMU Hash table and create the linear mapping  in hash__early_init_mmu()
1536 					    (PATB_SIZE_SHIFT - 12));  in hash__early_init_mmu_secondary()
1565 	if (!test_bit(PG_dcache_clean, &folio->flags.f) &&  in hash_page_do_lazy_icache()
1569 			set_bit(PG_dcache_clean, &folio->flags.f);  in hash_page_do_lazy_icache()
1582 		psizes = get_paca()->mm_ctx_low_slices_psize;  in get_paca_psize()
1585 		psizes = get_paca()->mm_ctx_high_slices_psize;  in get_paca_psize()
1606 	if ((get_paca_psize(addr) != MMU_PAGE_4K) && (current->mm == mm)) {  in demote_segment_4k()
1616  * This looks up a 2-bit protection code for a 4k subpage of a 64k page.
1619  * Result is 0: full permissions, _PAGE_RW: read-only,
1624 	struct subpage_prot_table *spt = mm_ctx_subpage_prot(&mm->context);  in subpage_protection()
1631 	if (ea >= spt->maxaddr)  in subpage_protection()
1634 		/* addresses below 4GB use spt->low_prot */  in subpage_protection()
1635 		sbpm = spt->low_prot;  in subpage_protection()
1637 		sbpm = spt->protptrs[ea >> SBP_L3_SHIFT];  in subpage_protection()
1641 	sbpp = sbpm[(ea >> SBP_L2_SHIFT) & (SBP_L2_COUNT - 1)];  in subpage_protection()
1644 	spp = sbpp[(ea >> PAGE_SHIFT) & (SBP_L1_COUNT - 1)];  in subpage_protection()
1646 	/* extract 2-bit bitfield for this 4k subpage */  in subpage_protection()
1647 	spp >>= 30 - 2 * ((ea >> 12) & 0xf);  in subpage_protection()
1650 	 * 0 -> full permission  in subpage_protection()
1651 	 * 1 -> Read only  in subpage_protection()
1652 	 * 2 -> no access.  in subpage_protection()
1673 		ea, access, current->comm);  in hash_failure_debug()
1686 	} else if (get_paca()->vmalloc_sllp !=  in check_paca_psize()
1688 		get_paca()->vmalloc_sllp =  in check_paca_psize()
1696  *  0 - handled
1697  *  1 - normal page fault
1698  * -1 - critical hash insertion error
1699  * -2 - access not permitted by subpage protection mechanism
1728 		vsid = get_user_vsid(&mm->context, ea, ssize);  in hash_page_mm()
1751 	DBG_LOW(" mm=%p, mm->pgdir=%p, vsid=%016lx\n", mm, mm->pgd, vsid);  in hash_page_mm()
1760 	pgdir = mm->pgd;  in hash_page_mm()
1773 	 * be hitting a special driver mapping, and need to align the  in hash_page_mm()
1776 	 * It could also be a hugepage mapping, in which case this is  in hash_page_mm()
1780 		ea &= ~((1ul << mmu_psize_defs[psize].shift) - 1);  in hash_page_mm()
1809 	 * Pre-check access permissions (will be re-checked atomically  in hash_page_mm()
1810 	 * in __hash_page_XX but this pre-check is a fast path  in hash_page_mm()
1836 		if (current->mm == mm)  in hash_page_mm()
1843 	DBG_LOW(" i-pte: %016lx\n", pte_val(*ptep));  in hash_page_mm()
1845 	DBG_LOW(" i-pte: %016lx %016lx\n", pte_val(*ptep),  in hash_page_mm()
1857 	 * If this PTE is non-cacheable and we have restrictions on  in hash_page_mm()
1866 			 * some driver did a non-cacheable mapping  in hash_page_mm()
1872 			       "non-cacheable mapping\n");  in hash_page_mm()
1882 	if (current->mm == mm)  in hash_page_mm()
1894 			rc = -2;  in hash_page_mm()
1904 	if (rc == -1)  in hash_page_mm()
1908 	DBG_LOW(" o-pte: %016lx\n", pte_val(*ptep));  in hash_page_mm()
1910 	DBG_LOW(" o-pte: %016lx %016lx\n", pte_val(*ptep),  in hash_page_mm()
1913 	DBG_LOW(" -> rc=%d\n", rc);  in hash_page_mm()
1924 	struct mm_struct *mm = current->mm;  in hash_page()
1939 	unsigned long ea = regs->dar;  in DEFINE_INTERRUPT_HANDLER()
1940 	unsigned long dsisr = regs->dsisr;  in DEFINE_INTERRUPT_HANDLER()
1956 		mm = current->mm;  in DEFINE_INTERRUPT_HANDLER()
1965 	 * kernel mode access kernel space.  in DEFINE_INTERRUPT_HANDLER()
1968 	 * 1) when kernel mode access user space  in DEFINE_INTERRUPT_HANDLER()
1982 			if (IS_ENABLED(CONFIG_PPC_SUBPAGE_PROT) && err == -2)  in DEFINE_INTERRUPT_HANDLER()
2001 	if (unlikely(psize != mm_ctx_user_psize(&mm->context)))  in should_hash_preload()
2027 	DBG_LOW("hash_preload(mm=%p, mm->pgdir=%p, ea=%016lx, access=%lx,"  in hash_preload()
2028 		" trap=%lx\n", mm, mm->pgd, ea, access, trap);  in hash_preload()
2031 	pgdir = mm->pgd;  in hash_preload()
2037 	vsid = get_user_vsid(&mm->context, ea, ssize);  in hash_preload()
2074 	if (mm_ctx_user_psize(&mm->context) == MMU_PAGE_64K)  in hash_preload()
2085 	if (rc == -1)  in hash_preload()
2087 				   mm_ctx_user_psize(&mm->context),  in hash_preload()
2088 				   mm_ctx_user_psize(&mm->context),  in hash_preload()
2107 	 * called with either mm->page_table_lock held or ptl lock held  in __update_mmu_cache()
2119 	 * double-faulting on execution of fresh text. We have to test  in __update_mmu_cache()
2125 	trap = current->thread.regs ? TRAP(current->thread.regs) : 0UL;  in __update_mmu_cache()
2137 	hash_preload(vma->vm_mm, ptep, address, is_exec, trap);  in __update_mmu_cache()
2147 	 * made read-only, which will flush_hash_page. BIG ISSUE here: if the  in tm_flush_hash_page()
2151 	if (local && cpu_has_feature(CPU_FTR_TM) && current->thread.regs &&  in tm_flush_hash_page()
2152 	    MSR_TM_ACTIVE(current->thread.regs->msr)) {  in tm_flush_hash_page()
2270 			flush_hash_page(batch->vpn[i], batch->pte[i],  in flush_hash_range()
2271 					batch->psize, batch->ssize, local);  in flush_hash_range()
2290 	if (unlikely(slot == -1)) {  in hpte_insert_repeating()
2295 		if (slot == -1) {  in hpte_insert_repeating()
2317 		if (last_group != -1UL) {  in hpt_clear_stress()
2320 				if (mmu_hash_ops.hpte_remove(last_group) == -1)  in hpt_clear_stress()
2323 			stress_hpt_struct[cpu].last_group[g] = -1;  in hpt_clear_stress()
2333 	last_group = stress_hpt_struct[cpu].last_group[stress_nr_groups() - 1];  in hpt_do_stress()
2337 	if (last_group != -1UL) {  in hpt_do_stress()
2345 			if (mmu_hash_ops.hpte_remove(last_group) == -1)  in hpt_do_stress()
2348 		stress_hpt_struct[cpu].last_group[stress_nr_groups() - 1] = -1;  in hpt_do_stress()
2360 		 * work for non-CI PTEs).  in hpt_do_stress()
2366 			(stress_nr_groups() - 1) * sizeof(unsigned long));  in hpt_do_stress()
2375 	 * We don't currently support the first MEMBLOCK not mapping 0  in hash__setup_initial_memory_limit()
2382 	 * non-virtualized 64-bit hash MMU systems don't have a limitation  in hash__setup_initial_memory_limit()
2383 	 * on real mode access.  in hash__setup_initial_memory_limit()
2391 	 * for virtual real mode addressing and so it doesn't make sense to  in hash__setup_initial_memory_limit()
2422 		return -ENODEV;  in hpt_order_set()
2460 		return randomize_page(mm->brk, SZ_32M);  in arch_randomize_brk()
2462 		return randomize_page(max_t(unsigned long, mm->brk, SZ_1T), SZ_1G);  in arch_randomize_brk()
2464 		return randomize_page(mm->brk, SZ_1G);  in arch_randomize_brk()