1 /* 2 * PPC Huge TLB Page Support for Kernel. 3 * 4 * Copyright (C) 2003 David Gibson, IBM Corporation. 5 * Copyright (C) 2011 Becky Bruce, Freescale Semiconductor 6 * 7 * Based on the IA-32 version: 8 * Copyright (C) 2002, Rohit Seth <rohit.seth@intel.com> 9 */ 10 11 #include <linux/mm.h> 12 #include <linux/io.h> 13 #include <linux/slab.h> 14 #include <linux/hugetlb.h> 15 #include <linux/export.h> 16 #include <linux/of_fdt.h> 17 #include <linux/memblock.h> 18 #include <linux/moduleparam.h> 19 #include <linux/swap.h> 20 #include <linux/swapops.h> 21 #include <linux/kmemleak.h> 22 #include <asm/pgalloc.h> 23 #include <asm/tlb.h> 24 #include <asm/setup.h> 25 #include <asm/hugetlb.h> 26 #include <asm/pte-walk.h> 27 #include <asm/firmware.h> 28 29 bool hugetlb_disabled = false; 30 31 #define PTE_T_ORDER (__builtin_ffs(sizeof(pte_basic_t)) - \ 32 __builtin_ffs(sizeof(void *))) 33 34 pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr, unsigned long sz) 35 { 36 /* 37 * Only called for hugetlbfs pages, hence can ignore THP and the 38 * irq disabled walk. 39 */ 40 return __find_linux_pte(mm->pgd, addr, NULL, NULL); 41 } 42 43 pte_t *huge_pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma, 44 unsigned long addr, unsigned long sz) 45 { 46 p4d_t *p4d; 47 pud_t *pud; 48 pmd_t *pmd; 49 50 addr &= ~(sz - 1); 51 52 p4d = p4d_offset(pgd_offset(mm, addr), addr); 53 if (!mm_pud_folded(mm) && sz >= P4D_SIZE) 54 return (pte_t *)p4d; 55 56 pud = pud_alloc(mm, p4d, addr); 57 if (!pud) 58 return NULL; 59 if (!mm_pmd_folded(mm) && sz >= PUD_SIZE) 60 return (pte_t *)pud; 61 62 pmd = pmd_alloc(mm, pud, addr); 63 if (!pmd) 64 return NULL; 65 66 if (sz >= PMD_SIZE) { 67 /* On 8xx, all hugepages are handled as contiguous PTEs */ 68 if (IS_ENABLED(CONFIG_PPC_8xx)) { 69 int i; 70 71 for (i = 0; i < sz / PMD_SIZE; i++) { 72 if (!pte_alloc_huge(mm, pmd + i, addr)) 73 return NULL; 74 } 75 } 76 return (pte_t *)pmd; 77 } 78 79 return pte_alloc_huge(mm, pmd, addr); 80 } 81 82 #ifdef CONFIG_PPC_BOOK3S_64 83 /* 84 * Tracks gpages after the device tree is scanned and before the 85 * huge_boot_pages list is ready on pseries. 86 */ 87 #define MAX_NUMBER_GPAGES 1024 88 __initdata static u64 gpage_freearray[MAX_NUMBER_GPAGES]; 89 __initdata static unsigned nr_gpages; 90 91 /* 92 * Build list of addresses of gigantic pages. This function is used in early 93 * boot before the buddy allocator is setup. 94 */ 95 void __init pseries_add_gpage(u64 addr, u64 page_size, unsigned long number_of_pages) 96 { 97 if (!addr) 98 return; 99 while (number_of_pages > 0) { 100 gpage_freearray[nr_gpages] = addr; 101 nr_gpages++; 102 number_of_pages--; 103 addr += page_size; 104 } 105 } 106 107 static int __init pseries_alloc_bootmem_huge_page(struct hstate *hstate) 108 { 109 struct huge_bootmem_page *m; 110 if (nr_gpages == 0) 111 return 0; 112 m = phys_to_virt(gpage_freearray[--nr_gpages]); 113 gpage_freearray[nr_gpages] = 0; 114 list_add(&m->list, &huge_boot_pages[0]); 115 m->hstate = hstate; 116 return 1; 117 } 118 119 bool __init hugetlb_node_alloc_supported(void) 120 { 121 return false; 122 } 123 #endif 124 125 126 int __init alloc_bootmem_huge_page(struct hstate *h, int nid) 127 { 128 129 #ifdef CONFIG_PPC_BOOK3S_64 130 if (firmware_has_feature(FW_FEATURE_LPAR) && !radix_enabled()) 131 return pseries_alloc_bootmem_huge_page(h); 132 #endif 133 return __alloc_bootmem_huge_page(h, nid); 134 } 135 136 bool __init arch_hugetlb_valid_size(unsigned long size) 137 { 138 int shift = __ffs(size); 139 int mmu_psize; 140 141 /* Check that it is a page size supported by the hardware and 142 * that it fits within pagetable and slice limits. */ 143 if (size <= PAGE_SIZE || !is_power_of_2(size)) 144 return false; 145 146 mmu_psize = check_and_get_huge_psize(shift); 147 if (mmu_psize < 0) 148 return false; 149 150 BUG_ON(mmu_psize_defs[mmu_psize].shift != shift); 151 152 return true; 153 } 154 155 static int __init add_huge_page_size(unsigned long long size) 156 { 157 int shift = __ffs(size); 158 159 if (!arch_hugetlb_valid_size((unsigned long)size)) 160 return -EINVAL; 161 162 hugetlb_add_hstate(shift - PAGE_SHIFT); 163 return 0; 164 } 165 166 static int __init hugetlbpage_init(void) 167 { 168 bool configured = false; 169 int psize; 170 171 if (hugetlb_disabled) { 172 pr_info("HugeTLB support is disabled!\n"); 173 return 0; 174 } 175 176 if (IS_ENABLED(CONFIG_PPC_BOOK3S_64) && !radix_enabled() && 177 !mmu_has_feature(MMU_FTR_16M_PAGE)) 178 return -ENODEV; 179 180 for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) { 181 unsigned shift; 182 183 if (!mmu_psize_defs[psize].shift) 184 continue; 185 186 shift = mmu_psize_to_shift(psize); 187 188 if (add_huge_page_size(1ULL << shift) < 0) 189 continue; 190 191 configured = true; 192 } 193 194 if (!configured) 195 pr_info("Failed to initialize. Disabling HugeTLB"); 196 197 return 0; 198 } 199 200 arch_initcall(hugetlbpage_init); 201 202 void __init gigantic_hugetlb_cma_reserve(void) 203 { 204 unsigned long order = 0; 205 206 if (radix_enabled()) 207 order = PUD_SHIFT - PAGE_SHIFT; 208 else if (!firmware_has_feature(FW_FEATURE_LPAR) && mmu_psize_defs[MMU_PAGE_16G].shift) 209 /* 210 * For pseries we do use ibm,expected#pages for reserving 16G pages. 211 */ 212 order = mmu_psize_to_shift(MMU_PAGE_16G) - PAGE_SHIFT; 213 214 if (order) 215 hugetlb_cma_reserve(order); 216 } 217