1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited 4 */ 5 #include <linux/init.h> 6 #include <linux/export.h> 7 #include <linux/mm.h> 8 #include <asm/pgalloc.h> 9 #include <asm/pgtable.h> 10 #include <asm/tlbflush.h> 11 12 pgd_t *pgd_alloc(struct mm_struct *mm) 13 { 14 pgd_t *init, *ret = NULL; 15 struct ptdesc *ptdesc = pagetable_alloc(GFP_KERNEL & ~__GFP_HIGHMEM, 0); 16 17 if (ptdesc) { 18 ret = (pgd_t *)ptdesc_address(ptdesc); 19 init = pgd_offset(&init_mm, 0UL); 20 pgd_init(ret); 21 memcpy(ret + USER_PTRS_PER_PGD, init + USER_PTRS_PER_PGD, 22 (PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t)); 23 } 24 25 return ret; 26 } 27 EXPORT_SYMBOL_GPL(pgd_alloc); 28 29 void pgd_init(void *addr) 30 { 31 unsigned long *p, *end; 32 unsigned long entry; 33 34 #if !defined(__PAGETABLE_PUD_FOLDED) 35 entry = (unsigned long)invalid_pud_table; 36 #elif !defined(__PAGETABLE_PMD_FOLDED) 37 entry = (unsigned long)invalid_pmd_table; 38 #else 39 entry = (unsigned long)invalid_pte_table; 40 #endif 41 42 p = (unsigned long *)addr; 43 end = p + PTRS_PER_PGD; 44 45 do { 46 p[0] = entry; 47 p[1] = entry; 48 p[2] = entry; 49 p[3] = entry; 50 p[4] = entry; 51 p += 8; 52 p[-3] = entry; 53 p[-2] = entry; 54 p[-1] = entry; 55 } while (p != end); 56 } 57 EXPORT_SYMBOL_GPL(pgd_init); 58 59 #ifndef __PAGETABLE_PMD_FOLDED 60 void pmd_init(void *addr) 61 { 62 unsigned long *p, *end; 63 unsigned long pagetable = (unsigned long)invalid_pte_table; 64 65 p = (unsigned long *)addr; 66 end = p + PTRS_PER_PMD; 67 68 do { 69 p[0] = pagetable; 70 p[1] = pagetable; 71 p[2] = pagetable; 72 p[3] = pagetable; 73 p[4] = pagetable; 74 p += 8; 75 p[-3] = pagetable; 76 p[-2] = pagetable; 77 p[-1] = pagetable; 78 } while (p != end); 79 } 80 EXPORT_SYMBOL_GPL(pmd_init); 81 #endif 82 83 #ifndef __PAGETABLE_PUD_FOLDED 84 void pud_init(void *addr) 85 { 86 unsigned long *p, *end; 87 unsigned long pagetable = (unsigned long)invalid_pmd_table; 88 89 p = (unsigned long *)addr; 90 end = p + PTRS_PER_PUD; 91 92 do { 93 p[0] = pagetable; 94 p[1] = pagetable; 95 p[2] = pagetable; 96 p[3] = pagetable; 97 p[4] = pagetable; 98 p += 8; 99 p[-3] = pagetable; 100 p[-2] = pagetable; 101 p[-1] = pagetable; 102 } while (p != end); 103 } 104 EXPORT_SYMBOL_GPL(pud_init); 105 #endif 106 107 pmd_t mk_pmd(struct page *page, pgprot_t prot) 108 { 109 pmd_t pmd; 110 111 pmd_val(pmd) = (page_to_pfn(page) << _PFN_SHIFT) | pgprot_val(prot); 112 113 return pmd; 114 } 115 116 void set_pmd_at(struct mm_struct *mm, unsigned long addr, 117 pmd_t *pmdp, pmd_t pmd) 118 { 119 *pmdp = pmd; 120 flush_tlb_all(); 121 } 122 123 void __init pagetable_init(void) 124 { 125 /* Initialize the entire pgd. */ 126 pgd_init(swapper_pg_dir); 127 pgd_init(invalid_pg_dir); 128 #ifndef __PAGETABLE_PUD_FOLDED 129 pud_init(invalid_pud_table); 130 #endif 131 #ifndef __PAGETABLE_PMD_FOLDED 132 pmd_init(invalid_pmd_table); 133 #endif 134 } 135