1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Copyright (C) 2012 Regents of the University of California 4 */ 5 6 #ifndef _ASM_RISCV_PGTABLE_64_H 7 #define _ASM_RISCV_PGTABLE_64_H 8 9 #include <linux/bits.h> 10 #include <linux/const.h> 11 #include <asm/errata_list.h> 12 13 extern bool pgtable_l4_enabled; 14 extern bool pgtable_l5_enabled; 15 16 #define PGDIR_SHIFT_L3 30 17 #define PGDIR_SHIFT_L4 39 18 #define PGDIR_SHIFT_L5 48 19 #define PGDIR_SIZE_L3 (_AC(1, UL) << PGDIR_SHIFT_L3) 20 21 #define PGDIR_SHIFT (pgtable_l5_enabled ? PGDIR_SHIFT_L5 : \ 22 (pgtable_l4_enabled ? PGDIR_SHIFT_L4 : PGDIR_SHIFT_L3)) 23 /* Size of region mapped by a page global directory */ 24 #define PGDIR_SIZE (_AC(1, UL) << PGDIR_SHIFT) 25 #define PGDIR_MASK (~(PGDIR_SIZE - 1)) 26 27 /* p4d is folded into pgd in case of 4-level page table */ 28 #define P4D_SHIFT_L3 30 29 #define P4D_SHIFT_L4 39 30 #define P4D_SHIFT_L5 39 31 #define P4D_SHIFT (pgtable_l5_enabled ? P4D_SHIFT_L5 : \ 32 (pgtable_l4_enabled ? P4D_SHIFT_L4 : P4D_SHIFT_L3)) 33 #define P4D_SIZE (_AC(1, UL) << P4D_SHIFT) 34 #define P4D_MASK (~(P4D_SIZE - 1)) 35 36 /* pud is folded into pgd in case of 3-level page table */ 37 #define PUD_SHIFT 30 38 #define PUD_SIZE (_AC(1, UL) << PUD_SHIFT) 39 #define PUD_MASK (~(PUD_SIZE - 1)) 40 41 #define PMD_SHIFT 21 42 /* Size of region mapped by a page middle directory */ 43 #define PMD_SIZE (_AC(1, UL) << PMD_SHIFT) 44 #define PMD_MASK (~(PMD_SIZE - 1)) 45 46 /* Page 4th Directory entry */ 47 typedef struct { 48 unsigned long p4d; 49 } p4d_t; 50 51 #define p4d_val(x) ((x).p4d) 52 #define __p4d(x) ((p4d_t) { (x) }) 53 #define PTRS_PER_P4D (PAGE_SIZE / sizeof(p4d_t)) 54 55 /* Page Upper Directory entry */ 56 typedef struct { 57 unsigned long pud; 58 } pud_t; 59 60 #define pud_val(x) ((x).pud) 61 #define __pud(x) ((pud_t) { (x) }) 62 #define PTRS_PER_PUD (PAGE_SIZE / sizeof(pud_t)) 63 64 /* Page Middle Directory entry */ 65 typedef struct { 66 unsigned long pmd; 67 } pmd_t; 68 69 #define pmd_val(x) ((x).pmd) 70 #define __pmd(x) ((pmd_t) { (x) }) 71 72 #define PTRS_PER_PMD (PAGE_SIZE / sizeof(pmd_t)) 73 74 /* 75 * rv64 PTE format: 76 * | 63 | 62 61 | 60 54 | 53 10 | 9 8 | 7 | 6 | 5 | 4 | 3 | 2 | 1 | 0 77 * N MT RSV PFN reserved for SW D A G U X W R V 78 */ 79 #define _PAGE_PFN_MASK GENMASK(53, 10) 80 81 /* 82 * [62:61] Svpbmt Memory Type definitions: 83 * 84 * 00 - PMA Normal Cacheable, No change to implied PMA memory type 85 * 01 - NC Non-cacheable, idempotent, weakly-ordered Main Memory 86 * 10 - IO Non-cacheable, non-idempotent, strongly-ordered I/O memory 87 * 11 - Rsvd Reserved for future standard use 88 */ 89 #define _PAGE_NOCACHE_SVPBMT (1UL << 61) 90 #define _PAGE_IO_SVPBMT (1UL << 62) 91 #define _PAGE_MTMASK_SVPBMT (_PAGE_NOCACHE_SVPBMT | _PAGE_IO_SVPBMT) 92 93 /* 94 * [63:59] T-Head Memory Type definitions: 95 * 96 * 00000 - NC Weakly-ordered, Non-cacheable, Non-bufferable, Non-shareable, Non-trustable 97 * 01110 - PMA Weakly-ordered, Cacheable, Bufferable, Shareable, Non-trustable 98 * 10000 - IO Strongly-ordered, Non-cacheable, Non-bufferable, Non-shareable, Non-trustable 99 */ 100 #define _PAGE_PMA_THEAD ((1UL << 62) | (1UL << 61) | (1UL << 60)) 101 #define _PAGE_NOCACHE_THEAD 0UL 102 #define _PAGE_IO_THEAD (1UL << 63) 103 #define _PAGE_MTMASK_THEAD (_PAGE_PMA_THEAD | _PAGE_IO_THEAD | (1UL << 59)) 104 105 static inline u64 riscv_page_mtmask(void) 106 { 107 u64 val; 108 109 ALT_SVPBMT(val, _PAGE_MTMASK); 110 return val; 111 } 112 113 static inline u64 riscv_page_nocache(void) 114 { 115 u64 val; 116 117 ALT_SVPBMT(val, _PAGE_NOCACHE); 118 return val; 119 } 120 121 static inline u64 riscv_page_io(void) 122 { 123 u64 val; 124 125 ALT_SVPBMT(val, _PAGE_IO); 126 return val; 127 } 128 129 #define _PAGE_NOCACHE riscv_page_nocache() 130 #define _PAGE_IO riscv_page_io() 131 #define _PAGE_MTMASK riscv_page_mtmask() 132 133 /* Set of bits to preserve across pte_modify() */ 134 #define _PAGE_CHG_MASK (~(unsigned long)(_PAGE_PRESENT | _PAGE_READ | \ 135 _PAGE_WRITE | _PAGE_EXEC | \ 136 _PAGE_USER | _PAGE_GLOBAL | \ 137 _PAGE_MTMASK)) 138 139 static inline int pud_present(pud_t pud) 140 { 141 return (pud_val(pud) & _PAGE_PRESENT); 142 } 143 144 static inline int pud_none(pud_t pud) 145 { 146 return (pud_val(pud) == 0); 147 } 148 149 static inline int pud_bad(pud_t pud) 150 { 151 return !pud_present(pud); 152 } 153 154 #define pud_leaf pud_leaf 155 static inline int pud_leaf(pud_t pud) 156 { 157 return pud_present(pud) && (pud_val(pud) & _PAGE_LEAF); 158 } 159 160 static inline int pud_user(pud_t pud) 161 { 162 return pud_val(pud) & _PAGE_USER; 163 } 164 165 static inline void set_pud(pud_t *pudp, pud_t pud) 166 { 167 *pudp = pud; 168 } 169 170 static inline void pud_clear(pud_t *pudp) 171 { 172 set_pud(pudp, __pud(0)); 173 } 174 175 static inline pud_t pfn_pud(unsigned long pfn, pgprot_t prot) 176 { 177 return __pud((pfn << _PAGE_PFN_SHIFT) | pgprot_val(prot)); 178 } 179 180 static inline unsigned long _pud_pfn(pud_t pud) 181 { 182 return __page_val_to_pfn(pud_val(pud)); 183 } 184 185 static inline pmd_t *pud_pgtable(pud_t pud) 186 { 187 return (pmd_t *)pfn_to_virt(__page_val_to_pfn(pud_val(pud))); 188 } 189 190 static inline struct page *pud_page(pud_t pud) 191 { 192 return pfn_to_page(__page_val_to_pfn(pud_val(pud))); 193 } 194 195 #define mm_p4d_folded mm_p4d_folded 196 static inline bool mm_p4d_folded(struct mm_struct *mm) 197 { 198 if (pgtable_l5_enabled) 199 return false; 200 201 return true; 202 } 203 204 #define mm_pud_folded mm_pud_folded 205 static inline bool mm_pud_folded(struct mm_struct *mm) 206 { 207 if (pgtable_l4_enabled) 208 return false; 209 210 return true; 211 } 212 213 #define pmd_index(addr) (((addr) >> PMD_SHIFT) & (PTRS_PER_PMD - 1)) 214 215 static inline pmd_t pfn_pmd(unsigned long pfn, pgprot_t prot) 216 { 217 unsigned long prot_val = pgprot_val(prot); 218 219 ALT_THEAD_PMA(prot_val); 220 221 return __pmd((pfn << _PAGE_PFN_SHIFT) | prot_val); 222 } 223 224 static inline unsigned long _pmd_pfn(pmd_t pmd) 225 { 226 return __page_val_to_pfn(pmd_val(pmd)); 227 } 228 229 #define mk_pmd(page, prot) pfn_pmd(page_to_pfn(page), prot) 230 231 #define pmd_ERROR(e) \ 232 pr_err("%s:%d: bad pmd %016lx.\n", __FILE__, __LINE__, pmd_val(e)) 233 234 #define pud_ERROR(e) \ 235 pr_err("%s:%d: bad pud %016lx.\n", __FILE__, __LINE__, pud_val(e)) 236 237 #define p4d_ERROR(e) \ 238 pr_err("%s:%d: bad p4d %016lx.\n", __FILE__, __LINE__, p4d_val(e)) 239 240 static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) 241 { 242 if (pgtable_l4_enabled) 243 *p4dp = p4d; 244 else 245 set_pud((pud_t *)p4dp, (pud_t){ p4d_val(p4d) }); 246 } 247 248 static inline int p4d_none(p4d_t p4d) 249 { 250 if (pgtable_l4_enabled) 251 return (p4d_val(p4d) == 0); 252 253 return 0; 254 } 255 256 static inline int p4d_present(p4d_t p4d) 257 { 258 if (pgtable_l4_enabled) 259 return (p4d_val(p4d) & _PAGE_PRESENT); 260 261 return 1; 262 } 263 264 static inline int p4d_bad(p4d_t p4d) 265 { 266 if (pgtable_l4_enabled) 267 return !p4d_present(p4d); 268 269 return 0; 270 } 271 272 static inline void p4d_clear(p4d_t *p4d) 273 { 274 if (pgtable_l4_enabled) 275 set_p4d(p4d, __p4d(0)); 276 } 277 278 static inline p4d_t pfn_p4d(unsigned long pfn, pgprot_t prot) 279 { 280 return __p4d((pfn << _PAGE_PFN_SHIFT) | pgprot_val(prot)); 281 } 282 283 static inline unsigned long _p4d_pfn(p4d_t p4d) 284 { 285 return __page_val_to_pfn(p4d_val(p4d)); 286 } 287 288 static inline pud_t *p4d_pgtable(p4d_t p4d) 289 { 290 if (pgtable_l4_enabled) 291 return (pud_t *)pfn_to_virt(__page_val_to_pfn(p4d_val(p4d))); 292 293 return (pud_t *)pud_pgtable((pud_t) { p4d_val(p4d) }); 294 } 295 #define p4d_page_vaddr(p4d) ((unsigned long)p4d_pgtable(p4d)) 296 297 static inline struct page *p4d_page(p4d_t p4d) 298 { 299 return pfn_to_page(__page_val_to_pfn(p4d_val(p4d))); 300 } 301 302 #define pud_index(addr) (((addr) >> PUD_SHIFT) & (PTRS_PER_PUD - 1)) 303 304 #define pud_offset pud_offset 305 static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address) 306 { 307 if (pgtable_l4_enabled) 308 return p4d_pgtable(*p4d) + pud_index(address); 309 310 return (pud_t *)p4d; 311 } 312 313 static inline void set_pgd(pgd_t *pgdp, pgd_t pgd) 314 { 315 if (pgtable_l5_enabled) 316 *pgdp = pgd; 317 else 318 set_p4d((p4d_t *)pgdp, (p4d_t){ pgd_val(pgd) }); 319 } 320 321 static inline int pgd_none(pgd_t pgd) 322 { 323 if (pgtable_l5_enabled) 324 return (pgd_val(pgd) == 0); 325 326 return 0; 327 } 328 329 static inline int pgd_present(pgd_t pgd) 330 { 331 if (pgtable_l5_enabled) 332 return (pgd_val(pgd) & _PAGE_PRESENT); 333 334 return 1; 335 } 336 337 static inline int pgd_bad(pgd_t pgd) 338 { 339 if (pgtable_l5_enabled) 340 return !pgd_present(pgd); 341 342 return 0; 343 } 344 345 static inline void pgd_clear(pgd_t *pgd) 346 { 347 if (pgtable_l5_enabled) 348 set_pgd(pgd, __pgd(0)); 349 } 350 351 static inline p4d_t *pgd_pgtable(pgd_t pgd) 352 { 353 if (pgtable_l5_enabled) 354 return (p4d_t *)pfn_to_virt(__page_val_to_pfn(pgd_val(pgd))); 355 356 return (p4d_t *)p4d_pgtable((p4d_t) { pgd_val(pgd) }); 357 } 358 #define pgd_page_vaddr(pgd) ((unsigned long)pgd_pgtable(pgd)) 359 360 static inline struct page *pgd_page(pgd_t pgd) 361 { 362 return pfn_to_page(__page_val_to_pfn(pgd_val(pgd))); 363 } 364 #define pgd_page(pgd) pgd_page(pgd) 365 366 #define p4d_index(addr) (((addr) >> P4D_SHIFT) & (PTRS_PER_P4D - 1)) 367 368 #define p4d_offset p4d_offset 369 static inline p4d_t *p4d_offset(pgd_t *pgd, unsigned long address) 370 { 371 if (pgtable_l5_enabled) 372 return pgd_pgtable(*pgd) + p4d_index(address); 373 374 return (p4d_t *)pgd; 375 } 376 377 #endif /* _ASM_RISCV_PGTABLE_64_H */ 378