1 #include <linux/gfp.h>
2 #include <linux/initrd.h>
3 #include <linux/ioport.h>
4 #include <linux/swap.h>
5 #include <linux/memblock.h>
6 #include <linux/swapfile.h>
7 #include <linux/swapops.h>
8 #include <linux/kmemleak.h>
9 #include <linux/sched/task.h>
10 #include <linux/execmem.h>
11
12 #include <asm/set_memory.h>
13 #include <asm/cpu_device_id.h>
14 #include <asm/e820/api.h>
15 #include <asm/init.h>
16 #include <asm/page.h>
17 #include <asm/page_types.h>
18 #include <asm/sections.h>
19 #include <asm/setup.h>
20 #include <asm/tlbflush.h>
21 #include <asm/tlb.h>
22 #include <asm/proto.h>
23 #include <asm/dma.h> /* for MAX_DMA_PFN */
24 #include <asm/kaslr.h>
25 #include <asm/hypervisor.h>
26 #include <asm/cpufeature.h>
27 #include <asm/pti.h>
28 #include <asm/text-patching.h>
29 #include <asm/memtype.h>
30 #include <asm/paravirt.h>
31 #include <asm/mmu_context.h>
32
33 /*
34 * We need to define the tracepoints somewhere, and tlb.c
35 * is only compiled when SMP=y.
36 */
37 #include <trace/events/tlb.h>
38
39 #include "mm_internal.h"
40
41 /*
42 * Tables translating between page_cache_type_t and pte encoding.
43 *
44 * The default values are defined statically as minimal supported mode;
45 * WC and WT fall back to UC-. pat_init() updates these values to support
46 * more cache modes, WC and WT, when it is safe to do so. See pat_init()
47 * for the details. Note, __early_ioremap() used during early boot-time
48 * takes pgprot_t (pte encoding) and does not use these tables.
49 *
50 * Index into __cachemode2pte_tbl[] is the cachemode.
51 *
52 * Index into __pte2cachemode_tbl[] are the caching attribute bits of the pte
53 * (_PAGE_PWT, _PAGE_PCD, _PAGE_PAT) at index bit positions 0, 1, 2.
54 */
55 static uint16_t __cachemode2pte_tbl[_PAGE_CACHE_MODE_NUM] = {
56 [_PAGE_CACHE_MODE_WB ] = 0 | 0 ,
57 [_PAGE_CACHE_MODE_WC ] = 0 | _PAGE_PCD,
58 [_PAGE_CACHE_MODE_UC_MINUS] = 0 | _PAGE_PCD,
59 [_PAGE_CACHE_MODE_UC ] = _PAGE_PWT | _PAGE_PCD,
60 [_PAGE_CACHE_MODE_WT ] = 0 | _PAGE_PCD,
61 [_PAGE_CACHE_MODE_WP ] = 0 | _PAGE_PCD,
62 };
63
cachemode2protval(enum page_cache_mode pcm)64 unsigned long cachemode2protval(enum page_cache_mode pcm)
65 {
66 if (likely(pcm == 0))
67 return 0;
68 return __cachemode2pte_tbl[pcm];
69 }
70 EXPORT_SYMBOL(cachemode2protval);
71
72 static uint8_t __pte2cachemode_tbl[8] = {
73 [__pte2cm_idx( 0 | 0 | 0 )] = _PAGE_CACHE_MODE_WB,
74 [__pte2cm_idx(_PAGE_PWT | 0 | 0 )] = _PAGE_CACHE_MODE_UC_MINUS,
75 [__pte2cm_idx( 0 | _PAGE_PCD | 0 )] = _PAGE_CACHE_MODE_UC_MINUS,
76 [__pte2cm_idx(_PAGE_PWT | _PAGE_PCD | 0 )] = _PAGE_CACHE_MODE_UC,
77 [__pte2cm_idx( 0 | 0 | _PAGE_PAT)] = _PAGE_CACHE_MODE_WB,
78 [__pte2cm_idx(_PAGE_PWT | 0 | _PAGE_PAT)] = _PAGE_CACHE_MODE_UC_MINUS,
79 [__pte2cm_idx(0 | _PAGE_PCD | _PAGE_PAT)] = _PAGE_CACHE_MODE_UC_MINUS,
80 [__pte2cm_idx(_PAGE_PWT | _PAGE_PCD | _PAGE_PAT)] = _PAGE_CACHE_MODE_UC,
81 };
82
83 /*
84 * Check that the write-protect PAT entry is set for write-protect.
85 * To do this without making assumptions how PAT has been set up (Xen has
86 * another layout than the kernel), translate the _PAGE_CACHE_MODE_WP cache
87 * mode via the __cachemode2pte_tbl[] into protection bits (those protection
88 * bits will select a cache mode of WP or better), and then translate the
89 * protection bits back into the cache mode using __pte2cm_idx() and the
90 * __pte2cachemode_tbl[] array. This will return the really used cache mode.
91 */
x86_has_pat_wp(void)92 bool x86_has_pat_wp(void)
93 {
94 uint16_t prot = __cachemode2pte_tbl[_PAGE_CACHE_MODE_WP];
95
96 return __pte2cachemode_tbl[__pte2cm_idx(prot)] == _PAGE_CACHE_MODE_WP;
97 }
98
pgprot2cachemode(pgprot_t pgprot)99 enum page_cache_mode pgprot2cachemode(pgprot_t pgprot)
100 {
101 unsigned long masked;
102
103 masked = pgprot_val(pgprot) & _PAGE_CACHE_MASK;
104 if (likely(masked == 0))
105 return 0;
106 return __pte2cachemode_tbl[__pte2cm_idx(masked)];
107 }
108
109 static unsigned long __initdata pgt_buf_start;
110 static unsigned long __initdata pgt_buf_end;
111 static unsigned long __initdata pgt_buf_top;
112
113 static unsigned long min_pfn_mapped;
114
115 static bool __initdata can_use_brk_pgt = true;
116
117 /*
118 * Pages returned are already directly mapped.
119 *
120 * Changing that is likely to break Xen, see commit:
121 *
122 * 279b706 x86,xen: introduce x86_init.mapping.pagetable_reserve
123 *
124 * for detailed information.
125 */
alloc_low_pages(unsigned int num)126 __ref void *alloc_low_pages(unsigned int num)
127 {
128 unsigned long pfn;
129 int i;
130
131 if (after_bootmem) {
132 unsigned int order;
133
134 order = get_order((unsigned long)num << PAGE_SHIFT);
135 return (void *)__get_free_pages(GFP_ATOMIC | __GFP_ZERO, order);
136 }
137
138 if ((pgt_buf_end + num) > pgt_buf_top || !can_use_brk_pgt) {
139 unsigned long ret = 0;
140
141 if (min_pfn_mapped < max_pfn_mapped) {
142 ret = memblock_phys_alloc_range(
143 PAGE_SIZE * num, PAGE_SIZE,
144 min_pfn_mapped << PAGE_SHIFT,
145 max_pfn_mapped << PAGE_SHIFT);
146 }
147 if (!ret && can_use_brk_pgt)
148 ret = __pa(extend_brk(PAGE_SIZE * num, PAGE_SIZE));
149
150 if (!ret)
151 panic("alloc_low_pages: can not alloc memory");
152
153 pfn = ret >> PAGE_SHIFT;
154 } else {
155 pfn = pgt_buf_end;
156 pgt_buf_end += num;
157 }
158
159 for (i = 0; i < num; i++) {
160 void *adr;
161
162 adr = __va((pfn + i) << PAGE_SHIFT);
163 clear_page(adr);
164 }
165
166 return __va(pfn << PAGE_SHIFT);
167 }
168
169 /*
170 * By default need to be able to allocate page tables below PGD firstly for
171 * the 0-ISA_END_ADDRESS range and secondly for the initial PMD_SIZE mapping.
172 * With KASLR memory randomization, depending on the machine e820 memory and the
173 * PUD alignment, twice that many pages may be needed when KASLR memory
174 * randomization is enabled.
175 */
176
177 #define INIT_PGD_PAGE_TABLES 4
178
179 #ifndef CONFIG_RANDOMIZE_MEMORY
180 #define INIT_PGD_PAGE_COUNT (2 * INIT_PGD_PAGE_TABLES)
181 #else
182 #define INIT_PGD_PAGE_COUNT (4 * INIT_PGD_PAGE_TABLES)
183 #endif
184
185 #define INIT_PGT_BUF_SIZE (INIT_PGD_PAGE_COUNT * PAGE_SIZE)
186 RESERVE_BRK(early_pgt_alloc, INIT_PGT_BUF_SIZE);
early_alloc_pgt_buf(void)187 void __init early_alloc_pgt_buf(void)
188 {
189 unsigned long tables = INIT_PGT_BUF_SIZE;
190 phys_addr_t base;
191
192 base = __pa(extend_brk(tables, PAGE_SIZE));
193
194 pgt_buf_start = base >> PAGE_SHIFT;
195 pgt_buf_end = pgt_buf_start;
196 pgt_buf_top = pgt_buf_start + (tables >> PAGE_SHIFT);
197 }
198
199 int after_bootmem;
200
201 early_param_on_off("gbpages", "nogbpages", direct_gbpages, CONFIG_X86_DIRECT_GBPAGES);
202
203 struct map_range {
204 unsigned long start;
205 unsigned long end;
206 unsigned page_size_mask;
207 };
208
209 static int page_size_mask;
210
211 /*
212 * Save some of cr4 feature set we're using (e.g. Pentium 4MB
213 * enable and PPro Global page enable), so that any CPU's that boot
214 * up after us can get the correct flags. Invoked on the boot CPU.
215 */
cr4_set_bits_and_update_boot(unsigned long mask)216 static inline void cr4_set_bits_and_update_boot(unsigned long mask)
217 {
218 mmu_cr4_features |= mask;
219 if (trampoline_cr4_features)
220 *trampoline_cr4_features = mmu_cr4_features;
221 cr4_set_bits(mask);
222 }
223
probe_page_size_mask(void)224 static void __init probe_page_size_mask(void)
225 {
226 /*
227 * For pagealloc debugging, identity mapping will use small pages.
228 * This will simplify cpa(), which otherwise needs to support splitting
229 * large pages into small in interrupt context, etc.
230 */
231 if (boot_cpu_has(X86_FEATURE_PSE) && !debug_pagealloc_enabled())
232 page_size_mask |= 1 << PG_LEVEL_2M;
233 else
234 direct_gbpages = 0;
235
236 /* Enable PSE if available */
237 if (boot_cpu_has(X86_FEATURE_PSE))
238 cr4_set_bits_and_update_boot(X86_CR4_PSE);
239
240 /* Enable PGE if available */
241 __supported_pte_mask &= ~_PAGE_GLOBAL;
242 if (boot_cpu_has(X86_FEATURE_PGE)) {
243 cr4_set_bits_and_update_boot(X86_CR4_PGE);
244 __supported_pte_mask |= _PAGE_GLOBAL;
245 }
246
247 /* By the default is everything supported: */
248 __default_kernel_pte_mask = __supported_pte_mask;
249 /* Except when with PTI where the kernel is mostly non-Global: */
250 if (cpu_feature_enabled(X86_FEATURE_PTI))
251 __default_kernel_pte_mask &= ~_PAGE_GLOBAL;
252
253 /* Enable 1 GB linear kernel mappings if available: */
254 if (direct_gbpages && boot_cpu_has(X86_FEATURE_GBPAGES)) {
255 printk(KERN_INFO "Using GB pages for direct mapping\n");
256 page_size_mask |= 1 << PG_LEVEL_1G;
257 } else {
258 direct_gbpages = 0;
259 }
260 }
261
262 /*
263 * INVLPG may not properly flush Global entries on
264 * these CPUs. New microcode fixes the issue.
265 */
266 static const struct x86_cpu_id invlpg_miss_ids[] = {
267 X86_MATCH_VFM(INTEL_ALDERLAKE, 0x2e),
268 X86_MATCH_VFM(INTEL_ALDERLAKE_L, 0x42c),
269 X86_MATCH_VFM(INTEL_ATOM_GRACEMONT, 0x11),
270 X86_MATCH_VFM(INTEL_RAPTORLAKE, 0x118),
271 X86_MATCH_VFM(INTEL_RAPTORLAKE_P, 0x4117),
272 X86_MATCH_VFM(INTEL_RAPTORLAKE_S, 0x2e),
273 {}
274 };
275
setup_pcid(void)276 static void setup_pcid(void)
277 {
278 const struct x86_cpu_id *invlpg_miss_match;
279
280 if (!IS_ENABLED(CONFIG_X86_64))
281 return;
282
283 if (!boot_cpu_has(X86_FEATURE_PCID))
284 return;
285
286 invlpg_miss_match = x86_match_cpu(invlpg_miss_ids);
287
288 if (invlpg_miss_match &&
289 boot_cpu_data.microcode < invlpg_miss_match->driver_data) {
290 pr_info("Incomplete global flushes, disabling PCID");
291 setup_clear_cpu_cap(X86_FEATURE_PCID);
292 return;
293 }
294
295 if (boot_cpu_has(X86_FEATURE_PGE)) {
296 /*
297 * This can't be cr4_set_bits_and_update_boot() -- the
298 * trampoline code can't handle CR4.PCIDE and it wouldn't
299 * do any good anyway. Despite the name,
300 * cr4_set_bits_and_update_boot() doesn't actually cause
301 * the bits in question to remain set all the way through
302 * the secondary boot asm.
303 *
304 * Instead, we brute-force it and set CR4.PCIDE manually in
305 * start_secondary().
306 */
307 cr4_set_bits(X86_CR4_PCIDE);
308 } else {
309 /*
310 * flush_tlb_all(), as currently implemented, won't work if
311 * PCID is on but PGE is not. Since that combination
312 * doesn't exist on real hardware, there's no reason to try
313 * to fully support it, but it's polite to avoid corrupting
314 * data if we're on an improperly configured VM.
315 */
316 setup_clear_cpu_cap(X86_FEATURE_PCID);
317 }
318 }
319
320 #ifdef CONFIG_X86_32
321 #define NR_RANGE_MR 3
322 #else /* CONFIG_X86_64 */
323 #define NR_RANGE_MR 5
324 #endif
325
save_mr(struct map_range * mr,int nr_range,unsigned long start_pfn,unsigned long end_pfn,unsigned long page_size_mask)326 static int __meminit save_mr(struct map_range *mr, int nr_range,
327 unsigned long start_pfn, unsigned long end_pfn,
328 unsigned long page_size_mask)
329 {
330 if (start_pfn < end_pfn) {
331 if (nr_range >= NR_RANGE_MR)
332 panic("run out of range for init_memory_mapping\n");
333 mr[nr_range].start = start_pfn<<PAGE_SHIFT;
334 mr[nr_range].end = end_pfn<<PAGE_SHIFT;
335 mr[nr_range].page_size_mask = page_size_mask;
336 nr_range++;
337 }
338
339 return nr_range;
340 }
341
342 /*
343 * adjust the page_size_mask for small range to go with
344 * big page size instead small one if nearby are ram too.
345 */
adjust_range_page_size_mask(struct map_range * mr,int nr_range)346 static void __ref adjust_range_page_size_mask(struct map_range *mr,
347 int nr_range)
348 {
349 int i;
350
351 for (i = 0; i < nr_range; i++) {
352 if ((page_size_mask & (1<<PG_LEVEL_2M)) &&
353 !(mr[i].page_size_mask & (1<<PG_LEVEL_2M))) {
354 unsigned long start = round_down(mr[i].start, PMD_SIZE);
355 unsigned long end = round_up(mr[i].end, PMD_SIZE);
356
357 #ifdef CONFIG_X86_32
358 if ((end >> PAGE_SHIFT) > max_low_pfn)
359 continue;
360 #endif
361
362 if (memblock_is_region_memory(start, end - start))
363 mr[i].page_size_mask |= 1<<PG_LEVEL_2M;
364 }
365 if ((page_size_mask & (1<<PG_LEVEL_1G)) &&
366 !(mr[i].page_size_mask & (1<<PG_LEVEL_1G))) {
367 unsigned long start = round_down(mr[i].start, PUD_SIZE);
368 unsigned long end = round_up(mr[i].end, PUD_SIZE);
369
370 if (memblock_is_region_memory(start, end - start))
371 mr[i].page_size_mask |= 1<<PG_LEVEL_1G;
372 }
373 }
374 }
375
page_size_string(struct map_range * mr)376 static const char *page_size_string(struct map_range *mr)
377 {
378 static const char str_1g[] = "1G";
379 static const char str_2m[] = "2M";
380 static const char str_4m[] = "4M";
381 static const char str_4k[] = "4k";
382
383 if (mr->page_size_mask & (1<<PG_LEVEL_1G))
384 return str_1g;
385 /*
386 * 32-bit without PAE has a 4M large page size.
387 * PG_LEVEL_2M is misnamed, but we can at least
388 * print out the right size in the string.
389 */
390 if (IS_ENABLED(CONFIG_X86_32) &&
391 !IS_ENABLED(CONFIG_X86_PAE) &&
392 mr->page_size_mask & (1<<PG_LEVEL_2M))
393 return str_4m;
394
395 if (mr->page_size_mask & (1<<PG_LEVEL_2M))
396 return str_2m;
397
398 return str_4k;
399 }
400
split_mem_range(struct map_range * mr,int nr_range,unsigned long start,unsigned long end)401 static int __meminit split_mem_range(struct map_range *mr, int nr_range,
402 unsigned long start,
403 unsigned long end)
404 {
405 unsigned long start_pfn, end_pfn, limit_pfn;
406 unsigned long pfn;
407 int i;
408
409 limit_pfn = PFN_DOWN(end);
410
411 /* head if not big page alignment ? */
412 pfn = start_pfn = PFN_DOWN(start);
413 #ifdef CONFIG_X86_32
414 /*
415 * Don't use a large page for the first 2/4MB of memory
416 * because there are often fixed size MTRRs in there
417 * and overlapping MTRRs into large pages can cause
418 * slowdowns.
419 */
420 if (pfn == 0)
421 end_pfn = PFN_DOWN(PMD_SIZE);
422 else
423 end_pfn = round_up(pfn, PFN_DOWN(PMD_SIZE));
424 #else /* CONFIG_X86_64 */
425 end_pfn = round_up(pfn, PFN_DOWN(PMD_SIZE));
426 #endif
427 if (end_pfn > limit_pfn)
428 end_pfn = limit_pfn;
429 if (start_pfn < end_pfn) {
430 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 0);
431 pfn = end_pfn;
432 }
433
434 /* big page (2M) range */
435 start_pfn = round_up(pfn, PFN_DOWN(PMD_SIZE));
436 #ifdef CONFIG_X86_32
437 end_pfn = round_down(limit_pfn, PFN_DOWN(PMD_SIZE));
438 #else /* CONFIG_X86_64 */
439 end_pfn = round_up(pfn, PFN_DOWN(PUD_SIZE));
440 if (end_pfn > round_down(limit_pfn, PFN_DOWN(PMD_SIZE)))
441 end_pfn = round_down(limit_pfn, PFN_DOWN(PMD_SIZE));
442 #endif
443
444 if (start_pfn < end_pfn) {
445 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn,
446 page_size_mask & (1<<PG_LEVEL_2M));
447 pfn = end_pfn;
448 }
449
450 #ifdef CONFIG_X86_64
451 /* big page (1G) range */
452 start_pfn = round_up(pfn, PFN_DOWN(PUD_SIZE));
453 end_pfn = round_down(limit_pfn, PFN_DOWN(PUD_SIZE));
454 if (start_pfn < end_pfn) {
455 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn,
456 page_size_mask &
457 ((1<<PG_LEVEL_2M)|(1<<PG_LEVEL_1G)));
458 pfn = end_pfn;
459 }
460
461 /* tail is not big page (1G) alignment */
462 start_pfn = round_up(pfn, PFN_DOWN(PMD_SIZE));
463 end_pfn = round_down(limit_pfn, PFN_DOWN(PMD_SIZE));
464 if (start_pfn < end_pfn) {
465 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn,
466 page_size_mask & (1<<PG_LEVEL_2M));
467 pfn = end_pfn;
468 }
469 #endif
470
471 /* tail is not big page (2M) alignment */
472 start_pfn = pfn;
473 end_pfn = limit_pfn;
474 nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 0);
475
476 if (!after_bootmem)
477 adjust_range_page_size_mask(mr, nr_range);
478
479 /* try to merge same page size and continuous */
480 for (i = 0; nr_range > 1 && i < nr_range - 1; i++) {
481 unsigned long old_start;
482 if (mr[i].end != mr[i+1].start ||
483 mr[i].page_size_mask != mr[i+1].page_size_mask)
484 continue;
485 /* move it */
486 old_start = mr[i].start;
487 memmove(&mr[i], &mr[i+1],
488 (nr_range - 1 - i) * sizeof(struct map_range));
489 mr[i--].start = old_start;
490 nr_range--;
491 }
492
493 for (i = 0; i < nr_range; i++)
494 pr_debug(" [mem %#010lx-%#010lx] page %s\n",
495 mr[i].start, mr[i].end - 1,
496 page_size_string(&mr[i]));
497
498 return nr_range;
499 }
500
501 struct range pfn_mapped[E820_MAX_ENTRIES];
502 int nr_pfn_mapped;
503
add_pfn_range_mapped(unsigned long start_pfn,unsigned long end_pfn)504 static void add_pfn_range_mapped(unsigned long start_pfn, unsigned long end_pfn)
505 {
506 nr_pfn_mapped = add_range_with_merge(pfn_mapped, E820_MAX_ENTRIES,
507 nr_pfn_mapped, start_pfn, end_pfn);
508 nr_pfn_mapped = clean_sort_range(pfn_mapped, E820_MAX_ENTRIES);
509
510 max_pfn_mapped = max(max_pfn_mapped, end_pfn);
511
512 if (start_pfn < (1UL<<(32-PAGE_SHIFT)))
513 max_low_pfn_mapped = max(max_low_pfn_mapped,
514 min(end_pfn, 1UL<<(32-PAGE_SHIFT)));
515 }
516
pfn_range_is_mapped(unsigned long start_pfn,unsigned long end_pfn)517 bool pfn_range_is_mapped(unsigned long start_pfn, unsigned long end_pfn)
518 {
519 int i;
520
521 for (i = 0; i < nr_pfn_mapped; i++)
522 if ((start_pfn >= pfn_mapped[i].start) &&
523 (end_pfn <= pfn_mapped[i].end))
524 return true;
525
526 return false;
527 }
528
529 /*
530 * Setup the direct mapping of the physical memory at PAGE_OFFSET.
531 * This runs before bootmem is initialized and gets pages directly from
532 * the physical memory. To access them they are temporarily mapped.
533 */
init_memory_mapping(unsigned long start,unsigned long end,pgprot_t prot)534 unsigned long __ref init_memory_mapping(unsigned long start,
535 unsigned long end, pgprot_t prot)
536 {
537 struct map_range mr[NR_RANGE_MR];
538 unsigned long ret = 0;
539 int nr_range, i;
540
541 pr_debug("init_memory_mapping: [mem %#010lx-%#010lx]\n",
542 start, end - 1);
543
544 memset(mr, 0, sizeof(mr));
545 nr_range = split_mem_range(mr, 0, start, end);
546
547 for (i = 0; i < nr_range; i++)
548 ret = kernel_physical_mapping_init(mr[i].start, mr[i].end,
549 mr[i].page_size_mask,
550 prot);
551
552 add_pfn_range_mapped(start >> PAGE_SHIFT, ret >> PAGE_SHIFT);
553
554 return ret >> PAGE_SHIFT;
555 }
556
557 /*
558 * We need to iterate through the E820 memory map and create direct mappings
559 * for only E820_TYPE_RAM and E820_KERN_RESERVED regions. We cannot simply
560 * create direct mappings for all pfns from [0 to max_low_pfn) and
561 * [4GB to max_pfn) because of possible memory holes in high addresses
562 * that cannot be marked as UC by fixed/variable range MTRRs.
563 * Depending on the alignment of E820 ranges, this may possibly result
564 * in using smaller size (i.e. 4K instead of 2M or 1G) page tables.
565 *
566 * init_mem_mapping() calls init_range_memory_mapping() with big range.
567 * That range would have hole in the middle or ends, and only ram parts
568 * will be mapped in init_range_memory_mapping().
569 */
init_range_memory_mapping(unsigned long r_start,unsigned long r_end)570 static unsigned long __init init_range_memory_mapping(
571 unsigned long r_start,
572 unsigned long r_end)
573 {
574 unsigned long start_pfn, end_pfn;
575 unsigned long mapped_ram_size = 0;
576 int i;
577
578 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, NULL) {
579 u64 start = clamp_val(PFN_PHYS(start_pfn), r_start, r_end);
580 u64 end = clamp_val(PFN_PHYS(end_pfn), r_start, r_end);
581 if (start >= end)
582 continue;
583
584 /*
585 * if it is overlapping with brk pgt, we need to
586 * alloc pgt buf from memblock instead.
587 */
588 can_use_brk_pgt = max(start, (u64)pgt_buf_end<<PAGE_SHIFT) >=
589 min(end, (u64)pgt_buf_top<<PAGE_SHIFT);
590 init_memory_mapping(start, end, PAGE_KERNEL);
591 mapped_ram_size += end - start;
592 can_use_brk_pgt = true;
593 }
594
595 return mapped_ram_size;
596 }
597
get_new_step_size(unsigned long step_size)598 static unsigned long __init get_new_step_size(unsigned long step_size)
599 {
600 /*
601 * Initial mapped size is PMD_SIZE (2M).
602 * We can not set step_size to be PUD_SIZE (1G) yet.
603 * In worse case, when we cross the 1G boundary, and
604 * PG_LEVEL_2M is not set, we will need 1+1+512 pages (2M + 8k)
605 * to map 1G range with PTE. Hence we use one less than the
606 * difference of page table level shifts.
607 *
608 * Don't need to worry about overflow in the top-down case, on 32bit,
609 * when step_size is 0, round_down() returns 0 for start, and that
610 * turns it into 0x100000000ULL.
611 * In the bottom-up case, round_up(x, 0) returns 0 though too, which
612 * needs to be taken into consideration by the code below.
613 */
614 return step_size << (PMD_SHIFT - PAGE_SHIFT - 1);
615 }
616
617 /**
618 * memory_map_top_down - Map [map_start, map_end) top down
619 * @map_start: start address of the target memory range
620 * @map_end: end address of the target memory range
621 *
622 * This function will setup direct mapping for memory range
623 * [map_start, map_end) in top-down. That said, the page tables
624 * will be allocated at the end of the memory, and we map the
625 * memory in top-down.
626 */
memory_map_top_down(unsigned long map_start,unsigned long map_end)627 static void __init memory_map_top_down(unsigned long map_start,
628 unsigned long map_end)
629 {
630 unsigned long real_end, last_start;
631 unsigned long step_size;
632 unsigned long addr;
633 unsigned long mapped_ram_size = 0;
634
635 /*
636 * Systems that have many reserved areas near top of the memory,
637 * e.g. QEMU with less than 1G RAM and EFI enabled, or Xen, will
638 * require lots of 4K mappings which may exhaust pgt_buf.
639 * Start with top-most PMD_SIZE range aligned at PMD_SIZE to ensure
640 * there is enough mapped memory that can be allocated from
641 * memblock.
642 */
643 addr = memblock_phys_alloc_range(PMD_SIZE, PMD_SIZE, map_start,
644 map_end);
645 if (!addr) {
646 pr_warn("Failed to release memory for alloc_low_pages()");
647 real_end = max(map_start, ALIGN_DOWN(map_end, PMD_SIZE));
648 } else {
649 memblock_phys_free(addr, PMD_SIZE);
650 real_end = addr + PMD_SIZE;
651 }
652
653 /* step_size need to be small so pgt_buf from BRK could cover it */
654 step_size = PMD_SIZE;
655 max_pfn_mapped = 0; /* will get exact value next */
656 min_pfn_mapped = real_end >> PAGE_SHIFT;
657 last_start = real_end;
658
659 /*
660 * We start from the top (end of memory) and go to the bottom.
661 * The memblock_find_in_range() gets us a block of RAM from the
662 * end of RAM in [min_pfn_mapped, max_pfn_mapped) used as new pages
663 * for page table.
664 */
665 while (last_start > map_start) {
666 unsigned long start;
667
668 if (last_start > step_size) {
669 start = round_down(last_start - 1, step_size);
670 if (start < map_start)
671 start = map_start;
672 } else
673 start = map_start;
674 mapped_ram_size += init_range_memory_mapping(start,
675 last_start);
676 last_start = start;
677 min_pfn_mapped = last_start >> PAGE_SHIFT;
678 if (mapped_ram_size >= step_size)
679 step_size = get_new_step_size(step_size);
680 }
681
682 if (real_end < map_end)
683 init_range_memory_mapping(real_end, map_end);
684 }
685
686 /**
687 * memory_map_bottom_up - Map [map_start, map_end) bottom up
688 * @map_start: start address of the target memory range
689 * @map_end: end address of the target memory range
690 *
691 * This function will setup direct mapping for memory range
692 * [map_start, map_end) in bottom-up. Since we have limited the
693 * bottom-up allocation above the kernel, the page tables will
694 * be allocated just above the kernel and we map the memory
695 * in [map_start, map_end) in bottom-up.
696 */
memory_map_bottom_up(unsigned long map_start,unsigned long map_end)697 static void __init memory_map_bottom_up(unsigned long map_start,
698 unsigned long map_end)
699 {
700 unsigned long next, start;
701 unsigned long mapped_ram_size = 0;
702 /* step_size need to be small so pgt_buf from BRK could cover it */
703 unsigned long step_size = PMD_SIZE;
704
705 start = map_start;
706 min_pfn_mapped = start >> PAGE_SHIFT;
707
708 /*
709 * We start from the bottom (@map_start) and go to the top (@map_end).
710 * The memblock_find_in_range() gets us a block of RAM from the
711 * end of RAM in [min_pfn_mapped, max_pfn_mapped) used as new pages
712 * for page table.
713 */
714 while (start < map_end) {
715 if (step_size && map_end - start > step_size) {
716 next = round_up(start + 1, step_size);
717 if (next > map_end)
718 next = map_end;
719 } else {
720 next = map_end;
721 }
722
723 mapped_ram_size += init_range_memory_mapping(start, next);
724 start = next;
725
726 if (mapped_ram_size >= step_size)
727 step_size = get_new_step_size(step_size);
728 }
729 }
730
731 /*
732 * The real mode trampoline, which is required for bootstrapping CPUs
733 * occupies only a small area under the low 1MB. See reserve_real_mode()
734 * for details.
735 *
736 * If KASLR is disabled the first PGD entry of the direct mapping is copied
737 * to map the real mode trampoline.
738 *
739 * If KASLR is enabled, copy only the PUD which covers the low 1MB
740 * area. This limits the randomization granularity to 1GB for both 4-level
741 * and 5-level paging.
742 */
init_trampoline(void)743 static void __init init_trampoline(void)
744 {
745 #ifdef CONFIG_X86_64
746 /*
747 * The code below will alias kernel page-tables in the user-range of the
748 * address space, including the Global bit. So global TLB entries will
749 * be created when using the trampoline page-table.
750 */
751 if (!kaslr_memory_enabled())
752 trampoline_pgd_entry = init_top_pgt[pgd_index(__PAGE_OFFSET)];
753 else
754 init_trampoline_kaslr();
755 #endif
756 }
757
init_mem_mapping(void)758 void __init init_mem_mapping(void)
759 {
760 unsigned long end;
761
762 pti_check_boottime_disable();
763 probe_page_size_mask();
764 setup_pcid();
765
766 #ifdef CONFIG_X86_64
767 end = max_pfn << PAGE_SHIFT;
768 #else
769 end = max_low_pfn << PAGE_SHIFT;
770 #endif
771
772 /* the ISA range is always mapped regardless of memory holes */
773 init_memory_mapping(0, ISA_END_ADDRESS, PAGE_KERNEL);
774
775 /* Init the trampoline, possibly with KASLR memory offset */
776 init_trampoline();
777
778 /*
779 * If the allocation is in bottom-up direction, we setup direct mapping
780 * in bottom-up, otherwise we setup direct mapping in top-down.
781 */
782 if (memblock_bottom_up()) {
783 unsigned long kernel_end = __pa_symbol(_end);
784
785 /*
786 * we need two separate calls here. This is because we want to
787 * allocate page tables above the kernel. So we first map
788 * [kernel_end, end) to make memory above the kernel be mapped
789 * as soon as possible. And then use page tables allocated above
790 * the kernel to map [ISA_END_ADDRESS, kernel_end).
791 */
792 memory_map_bottom_up(kernel_end, end);
793 memory_map_bottom_up(ISA_END_ADDRESS, kernel_end);
794 } else {
795 memory_map_top_down(ISA_END_ADDRESS, end);
796 }
797
798 #ifdef CONFIG_X86_64
799 if (max_pfn > max_low_pfn) {
800 /* can we preserve max_low_pfn ?*/
801 max_low_pfn = max_pfn;
802 }
803 #else
804 early_ioremap_page_table_range_init();
805 #endif
806
807 load_cr3(swapper_pg_dir);
808 __flush_tlb_all();
809
810 x86_init.hyper.init_mem_mapping();
811
812 early_memtest(0, max_pfn_mapped << PAGE_SHIFT);
813 }
814
815 /*
816 * Initialize an mm_struct to be used during poking and a pointer to be used
817 * during patching.
818 */
poking_init(void)819 void __init poking_init(void)
820 {
821 spinlock_t *ptl;
822 pte_t *ptep;
823
824 text_poke_mm = mm_alloc();
825 BUG_ON(!text_poke_mm);
826
827 /* Xen PV guests need the PGD to be pinned. */
828 paravirt_enter_mmap(text_poke_mm);
829
830 set_notrack_mm(text_poke_mm);
831
832 /*
833 * Randomize the poking address, but make sure that the following page
834 * will be mapped at the same PMD. We need 2 pages, so find space for 3,
835 * and adjust the address if the PMD ends after the first one.
836 */
837 text_poke_mm_addr = TASK_UNMAPPED_BASE;
838 if (IS_ENABLED(CONFIG_RANDOMIZE_BASE))
839 text_poke_mm_addr += (kaslr_get_random_long("Poking") & PAGE_MASK) %
840 (TASK_SIZE - TASK_UNMAPPED_BASE - 3 * PAGE_SIZE);
841
842 if (((text_poke_mm_addr + PAGE_SIZE) & ~PMD_MASK) == 0)
843 text_poke_mm_addr += PAGE_SIZE;
844
845 /*
846 * We need to trigger the allocation of the page-tables that will be
847 * needed for poking now. Later, poking may be performed in an atomic
848 * section, which might cause allocation to fail.
849 */
850 ptep = get_locked_pte(text_poke_mm, text_poke_mm_addr, &ptl);
851 BUG_ON(!ptep);
852 pte_unmap_unlock(ptep, ptl);
853 }
854
855 /*
856 * devmem_is_allowed() checks to see if /dev/mem access to a certain address
857 * is valid. The argument is a physical page number.
858 *
859 * On x86, access has to be given to the first megabyte of RAM because that
860 * area traditionally contains BIOS code and data regions used by X, dosemu,
861 * and similar apps. Since they map the entire memory range, the whole range
862 * must be allowed (for mapping), but any areas that would otherwise be
863 * disallowed are flagged as being "zero filled" instead of rejected.
864 * Access has to be given to non-kernel-ram areas as well, these contain the
865 * PCI mmio resources as well as potential bios/acpi data regions.
866 */
devmem_is_allowed(unsigned long pagenr)867 int devmem_is_allowed(unsigned long pagenr)
868 {
869 if (region_intersects(PFN_PHYS(pagenr), PAGE_SIZE,
870 IORESOURCE_SYSTEM_RAM, IORES_DESC_NONE)
871 != REGION_DISJOINT) {
872 /*
873 * For disallowed memory regions in the low 1MB range,
874 * request that the page be shown as all zeros.
875 */
876 if (pagenr < 256)
877 return 2;
878
879 return 0;
880 }
881
882 /*
883 * This must follow RAM test, since System RAM is considered a
884 * restricted resource under CONFIG_STRICT_DEVMEM.
885 */
886 if (iomem_is_exclusive(pagenr << PAGE_SHIFT)) {
887 /* Low 1MB bypasses iomem restrictions. */
888 if (pagenr < 256)
889 return 1;
890
891 return 0;
892 }
893
894 return 1;
895 }
896
free_init_pages(const char * what,unsigned long begin,unsigned long end)897 void free_init_pages(const char *what, unsigned long begin, unsigned long end)
898 {
899 unsigned long begin_aligned, end_aligned;
900
901 /* Make sure boundaries are page aligned */
902 begin_aligned = PAGE_ALIGN(begin);
903 end_aligned = end & PAGE_MASK;
904
905 if (WARN_ON(begin_aligned != begin || end_aligned != end)) {
906 begin = begin_aligned;
907 end = end_aligned;
908 }
909
910 if (begin >= end)
911 return;
912
913 /*
914 * If debugging page accesses then do not free this memory but
915 * mark them not present - any buggy init-section access will
916 * create a kernel page fault:
917 */
918 if (debug_pagealloc_enabled()) {
919 pr_info("debug: unmapping init [mem %#010lx-%#010lx]\n",
920 begin, end - 1);
921 /*
922 * Inform kmemleak about the hole in the memory since the
923 * corresponding pages will be unmapped.
924 */
925 kmemleak_free_part((void *)begin, end - begin);
926 set_memory_np(begin, (end - begin) >> PAGE_SHIFT);
927 } else {
928 /*
929 * We just marked the kernel text read only above, now that
930 * we are going to free part of that, we need to make that
931 * writeable and non-executable first.
932 */
933 set_memory_nx(begin, (end - begin) >> PAGE_SHIFT);
934 set_memory_rw(begin, (end - begin) >> PAGE_SHIFT);
935
936 free_reserved_area((void *)begin, (void *)end,
937 POISON_FREE_INITMEM, what);
938 }
939 }
940
941 /*
942 * begin/end can be in the direct map or the "high kernel mapping"
943 * used for the kernel image only. free_init_pages() will do the
944 * right thing for either kind of address.
945 */
free_kernel_image_pages(const char * what,void * begin,void * end)946 void free_kernel_image_pages(const char *what, void *begin, void *end)
947 {
948 unsigned long begin_ul = (unsigned long)begin;
949 unsigned long end_ul = (unsigned long)end;
950 unsigned long len_pages = (end_ul - begin_ul) >> PAGE_SHIFT;
951
952 free_init_pages(what, begin_ul, end_ul);
953
954 /*
955 * PTI maps some of the kernel into userspace. For performance,
956 * this includes some kernel areas that do not contain secrets.
957 * Those areas might be adjacent to the parts of the kernel image
958 * being freed, which may contain secrets. Remove the "high kernel
959 * image mapping" for these freed areas, ensuring they are not even
960 * potentially vulnerable to Meltdown regardless of the specific
961 * optimizations PTI is currently using.
962 *
963 * The "noalias" prevents unmapping the direct map alias which is
964 * needed to access the freed pages.
965 *
966 * This is only valid for 64bit kernels. 32bit has only one mapping
967 * which can't be treated in this way for obvious reasons.
968 */
969 if (IS_ENABLED(CONFIG_X86_64) && cpu_feature_enabled(X86_FEATURE_PTI))
970 set_memory_np_noalias(begin_ul, len_pages);
971 }
972
free_initmem(void)973 void __ref free_initmem(void)
974 {
975 e820__reallocate_tables();
976
977 mem_encrypt_free_decrypted_mem();
978
979 free_kernel_image_pages("unused kernel image (initmem)",
980 &__init_begin, &__init_end);
981 }
982
983 #ifdef CONFIG_BLK_DEV_INITRD
free_initrd_mem(unsigned long start,unsigned long end)984 void __init free_initrd_mem(unsigned long start, unsigned long end)
985 {
986 /*
987 * end could be not aligned, and We can not align that,
988 * decompressor could be confused by aligned initrd_end
989 * We already reserve the end partial page before in
990 * - i386_start_kernel()
991 * - x86_64_start_kernel()
992 * - relocate_initrd()
993 * So here We can do PAGE_ALIGN() safely to get partial page to be freed
994 */
995 free_init_pages("initrd", start, PAGE_ALIGN(end));
996 }
997 #endif
998
zone_sizes_init(void)999 void __init zone_sizes_init(void)
1000 {
1001 unsigned long max_zone_pfns[MAX_NR_ZONES];
1002
1003 memset(max_zone_pfns, 0, sizeof(max_zone_pfns));
1004
1005 #ifdef CONFIG_ZONE_DMA
1006 max_zone_pfns[ZONE_DMA] = min(MAX_DMA_PFN, max_low_pfn);
1007 #endif
1008 #ifdef CONFIG_ZONE_DMA32
1009 max_zone_pfns[ZONE_DMA32] = min(MAX_DMA32_PFN, max_low_pfn);
1010 #endif
1011 max_zone_pfns[ZONE_NORMAL] = max_low_pfn;
1012 #ifdef CONFIG_HIGHMEM
1013 max_zone_pfns[ZONE_HIGHMEM] = max_pfn;
1014 #endif
1015
1016 free_area_init(max_zone_pfns);
1017 }
1018
1019 __visible DEFINE_PER_CPU_ALIGNED(struct tlb_state, cpu_tlbstate) = {
1020 .loaded_mm = &init_mm,
1021 .next_asid = 1,
1022 .cr4 = ~0UL, /* fail hard if we screw up cr4 shadow initialization */
1023 };
1024
1025 #ifdef CONFIG_ADDRESS_MASKING
1026 DEFINE_PER_CPU(u64, tlbstate_untag_mask);
1027 EXPORT_PER_CPU_SYMBOL(tlbstate_untag_mask);
1028 #endif
1029
update_cache_mode_entry(unsigned entry,enum page_cache_mode cache)1030 void update_cache_mode_entry(unsigned entry, enum page_cache_mode cache)
1031 {
1032 /* entry 0 MUST be WB (hardwired to speed up translations) */
1033 BUG_ON(!entry && cache != _PAGE_CACHE_MODE_WB);
1034
1035 __cachemode2pte_tbl[cache] = __cm_idx2pte(entry);
1036 __pte2cachemode_tbl[entry] = cache;
1037 }
1038
1039 #ifdef CONFIG_SWAP
arch_max_swapfile_size(void)1040 unsigned long arch_max_swapfile_size(void)
1041 {
1042 unsigned long pages;
1043
1044 pages = generic_max_swapfile_size();
1045
1046 if (boot_cpu_has_bug(X86_BUG_L1TF) && l1tf_mitigation != L1TF_MITIGATION_OFF) {
1047 /* Limit the swap file size to MAX_PA/2 for L1TF workaround */
1048 unsigned long long l1tf_limit = l1tf_pfn_limit();
1049 /*
1050 * We encode swap offsets also with 3 bits below those for pfn
1051 * which makes the usable limit higher.
1052 */
1053 #if CONFIG_PGTABLE_LEVELS > 2
1054 l1tf_limit <<= PAGE_SHIFT - SWP_OFFSET_FIRST_BIT;
1055 #endif
1056 pages = min_t(unsigned long long, l1tf_limit, pages);
1057 }
1058 return pages;
1059 }
1060 #endif
1061
1062 #ifdef CONFIG_EXECMEM
1063 static struct execmem_info execmem_info __ro_after_init;
1064
1065 #ifdef CONFIG_ARCH_HAS_EXECMEM_ROX
execmem_fill_trapping_insns(void * ptr,size_t size)1066 void execmem_fill_trapping_insns(void *ptr, size_t size)
1067 {
1068 memset(ptr, INT3_INSN_OPCODE, size);
1069 }
1070 #endif
1071
execmem_arch_setup(void)1072 struct execmem_info __init *execmem_arch_setup(void)
1073 {
1074 unsigned long start, offset = 0;
1075 enum execmem_range_flags flags;
1076 pgprot_t pgprot;
1077
1078 if (kaslr_enabled())
1079 offset = get_random_u32_inclusive(1, 1024) * PAGE_SIZE;
1080
1081 start = MODULES_VADDR + offset;
1082
1083 if (IS_ENABLED(CONFIG_ARCH_HAS_EXECMEM_ROX) &&
1084 cpu_feature_enabled(X86_FEATURE_PSE)) {
1085 pgprot = PAGE_KERNEL_ROX;
1086 flags = EXECMEM_KASAN_SHADOW | EXECMEM_ROX_CACHE;
1087 } else {
1088 pgprot = PAGE_KERNEL;
1089 flags = EXECMEM_KASAN_SHADOW;
1090 }
1091
1092 execmem_info = (struct execmem_info){
1093 .ranges = {
1094 [EXECMEM_MODULE_TEXT] = {
1095 .flags = flags,
1096 .start = start,
1097 .end = MODULES_END,
1098 .pgprot = pgprot,
1099 .alignment = MODULE_ALIGN,
1100 },
1101 [EXECMEM_KPROBES] = {
1102 .flags = flags,
1103 .start = start,
1104 .end = MODULES_END,
1105 .pgprot = PAGE_KERNEL_ROX,
1106 .alignment = MODULE_ALIGN,
1107 },
1108 [EXECMEM_FTRACE] = {
1109 .flags = flags,
1110 .start = start,
1111 .end = MODULES_END,
1112 .pgprot = pgprot,
1113 .alignment = MODULE_ALIGN,
1114 },
1115 [EXECMEM_BPF] = {
1116 .flags = EXECMEM_KASAN_SHADOW,
1117 .start = start,
1118 .end = MODULES_END,
1119 .pgprot = PAGE_KERNEL,
1120 .alignment = MODULE_ALIGN,
1121 },
1122 [EXECMEM_MODULE_DATA] = {
1123 .flags = EXECMEM_KASAN_SHADOW,
1124 .start = start,
1125 .end = MODULES_END,
1126 .pgprot = PAGE_KERNEL,
1127 .alignment = MODULE_ALIGN,
1128 },
1129 },
1130 };
1131
1132 return &execmem_info;
1133 }
1134 #endif /* CONFIG_EXECMEM */
1135