xref: /linux/arch/mips/mm/init.c (revision a49468240e89628236b738b5ab9416eae8f90c15)
1 /*
2  * This file is subject to the terms and conditions of the GNU General Public
3  * License.  See the file "COPYING" in the main directory of this archive
4  * for more details.
5  *
6  * Copyright (C) 1994 - 2000 Ralf Baechle
7  * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
8  * Kevin D. Kissell, kevink@mips.com and Carsten Langgaard, carstenl@mips.com
9  * Copyright (C) 2000 MIPS Technologies, Inc.  All rights reserved.
10  */
11 #include <linux/bug.h>
12 #include <linux/init.h>
13 #include <linux/export.h>
14 #include <linux/signal.h>
15 #include <linux/sched.h>
16 #include <linux/smp.h>
17 #include <linux/kernel.h>
18 #include <linux/errno.h>
19 #include <linux/string.h>
20 #include <linux/types.h>
21 #include <linux/pagemap.h>
22 #include <linux/ptrace.h>
23 #include <linux/mman.h>
24 #include <linux/mm.h>
25 #include <linux/memblock.h>
26 #include <linux/highmem.h>
27 #include <linux/swap.h>
28 #include <linux/proc_fs.h>
29 #include <linux/pfn.h>
30 #include <linux/hardirq.h>
31 #include <linux/gfp.h>
32 #include <linux/kcore.h>
33 #include <linux/initrd.h>
34 #include <linux/execmem.h>
35 
36 #include <asm/bootinfo.h>
37 #include <asm/cachectl.h>
38 #include <asm/cpu.h>
39 #include <asm/dma.h>
40 #include <asm/maar.h>
41 #include <asm/mmu_context.h>
42 #include <asm/mmzone.h>
43 #include <asm/sections.h>
44 #include <asm/pgalloc.h>
45 #include <asm/tlb.h>
46 #include <asm/fixmap.h>
47 
48 /*
49  * We have up to 8 empty zeroed pages so we can map one of the right colour
50  * when needed.	 This is necessary only on R4000 / R4400 SC and MC versions
51  * where we have to avoid VCED / VECI exceptions for good performance at
52  * any price.  Since page is never written to after the initialization we
53  * don't have to care about aliases on other CPUs.
54  */
55 unsigned long empty_zero_page, zero_page_mask;
56 EXPORT_SYMBOL_GPL(empty_zero_page);
57 EXPORT_SYMBOL(zero_page_mask);
58 
59 /*
60  * Not static inline because used by IP27 special magic initialization code
61  */
setup_zero_pages(void)62 void setup_zero_pages(void)
63 {
64 	unsigned int order, i;
65 	struct page *page;
66 
67 	if (cpu_has_vce)
68 		order = 3;
69 	else
70 		order = 0;
71 
72 	empty_zero_page = __get_free_pages(GFP_KERNEL | __GFP_ZERO, order);
73 	if (!empty_zero_page)
74 		panic("Oh boy, that early out of memory?");
75 
76 	page = virt_to_page((void *)empty_zero_page);
77 	split_page(page, order);
78 	for (i = 0; i < (1 << order); i++, page++)
79 		mark_page_reserved(page);
80 
81 	zero_page_mask = ((PAGE_SIZE << order) - 1) & PAGE_MASK;
82 }
83 
__kmap_pgprot(struct page * page,unsigned long addr,pgprot_t prot)84 static void *__kmap_pgprot(struct page *page, unsigned long addr, pgprot_t prot)
85 {
86 	enum fixed_addresses idx;
87 	unsigned int old_mmid;
88 	unsigned long vaddr, flags, entrylo;
89 	unsigned long old_ctx;
90 	pte_t pte;
91 	int tlbidx;
92 
93 	BUG_ON(folio_test_dcache_dirty(page_folio(page)));
94 
95 	preempt_disable();
96 	pagefault_disable();
97 	idx = (addr >> PAGE_SHIFT) & (FIX_N_COLOURS - 1);
98 	idx += in_interrupt() ? FIX_N_COLOURS : 0;
99 	vaddr = __fix_to_virt(FIX_CMAP_END - idx);
100 	pte = mk_pte(page, prot);
101 #if defined(CONFIG_XPA)
102 	entrylo = pte_to_entrylo(pte.pte_high);
103 #elif defined(CONFIG_PHYS_ADDR_T_64BIT) && defined(CONFIG_CPU_MIPS32)
104 	entrylo = pte.pte_high;
105 #else
106 	entrylo = pte_to_entrylo(pte_val(pte));
107 #endif
108 
109 	local_irq_save(flags);
110 	old_ctx = read_c0_entryhi();
111 	write_c0_entryhi(vaddr & (PAGE_MASK << 1));
112 	write_c0_entrylo0(entrylo);
113 	write_c0_entrylo1(entrylo);
114 	if (cpu_has_mmid) {
115 		old_mmid = read_c0_memorymapid();
116 		write_c0_memorymapid(MMID_KERNEL_WIRED);
117 	}
118 #ifdef CONFIG_XPA
119 	if (cpu_has_xpa) {
120 		entrylo = (pte.pte_low & _PFNX_MASK);
121 		writex_c0_entrylo0(entrylo);
122 		writex_c0_entrylo1(entrylo);
123 	}
124 #endif
125 	tlbidx = num_wired_entries();
126 	write_c0_wired(tlbidx + 1);
127 	write_c0_index(tlbidx);
128 	mtc0_tlbw_hazard();
129 	tlb_write_indexed();
130 	tlbw_use_hazard();
131 	write_c0_entryhi(old_ctx);
132 	if (cpu_has_mmid)
133 		write_c0_memorymapid(old_mmid);
134 	local_irq_restore(flags);
135 
136 	return (void*) vaddr;
137 }
138 
kmap_coherent(struct page * page,unsigned long addr)139 void *kmap_coherent(struct page *page, unsigned long addr)
140 {
141 	return __kmap_pgprot(page, addr, PAGE_KERNEL);
142 }
143 
kmap_noncoherent(struct page * page,unsigned long addr)144 void *kmap_noncoherent(struct page *page, unsigned long addr)
145 {
146 	return __kmap_pgprot(page, addr, PAGE_KERNEL_NC);
147 }
148 
kunmap_coherent(void)149 void kunmap_coherent(void)
150 {
151 	unsigned int wired;
152 	unsigned long flags, old_ctx;
153 
154 	local_irq_save(flags);
155 	old_ctx = read_c0_entryhi();
156 	wired = num_wired_entries() - 1;
157 	write_c0_wired(wired);
158 	write_c0_index(wired);
159 	write_c0_entryhi(UNIQUE_ENTRYHI(wired));
160 	write_c0_entrylo0(0);
161 	write_c0_entrylo1(0);
162 	mtc0_tlbw_hazard();
163 	tlb_write_indexed();
164 	tlbw_use_hazard();
165 	write_c0_entryhi(old_ctx);
166 	local_irq_restore(flags);
167 	pagefault_enable();
168 	preempt_enable();
169 }
170 
copy_user_highpage(struct page * to,struct page * from,unsigned long vaddr,struct vm_area_struct * vma)171 void copy_user_highpage(struct page *to, struct page *from,
172 	unsigned long vaddr, struct vm_area_struct *vma)
173 {
174 	struct folio *src = page_folio(from);
175 	void *vfrom, *vto;
176 
177 	vto = kmap_atomic(to);
178 	if (cpu_has_dc_aliases &&
179 	    folio_mapped(src) && !folio_test_dcache_dirty(src)) {
180 		vfrom = kmap_coherent(from, vaddr);
181 		copy_page(vto, vfrom);
182 		kunmap_coherent();
183 	} else {
184 		vfrom = kmap_atomic(from);
185 		copy_page(vto, vfrom);
186 		kunmap_atomic(vfrom);
187 	}
188 	if ((!cpu_has_ic_fills_f_dc) ||
189 	    pages_do_alias((unsigned long)vto, vaddr & PAGE_MASK))
190 		flush_data_cache_page((unsigned long)vto);
191 	kunmap_atomic(vto);
192 	/* Make sure this page is cleared on other CPU's too before using it */
193 	smp_wmb();
194 }
195 
copy_to_user_page(struct vm_area_struct * vma,struct page * page,unsigned long vaddr,void * dst,const void * src,unsigned long len)196 void copy_to_user_page(struct vm_area_struct *vma,
197 	struct page *page, unsigned long vaddr, void *dst, const void *src,
198 	unsigned long len)
199 {
200 	struct folio *folio = page_folio(page);
201 
202 	if (cpu_has_dc_aliases &&
203 	    folio_mapped(folio) && !folio_test_dcache_dirty(folio)) {
204 		void *vto = kmap_coherent(page, vaddr) + (vaddr & ~PAGE_MASK);
205 		memcpy(vto, src, len);
206 		kunmap_coherent();
207 	} else {
208 		memcpy(dst, src, len);
209 		if (cpu_has_dc_aliases)
210 			folio_set_dcache_dirty(folio);
211 	}
212 	if (vma->vm_flags & VM_EXEC)
213 		flush_cache_page(vma, vaddr, page_to_pfn(page));
214 }
215 
copy_from_user_page(struct vm_area_struct * vma,struct page * page,unsigned long vaddr,void * dst,const void * src,unsigned long len)216 void copy_from_user_page(struct vm_area_struct *vma,
217 	struct page *page, unsigned long vaddr, void *dst, const void *src,
218 	unsigned long len)
219 {
220 	struct folio *folio = page_folio(page);
221 
222 	if (cpu_has_dc_aliases &&
223 	    folio_mapped(folio) && !folio_test_dcache_dirty(folio)) {
224 		void *vfrom = kmap_coherent(page, vaddr) + (vaddr & ~PAGE_MASK);
225 		memcpy(dst, vfrom, len);
226 		kunmap_coherent();
227 	} else {
228 		memcpy(dst, src, len);
229 		if (cpu_has_dc_aliases)
230 			folio_set_dcache_dirty(folio);
231 	}
232 }
233 EXPORT_SYMBOL_GPL(copy_from_user_page);
234 
fixrange_init(unsigned long start,unsigned long end,pgd_t * pgd_base)235 void __init fixrange_init(unsigned long start, unsigned long end,
236 	pgd_t *pgd_base)
237 {
238 #ifdef CONFIG_HIGHMEM
239 	pgd_t *pgd;
240 	pud_t *pud;
241 	pmd_t *pmd;
242 	pte_t *pte;
243 	int i, j, k;
244 	unsigned long vaddr;
245 
246 	vaddr = start;
247 	i = pgd_index(vaddr);
248 	j = pud_index(vaddr);
249 	k = pmd_index(vaddr);
250 	pgd = pgd_base + i;
251 
252 	for ( ; (i < PTRS_PER_PGD) && (vaddr < end); pgd++, i++) {
253 		pud = (pud_t *)pgd;
254 		for ( ; (j < PTRS_PER_PUD) && (vaddr < end); pud++, j++) {
255 			pmd = (pmd_t *)pud;
256 			for (; (k < PTRS_PER_PMD) && (vaddr < end); pmd++, k++) {
257 				if (pmd_none(*pmd)) {
258 					pte = (pte_t *) memblock_alloc_low(PAGE_SIZE,
259 									   PAGE_SIZE);
260 					if (!pte)
261 						panic("%s: Failed to allocate %lu bytes align=%lx\n",
262 						      __func__, PAGE_SIZE,
263 						      PAGE_SIZE);
264 
265 					set_pmd(pmd, __pmd((unsigned long)pte));
266 					BUG_ON(pte != pte_offset_kernel(pmd, 0));
267 				}
268 				vaddr += PMD_SIZE;
269 			}
270 			k = 0;
271 		}
272 		j = 0;
273 	}
274 #endif
275 }
276 
277 struct maar_walk_info {
278 	struct maar_config cfg[16];
279 	unsigned int num_cfg;
280 };
281 
maar_res_walk(unsigned long start_pfn,unsigned long nr_pages,void * data)282 static int maar_res_walk(unsigned long start_pfn, unsigned long nr_pages,
283 			 void *data)
284 {
285 	struct maar_walk_info *wi = data;
286 	struct maar_config *cfg = &wi->cfg[wi->num_cfg];
287 	unsigned int maar_align;
288 
289 	/* MAAR registers hold physical addresses right shifted by 4 bits */
290 	maar_align = BIT(MIPS_MAAR_ADDR_SHIFT + 4);
291 
292 	/* Fill in the MAAR config entry */
293 	cfg->lower = ALIGN(PFN_PHYS(start_pfn), maar_align);
294 	cfg->upper = ALIGN_DOWN(PFN_PHYS(start_pfn + nr_pages), maar_align) - 1;
295 	cfg->attrs = MIPS_MAAR_S;
296 
297 	/* Ensure we don't overflow the cfg array */
298 	if (!WARN_ON(wi->num_cfg >= ARRAY_SIZE(wi->cfg)))
299 		wi->num_cfg++;
300 
301 	return 0;
302 }
303 
304 
platform_maar_init(unsigned num_pairs)305 unsigned __weak platform_maar_init(unsigned num_pairs)
306 {
307 	unsigned int num_configured;
308 	struct maar_walk_info wi;
309 
310 	wi.num_cfg = 0;
311 	walk_system_ram_range(0, max_pfn, &wi, maar_res_walk);
312 
313 	num_configured = maar_config(wi.cfg, wi.num_cfg, num_pairs);
314 	if (num_configured < wi.num_cfg)
315 		pr_warn("Not enough MAAR pairs (%u) for all memory regions (%u)\n",
316 			num_pairs, wi.num_cfg);
317 
318 	return num_configured;
319 }
320 
maar_init(void)321 void maar_init(void)
322 {
323 	unsigned num_maars, used, i;
324 	phys_addr_t lower, upper, attr;
325 	static struct {
326 		struct maar_config cfgs[3];
327 		unsigned used;
328 	} recorded = { { { 0 } }, 0 };
329 
330 	if (!cpu_has_maar)
331 		return;
332 
333 	/* Detect the number of MAARs */
334 	write_c0_maari(~0);
335 	back_to_back_c0_hazard();
336 	num_maars = read_c0_maari() + 1;
337 
338 	/* MAARs should be in pairs */
339 	WARN_ON(num_maars % 2);
340 
341 	/* Set MAARs using values we recorded already */
342 	if (recorded.used) {
343 		used = maar_config(recorded.cfgs, recorded.used, num_maars / 2);
344 		BUG_ON(used != recorded.used);
345 	} else {
346 		/* Configure the required MAARs */
347 		used = platform_maar_init(num_maars / 2);
348 	}
349 
350 	/* Disable any further MAARs */
351 	for (i = (used * 2); i < num_maars; i++) {
352 		write_c0_maari(i);
353 		back_to_back_c0_hazard();
354 		write_c0_maar(0);
355 		back_to_back_c0_hazard();
356 	}
357 
358 	if (recorded.used)
359 		return;
360 
361 	pr_info("MAAR configuration:\n");
362 	for (i = 0; i < num_maars; i += 2) {
363 		write_c0_maari(i);
364 		back_to_back_c0_hazard();
365 		upper = read_c0_maar();
366 #ifdef CONFIG_XPA
367 		upper |= (phys_addr_t)readx_c0_maar() << MIPS_MAARX_ADDR_SHIFT;
368 #endif
369 
370 		write_c0_maari(i + 1);
371 		back_to_back_c0_hazard();
372 		lower = read_c0_maar();
373 #ifdef CONFIG_XPA
374 		lower |= (phys_addr_t)readx_c0_maar() << MIPS_MAARX_ADDR_SHIFT;
375 #endif
376 
377 		attr = lower & upper;
378 		lower = (lower & MIPS_MAAR_ADDR) << 4;
379 		upper = ((upper & MIPS_MAAR_ADDR) << 4) | 0xffff;
380 
381 		pr_info("  [%d]: ", i / 2);
382 		if ((attr & MIPS_MAAR_V) != MIPS_MAAR_V) {
383 			pr_cont("disabled\n");
384 			continue;
385 		}
386 
387 		pr_cont("%pa-%pa", &lower, &upper);
388 
389 		if (attr & MIPS_MAAR_S)
390 			pr_cont(" speculate");
391 
392 		pr_cont("\n");
393 
394 		/* Record the setup for use on secondary CPUs */
395 		if (used <= ARRAY_SIZE(recorded.cfgs)) {
396 			recorded.cfgs[recorded.used].lower = lower;
397 			recorded.cfgs[recorded.used].upper = upper;
398 			recorded.cfgs[recorded.used].attrs = attr;
399 			recorded.used++;
400 		}
401 	}
402 }
403 
404 #ifndef CONFIG_NUMA
paging_init(void)405 void __init paging_init(void)
406 {
407 	unsigned long max_zone_pfns[MAX_NR_ZONES];
408 
409 	pagetable_init();
410 
411 #ifdef CONFIG_ZONE_DMA
412 	max_zone_pfns[ZONE_DMA] = MAX_DMA_PFN;
413 #endif
414 #ifdef CONFIG_ZONE_DMA32
415 	max_zone_pfns[ZONE_DMA32] = MAX_DMA32_PFN;
416 #endif
417 	max_zone_pfns[ZONE_NORMAL] = max_low_pfn;
418 #ifdef CONFIG_HIGHMEM
419 	max_zone_pfns[ZONE_HIGHMEM] = highend_pfn;
420 
421 	if (cpu_has_dc_aliases && max_low_pfn != highend_pfn) {
422 		printk(KERN_WARNING "This processor doesn't support highmem."
423 		       " %ldk highmem ignored\n",
424 		       (highend_pfn - max_low_pfn) << (PAGE_SHIFT - 10));
425 		max_zone_pfns[ZONE_HIGHMEM] = max_low_pfn;
426 
427 		max_mapnr = max_low_pfn;
428 	} else if (highend_pfn) {
429 		max_mapnr = highend_pfn;
430 	} else {
431 		max_mapnr = max_low_pfn;
432 	}
433 #else
434 	max_mapnr = max_low_pfn;
435 #endif
436 	high_memory = (void *) __va(max_low_pfn << PAGE_SHIFT);
437 
438 	free_area_init(max_zone_pfns);
439 }
440 
441 #ifdef CONFIG_64BIT
442 static struct kcore_list kcore_kseg0;
443 #endif
444 
mem_init_free_highmem(void)445 static inline void __init mem_init_free_highmem(void)
446 {
447 #ifdef CONFIG_HIGHMEM
448 	unsigned long tmp;
449 
450 	if (cpu_has_dc_aliases)
451 		return;
452 
453 	for (tmp = highstart_pfn; tmp < highend_pfn; tmp++) {
454 		struct page *page = pfn_to_page(tmp);
455 
456 		if (!memblock_is_memory(PFN_PHYS(tmp)))
457 			SetPageReserved(page);
458 		else
459 			free_highmem_page(page);
460 	}
461 #endif
462 }
463 
mem_init(void)464 void __init mem_init(void)
465 {
466 	/*
467 	 * When PFN_PTE_SHIFT is greater than PAGE_SHIFT we won't have enough PTE
468 	 * bits to hold a full 32b physical address on MIPS32 systems.
469 	 */
470 	BUILD_BUG_ON(IS_ENABLED(CONFIG_32BIT) && (PFN_PTE_SHIFT > PAGE_SHIFT));
471 
472 	maar_init();
473 	memblock_free_all();
474 	setup_zero_pages();	/* Setup zeroed pages.  */
475 	mem_init_free_highmem();
476 
477 #ifdef CONFIG_64BIT
478 	if ((unsigned long) &_text > (unsigned long) CKSEG0)
479 		/* The -4 is a hack so that user tools don't have to handle
480 		   the overflow.  */
481 		kclist_add(&kcore_kseg0, (void *) CKSEG0,
482 				0x80000000 - 4, KCORE_TEXT);
483 #endif
484 }
485 #endif /* !CONFIG_NUMA */
486 
free_init_pages(const char * what,unsigned long begin,unsigned long end)487 void free_init_pages(const char *what, unsigned long begin, unsigned long end)
488 {
489 	unsigned long pfn;
490 
491 	for (pfn = PFN_UP(begin); pfn < PFN_DOWN(end); pfn++) {
492 		struct page *page = pfn_to_page(pfn);
493 		void *addr = phys_to_virt(PFN_PHYS(pfn));
494 
495 		memset(addr, POISON_FREE_INITMEM, PAGE_SIZE);
496 		free_reserved_page(page);
497 	}
498 	printk(KERN_INFO "Freeing %s: %ldk freed\n", what, (end - begin) >> 10);
499 }
500 
501 void (*free_init_pages_eva)(void *begin, void *end) = NULL;
502 
prom_free_prom_memory(void)503 void __weak __init prom_free_prom_memory(void)
504 {
505 	/* nothing to do */
506 }
507 
free_initmem(void)508 void __ref free_initmem(void)
509 {
510 	prom_free_prom_memory();
511 	/*
512 	 * Let the platform define a specific function to free the
513 	 * init section since EVA may have used any possible mapping
514 	 * between virtual and physical addresses.
515 	 */
516 	if (free_init_pages_eva)
517 		free_init_pages_eva((void *)&__init_begin, (void *)&__init_end);
518 	else
519 		free_initmem_default(POISON_FREE_INITMEM);
520 }
521 
522 #ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA
523 unsigned long __per_cpu_offset[NR_CPUS] __read_mostly;
524 EXPORT_SYMBOL(__per_cpu_offset);
525 
pcpu_cpu_distance(unsigned int from,unsigned int to)526 static int __init pcpu_cpu_distance(unsigned int from, unsigned int to)
527 {
528 	return node_distance(cpu_to_node(from), cpu_to_node(to));
529 }
530 
pcpu_cpu_to_node(int cpu)531 static int __init pcpu_cpu_to_node(int cpu)
532 {
533 	return cpu_to_node(cpu);
534 }
535 
setup_per_cpu_areas(void)536 void __init setup_per_cpu_areas(void)
537 {
538 	unsigned long delta;
539 	unsigned int cpu;
540 	int rc;
541 
542 	/*
543 	 * Always reserve area for module percpu variables.  That's
544 	 * what the legacy allocator did.
545 	 */
546 	rc = pcpu_embed_first_chunk(PERCPU_MODULE_RESERVE,
547 				    PERCPU_DYNAMIC_RESERVE, PAGE_SIZE,
548 				    pcpu_cpu_distance,
549 				    pcpu_cpu_to_node);
550 	if (rc < 0)
551 		panic("Failed to initialize percpu areas.");
552 
553 	delta = (unsigned long)pcpu_base_addr - (unsigned long)__per_cpu_start;
554 	for_each_possible_cpu(cpu)
555 		__per_cpu_offset[cpu] = delta + pcpu_unit_offsets[cpu];
556 }
557 #endif
558 
559 #ifndef CONFIG_MIPS_PGD_C0_CONTEXT
560 unsigned long pgd_current[NR_CPUS];
561 #endif
562 
563 /*
564  * Align swapper_pg_dir in to 64K, allows its address to be loaded
565  * with a single LUI instruction in the TLB handlers.  If we used
566  * __aligned(64K), its size would get rounded up to the alignment
567  * size, and waste space.  So we place it in its own section and align
568  * it in the linker script.
569  */
570 pgd_t swapper_pg_dir[PTRS_PER_PGD] __section(".bss..swapper_pg_dir");
571 #ifndef __PAGETABLE_PUD_FOLDED
572 pud_t invalid_pud_table[PTRS_PER_PUD] __page_aligned_bss;
573 #endif
574 #ifndef __PAGETABLE_PMD_FOLDED
575 pmd_t invalid_pmd_table[PTRS_PER_PMD] __page_aligned_bss;
576 EXPORT_SYMBOL_GPL(invalid_pmd_table);
577 #endif
578 pte_t invalid_pte_table[PTRS_PER_PTE] __page_aligned_bss;
579 EXPORT_SYMBOL(invalid_pte_table);
580 
581 #ifdef CONFIG_EXECMEM
582 #ifdef MODULES_VADDR
583 static struct execmem_info execmem_info __ro_after_init;
584 
execmem_arch_setup(void)585 struct execmem_info __init *execmem_arch_setup(void)
586 {
587 	execmem_info = (struct execmem_info){
588 		.ranges = {
589 			[EXECMEM_DEFAULT] = {
590 				.start	= MODULES_VADDR,
591 				.end	= MODULES_END,
592 				.pgprot	= PAGE_KERNEL,
593 				.alignment = 1,
594 			},
595 		},
596 	};
597 
598 	return &execmem_info;
599 }
600 #endif
601 #endif /* CONFIG_EXECMEM */
602