xref: /linux/arch/powerpc/mm/mem.c (revision 42fda66387daa53538ae13a2c858396aaf037158) 
1 /*
2  *  PowerPC version
3  *    Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
4  *
5  *  Modifications by Paul Mackerras (PowerMac) (paulus@cs.anu.edu.au)
6  *  and Cort Dougan (PReP) (cort@cs.nmt.edu)
7  *    Copyright (C) 1996 Paul Mackerras
8  *  PPC44x/36-bit changes by Matt Porter (mporter@mvista.com)
9  *
10  *  Derived from "arch/i386/mm/init.c"
11  *    Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
12  *
13  *  This program is free software; you can redistribute it and/or
14  *  modify it under the terms of the GNU General Public License
15  *  as published by the Free Software Foundation; either version
16  *  2 of the License, or (at your option) any later version.
17  *
18  */
19 
20 #include <linux/module.h>
21 #include <linux/sched.h>
22 #include <linux/kernel.h>
23 #include <linux/errno.h>
24 #include <linux/string.h>
25 #include <linux/types.h>
26 #include <linux/mm.h>
27 #include <linux/stddef.h>
28 #include <linux/init.h>
29 #include <linux/bootmem.h>
30 #include <linux/highmem.h>
31 #include <linux/initrd.h>
32 #include <linux/pagemap.h>
33 #include <linux/suspend.h>
34 
35 #include <asm/pgalloc.h>
36 #include <asm/prom.h>
37 #include <asm/io.h>
38 #include <asm/mmu_context.h>
39 #include <asm/pgtable.h>
40 #include <asm/mmu.h>
41 #include <asm/smp.h>
42 #include <asm/machdep.h>
43 #include <asm/btext.h>
44 #include <asm/tlb.h>
45 #include <asm/lmb.h>
46 #include <asm/sections.h>
47 #include <asm/vdso.h>
48 
49 #include "mmu_decl.h"
50 
51 #ifndef CPU_FTR_COHERENT_ICACHE
52 #define CPU_FTR_COHERENT_ICACHE	0	/* XXX for now */
53 #define CPU_FTR_NOEXECUTE	0
54 #endif
55 
56 int init_bootmem_done;
57 int mem_init_done;
58 unsigned long memory_limit;
59 
60 int page_is_ram(unsigned long pfn)
61 {
62 	unsigned long paddr = (pfn << PAGE_SHIFT);
63 
64 #ifndef CONFIG_PPC64	/* XXX for now */
65 	return paddr < __pa(high_memory);
66 #else
67 	int i;
68 	for (i=0; i < lmb.memory.cnt; i++) {
69 		unsigned long base;
70 
71 		base = lmb.memory.region[i].base;
72 
73 		if ((paddr >= base) &&
74 			(paddr < (base + lmb.memory.region[i].size))) {
75 			return 1;
76 		}
77 	}
78 
79 	return 0;
80 #endif
81 }
82 
83 pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
84 			      unsigned long size, pgprot_t vma_prot)
85 {
86 	if (ppc_md.phys_mem_access_prot)
87 		return ppc_md.phys_mem_access_prot(file, pfn, size, vma_prot);
88 
89 	if (!page_is_ram(pfn))
90 		vma_prot = __pgprot(pgprot_val(vma_prot)
91 				    | _PAGE_GUARDED | _PAGE_NO_CACHE);
92 	return vma_prot;
93 }
94 EXPORT_SYMBOL(phys_mem_access_prot);
95 
96 #ifdef CONFIG_MEMORY_HOTPLUG
97 
98 void online_page(struct page *page)
99 {
100 	ClearPageReserved(page);
101 	init_page_count(page);
102 	__free_page(page);
103 	totalram_pages++;
104 	num_physpages++;
105 }
106 
107 #ifdef CONFIG_NUMA
108 int memory_add_physaddr_to_nid(u64 start)
109 {
110 	return hot_add_scn_to_nid(start);
111 }
112 #endif
113 
114 int __devinit arch_add_memory(int nid, u64 start, u64 size)
115 {
116 	struct pglist_data *pgdata;
117 	struct zone *zone;
118 	unsigned long start_pfn = start >> PAGE_SHIFT;
119 	unsigned long nr_pages = size >> PAGE_SHIFT;
120 
121 	pgdata = NODE_DATA(nid);
122 
123 	start = (unsigned long)__va(start);
124 	create_section_mapping(start, start + size);
125 
126 	/* this should work for most non-highmem platforms */
127 	zone = pgdata->node_zones;
128 
129 	return __add_pages(zone, start_pfn, nr_pages);
130 }
131 
132 #endif /* CONFIG_MEMORY_HOTPLUG */
133 
134 void show_mem(void)
135 {
136 	unsigned long total = 0, reserved = 0;
137 	unsigned long shared = 0, cached = 0;
138 	unsigned long highmem = 0;
139 	struct page *page;
140 	pg_data_t *pgdat;
141 	unsigned long i;
142 
143 	printk("Mem-info:\n");
144 	show_free_areas();
145 	printk("Free swap:       %6ldkB\n", nr_swap_pages<<(PAGE_SHIFT-10));
146 	for_each_online_pgdat(pgdat) {
147 		unsigned long flags;
148 		pgdat_resize_lock(pgdat, &flags);
149 		for (i = 0; i < pgdat->node_spanned_pages; i++) {
150 			if (!pfn_valid(pgdat->node_start_pfn + i))
151 				continue;
152 			page = pgdat_page_nr(pgdat, i);
153 			total++;
154 			if (PageHighMem(page))
155 				highmem++;
156 			if (PageReserved(page))
157 				reserved++;
158 			else if (PageSwapCache(page))
159 				cached++;
160 			else if (page_count(page))
161 				shared += page_count(page) - 1;
162 		}
163 		pgdat_resize_unlock(pgdat, &flags);
164 	}
165 	printk("%ld pages of RAM\n", total);
166 #ifdef CONFIG_HIGHMEM
167 	printk("%ld pages of HIGHMEM\n", highmem);
168 #endif
169 	printk("%ld reserved pages\n", reserved);
170 	printk("%ld pages shared\n", shared);
171 	printk("%ld pages swap cached\n", cached);
172 }
173 
174 /*
175  * Initialize the bootmem system and give it all the memory we
176  * have available.  If we are using highmem, we only put the
177  * lowmem into the bootmem system.
178  */
179 #ifndef CONFIG_NEED_MULTIPLE_NODES
180 void __init do_init_bootmem(void)
181 {
182 	unsigned long i;
183 	unsigned long start, bootmap_pages;
184 	unsigned long total_pages;
185 	int boot_mapsize;
186 
187 	max_pfn = total_pages = lmb_end_of_DRAM() >> PAGE_SHIFT;
188 #ifdef CONFIG_HIGHMEM
189 	total_pages = total_lowmem >> PAGE_SHIFT;
190 #endif
191 
192 	/*
193 	 * Find an area to use for the bootmem bitmap.  Calculate the size of
194 	 * bitmap required as (Total Memory) / PAGE_SIZE / BITS_PER_BYTE.
195 	 * Add 1 additional page in case the address isn't page-aligned.
196 	 */
197 	bootmap_pages = bootmem_bootmap_pages(total_pages);
198 
199 	start = lmb_alloc(bootmap_pages << PAGE_SHIFT, PAGE_SIZE);
200 
201 	boot_mapsize = init_bootmem(start >> PAGE_SHIFT, total_pages);
202 
203 	/* Add active regions with valid PFNs */
204 	for (i = 0; i < lmb.memory.cnt; i++) {
205 		unsigned long start_pfn, end_pfn;
206 		start_pfn = lmb.memory.region[i].base >> PAGE_SHIFT;
207 		end_pfn = start_pfn + lmb_size_pages(&lmb.memory, i);
208 		add_active_range(0, start_pfn, end_pfn);
209 	}
210 
211 	/* Add all physical memory to the bootmem map, mark each area
212 	 * present.
213 	 */
214 #ifdef CONFIG_HIGHMEM
215 	free_bootmem_with_active_regions(0, total_lowmem >> PAGE_SHIFT);
216 #else
217 	free_bootmem_with_active_regions(0, max_pfn);
218 #endif
219 
220 	/* reserve the sections we're already using */
221 	for (i = 0; i < lmb.reserved.cnt; i++)
222 		reserve_bootmem(lmb.reserved.region[i].base,
223 				lmb_size_bytes(&lmb.reserved, i));
224 
225 	/* XXX need to clip this if using highmem? */
226 	sparse_memory_present_with_active_regions(0);
227 
228 	init_bootmem_done = 1;
229 }
230 
231 /* mark pages that don't exist as nosave */
232 static int __init mark_nonram_nosave(void)
233 {
234 	unsigned long lmb_next_region_start_pfn,
235 		      lmb_region_max_pfn;
236 	int i;
237 
238 	for (i = 0; i < lmb.memory.cnt - 1; i++) {
239 		lmb_region_max_pfn =
240 			(lmb.memory.region[i].base >> PAGE_SHIFT) +
241 			(lmb.memory.region[i].size >> PAGE_SHIFT);
242 		lmb_next_region_start_pfn =
243 			lmb.memory.region[i+1].base >> PAGE_SHIFT;
244 
245 		if (lmb_region_max_pfn < lmb_next_region_start_pfn)
246 			register_nosave_region(lmb_region_max_pfn,
247 					       lmb_next_region_start_pfn);
248 	}
249 
250 	return 0;
251 }
252 
253 /*
254  * paging_init() sets up the page tables - in fact we've already done this.
255  */
256 void __init paging_init(void)
257 {
258 	unsigned long total_ram = lmb_phys_mem_size();
259 	unsigned long top_of_ram = lmb_end_of_DRAM();
260 	unsigned long max_zone_pfns[MAX_NR_ZONES];
261 
262 #ifdef CONFIG_HIGHMEM
263 	map_page(PKMAP_BASE, 0, 0);	/* XXX gross */
264 	pkmap_page_table = pte_offset_kernel(pmd_offset(pud_offset(pgd_offset_k
265 			(PKMAP_BASE), PKMAP_BASE), PKMAP_BASE), PKMAP_BASE);
266 	map_page(KMAP_FIX_BEGIN, 0, 0);	/* XXX gross */
267 	kmap_pte = pte_offset_kernel(pmd_offset(pud_offset(pgd_offset_k
268 			(KMAP_FIX_BEGIN), KMAP_FIX_BEGIN), KMAP_FIX_BEGIN),
269 			 KMAP_FIX_BEGIN);
270 	kmap_prot = PAGE_KERNEL;
271 #endif /* CONFIG_HIGHMEM */
272 
273 	printk(KERN_DEBUG "Top of RAM: 0x%lx, Total RAM: 0x%lx\n",
274 	       top_of_ram, total_ram);
275 	printk(KERN_DEBUG "Memory hole size: %ldMB\n",
276 	       (top_of_ram - total_ram) >> 20);
277 	memset(max_zone_pfns, 0, sizeof(max_zone_pfns));
278 #ifdef CONFIG_HIGHMEM
279 	max_zone_pfns[ZONE_DMA] = total_lowmem >> PAGE_SHIFT;
280 	max_zone_pfns[ZONE_HIGHMEM] = top_of_ram >> PAGE_SHIFT;
281 #else
282 	max_zone_pfns[ZONE_DMA] = top_of_ram >> PAGE_SHIFT;
283 #endif
284 	free_area_init_nodes(max_zone_pfns);
285 
286 	mark_nonram_nosave();
287 }
288 #endif /* ! CONFIG_NEED_MULTIPLE_NODES */
289 
290 void __init mem_init(void)
291 {
292 #ifdef CONFIG_NEED_MULTIPLE_NODES
293 	int nid;
294 #endif
295 	pg_data_t *pgdat;
296 	unsigned long i;
297 	struct page *page;
298 	unsigned long reservedpages = 0, codesize, initsize, datasize, bsssize;
299 
300 	num_physpages = lmb.memory.size >> PAGE_SHIFT;
301 	high_memory = (void *) __va(max_low_pfn * PAGE_SIZE);
302 
303 #ifdef CONFIG_NEED_MULTIPLE_NODES
304         for_each_online_node(nid) {
305 		if (NODE_DATA(nid)->node_spanned_pages != 0) {
306 			printk("freeing bootmem node %d\n", nid);
307 			totalram_pages +=
308 				free_all_bootmem_node(NODE_DATA(nid));
309 		}
310 	}
311 #else
312 	max_mapnr = max_pfn;
313 	totalram_pages += free_all_bootmem();
314 #endif
315 	for_each_online_pgdat(pgdat) {
316 		for (i = 0; i < pgdat->node_spanned_pages; i++) {
317 			if (!pfn_valid(pgdat->node_start_pfn + i))
318 				continue;
319 			page = pgdat_page_nr(pgdat, i);
320 			if (PageReserved(page))
321 				reservedpages++;
322 		}
323 	}
324 
325 	codesize = (unsigned long)&_sdata - (unsigned long)&_stext;
326 	datasize = (unsigned long)&_edata - (unsigned long)&_sdata;
327 	initsize = (unsigned long)&__init_end - (unsigned long)&__init_begin;
328 	bsssize = (unsigned long)&__bss_stop - (unsigned long)&__bss_start;
329 
330 #ifdef CONFIG_HIGHMEM
331 	{
332 		unsigned long pfn, highmem_mapnr;
333 
334 		highmem_mapnr = total_lowmem >> PAGE_SHIFT;
335 		for (pfn = highmem_mapnr; pfn < max_mapnr; ++pfn) {
336 			struct page *page = pfn_to_page(pfn);
337 
338 			ClearPageReserved(page);
339 			init_page_count(page);
340 			__free_page(page);
341 			totalhigh_pages++;
342 		}
343 		totalram_pages += totalhigh_pages;
344 		printk(KERN_DEBUG "High memory: %luk\n",
345 		       totalhigh_pages << (PAGE_SHIFT-10));
346 	}
347 #endif /* CONFIG_HIGHMEM */
348 
349 	printk(KERN_INFO "Memory: %luk/%luk available (%luk kernel code, "
350 	       "%luk reserved, %luk data, %luk bss, %luk init)\n",
351 		(unsigned long)nr_free_pages() << (PAGE_SHIFT-10),
352 		num_physpages << (PAGE_SHIFT-10),
353 		codesize >> 10,
354 		reservedpages << (PAGE_SHIFT-10),
355 		datasize >> 10,
356 		bsssize >> 10,
357 		initsize >> 10);
358 
359 	mem_init_done = 1;
360 }
361 
362 /*
363  * This is called when a page has been modified by the kernel.
364  * It just marks the page as not i-cache clean.  We do the i-cache
365  * flush later when the page is given to a user process, if necessary.
366  */
367 void flush_dcache_page(struct page *page)
368 {
369 	if (cpu_has_feature(CPU_FTR_COHERENT_ICACHE))
370 		return;
371 	/* avoid an atomic op if possible */
372 	if (test_bit(PG_arch_1, &page->flags))
373 		clear_bit(PG_arch_1, &page->flags);
374 }
375 EXPORT_SYMBOL(flush_dcache_page);
376 
377 void flush_dcache_icache_page(struct page *page)
378 {
379 #ifdef CONFIG_BOOKE
380 	void *start = kmap_atomic(page, KM_PPC_SYNC_ICACHE);
381 	__flush_dcache_icache(start);
382 	kunmap_atomic(start, KM_PPC_SYNC_ICACHE);
383 #elif defined(CONFIG_8xx) || defined(CONFIG_PPC64)
384 	/* On 8xx there is no need to kmap since highmem is not supported */
385 	__flush_dcache_icache(page_address(page));
386 #else
387 	__flush_dcache_icache_phys(page_to_pfn(page) << PAGE_SHIFT);
388 #endif
389 
390 }
391 void clear_user_page(void *page, unsigned long vaddr, struct page *pg)
392 {
393 	clear_page(page);
394 
395 	/*
396 	 * We shouldnt have to do this, but some versions of glibc
397 	 * require it (ld.so assumes zero filled pages are icache clean)
398 	 * - Anton
399 	 */
400 	flush_dcache_page(pg);
401 }
402 EXPORT_SYMBOL(clear_user_page);
403 
404 void copy_user_page(void *vto, void *vfrom, unsigned long vaddr,
405 		    struct page *pg)
406 {
407 	copy_page(vto, vfrom);
408 
409 	/*
410 	 * We should be able to use the following optimisation, however
411 	 * there are two problems.
412 	 * Firstly a bug in some versions of binutils meant PLT sections
413 	 * were not marked executable.
414 	 * Secondly the first word in the GOT section is blrl, used
415 	 * to establish the GOT address. Until recently the GOT was
416 	 * not marked executable.
417 	 * - Anton
418 	 */
419 #if 0
420 	if (!vma->vm_file && ((vma->vm_flags & VM_EXEC) == 0))
421 		return;
422 #endif
423 
424 	flush_dcache_page(pg);
425 }
426 
427 void flush_icache_user_range(struct vm_area_struct *vma, struct page *page,
428 			     unsigned long addr, int len)
429 {
430 	unsigned long maddr;
431 
432 	maddr = (unsigned long) kmap(page) + (addr & ~PAGE_MASK);
433 	flush_icache_range(maddr, maddr + len);
434 	kunmap(page);
435 }
436 EXPORT_SYMBOL(flush_icache_user_range);
437 
438 /*
439  * This is called at the end of handling a user page fault, when the
440  * fault has been handled by updating a PTE in the linux page tables.
441  * We use it to preload an HPTE into the hash table corresponding to
442  * the updated linux PTE.
443  *
444  * This must always be called with the pte lock held.
445  */
446 void update_mmu_cache(struct vm_area_struct *vma, unsigned long address,
447 		      pte_t pte)
448 {
449 #ifdef CONFIG_PPC_STD_MMU
450 	unsigned long access = 0, trap;
451 #endif
452 	unsigned long pfn = pte_pfn(pte);
453 
454 	/* handle i-cache coherency */
455 	if (!cpu_has_feature(CPU_FTR_COHERENT_ICACHE) &&
456 	    !cpu_has_feature(CPU_FTR_NOEXECUTE) &&
457 	    pfn_valid(pfn)) {
458 		struct page *page = pfn_to_page(pfn);
459 #ifdef CONFIG_8xx
460 		/* On 8xx, cache control instructions (particularly
461 		 * "dcbst" from flush_dcache_icache) fault as write
462 		 * operation if there is an unpopulated TLB entry
463 		 * for the address in question. To workaround that,
464 		 * we invalidate the TLB here, thus avoiding dcbst
465 		 * misbehaviour.
466 		 */
467 		_tlbie(address);
468 #endif
469 		if (!PageReserved(page)
470 		    && !test_bit(PG_arch_1, &page->flags)) {
471 			if (vma->vm_mm == current->active_mm) {
472 				__flush_dcache_icache((void *) address);
473 			} else
474 				flush_dcache_icache_page(page);
475 			set_bit(PG_arch_1, &page->flags);
476 		}
477 	}
478 
479 #ifdef CONFIG_PPC_STD_MMU
480 	/* We only want HPTEs for linux PTEs that have _PAGE_ACCESSED set */
481 	if (!pte_young(pte) || address >= TASK_SIZE)
482 		return;
483 
484 	/* We try to figure out if we are coming from an instruction
485 	 * access fault and pass that down to __hash_page so we avoid
486 	 * double-faulting on execution of fresh text. We have to test
487 	 * for regs NULL since init will get here first thing at boot
488 	 *
489 	 * We also avoid filling the hash if not coming from a fault
490 	 */
491 	if (current->thread.regs == NULL)
492 		return;
493 	trap = TRAP(current->thread.regs);
494 	if (trap == 0x400)
495 		access |= _PAGE_EXEC;
496 	else if (trap != 0x300)
497 		return;
498 	hash_preload(vma->vm_mm, address, access, trap);
499 #endif /* CONFIG_PPC_STD_MMU */
500 }
501