1 /*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 1996 David S. Miller (davem@davemloft.net)
7 * Copyright (C) 1997, 1998, 1999, 2000 Ralf Baechle ralf@gnu.org
8 * Carsten Langgaard, carstenl@mips.com
9 * Copyright (C) 2002 MIPS Technologies, Inc. All rights reserved.
10 */
11 #include <linux/cpu_pm.h>
12 #include <linux/init.h>
13 #include <linux/sched.h>
14 #include <linux/smp.h>
15 #include <linux/memblock.h>
16 #include <linux/minmax.h>
17 #include <linux/mm.h>
18 #include <linux/hugetlb.h>
19 #include <linux/export.h>
20 #include <linux/sort.h>
21
22 #include <asm/cpu.h>
23 #include <asm/cpu-type.h>
24 #include <asm/bootinfo.h>
25 #include <asm/hazards.h>
26 #include <asm/mmu_context.h>
27 #include <asm/tlb.h>
28 #include <asm/tlbdebug.h>
29 #include <asm/tlbex.h>
30 #include <asm/tlbmisc.h>
31 #include <asm/setup.h>
32
33 /*
34 * LOONGSON-2 has a 4 entry itlb which is a subset of jtlb, LOONGSON-3 has
35 * a 4 entry itlb and a 4 entry dtlb which are subsets of jtlb. Unfortunately,
36 * itlb/dtlb are not totally transparent to software.
37 */
flush_micro_tlb(void)38 static inline void flush_micro_tlb(void)
39 {
40 switch (current_cpu_type()) {
41 case CPU_LOONGSON2EF:
42 write_c0_diag(LOONGSON_DIAG_ITLB);
43 break;
44 case CPU_LOONGSON64:
45 write_c0_diag(LOONGSON_DIAG_ITLB | LOONGSON_DIAG_DTLB);
46 break;
47 default:
48 break;
49 }
50 }
51
flush_micro_tlb_vm(struct vm_area_struct * vma)52 static inline void flush_micro_tlb_vm(struct vm_area_struct *vma)
53 {
54 if (vma->vm_flags & VM_EXEC)
55 flush_micro_tlb();
56 }
57
local_flush_tlb_all(void)58 void local_flush_tlb_all(void)
59 {
60 unsigned long flags;
61 unsigned long old_ctx;
62 int entry, ftlbhighset;
63
64 local_irq_save(flags);
65 /* Save old context and create impossible VPN2 value */
66 old_ctx = read_c0_entryhi();
67 htw_stop();
68 write_c0_entrylo0(0);
69 write_c0_entrylo1(0);
70
71 entry = num_wired_entries();
72
73 /*
74 * Blast 'em all away.
75 * If there are any wired entries, fall back to iterating
76 */
77 if (cpu_has_tlbinv && !entry) {
78 if (current_cpu_data.tlbsizevtlb) {
79 write_c0_index(0);
80 mtc0_tlbw_hazard();
81 tlbinvf(); /* invalidate VTLB */
82 }
83 ftlbhighset = current_cpu_data.tlbsizevtlb +
84 current_cpu_data.tlbsizeftlbsets;
85 for (entry = current_cpu_data.tlbsizevtlb;
86 entry < ftlbhighset;
87 entry++) {
88 write_c0_index(entry);
89 mtc0_tlbw_hazard();
90 tlbinvf(); /* invalidate one FTLB set */
91 }
92 } else {
93 while (entry < current_cpu_data.tlbsize) {
94 /* Make sure all entries differ. */
95 write_c0_entryhi(UNIQUE_ENTRYHI(entry));
96 write_c0_index(entry);
97 mtc0_tlbw_hazard();
98 tlb_write_indexed();
99 entry++;
100 }
101 }
102 tlbw_use_hazard();
103 write_c0_entryhi(old_ctx);
104 htw_start();
105 flush_micro_tlb();
106 local_irq_restore(flags);
107 }
108 EXPORT_SYMBOL(local_flush_tlb_all);
109
local_flush_tlb_range(struct vm_area_struct * vma,unsigned long start,unsigned long end)110 void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
111 unsigned long end)
112 {
113 struct mm_struct *mm = vma->vm_mm;
114 int cpu = smp_processor_id();
115
116 if (cpu_context(cpu, mm) != 0) {
117 unsigned long size, flags;
118
119 local_irq_save(flags);
120 start = round_down(start, PAGE_SIZE << 1);
121 end = round_up(end, PAGE_SIZE << 1);
122 size = (end - start) >> (PAGE_SHIFT + 1);
123 if (size <= (current_cpu_data.tlbsizeftlbsets ?
124 current_cpu_data.tlbsize / 8 :
125 current_cpu_data.tlbsize / 2)) {
126 unsigned long old_entryhi, old_mmid;
127 int newpid = cpu_asid(cpu, mm);
128
129 old_entryhi = read_c0_entryhi();
130 if (cpu_has_mmid) {
131 old_mmid = read_c0_memorymapid();
132 write_c0_memorymapid(newpid);
133 }
134
135 htw_stop();
136 while (start < end) {
137 int idx;
138
139 if (cpu_has_mmid)
140 write_c0_entryhi(start);
141 else
142 write_c0_entryhi(start | newpid);
143 start += (PAGE_SIZE << 1);
144 mtc0_tlbw_hazard();
145 tlb_probe();
146 tlb_probe_hazard();
147 idx = read_c0_index();
148 write_c0_entrylo0(0);
149 write_c0_entrylo1(0);
150 if (idx < 0)
151 continue;
152 /* Make sure all entries differ. */
153 write_c0_entryhi(UNIQUE_ENTRYHI(idx));
154 mtc0_tlbw_hazard();
155 tlb_write_indexed();
156 }
157 tlbw_use_hazard();
158 write_c0_entryhi(old_entryhi);
159 if (cpu_has_mmid)
160 write_c0_memorymapid(old_mmid);
161 htw_start();
162 } else {
163 drop_mmu_context(mm);
164 }
165 flush_micro_tlb();
166 local_irq_restore(flags);
167 }
168 }
169
local_flush_tlb_kernel_range(unsigned long start,unsigned long end)170 void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
171 {
172 unsigned long size, flags;
173
174 local_irq_save(flags);
175 size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
176 size = (size + 1) >> 1;
177 if (size <= (current_cpu_data.tlbsizeftlbsets ?
178 current_cpu_data.tlbsize / 8 :
179 current_cpu_data.tlbsize / 2)) {
180 int pid = read_c0_entryhi();
181
182 start &= (PAGE_MASK << 1);
183 end += ((PAGE_SIZE << 1) - 1);
184 end &= (PAGE_MASK << 1);
185 htw_stop();
186
187 while (start < end) {
188 int idx;
189
190 write_c0_entryhi(start);
191 start += (PAGE_SIZE << 1);
192 mtc0_tlbw_hazard();
193 tlb_probe();
194 tlb_probe_hazard();
195 idx = read_c0_index();
196 write_c0_entrylo0(0);
197 write_c0_entrylo1(0);
198 if (idx < 0)
199 continue;
200 /* Make sure all entries differ. */
201 write_c0_entryhi(UNIQUE_ENTRYHI(idx));
202 mtc0_tlbw_hazard();
203 tlb_write_indexed();
204 }
205 tlbw_use_hazard();
206 write_c0_entryhi(pid);
207 htw_start();
208 } else {
209 local_flush_tlb_all();
210 }
211 flush_micro_tlb();
212 local_irq_restore(flags);
213 }
214
local_flush_tlb_page(struct vm_area_struct * vma,unsigned long page)215 void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
216 {
217 int cpu = smp_processor_id();
218
219 if (cpu_context(cpu, vma->vm_mm) != 0) {
220 unsigned long old_mmid;
221 unsigned long flags, old_entryhi;
222 int idx;
223
224 page &= (PAGE_MASK << 1);
225 local_irq_save(flags);
226 old_entryhi = read_c0_entryhi();
227 htw_stop();
228 if (cpu_has_mmid) {
229 old_mmid = read_c0_memorymapid();
230 write_c0_entryhi(page);
231 write_c0_memorymapid(cpu_asid(cpu, vma->vm_mm));
232 } else {
233 write_c0_entryhi(page | cpu_asid(cpu, vma->vm_mm));
234 }
235 mtc0_tlbw_hazard();
236 tlb_probe();
237 tlb_probe_hazard();
238 idx = read_c0_index();
239 write_c0_entrylo0(0);
240 write_c0_entrylo1(0);
241 if (idx < 0)
242 goto finish;
243 /* Make sure all entries differ. */
244 write_c0_entryhi(UNIQUE_ENTRYHI(idx));
245 mtc0_tlbw_hazard();
246 tlb_write_indexed();
247 tlbw_use_hazard();
248
249 finish:
250 write_c0_entryhi(old_entryhi);
251 if (cpu_has_mmid)
252 write_c0_memorymapid(old_mmid);
253 htw_start();
254 flush_micro_tlb_vm(vma);
255 local_irq_restore(flags);
256 }
257 }
258
259 /*
260 * This one is only used for pages with the global bit set so we don't care
261 * much about the ASID.
262 */
local_flush_tlb_one(unsigned long page)263 void local_flush_tlb_one(unsigned long page)
264 {
265 unsigned long flags;
266 int oldpid, idx;
267
268 local_irq_save(flags);
269 oldpid = read_c0_entryhi();
270 htw_stop();
271 page &= (PAGE_MASK << 1);
272 write_c0_entryhi(page);
273 mtc0_tlbw_hazard();
274 tlb_probe();
275 tlb_probe_hazard();
276 idx = read_c0_index();
277 write_c0_entrylo0(0);
278 write_c0_entrylo1(0);
279 if (idx >= 0) {
280 /* Make sure all entries differ. */
281 write_c0_entryhi(UNIQUE_ENTRYHI(idx));
282 mtc0_tlbw_hazard();
283 tlb_write_indexed();
284 tlbw_use_hazard();
285 }
286 write_c0_entryhi(oldpid);
287 htw_start();
288 flush_micro_tlb();
289 local_irq_restore(flags);
290 }
291
292 /*
293 * We will need multiple versions of update_mmu_cache(), one that just
294 * updates the TLB with the new pte(s), and another which also checks
295 * for the R4k "end of page" hardware bug and does the needy.
296 */
__update_tlb(struct vm_area_struct * vma,unsigned long address,pte_t pte)297 void __update_tlb(struct vm_area_struct * vma, unsigned long address, pte_t pte)
298 {
299 unsigned long flags;
300 pgd_t *pgdp;
301 p4d_t *p4dp;
302 pud_t *pudp;
303 pmd_t *pmdp;
304 pte_t *ptep, *ptemap = NULL;
305 int idx, pid;
306
307 /*
308 * Handle debugger faulting in for debuggee.
309 */
310 if (current->active_mm != vma->vm_mm)
311 return;
312
313 local_irq_save(flags);
314
315 htw_stop();
316 address &= (PAGE_MASK << 1);
317 if (cpu_has_mmid) {
318 write_c0_entryhi(address);
319 } else {
320 pid = read_c0_entryhi() & cpu_asid_mask(¤t_cpu_data);
321 write_c0_entryhi(address | pid);
322 }
323 pgdp = pgd_offset(vma->vm_mm, address);
324 mtc0_tlbw_hazard();
325 tlb_probe();
326 tlb_probe_hazard();
327 p4dp = p4d_offset(pgdp, address);
328 pudp = pud_offset(p4dp, address);
329 pmdp = pmd_offset(pudp, address);
330 idx = read_c0_index();
331 #ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
332 /* this could be a huge page */
333 if (pmd_leaf(*pmdp)) {
334 unsigned long lo;
335 write_c0_pagemask(PM_HUGE_MASK);
336 ptep = (pte_t *)pmdp;
337 lo = pte_to_entrylo(pte_val(*ptep));
338 write_c0_entrylo0(lo);
339 write_c0_entrylo1(lo + (HPAGE_SIZE >> 7));
340
341 mtc0_tlbw_hazard();
342 if (idx < 0)
343 tlb_write_random();
344 else
345 tlb_write_indexed();
346 tlbw_use_hazard();
347 write_c0_pagemask(PM_DEFAULT_MASK);
348 } else
349 #endif
350 {
351 ptemap = ptep = pte_offset_map(pmdp, address);
352 /*
353 * update_mmu_cache() is called between pte_offset_map_lock()
354 * and pte_unmap_unlock(), so we can assume that ptep is not
355 * NULL here: and what should be done below if it were NULL?
356 */
357
358 #if defined(CONFIG_PHYS_ADDR_T_64BIT) && defined(CONFIG_CPU_MIPS32)
359 #ifdef CONFIG_XPA
360 write_c0_entrylo0(pte_to_entrylo(ptep->pte_high));
361 if (cpu_has_xpa)
362 writex_c0_entrylo0(ptep->pte_low & _PFNX_MASK);
363 ptep++;
364 write_c0_entrylo1(pte_to_entrylo(ptep->pte_high));
365 if (cpu_has_xpa)
366 writex_c0_entrylo1(ptep->pte_low & _PFNX_MASK);
367 #else
368 write_c0_entrylo0(ptep->pte_high);
369 ptep++;
370 write_c0_entrylo1(ptep->pte_high);
371 #endif
372 #else
373 write_c0_entrylo0(pte_to_entrylo(pte_val(*ptep++)));
374 write_c0_entrylo1(pte_to_entrylo(pte_val(*ptep)));
375 #endif
376 mtc0_tlbw_hazard();
377 if (idx < 0)
378 tlb_write_random();
379 else
380 tlb_write_indexed();
381 }
382 tlbw_use_hazard();
383 htw_start();
384 flush_micro_tlb_vm(vma);
385
386 if (ptemap)
387 pte_unmap(ptemap);
388 local_irq_restore(flags);
389 }
390
add_wired_entry(unsigned long entrylo0,unsigned long entrylo1,unsigned long entryhi,unsigned long pagemask)391 void add_wired_entry(unsigned long entrylo0, unsigned long entrylo1,
392 unsigned long entryhi, unsigned long pagemask)
393 {
394 #ifdef CONFIG_XPA
395 panic("Broken for XPA kernels");
396 #else
397 unsigned int old_mmid;
398 unsigned long flags;
399 unsigned long wired;
400 unsigned long old_pagemask;
401 unsigned long old_ctx;
402
403 local_irq_save(flags);
404 if (cpu_has_mmid) {
405 old_mmid = read_c0_memorymapid();
406 write_c0_memorymapid(MMID_KERNEL_WIRED);
407 }
408 /* Save old context and create impossible VPN2 value */
409 old_ctx = read_c0_entryhi();
410 htw_stop();
411 old_pagemask = read_c0_pagemask();
412 wired = num_wired_entries();
413 write_c0_wired(wired + 1);
414 write_c0_index(wired);
415 tlbw_use_hazard(); /* What is the hazard here? */
416 write_c0_pagemask(pagemask);
417 write_c0_entryhi(entryhi);
418 write_c0_entrylo0(entrylo0);
419 write_c0_entrylo1(entrylo1);
420 mtc0_tlbw_hazard();
421 tlb_write_indexed();
422 tlbw_use_hazard();
423
424 write_c0_entryhi(old_ctx);
425 if (cpu_has_mmid)
426 write_c0_memorymapid(old_mmid);
427 tlbw_use_hazard(); /* What is the hazard here? */
428 htw_start();
429 write_c0_pagemask(old_pagemask);
430 local_flush_tlb_all();
431 local_irq_restore(flags);
432 #endif
433 }
434
435 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
436
has_transparent_hugepage(void)437 int has_transparent_hugepage(void)
438 {
439 static unsigned int mask = -1;
440
441 if (mask == -1) { /* first call comes during __init */
442 unsigned long flags;
443
444 local_irq_save(flags);
445 write_c0_pagemask(PM_HUGE_MASK);
446 back_to_back_c0_hazard();
447 mask = read_c0_pagemask();
448 write_c0_pagemask(PM_DEFAULT_MASK);
449 local_irq_restore(flags);
450 }
451 return mask == PM_HUGE_MASK;
452 }
453 EXPORT_SYMBOL(has_transparent_hugepage);
454
455 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
456
457 /*
458 * Used for loading TLB entries before trap_init() has started, when we
459 * don't actually want to add a wired entry which remains throughout the
460 * lifetime of the system
461 */
462
463 int temp_tlb_entry;
464
465 #ifndef CONFIG_64BIT
add_temporary_entry(unsigned long entrylo0,unsigned long entrylo1,unsigned long entryhi,unsigned long pagemask)466 __init int add_temporary_entry(unsigned long entrylo0, unsigned long entrylo1,
467 unsigned long entryhi, unsigned long pagemask)
468 {
469 int ret = 0;
470 unsigned long flags;
471 unsigned long wired;
472 unsigned long old_pagemask;
473 unsigned long old_ctx;
474
475 local_irq_save(flags);
476 /* Save old context and create impossible VPN2 value */
477 htw_stop();
478 old_ctx = read_c0_entryhi();
479 old_pagemask = read_c0_pagemask();
480 wired = num_wired_entries();
481 if (--temp_tlb_entry < wired) {
482 printk(KERN_WARNING
483 "No TLB space left for add_temporary_entry\n");
484 ret = -ENOSPC;
485 goto out;
486 }
487
488 write_c0_index(temp_tlb_entry);
489 write_c0_pagemask(pagemask);
490 write_c0_entryhi(entryhi);
491 write_c0_entrylo0(entrylo0);
492 write_c0_entrylo1(entrylo1);
493 mtc0_tlbw_hazard();
494 tlb_write_indexed();
495 tlbw_use_hazard();
496
497 write_c0_entryhi(old_ctx);
498 write_c0_pagemask(old_pagemask);
499 htw_start();
500 out:
501 local_irq_restore(flags);
502 return ret;
503 }
504 #endif
505
506 static int ntlb;
set_ntlb(char * str)507 static int __init set_ntlb(char *str)
508 {
509 get_option(&str, &ntlb);
510 return 1;
511 }
512
513 __setup("ntlb=", set_ntlb);
514
515
516 /* The start bit position of VPN2 and Mask in EntryHi/PageMask registers. */
517 #define VPN2_SHIFT 13
518
519 /* Read full EntryHi even with CONFIG_32BIT. */
read_c0_entryhi_native(void)520 static inline unsigned long long read_c0_entryhi_native(void)
521 {
522 return cpu_has_64bits ? read_c0_entryhi_64() : read_c0_entryhi();
523 }
524
525 /* Write full EntryHi even with CONFIG_32BIT. */
write_c0_entryhi_native(unsigned long long v)526 static inline void write_c0_entryhi_native(unsigned long long v)
527 {
528 if (cpu_has_64bits)
529 write_c0_entryhi_64(v);
530 else
531 write_c0_entryhi(v);
532 }
533
534 /* TLB entry state for uniquification. */
535 struct tlbent {
536 unsigned long long wired:1;
537 unsigned long long global:1;
538 unsigned long long asid:10;
539 unsigned long long vpn:51;
540 unsigned long long pagesz:5;
541 unsigned long long index:14;
542 };
543
544 /*
545 * Comparison function for TLB entry sorting. Place wired entries first,
546 * then global entries, then order by the increasing VPN/ASID and the
547 * decreasing page size. This lets us avoid clashes with wired entries
548 * easily and get entries for larger pages out of the way first.
549 *
550 * We could group bits so as to reduce the number of comparisons, but this
551 * is seldom executed and not performance-critical, so prefer legibility.
552 */
r4k_entry_cmp(const void * a,const void * b)553 static int r4k_entry_cmp(const void *a, const void *b)
554 {
555 struct tlbent ea = *(struct tlbent *)a, eb = *(struct tlbent *)b;
556
557 if (ea.wired > eb.wired)
558 return -1;
559 else if (ea.wired < eb.wired)
560 return 1;
561 else if (ea.global > eb.global)
562 return -1;
563 else if (ea.global < eb.global)
564 return 1;
565 else if (ea.vpn < eb.vpn)
566 return -1;
567 else if (ea.vpn > eb.vpn)
568 return 1;
569 else if (ea.asid < eb.asid)
570 return -1;
571 else if (ea.asid > eb.asid)
572 return 1;
573 else if (ea.pagesz > eb.pagesz)
574 return -1;
575 else if (ea.pagesz < eb.pagesz)
576 return 1;
577 else
578 return 0;
579 }
580
581 /*
582 * Fetch all the TLB entries. Mask individual VPN values retrieved with
583 * the corresponding page mask and ignoring any 1KiB extension as we'll
584 * be using 4KiB pages for uniquification.
585 */
r4k_tlb_uniquify_read(struct tlbent * tlb_vpns,int tlbsize)586 static void __ref r4k_tlb_uniquify_read(struct tlbent *tlb_vpns, int tlbsize)
587 {
588 int start = num_wired_entries();
589 unsigned long long vpn_mask;
590 bool global;
591 int i;
592
593 vpn_mask = GENMASK(current_cpu_data.vmbits - 1, VPN2_SHIFT);
594 vpn_mask |= cpu_has_64bits ? 3ULL << 62 : 1 << 31;
595
596 for (i = 0; i < tlbsize; i++) {
597 unsigned long long entryhi, vpn, mask, asid;
598 unsigned int pagesz;
599
600 write_c0_index(i);
601 mtc0_tlbr_hazard();
602 tlb_read();
603 tlb_read_hazard();
604
605 global = !!(read_c0_entrylo0() & ENTRYLO_G);
606 entryhi = read_c0_entryhi_native();
607 mask = read_c0_pagemask();
608
609 asid = entryhi & cpu_asid_mask(¤t_cpu_data);
610 vpn = (entryhi & vpn_mask & ~mask) >> VPN2_SHIFT;
611 pagesz = ilog2((mask >> VPN2_SHIFT) + 1);
612
613 tlb_vpns[i].global = global;
614 tlb_vpns[i].asid = global ? 0 : asid;
615 tlb_vpns[i].vpn = vpn;
616 tlb_vpns[i].pagesz = pagesz;
617 tlb_vpns[i].wired = i < start;
618 tlb_vpns[i].index = i;
619 }
620 }
621
622 /*
623 * Write unique values to all but the wired TLB entries each, using
624 * the 4KiB page size. This size might not be supported with R6, but
625 * EHINV is mandatory for R6, so we won't ever be called in that case.
626 *
627 * A sorted table is supplied with any wired entries at the beginning,
628 * followed by any global entries, and then finally regular entries.
629 * We start at the VPN and ASID values of zero and only assign user
630 * addresses, therefore guaranteeing no clash with addresses produced
631 * by UNIQUE_ENTRYHI. We avoid any VPN values used by wired or global
632 * entries, by increasing the VPN value beyond the span of such entry.
633 *
634 * When a VPN/ASID clash is found with a regular entry we increment the
635 * ASID instead until no VPN/ASID clash has been found or the ASID space
636 * has been exhausted, in which case we increase the VPN value beyond
637 * the span of the largest clashing entry.
638 *
639 * We do not need to be concerned about FTLB or MMID configurations as
640 * those are required to implement the EHINV feature.
641 */
r4k_tlb_uniquify_write(struct tlbent * tlb_vpns,int tlbsize)642 static void __ref r4k_tlb_uniquify_write(struct tlbent *tlb_vpns, int tlbsize)
643 {
644 unsigned long long asid, vpn, vpn_size, pagesz;
645 int widx, gidx, idx, sidx, lidx, i;
646
647 vpn_size = 1ULL << (current_cpu_data.vmbits - VPN2_SHIFT);
648 pagesz = ilog2((PM_4K >> VPN2_SHIFT) + 1);
649
650 write_c0_pagemask(PM_4K);
651 write_c0_entrylo0(0);
652 write_c0_entrylo1(0);
653
654 asid = 0;
655 vpn = 0;
656 widx = 0;
657 gidx = 0;
658 for (sidx = 0; sidx < tlbsize && tlb_vpns[sidx].wired; sidx++)
659 ;
660 for (lidx = sidx; lidx < tlbsize && tlb_vpns[lidx].global; lidx++)
661 ;
662 idx = gidx = sidx + 1;
663 for (i = sidx; i < tlbsize; i++) {
664 unsigned long long entryhi, vpn_pagesz = 0;
665
666 while (1) {
667 if (WARN_ON(vpn >= vpn_size)) {
668 dump_tlb_all();
669 /* Pray local_flush_tlb_all() will cope. */
670 return;
671 }
672
673 /* VPN must be below the next wired entry. */
674 if (widx < sidx && vpn >= tlb_vpns[widx].vpn) {
675 vpn = max(vpn,
676 (tlb_vpns[widx].vpn +
677 (1ULL << tlb_vpns[widx].pagesz)));
678 asid = 0;
679 widx++;
680 continue;
681 }
682 /* VPN must be below the next global entry. */
683 if (gidx < lidx && vpn >= tlb_vpns[gidx].vpn) {
684 vpn = max(vpn,
685 (tlb_vpns[gidx].vpn +
686 (1ULL << tlb_vpns[gidx].pagesz)));
687 asid = 0;
688 gidx++;
689 continue;
690 }
691 /* Try to find a free ASID so as to conserve VPNs. */
692 if (idx < tlbsize && vpn == tlb_vpns[idx].vpn &&
693 asid == tlb_vpns[idx].asid) {
694 unsigned long long idx_pagesz;
695
696 idx_pagesz = tlb_vpns[idx].pagesz;
697 vpn_pagesz = max(vpn_pagesz, idx_pagesz);
698 do
699 idx++;
700 while (idx < tlbsize &&
701 vpn == tlb_vpns[idx].vpn &&
702 asid == tlb_vpns[idx].asid);
703 asid++;
704 if (asid > cpu_asid_mask(¤t_cpu_data)) {
705 vpn += vpn_pagesz;
706 asid = 0;
707 vpn_pagesz = 0;
708 }
709 continue;
710 }
711 /* VPN mustn't be above the next regular entry. */
712 if (idx < tlbsize && vpn > tlb_vpns[idx].vpn) {
713 vpn = max(vpn,
714 (tlb_vpns[idx].vpn +
715 (1ULL << tlb_vpns[idx].pagesz)));
716 asid = 0;
717 idx++;
718 continue;
719 }
720 break;
721 }
722
723 entryhi = (vpn << VPN2_SHIFT) | asid;
724 write_c0_entryhi_native(entryhi);
725 write_c0_index(tlb_vpns[i].index);
726 mtc0_tlbw_hazard();
727 tlb_write_indexed();
728
729 tlb_vpns[i].asid = asid;
730 tlb_vpns[i].vpn = vpn;
731 tlb_vpns[i].pagesz = pagesz;
732
733 asid++;
734 if (asid > cpu_asid_mask(¤t_cpu_data)) {
735 vpn += 1ULL << pagesz;
736 asid = 0;
737 }
738 }
739 }
740
741 /*
742 * Initialise all TLB entries with unique values that do not clash with
743 * what we have been handed over and what we'll be using ourselves.
744 */
r4k_tlb_uniquify(void)745 static void __ref r4k_tlb_uniquify(void)
746 {
747 int tlbsize = current_cpu_data.tlbsize;
748 bool use_slab = slab_is_available();
749 phys_addr_t tlb_vpn_size;
750 struct tlbent *tlb_vpns;
751
752 tlb_vpn_size = tlbsize * sizeof(*tlb_vpns);
753 tlb_vpns = (use_slab ?
754 kmalloc(tlb_vpn_size, GFP_ATOMIC) :
755 memblock_alloc_raw(tlb_vpn_size, sizeof(*tlb_vpns)));
756 if (WARN_ON(!tlb_vpns))
757 return; /* Pray local_flush_tlb_all() is good enough. */
758
759 htw_stop();
760
761 r4k_tlb_uniquify_read(tlb_vpns, tlbsize);
762
763 sort(tlb_vpns, tlbsize, sizeof(*tlb_vpns), r4k_entry_cmp, NULL);
764
765 r4k_tlb_uniquify_write(tlb_vpns, tlbsize);
766
767 write_c0_pagemask(PM_DEFAULT_MASK);
768
769 tlbw_use_hazard();
770 htw_start();
771 flush_micro_tlb();
772 if (use_slab)
773 kfree(tlb_vpns);
774 else
775 memblock_free(tlb_vpns, tlb_vpn_size);
776 }
777
778 /*
779 * Configure TLB (for init or after a CPU has been powered off).
780 */
r4k_tlb_configure(void)781 static void r4k_tlb_configure(void)
782 {
783 /*
784 * You should never change this register:
785 * - On R4600 1.7 the tlbp never hits for pages smaller than
786 * the value in the c0_pagemask register.
787 * - The entire mm handling assumes the c0_pagemask register to
788 * be set to fixed-size pages.
789 */
790 write_c0_pagemask(PM_DEFAULT_MASK);
791 back_to_back_c0_hazard();
792 if (read_c0_pagemask() != PM_DEFAULT_MASK)
793 panic("MMU doesn't support PAGE_SIZE=0x%lx", PAGE_SIZE);
794
795 write_c0_wired(0);
796 if (current_cpu_type() == CPU_R10000 ||
797 current_cpu_type() == CPU_R12000 ||
798 current_cpu_type() == CPU_R14000 ||
799 current_cpu_type() == CPU_R16000)
800 write_c0_framemask(0);
801
802 if (cpu_has_rixi) {
803 /*
804 * Enable the no read, no exec bits, and enable large physical
805 * address.
806 */
807 #ifdef CONFIG_64BIT
808 set_c0_pagegrain(PG_RIE | PG_XIE | PG_ELPA);
809 #else
810 set_c0_pagegrain(PG_RIE | PG_XIE);
811 #endif
812 }
813
814 temp_tlb_entry = current_cpu_data.tlbsize - 1;
815
816 /* From this point on the ARC firmware is dead. */
817 if (!cpu_has_tlbinv)
818 r4k_tlb_uniquify();
819 local_flush_tlb_all();
820
821 /* Did I tell you that ARC SUCKS? */
822 }
823
tlb_init(void)824 void tlb_init(void)
825 {
826 r4k_tlb_configure();
827
828 if (ntlb) {
829 if (ntlb > 1 && ntlb <= current_cpu_data.tlbsize) {
830 int wired = current_cpu_data.tlbsize - ntlb;
831 write_c0_wired(wired);
832 write_c0_index(wired-1);
833 printk("Restricting TLB to %d entries\n", ntlb);
834 } else
835 printk("Ignoring invalid argument ntlb=%d\n", ntlb);
836 }
837
838 build_tlb_refill_handler();
839 }
840
r4k_tlb_pm_notifier(struct notifier_block * self,unsigned long cmd,void * v)841 static int r4k_tlb_pm_notifier(struct notifier_block *self, unsigned long cmd,
842 void *v)
843 {
844 switch (cmd) {
845 case CPU_PM_ENTER_FAILED:
846 case CPU_PM_EXIT:
847 r4k_tlb_configure();
848 break;
849 }
850
851 return NOTIFY_OK;
852 }
853
854 static struct notifier_block r4k_tlb_pm_notifier_block = {
855 .notifier_call = r4k_tlb_pm_notifier,
856 };
857
r4k_tlb_init_pm(void)858 static int __init r4k_tlb_init_pm(void)
859 {
860 return cpu_pm_register_notifier(&r4k_tlb_pm_notifier_block);
861 }
862 arch_initcall(r4k_tlb_init_pm);
863