1 /* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * Copyright (C) 1996 David S. Miller (davem@davemloft.net) 7 * Copyright (C) 1997, 1998, 1999, 2000 Ralf Baechle ralf@gnu.org 8 * Carsten Langgaard, carstenl@mips.com 9 * Copyright (C) 2002 MIPS Technologies, Inc. All rights reserved. 10 */ 11 #include <linux/cpu_pm.h> 12 #include <linux/init.h> 13 #include <linux/sched.h> 14 #include <linux/smp.h> 15 #include <linux/mm.h> 16 #include <linux/hugetlb.h> 17 #include <linux/export.h> 18 #include <linux/sort.h> 19 20 #include <asm/cpu.h> 21 #include <asm/cpu-type.h> 22 #include <asm/bootinfo.h> 23 #include <asm/hazards.h> 24 #include <asm/mmu_context.h> 25 #include <asm/tlb.h> 26 #include <asm/tlbex.h> 27 #include <asm/tlbmisc.h> 28 #include <asm/setup.h> 29 30 /* 31 * LOONGSON-2 has a 4 entry itlb which is a subset of jtlb, LOONGSON-3 has 32 * a 4 entry itlb and a 4 entry dtlb which are subsets of jtlb. Unfortunately, 33 * itlb/dtlb are not totally transparent to software. 34 */ 35 static inline void flush_micro_tlb(void) 36 { 37 switch (current_cpu_type()) { 38 case CPU_LOONGSON2EF: 39 write_c0_diag(LOONGSON_DIAG_ITLB); 40 break; 41 case CPU_LOONGSON64: 42 write_c0_diag(LOONGSON_DIAG_ITLB | LOONGSON_DIAG_DTLB); 43 break; 44 default: 45 break; 46 } 47 } 48 49 static inline void flush_micro_tlb_vm(struct vm_area_struct *vma) 50 { 51 if (vma->vm_flags & VM_EXEC) 52 flush_micro_tlb(); 53 } 54 55 void local_flush_tlb_all(void) 56 { 57 unsigned long flags; 58 unsigned long old_ctx; 59 int entry, ftlbhighset; 60 61 local_irq_save(flags); 62 /* Save old context and create impossible VPN2 value */ 63 old_ctx = read_c0_entryhi(); 64 htw_stop(); 65 write_c0_entrylo0(0); 66 write_c0_entrylo1(0); 67 68 entry = num_wired_entries(); 69 70 /* 71 * Blast 'em all away. 72 * If there are any wired entries, fall back to iterating 73 */ 74 if (cpu_has_tlbinv && !entry) { 75 if (current_cpu_data.tlbsizevtlb) { 76 write_c0_index(0); 77 mtc0_tlbw_hazard(); 78 tlbinvf(); /* invalidate VTLB */ 79 } 80 ftlbhighset = current_cpu_data.tlbsizevtlb + 81 current_cpu_data.tlbsizeftlbsets; 82 for (entry = current_cpu_data.tlbsizevtlb; 83 entry < ftlbhighset; 84 entry++) { 85 write_c0_index(entry); 86 mtc0_tlbw_hazard(); 87 tlbinvf(); /* invalidate one FTLB set */ 88 } 89 } else { 90 while (entry < current_cpu_data.tlbsize) { 91 /* Make sure all entries differ. */ 92 write_c0_entryhi(UNIQUE_ENTRYHI(entry)); 93 write_c0_index(entry); 94 mtc0_tlbw_hazard(); 95 tlb_write_indexed(); 96 entry++; 97 } 98 } 99 tlbw_use_hazard(); 100 write_c0_entryhi(old_ctx); 101 htw_start(); 102 flush_micro_tlb(); 103 local_irq_restore(flags); 104 } 105 EXPORT_SYMBOL(local_flush_tlb_all); 106 107 void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, 108 unsigned long end) 109 { 110 struct mm_struct *mm = vma->vm_mm; 111 int cpu = smp_processor_id(); 112 113 if (cpu_context(cpu, mm) != 0) { 114 unsigned long size, flags; 115 116 local_irq_save(flags); 117 start = round_down(start, PAGE_SIZE << 1); 118 end = round_up(end, PAGE_SIZE << 1); 119 size = (end - start) >> (PAGE_SHIFT + 1); 120 if (size <= (current_cpu_data.tlbsizeftlbsets ? 121 current_cpu_data.tlbsize / 8 : 122 current_cpu_data.tlbsize / 2)) { 123 unsigned long old_entryhi, old_mmid; 124 int newpid = cpu_asid(cpu, mm); 125 126 old_entryhi = read_c0_entryhi(); 127 if (cpu_has_mmid) { 128 old_mmid = read_c0_memorymapid(); 129 write_c0_memorymapid(newpid); 130 } 131 132 htw_stop(); 133 while (start < end) { 134 int idx; 135 136 if (cpu_has_mmid) 137 write_c0_entryhi(start); 138 else 139 write_c0_entryhi(start | newpid); 140 start += (PAGE_SIZE << 1); 141 mtc0_tlbw_hazard(); 142 tlb_probe(); 143 tlb_probe_hazard(); 144 idx = read_c0_index(); 145 write_c0_entrylo0(0); 146 write_c0_entrylo1(0); 147 if (idx < 0) 148 continue; 149 /* Make sure all entries differ. */ 150 write_c0_entryhi(UNIQUE_ENTRYHI(idx)); 151 mtc0_tlbw_hazard(); 152 tlb_write_indexed(); 153 } 154 tlbw_use_hazard(); 155 write_c0_entryhi(old_entryhi); 156 if (cpu_has_mmid) 157 write_c0_memorymapid(old_mmid); 158 htw_start(); 159 } else { 160 drop_mmu_context(mm); 161 } 162 flush_micro_tlb(); 163 local_irq_restore(flags); 164 } 165 } 166 167 void local_flush_tlb_kernel_range(unsigned long start, unsigned long end) 168 { 169 unsigned long size, flags; 170 171 local_irq_save(flags); 172 size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT; 173 size = (size + 1) >> 1; 174 if (size <= (current_cpu_data.tlbsizeftlbsets ? 175 current_cpu_data.tlbsize / 8 : 176 current_cpu_data.tlbsize / 2)) { 177 int pid = read_c0_entryhi(); 178 179 start &= (PAGE_MASK << 1); 180 end += ((PAGE_SIZE << 1) - 1); 181 end &= (PAGE_MASK << 1); 182 htw_stop(); 183 184 while (start < end) { 185 int idx; 186 187 write_c0_entryhi(start); 188 start += (PAGE_SIZE << 1); 189 mtc0_tlbw_hazard(); 190 tlb_probe(); 191 tlb_probe_hazard(); 192 idx = read_c0_index(); 193 write_c0_entrylo0(0); 194 write_c0_entrylo1(0); 195 if (idx < 0) 196 continue; 197 /* Make sure all entries differ. */ 198 write_c0_entryhi(UNIQUE_ENTRYHI(idx)); 199 mtc0_tlbw_hazard(); 200 tlb_write_indexed(); 201 } 202 tlbw_use_hazard(); 203 write_c0_entryhi(pid); 204 htw_start(); 205 } else { 206 local_flush_tlb_all(); 207 } 208 flush_micro_tlb(); 209 local_irq_restore(flags); 210 } 211 212 void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page) 213 { 214 int cpu = smp_processor_id(); 215 216 if (cpu_context(cpu, vma->vm_mm) != 0) { 217 unsigned long old_mmid; 218 unsigned long flags, old_entryhi; 219 int idx; 220 221 page &= (PAGE_MASK << 1); 222 local_irq_save(flags); 223 old_entryhi = read_c0_entryhi(); 224 htw_stop(); 225 if (cpu_has_mmid) { 226 old_mmid = read_c0_memorymapid(); 227 write_c0_entryhi(page); 228 write_c0_memorymapid(cpu_asid(cpu, vma->vm_mm)); 229 } else { 230 write_c0_entryhi(page | cpu_asid(cpu, vma->vm_mm)); 231 } 232 mtc0_tlbw_hazard(); 233 tlb_probe(); 234 tlb_probe_hazard(); 235 idx = read_c0_index(); 236 write_c0_entrylo0(0); 237 write_c0_entrylo1(0); 238 if (idx < 0) 239 goto finish; 240 /* Make sure all entries differ. */ 241 write_c0_entryhi(UNIQUE_ENTRYHI(idx)); 242 mtc0_tlbw_hazard(); 243 tlb_write_indexed(); 244 tlbw_use_hazard(); 245 246 finish: 247 write_c0_entryhi(old_entryhi); 248 if (cpu_has_mmid) 249 write_c0_memorymapid(old_mmid); 250 htw_start(); 251 flush_micro_tlb_vm(vma); 252 local_irq_restore(flags); 253 } 254 } 255 256 /* 257 * This one is only used for pages with the global bit set so we don't care 258 * much about the ASID. 259 */ 260 void local_flush_tlb_one(unsigned long page) 261 { 262 unsigned long flags; 263 int oldpid, idx; 264 265 local_irq_save(flags); 266 oldpid = read_c0_entryhi(); 267 htw_stop(); 268 page &= (PAGE_MASK << 1); 269 write_c0_entryhi(page); 270 mtc0_tlbw_hazard(); 271 tlb_probe(); 272 tlb_probe_hazard(); 273 idx = read_c0_index(); 274 write_c0_entrylo0(0); 275 write_c0_entrylo1(0); 276 if (idx >= 0) { 277 /* Make sure all entries differ. */ 278 write_c0_entryhi(UNIQUE_ENTRYHI(idx)); 279 mtc0_tlbw_hazard(); 280 tlb_write_indexed(); 281 tlbw_use_hazard(); 282 } 283 write_c0_entryhi(oldpid); 284 htw_start(); 285 flush_micro_tlb(); 286 local_irq_restore(flags); 287 } 288 289 /* 290 * We will need multiple versions of update_mmu_cache(), one that just 291 * updates the TLB with the new pte(s), and another which also checks 292 * for the R4k "end of page" hardware bug and does the needy. 293 */ 294 void __update_tlb(struct vm_area_struct * vma, unsigned long address, pte_t pte) 295 { 296 unsigned long flags; 297 pgd_t *pgdp; 298 p4d_t *p4dp; 299 pud_t *pudp; 300 pmd_t *pmdp; 301 pte_t *ptep, *ptemap = NULL; 302 int idx, pid; 303 304 /* 305 * Handle debugger faulting in for debuggee. 306 */ 307 if (current->active_mm != vma->vm_mm) 308 return; 309 310 local_irq_save(flags); 311 312 htw_stop(); 313 address &= (PAGE_MASK << 1); 314 if (cpu_has_mmid) { 315 write_c0_entryhi(address); 316 } else { 317 pid = read_c0_entryhi() & cpu_asid_mask(¤t_cpu_data); 318 write_c0_entryhi(address | pid); 319 } 320 pgdp = pgd_offset(vma->vm_mm, address); 321 mtc0_tlbw_hazard(); 322 tlb_probe(); 323 tlb_probe_hazard(); 324 p4dp = p4d_offset(pgdp, address); 325 pudp = pud_offset(p4dp, address); 326 pmdp = pmd_offset(pudp, address); 327 idx = read_c0_index(); 328 #ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT 329 /* this could be a huge page */ 330 if (pmd_leaf(*pmdp)) { 331 unsigned long lo; 332 write_c0_pagemask(PM_HUGE_MASK); 333 ptep = (pte_t *)pmdp; 334 lo = pte_to_entrylo(pte_val(*ptep)); 335 write_c0_entrylo0(lo); 336 write_c0_entrylo1(lo + (HPAGE_SIZE >> 7)); 337 338 mtc0_tlbw_hazard(); 339 if (idx < 0) 340 tlb_write_random(); 341 else 342 tlb_write_indexed(); 343 tlbw_use_hazard(); 344 write_c0_pagemask(PM_DEFAULT_MASK); 345 } else 346 #endif 347 { 348 ptemap = ptep = pte_offset_map(pmdp, address); 349 /* 350 * update_mmu_cache() is called between pte_offset_map_lock() 351 * and pte_unmap_unlock(), so we can assume that ptep is not 352 * NULL here: and what should be done below if it were NULL? 353 */ 354 355 #if defined(CONFIG_PHYS_ADDR_T_64BIT) && defined(CONFIG_CPU_MIPS32) 356 #ifdef CONFIG_XPA 357 write_c0_entrylo0(pte_to_entrylo(ptep->pte_high)); 358 if (cpu_has_xpa) 359 writex_c0_entrylo0(ptep->pte_low & _PFNX_MASK); 360 ptep++; 361 write_c0_entrylo1(pte_to_entrylo(ptep->pte_high)); 362 if (cpu_has_xpa) 363 writex_c0_entrylo1(ptep->pte_low & _PFNX_MASK); 364 #else 365 write_c0_entrylo0(ptep->pte_high); 366 ptep++; 367 write_c0_entrylo1(ptep->pte_high); 368 #endif 369 #else 370 write_c0_entrylo0(pte_to_entrylo(pte_val(*ptep++))); 371 write_c0_entrylo1(pte_to_entrylo(pte_val(*ptep))); 372 #endif 373 mtc0_tlbw_hazard(); 374 if (idx < 0) 375 tlb_write_random(); 376 else 377 tlb_write_indexed(); 378 } 379 tlbw_use_hazard(); 380 htw_start(); 381 flush_micro_tlb_vm(vma); 382 383 if (ptemap) 384 pte_unmap(ptemap); 385 local_irq_restore(flags); 386 } 387 388 void add_wired_entry(unsigned long entrylo0, unsigned long entrylo1, 389 unsigned long entryhi, unsigned long pagemask) 390 { 391 #ifdef CONFIG_XPA 392 panic("Broken for XPA kernels"); 393 #else 394 unsigned int old_mmid; 395 unsigned long flags; 396 unsigned long wired; 397 unsigned long old_pagemask; 398 unsigned long old_ctx; 399 400 local_irq_save(flags); 401 if (cpu_has_mmid) { 402 old_mmid = read_c0_memorymapid(); 403 write_c0_memorymapid(MMID_KERNEL_WIRED); 404 } 405 /* Save old context and create impossible VPN2 value */ 406 old_ctx = read_c0_entryhi(); 407 htw_stop(); 408 old_pagemask = read_c0_pagemask(); 409 wired = num_wired_entries(); 410 write_c0_wired(wired + 1); 411 write_c0_index(wired); 412 tlbw_use_hazard(); /* What is the hazard here? */ 413 write_c0_pagemask(pagemask); 414 write_c0_entryhi(entryhi); 415 write_c0_entrylo0(entrylo0); 416 write_c0_entrylo1(entrylo1); 417 mtc0_tlbw_hazard(); 418 tlb_write_indexed(); 419 tlbw_use_hazard(); 420 421 write_c0_entryhi(old_ctx); 422 if (cpu_has_mmid) 423 write_c0_memorymapid(old_mmid); 424 tlbw_use_hazard(); /* What is the hazard here? */ 425 htw_start(); 426 write_c0_pagemask(old_pagemask); 427 local_flush_tlb_all(); 428 local_irq_restore(flags); 429 #endif 430 } 431 432 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 433 434 int has_transparent_hugepage(void) 435 { 436 static unsigned int mask = -1; 437 438 if (mask == -1) { /* first call comes during __init */ 439 unsigned long flags; 440 441 local_irq_save(flags); 442 write_c0_pagemask(PM_HUGE_MASK); 443 back_to_back_c0_hazard(); 444 mask = read_c0_pagemask(); 445 write_c0_pagemask(PM_DEFAULT_MASK); 446 local_irq_restore(flags); 447 } 448 return mask == PM_HUGE_MASK; 449 } 450 EXPORT_SYMBOL(has_transparent_hugepage); 451 452 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 453 454 /* 455 * Used for loading TLB entries before trap_init() has started, when we 456 * don't actually want to add a wired entry which remains throughout the 457 * lifetime of the system 458 */ 459 460 int temp_tlb_entry; 461 462 #ifndef CONFIG_64BIT 463 __init int add_temporary_entry(unsigned long entrylo0, unsigned long entrylo1, 464 unsigned long entryhi, unsigned long pagemask) 465 { 466 int ret = 0; 467 unsigned long flags; 468 unsigned long wired; 469 unsigned long old_pagemask; 470 unsigned long old_ctx; 471 472 local_irq_save(flags); 473 /* Save old context and create impossible VPN2 value */ 474 htw_stop(); 475 old_ctx = read_c0_entryhi(); 476 old_pagemask = read_c0_pagemask(); 477 wired = num_wired_entries(); 478 if (--temp_tlb_entry < wired) { 479 printk(KERN_WARNING 480 "No TLB space left for add_temporary_entry\n"); 481 ret = -ENOSPC; 482 goto out; 483 } 484 485 write_c0_index(temp_tlb_entry); 486 write_c0_pagemask(pagemask); 487 write_c0_entryhi(entryhi); 488 write_c0_entrylo0(entrylo0); 489 write_c0_entrylo1(entrylo1); 490 mtc0_tlbw_hazard(); 491 tlb_write_indexed(); 492 tlbw_use_hazard(); 493 494 write_c0_entryhi(old_ctx); 495 write_c0_pagemask(old_pagemask); 496 htw_start(); 497 out: 498 local_irq_restore(flags); 499 return ret; 500 } 501 #endif 502 503 static int ntlb; 504 static int __init set_ntlb(char *str) 505 { 506 get_option(&str, &ntlb); 507 return 1; 508 } 509 510 __setup("ntlb=", set_ntlb); 511 512 513 /* Comparison function for EntryHi VPN fields. */ 514 static int r4k_vpn_cmp(const void *a, const void *b) 515 { 516 long v = *(unsigned long *)a - *(unsigned long *)b; 517 int s = sizeof(long) > sizeof(int) ? sizeof(long) * 8 - 1: 0; 518 return s ? (v != 0) | v >> s : v; 519 } 520 521 /* 522 * Initialise all TLB entries with unique values that do not clash with 523 * what we have been handed over and what we'll be using ourselves. 524 */ 525 static void r4k_tlb_uniquify(void) 526 { 527 unsigned long tlb_vpns[1 << MIPS_CONF1_TLBS_SIZE]; 528 int tlbsize = current_cpu_data.tlbsize; 529 int start = num_wired_entries(); 530 unsigned long vpn_mask; 531 int cnt, ent, idx, i; 532 533 vpn_mask = GENMASK(cpu_vmbits - 1, 13); 534 vpn_mask |= IS_ENABLED(CONFIG_64BIT) ? 3ULL << 62 : 1 << 31; 535 536 htw_stop(); 537 538 for (i = start, cnt = 0; i < tlbsize; i++, cnt++) { 539 unsigned long vpn; 540 541 write_c0_index(i); 542 mtc0_tlbr_hazard(); 543 tlb_read(); 544 tlb_read_hazard(); 545 vpn = read_c0_entryhi(); 546 vpn &= vpn_mask & PAGE_MASK; 547 tlb_vpns[cnt] = vpn; 548 549 /* Prevent any large pages from overlapping regular ones. */ 550 write_c0_pagemask(read_c0_pagemask() & PM_DEFAULT_MASK); 551 mtc0_tlbw_hazard(); 552 tlb_write_indexed(); 553 tlbw_use_hazard(); 554 } 555 556 sort(tlb_vpns, cnt, sizeof(tlb_vpns[0]), r4k_vpn_cmp, NULL); 557 558 write_c0_pagemask(PM_DEFAULT_MASK); 559 write_c0_entrylo0(0); 560 write_c0_entrylo1(0); 561 562 idx = 0; 563 ent = tlbsize; 564 for (i = start; i < tlbsize; i++) 565 while (1) { 566 unsigned long entryhi, vpn; 567 568 entryhi = UNIQUE_ENTRYHI(ent); 569 vpn = entryhi & vpn_mask & PAGE_MASK; 570 571 if (idx >= cnt || vpn < tlb_vpns[idx]) { 572 write_c0_entryhi(entryhi); 573 write_c0_index(i); 574 mtc0_tlbw_hazard(); 575 tlb_write_indexed(); 576 ent++; 577 break; 578 } else if (vpn == tlb_vpns[idx]) { 579 ent++; 580 } else { 581 idx++; 582 } 583 } 584 585 tlbw_use_hazard(); 586 htw_start(); 587 flush_micro_tlb(); 588 } 589 590 /* 591 * Configure TLB (for init or after a CPU has been powered off). 592 */ 593 static void r4k_tlb_configure(void) 594 { 595 /* 596 * You should never change this register: 597 * - On R4600 1.7 the tlbp never hits for pages smaller than 598 * the value in the c0_pagemask register. 599 * - The entire mm handling assumes the c0_pagemask register to 600 * be set to fixed-size pages. 601 */ 602 write_c0_pagemask(PM_DEFAULT_MASK); 603 back_to_back_c0_hazard(); 604 if (read_c0_pagemask() != PM_DEFAULT_MASK) 605 panic("MMU doesn't support PAGE_SIZE=0x%lx", PAGE_SIZE); 606 607 write_c0_wired(0); 608 if (current_cpu_type() == CPU_R10000 || 609 current_cpu_type() == CPU_R12000 || 610 current_cpu_type() == CPU_R14000 || 611 current_cpu_type() == CPU_R16000) 612 write_c0_framemask(0); 613 614 if (cpu_has_rixi) { 615 /* 616 * Enable the no read, no exec bits, and enable large physical 617 * address. 618 */ 619 #ifdef CONFIG_64BIT 620 set_c0_pagegrain(PG_RIE | PG_XIE | PG_ELPA); 621 #else 622 set_c0_pagegrain(PG_RIE | PG_XIE); 623 #endif 624 } 625 626 temp_tlb_entry = current_cpu_data.tlbsize - 1; 627 628 /* From this point on the ARC firmware is dead. */ 629 r4k_tlb_uniquify(); 630 local_flush_tlb_all(); 631 632 /* Did I tell you that ARC SUCKS? */ 633 } 634 635 void tlb_init(void) 636 { 637 r4k_tlb_configure(); 638 639 if (ntlb) { 640 if (ntlb > 1 && ntlb <= current_cpu_data.tlbsize) { 641 int wired = current_cpu_data.tlbsize - ntlb; 642 write_c0_wired(wired); 643 write_c0_index(wired-1); 644 printk("Restricting TLB to %d entries\n", ntlb); 645 } else 646 printk("Ignoring invalid argument ntlb=%d\n", ntlb); 647 } 648 649 build_tlb_refill_handler(); 650 } 651 652 static int r4k_tlb_pm_notifier(struct notifier_block *self, unsigned long cmd, 653 void *v) 654 { 655 switch (cmd) { 656 case CPU_PM_ENTER_FAILED: 657 case CPU_PM_EXIT: 658 r4k_tlb_configure(); 659 break; 660 } 661 662 return NOTIFY_OK; 663 } 664 665 static struct notifier_block r4k_tlb_pm_notifier_block = { 666 .notifier_call = r4k_tlb_pm_notifier, 667 }; 668 669 static int __init r4k_tlb_init_pm(void) 670 { 671 return cpu_pm_register_notifier(&r4k_tlb_pm_notifier_block); 672 } 673 arch_initcall(r4k_tlb_init_pm); 674