1 /* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * Copyright (C) 1996 David S. Miller (dm@engr.sgi.com) 7 * Copyright (C) 1997, 1998, 1999, 2000 Ralf Baechle ralf@gnu.org 8 * Carsten Langgaard, carstenl@mips.com 9 * Copyright (C) 2002 MIPS Technologies, Inc. All rights reserved. 10 */ 11 #include <linux/init.h> 12 #include <linux/sched.h> 13 #include <linux/mm.h> 14 15 #include <asm/cpu.h> 16 #include <asm/bootinfo.h> 17 #include <asm/mmu_context.h> 18 #include <asm/pgtable.h> 19 #include <asm/system.h> 20 21 extern void build_tlb_refill_handler(void); 22 23 /* 24 * Make sure all entries differ. If they're not different 25 * MIPS32 will take revenge ... 26 */ 27 #define UNIQUE_ENTRYHI(idx) (CKSEG0 + ((idx) << (PAGE_SHIFT + 1))) 28 29 /* CP0 hazard avoidance. */ 30 #define BARRIER __asm__ __volatile__(".set noreorder\n\t" \ 31 "nop; nop; nop; nop; nop; nop;\n\t" \ 32 ".set reorder\n\t") 33 34 /* Atomicity and interruptability */ 35 #ifdef CONFIG_MIPS_MT_SMTC 36 37 #include <asm/smtc.h> 38 #include <asm/mipsmtregs.h> 39 40 #define ENTER_CRITICAL(flags) \ 41 { \ 42 unsigned int mvpflags; \ 43 local_irq_save(flags);\ 44 mvpflags = dvpe() 45 #define EXIT_CRITICAL(flags) \ 46 evpe(mvpflags); \ 47 local_irq_restore(flags); \ 48 } 49 #else 50 51 #define ENTER_CRITICAL(flags) local_irq_save(flags) 52 #define EXIT_CRITICAL(flags) local_irq_restore(flags) 53 54 #endif /* CONFIG_MIPS_MT_SMTC */ 55 56 void local_flush_tlb_all(void) 57 { 58 unsigned long flags; 59 unsigned long old_ctx; 60 int entry; 61 62 ENTER_CRITICAL(flags); 63 /* Save old context and create impossible VPN2 value */ 64 old_ctx = read_c0_entryhi(); 65 write_c0_entrylo0(0); 66 write_c0_entrylo1(0); 67 68 entry = read_c0_wired(); 69 70 /* Blast 'em all away. */ 71 while (entry < current_cpu_data.tlbsize) { 72 /* Make sure all entries differ. */ 73 write_c0_entryhi(UNIQUE_ENTRYHI(entry)); 74 write_c0_index(entry); 75 mtc0_tlbw_hazard(); 76 tlb_write_indexed(); 77 entry++; 78 } 79 tlbw_use_hazard(); 80 write_c0_entryhi(old_ctx); 81 EXIT_CRITICAL(flags); 82 } 83 84 /* All entries common to a mm share an asid. To effectively flush 85 these entries, we just bump the asid. */ 86 void local_flush_tlb_mm(struct mm_struct *mm) 87 { 88 int cpu; 89 90 preempt_disable(); 91 92 cpu = smp_processor_id(); 93 94 if (cpu_context(cpu, mm) != 0) { 95 drop_mmu_context(mm, cpu); 96 } 97 98 preempt_enable(); 99 } 100 101 void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, 102 unsigned long end) 103 { 104 struct mm_struct *mm = vma->vm_mm; 105 int cpu = smp_processor_id(); 106 107 if (cpu_context(cpu, mm) != 0) { 108 unsigned long flags; 109 int size; 110 111 ENTER_CRITICAL(flags); 112 size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT; 113 size = (size + 1) >> 1; 114 local_irq_save(flags); 115 if (size <= current_cpu_data.tlbsize/2) { 116 int oldpid = read_c0_entryhi(); 117 int newpid = cpu_asid(cpu, mm); 118 119 start &= (PAGE_MASK << 1); 120 end += ((PAGE_SIZE << 1) - 1); 121 end &= (PAGE_MASK << 1); 122 while (start < end) { 123 int idx; 124 125 write_c0_entryhi(start | newpid); 126 start += (PAGE_SIZE << 1); 127 mtc0_tlbw_hazard(); 128 tlb_probe(); 129 BARRIER; 130 idx = read_c0_index(); 131 write_c0_entrylo0(0); 132 write_c0_entrylo1(0); 133 if (idx < 0) 134 continue; 135 /* Make sure all entries differ. */ 136 write_c0_entryhi(UNIQUE_ENTRYHI(idx)); 137 mtc0_tlbw_hazard(); 138 tlb_write_indexed(); 139 } 140 tlbw_use_hazard(); 141 write_c0_entryhi(oldpid); 142 } else { 143 drop_mmu_context(mm, cpu); 144 } 145 EXIT_CRITICAL(flags); 146 } 147 } 148 149 void local_flush_tlb_kernel_range(unsigned long start, unsigned long end) 150 { 151 unsigned long flags; 152 int size; 153 154 ENTER_CRITICAL(flags); 155 size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT; 156 size = (size + 1) >> 1; 157 if (size <= current_cpu_data.tlbsize / 2) { 158 int pid = read_c0_entryhi(); 159 160 start &= (PAGE_MASK << 1); 161 end += ((PAGE_SIZE << 1) - 1); 162 end &= (PAGE_MASK << 1); 163 164 while (start < end) { 165 int idx; 166 167 write_c0_entryhi(start); 168 start += (PAGE_SIZE << 1); 169 mtc0_tlbw_hazard(); 170 tlb_probe(); 171 BARRIER; 172 idx = read_c0_index(); 173 write_c0_entrylo0(0); 174 write_c0_entrylo1(0); 175 if (idx < 0) 176 continue; 177 /* Make sure all entries differ. */ 178 write_c0_entryhi(UNIQUE_ENTRYHI(idx)); 179 mtc0_tlbw_hazard(); 180 tlb_write_indexed(); 181 } 182 tlbw_use_hazard(); 183 write_c0_entryhi(pid); 184 } else { 185 local_flush_tlb_all(); 186 } 187 EXIT_CRITICAL(flags); 188 } 189 190 void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page) 191 { 192 int cpu = smp_processor_id(); 193 194 if (cpu_context(cpu, vma->vm_mm) != 0) { 195 unsigned long flags; 196 int oldpid, newpid, idx; 197 198 newpid = cpu_asid(cpu, vma->vm_mm); 199 page &= (PAGE_MASK << 1); 200 ENTER_CRITICAL(flags); 201 oldpid = read_c0_entryhi(); 202 write_c0_entryhi(page | newpid); 203 mtc0_tlbw_hazard(); 204 tlb_probe(); 205 BARRIER; 206 idx = read_c0_index(); 207 write_c0_entrylo0(0); 208 write_c0_entrylo1(0); 209 if (idx < 0) 210 goto finish; 211 /* Make sure all entries differ. */ 212 write_c0_entryhi(UNIQUE_ENTRYHI(idx)); 213 mtc0_tlbw_hazard(); 214 tlb_write_indexed(); 215 tlbw_use_hazard(); 216 217 finish: 218 write_c0_entryhi(oldpid); 219 EXIT_CRITICAL(flags); 220 } 221 } 222 223 /* 224 * This one is only used for pages with the global bit set so we don't care 225 * much about the ASID. 226 */ 227 void local_flush_tlb_one(unsigned long page) 228 { 229 unsigned long flags; 230 int oldpid, idx; 231 232 ENTER_CRITICAL(flags); 233 oldpid = read_c0_entryhi(); 234 page &= (PAGE_MASK << 1); 235 write_c0_entryhi(page); 236 mtc0_tlbw_hazard(); 237 tlb_probe(); 238 BARRIER; 239 idx = read_c0_index(); 240 write_c0_entrylo0(0); 241 write_c0_entrylo1(0); 242 if (idx >= 0) { 243 /* Make sure all entries differ. */ 244 write_c0_entryhi(UNIQUE_ENTRYHI(idx)); 245 mtc0_tlbw_hazard(); 246 tlb_write_indexed(); 247 tlbw_use_hazard(); 248 } 249 write_c0_entryhi(oldpid); 250 251 EXIT_CRITICAL(flags); 252 } 253 254 /* 255 * We will need multiple versions of update_mmu_cache(), one that just 256 * updates the TLB with the new pte(s), and another which also checks 257 * for the R4k "end of page" hardware bug and does the needy. 258 */ 259 void __update_tlb(struct vm_area_struct * vma, unsigned long address, pte_t pte) 260 { 261 unsigned long flags; 262 pgd_t *pgdp; 263 pud_t *pudp; 264 pmd_t *pmdp; 265 pte_t *ptep; 266 int idx, pid; 267 268 /* 269 * Handle debugger faulting in for debugee. 270 */ 271 if (current->active_mm != vma->vm_mm) 272 return; 273 274 ENTER_CRITICAL(flags); 275 276 pid = read_c0_entryhi() & ASID_MASK; 277 address &= (PAGE_MASK << 1); 278 write_c0_entryhi(address | pid); 279 pgdp = pgd_offset(vma->vm_mm, address); 280 mtc0_tlbw_hazard(); 281 tlb_probe(); 282 BARRIER; 283 pudp = pud_offset(pgdp, address); 284 pmdp = pmd_offset(pudp, address); 285 idx = read_c0_index(); 286 ptep = pte_offset_map(pmdp, address); 287 288 #if defined(CONFIG_64BIT_PHYS_ADDR) && defined(CONFIG_CPU_MIPS32_R1) 289 write_c0_entrylo0(ptep->pte_high); 290 ptep++; 291 write_c0_entrylo1(ptep->pte_high); 292 #else 293 write_c0_entrylo0(pte_val(*ptep++) >> 6); 294 write_c0_entrylo1(pte_val(*ptep) >> 6); 295 #endif 296 mtc0_tlbw_hazard(); 297 if (idx < 0) 298 tlb_write_random(); 299 else 300 tlb_write_indexed(); 301 tlbw_use_hazard(); 302 EXIT_CRITICAL(flags); 303 } 304 305 #if 0 306 static void r4k_update_mmu_cache_hwbug(struct vm_area_struct * vma, 307 unsigned long address, pte_t pte) 308 { 309 unsigned long flags; 310 unsigned int asid; 311 pgd_t *pgdp; 312 pmd_t *pmdp; 313 pte_t *ptep; 314 int idx; 315 316 ENTER_CRITICAL(flags); 317 address &= (PAGE_MASK << 1); 318 asid = read_c0_entryhi() & ASID_MASK; 319 write_c0_entryhi(address | asid); 320 pgdp = pgd_offset(vma->vm_mm, address); 321 mtc0_tlbw_hazard(); 322 tlb_probe(); 323 BARRIER; 324 pmdp = pmd_offset(pgdp, address); 325 idx = read_c0_index(); 326 ptep = pte_offset_map(pmdp, address); 327 write_c0_entrylo0(pte_val(*ptep++) >> 6); 328 write_c0_entrylo1(pte_val(*ptep) >> 6); 329 mtc0_tlbw_hazard(); 330 if (idx < 0) 331 tlb_write_random(); 332 else 333 tlb_write_indexed(); 334 tlbw_use_hazard(); 335 EXIT_CRITICAL(flags); 336 } 337 #endif 338 339 void __init add_wired_entry(unsigned long entrylo0, unsigned long entrylo1, 340 unsigned long entryhi, unsigned long pagemask) 341 { 342 unsigned long flags; 343 unsigned long wired; 344 unsigned long old_pagemask; 345 unsigned long old_ctx; 346 347 ENTER_CRITICAL(flags); 348 /* Save old context and create impossible VPN2 value */ 349 old_ctx = read_c0_entryhi(); 350 old_pagemask = read_c0_pagemask(); 351 wired = read_c0_wired(); 352 write_c0_wired(wired + 1); 353 write_c0_index(wired); 354 BARRIER; 355 write_c0_pagemask(pagemask); 356 write_c0_entryhi(entryhi); 357 write_c0_entrylo0(entrylo0); 358 write_c0_entrylo1(entrylo1); 359 mtc0_tlbw_hazard(); 360 tlb_write_indexed(); 361 tlbw_use_hazard(); 362 363 write_c0_entryhi(old_ctx); 364 BARRIER; 365 write_c0_pagemask(old_pagemask); 366 local_flush_tlb_all(); 367 EXIT_CRITICAL(flags); 368 } 369 370 /* 371 * Used for loading TLB entries before trap_init() has started, when we 372 * don't actually want to add a wired entry which remains throughout the 373 * lifetime of the system 374 */ 375 376 static int temp_tlb_entry __initdata; 377 378 __init int add_temporary_entry(unsigned long entrylo0, unsigned long entrylo1, 379 unsigned long entryhi, unsigned long pagemask) 380 { 381 int ret = 0; 382 unsigned long flags; 383 unsigned long wired; 384 unsigned long old_pagemask; 385 unsigned long old_ctx; 386 387 ENTER_CRITICAL(flags); 388 /* Save old context and create impossible VPN2 value */ 389 old_ctx = read_c0_entryhi(); 390 old_pagemask = read_c0_pagemask(); 391 wired = read_c0_wired(); 392 if (--temp_tlb_entry < wired) { 393 printk(KERN_WARNING 394 "No TLB space left for add_temporary_entry\n"); 395 ret = -ENOSPC; 396 goto out; 397 } 398 399 write_c0_index(temp_tlb_entry); 400 write_c0_pagemask(pagemask); 401 write_c0_entryhi(entryhi); 402 write_c0_entrylo0(entrylo0); 403 write_c0_entrylo1(entrylo1); 404 mtc0_tlbw_hazard(); 405 tlb_write_indexed(); 406 tlbw_use_hazard(); 407 408 write_c0_entryhi(old_ctx); 409 write_c0_pagemask(old_pagemask); 410 out: 411 EXIT_CRITICAL(flags); 412 return ret; 413 } 414 415 static void __init probe_tlb(unsigned long config) 416 { 417 struct cpuinfo_mips *c = ¤t_cpu_data; 418 unsigned int reg; 419 420 /* 421 * If this isn't a MIPS32 / MIPS64 compliant CPU. Config 1 register 422 * is not supported, we assume R4k style. Cpu probing already figured 423 * out the number of tlb entries. 424 */ 425 if ((c->processor_id & 0xff0000) == PRID_COMP_LEGACY) 426 return; 427 #ifdef CONFIG_MIPS_MT_SMTC 428 /* 429 * If TLB is shared in SMTC system, total size already 430 * has been calculated and written into cpu_data tlbsize 431 */ 432 if((smtc_status & SMTC_TLB_SHARED) == SMTC_TLB_SHARED) 433 return; 434 #endif /* CONFIG_MIPS_MT_SMTC */ 435 436 reg = read_c0_config1(); 437 if (!((config >> 7) & 3)) 438 panic("No TLB present"); 439 440 c->tlbsize = ((reg >> 25) & 0x3f) + 1; 441 } 442 443 static int __initdata ntlb = 0; 444 static int __init set_ntlb(char *str) 445 { 446 get_option(&str, &ntlb); 447 return 1; 448 } 449 450 __setup("ntlb=", set_ntlb); 451 452 void __init tlb_init(void) 453 { 454 unsigned int config = read_c0_config(); 455 456 /* 457 * You should never change this register: 458 * - On R4600 1.7 the tlbp never hits for pages smaller than 459 * the value in the c0_pagemask register. 460 * - The entire mm handling assumes the c0_pagemask register to 461 * be set for 4kb pages. 462 */ 463 probe_tlb(config); 464 write_c0_pagemask(PM_DEFAULT_MASK); 465 write_c0_wired(0); 466 write_c0_framemask(0); 467 temp_tlb_entry = current_cpu_data.tlbsize - 1; 468 469 /* From this point on the ARC firmware is dead. */ 470 local_flush_tlb_all(); 471 472 /* Did I tell you that ARC SUCKS? */ 473 474 if (ntlb) { 475 if (ntlb > 1 && ntlb <= current_cpu_data.tlbsize) { 476 int wired = current_cpu_data.tlbsize - ntlb; 477 write_c0_wired(wired); 478 write_c0_index(wired-1); 479 printk ("Restricting TLB to %d entries\n", ntlb); 480 } else 481 printk("Ignoring invalid argument ntlb=%d\n", ntlb); 482 } 483 484 build_tlb_refill_handler(); 485 } 486