1 /* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * Synthesize TLB refill handlers at runtime. 7 * 8 * Copyright (C) 2004, 2005, 2006, 2008 Thiemo Seufer 9 * Copyright (C) 2005, 2007, 2008, 2009 Maciej W. Rozycki 10 * Copyright (C) 2006 Ralf Baechle (ralf@linux-mips.org) 11 * Copyright (C) 2008, 2009 Cavium Networks, Inc. 12 * Copyright (C) 2011 MIPS Technologies, Inc. 13 * 14 * ... and the days got worse and worse and now you see 15 * I've gone completly out of my mind. 16 * 17 * They're coming to take me a away haha 18 * they're coming to take me a away hoho hihi haha 19 * to the funny farm where code is beautiful all the time ... 20 * 21 * (Condolences to Napoleon XIV) 22 */ 23 24 #include <linux/bug.h> 25 #include <linux/kernel.h> 26 #include <linux/types.h> 27 #include <linux/smp.h> 28 #include <linux/string.h> 29 #include <linux/init.h> 30 #include <linux/cache.h> 31 32 #include <asm/cacheflush.h> 33 #include <asm/pgtable.h> 34 #include <asm/war.h> 35 #include <asm/uasm.h> 36 #include <asm/setup.h> 37 38 /* 39 * TLB load/store/modify handlers. 40 * 41 * Only the fastpath gets synthesized at runtime, the slowpath for 42 * do_page_fault remains normal asm. 43 */ 44 extern void tlb_do_page_fault_0(void); 45 extern void tlb_do_page_fault_1(void); 46 47 struct work_registers { 48 int r1; 49 int r2; 50 int r3; 51 }; 52 53 struct tlb_reg_save { 54 unsigned long a; 55 unsigned long b; 56 } ____cacheline_aligned_in_smp; 57 58 static struct tlb_reg_save handler_reg_save[NR_CPUS]; 59 60 static inline int r45k_bvahwbug(void) 61 { 62 /* XXX: We should probe for the presence of this bug, but we don't. */ 63 return 0; 64 } 65 66 static inline int r4k_250MHZhwbug(void) 67 { 68 /* XXX: We should probe for the presence of this bug, but we don't. */ 69 return 0; 70 } 71 72 static inline int __maybe_unused bcm1250_m3_war(void) 73 { 74 return BCM1250_M3_WAR; 75 } 76 77 static inline int __maybe_unused r10000_llsc_war(void) 78 { 79 return R10000_LLSC_WAR; 80 } 81 82 static int use_bbit_insns(void) 83 { 84 switch (current_cpu_type()) { 85 case CPU_CAVIUM_OCTEON: 86 case CPU_CAVIUM_OCTEON_PLUS: 87 case CPU_CAVIUM_OCTEON2: 88 return 1; 89 default: 90 return 0; 91 } 92 } 93 94 static int use_lwx_insns(void) 95 { 96 switch (current_cpu_type()) { 97 case CPU_CAVIUM_OCTEON2: 98 return 1; 99 default: 100 return 0; 101 } 102 } 103 #if defined(CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE) && \ 104 CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE > 0 105 static bool scratchpad_available(void) 106 { 107 return true; 108 } 109 static int scratchpad_offset(int i) 110 { 111 /* 112 * CVMSEG starts at address -32768 and extends for 113 * CAVIUM_OCTEON_CVMSEG_SIZE 128 byte cache lines. 114 */ 115 i += 1; /* Kernel use starts at the top and works down. */ 116 return CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE * 128 - (8 * i) - 32768; 117 } 118 #else 119 static bool scratchpad_available(void) 120 { 121 return false; 122 } 123 static int scratchpad_offset(int i) 124 { 125 BUG(); 126 /* Really unreachable, but evidently some GCC want this. */ 127 return 0; 128 } 129 #endif 130 /* 131 * Found by experiment: At least some revisions of the 4kc throw under 132 * some circumstances a machine check exception, triggered by invalid 133 * values in the index register. Delaying the tlbp instruction until 134 * after the next branch, plus adding an additional nop in front of 135 * tlbwi/tlbwr avoids the invalid index register values. Nobody knows 136 * why; it's not an issue caused by the core RTL. 137 * 138 */ 139 static int __cpuinit m4kc_tlbp_war(void) 140 { 141 return (current_cpu_data.processor_id & 0xffff00) == 142 (PRID_COMP_MIPS | PRID_IMP_4KC); 143 } 144 145 /* Handle labels (which must be positive integers). */ 146 enum label_id { 147 label_second_part = 1, 148 label_leave, 149 label_vmalloc, 150 label_vmalloc_done, 151 label_tlbw_hazard, 152 label_split, 153 label_tlbl_goaround1, 154 label_tlbl_goaround2, 155 label_nopage_tlbl, 156 label_nopage_tlbs, 157 label_nopage_tlbm, 158 label_smp_pgtable_change, 159 label_r3000_write_probe_fail, 160 label_large_segbits_fault, 161 #ifdef CONFIG_HUGETLB_PAGE 162 label_tlb_huge_update, 163 #endif 164 }; 165 166 UASM_L_LA(_second_part) 167 UASM_L_LA(_leave) 168 UASM_L_LA(_vmalloc) 169 UASM_L_LA(_vmalloc_done) 170 UASM_L_LA(_tlbw_hazard) 171 UASM_L_LA(_split) 172 UASM_L_LA(_tlbl_goaround1) 173 UASM_L_LA(_tlbl_goaround2) 174 UASM_L_LA(_nopage_tlbl) 175 UASM_L_LA(_nopage_tlbs) 176 UASM_L_LA(_nopage_tlbm) 177 UASM_L_LA(_smp_pgtable_change) 178 UASM_L_LA(_r3000_write_probe_fail) 179 UASM_L_LA(_large_segbits_fault) 180 #ifdef CONFIG_HUGETLB_PAGE 181 UASM_L_LA(_tlb_huge_update) 182 #endif 183 184 /* 185 * For debug purposes. 186 */ 187 static inline void dump_handler(const u32 *handler, int count) 188 { 189 int i; 190 191 pr_debug("\t.set push\n"); 192 pr_debug("\t.set noreorder\n"); 193 194 for (i = 0; i < count; i++) 195 pr_debug("\t%p\t.word 0x%08x\n", &handler[i], handler[i]); 196 197 pr_debug("\t.set pop\n"); 198 } 199 200 /* The only general purpose registers allowed in TLB handlers. */ 201 #define K0 26 202 #define K1 27 203 204 /* Some CP0 registers */ 205 #define C0_INDEX 0, 0 206 #define C0_ENTRYLO0 2, 0 207 #define C0_TCBIND 2, 2 208 #define C0_ENTRYLO1 3, 0 209 #define C0_CONTEXT 4, 0 210 #define C0_PAGEMASK 5, 0 211 #define C0_BADVADDR 8, 0 212 #define C0_ENTRYHI 10, 0 213 #define C0_EPC 14, 0 214 #define C0_XCONTEXT 20, 0 215 216 #ifdef CONFIG_64BIT 217 # define GET_CONTEXT(buf, reg) UASM_i_MFC0(buf, reg, C0_XCONTEXT) 218 #else 219 # define GET_CONTEXT(buf, reg) UASM_i_MFC0(buf, reg, C0_CONTEXT) 220 #endif 221 222 /* The worst case length of the handler is around 18 instructions for 223 * R3000-style TLBs and up to 63 instructions for R4000-style TLBs. 224 * Maximum space available is 32 instructions for R3000 and 64 225 * instructions for R4000. 226 * 227 * We deliberately chose a buffer size of 128, so we won't scribble 228 * over anything important on overflow before we panic. 229 */ 230 static u32 tlb_handler[128] __cpuinitdata; 231 232 /* simply assume worst case size for labels and relocs */ 233 static struct uasm_label labels[128] __cpuinitdata; 234 static struct uasm_reloc relocs[128] __cpuinitdata; 235 236 #ifdef CONFIG_64BIT 237 static int check_for_high_segbits __cpuinitdata; 238 #endif 239 240 static int check_for_high_segbits __cpuinitdata; 241 242 static unsigned int kscratch_used_mask __cpuinitdata; 243 244 static int __cpuinit allocate_kscratch(void) 245 { 246 int r; 247 unsigned int a = cpu_data[0].kscratch_mask & ~kscratch_used_mask; 248 249 r = ffs(a); 250 251 if (r == 0) 252 return -1; 253 254 r--; /* make it zero based */ 255 256 kscratch_used_mask |= (1 << r); 257 258 return r; 259 } 260 261 static int scratch_reg __cpuinitdata; 262 static int pgd_reg __cpuinitdata; 263 enum vmalloc64_mode {not_refill, refill_scratch, refill_noscratch}; 264 265 static struct work_registers __cpuinit build_get_work_registers(u32 **p) 266 { 267 struct work_registers r; 268 269 int smp_processor_id_reg; 270 int smp_processor_id_sel; 271 int smp_processor_id_shift; 272 273 if (scratch_reg > 0) { 274 /* Save in CPU local C0_KScratch? */ 275 UASM_i_MTC0(p, 1, 31, scratch_reg); 276 r.r1 = K0; 277 r.r2 = K1; 278 r.r3 = 1; 279 return r; 280 } 281 282 if (num_possible_cpus() > 1) { 283 #ifdef CONFIG_MIPS_PGD_C0_CONTEXT 284 smp_processor_id_shift = 51; 285 smp_processor_id_reg = 20; /* XContext */ 286 smp_processor_id_sel = 0; 287 #else 288 # ifdef CONFIG_32BIT 289 smp_processor_id_shift = 25; 290 smp_processor_id_reg = 4; /* Context */ 291 smp_processor_id_sel = 0; 292 # endif 293 # ifdef CONFIG_64BIT 294 smp_processor_id_shift = 26; 295 smp_processor_id_reg = 4; /* Context */ 296 smp_processor_id_sel = 0; 297 # endif 298 #endif 299 /* Get smp_processor_id */ 300 UASM_i_MFC0(p, K0, smp_processor_id_reg, smp_processor_id_sel); 301 UASM_i_SRL_SAFE(p, K0, K0, smp_processor_id_shift); 302 303 /* handler_reg_save index in K0 */ 304 UASM_i_SLL(p, K0, K0, ilog2(sizeof(struct tlb_reg_save))); 305 306 UASM_i_LA(p, K1, (long)&handler_reg_save); 307 UASM_i_ADDU(p, K0, K0, K1); 308 } else { 309 UASM_i_LA(p, K0, (long)&handler_reg_save); 310 } 311 /* K0 now points to save area, save $1 and $2 */ 312 UASM_i_SW(p, 1, offsetof(struct tlb_reg_save, a), K0); 313 UASM_i_SW(p, 2, offsetof(struct tlb_reg_save, b), K0); 314 315 r.r1 = K1; 316 r.r2 = 1; 317 r.r3 = 2; 318 return r; 319 } 320 321 static void __cpuinit build_restore_work_registers(u32 **p) 322 { 323 if (scratch_reg > 0) { 324 UASM_i_MFC0(p, 1, 31, scratch_reg); 325 return; 326 } 327 /* K0 already points to save area, restore $1 and $2 */ 328 UASM_i_LW(p, 1, offsetof(struct tlb_reg_save, a), K0); 329 UASM_i_LW(p, 2, offsetof(struct tlb_reg_save, b), K0); 330 } 331 332 #ifndef CONFIG_MIPS_PGD_C0_CONTEXT 333 334 /* 335 * CONFIG_MIPS_PGD_C0_CONTEXT implies 64 bit and lack of pgd_current, 336 * we cannot do r3000 under these circumstances. 337 * 338 * Declare pgd_current here instead of including mmu_context.h to avoid type 339 * conflicts for tlbmiss_handler_setup_pgd 340 */ 341 extern unsigned long pgd_current[]; 342 343 /* 344 * The R3000 TLB handler is simple. 345 */ 346 static void __cpuinit build_r3000_tlb_refill_handler(void) 347 { 348 long pgdc = (long)pgd_current; 349 u32 *p; 350 351 memset(tlb_handler, 0, sizeof(tlb_handler)); 352 p = tlb_handler; 353 354 uasm_i_mfc0(&p, K0, C0_BADVADDR); 355 uasm_i_lui(&p, K1, uasm_rel_hi(pgdc)); /* cp0 delay */ 356 uasm_i_lw(&p, K1, uasm_rel_lo(pgdc), K1); 357 uasm_i_srl(&p, K0, K0, 22); /* load delay */ 358 uasm_i_sll(&p, K0, K0, 2); 359 uasm_i_addu(&p, K1, K1, K0); 360 uasm_i_mfc0(&p, K0, C0_CONTEXT); 361 uasm_i_lw(&p, K1, 0, K1); /* cp0 delay */ 362 uasm_i_andi(&p, K0, K0, 0xffc); /* load delay */ 363 uasm_i_addu(&p, K1, K1, K0); 364 uasm_i_lw(&p, K0, 0, K1); 365 uasm_i_nop(&p); /* load delay */ 366 uasm_i_mtc0(&p, K0, C0_ENTRYLO0); 367 uasm_i_mfc0(&p, K1, C0_EPC); /* cp0 delay */ 368 uasm_i_tlbwr(&p); /* cp0 delay */ 369 uasm_i_jr(&p, K1); 370 uasm_i_rfe(&p); /* branch delay */ 371 372 if (p > tlb_handler + 32) 373 panic("TLB refill handler space exceeded"); 374 375 pr_debug("Wrote TLB refill handler (%u instructions).\n", 376 (unsigned int)(p - tlb_handler)); 377 378 memcpy((void *)ebase, tlb_handler, 0x80); 379 380 dump_handler((u32 *)ebase, 32); 381 } 382 #endif /* CONFIG_MIPS_PGD_C0_CONTEXT */ 383 384 /* 385 * The R4000 TLB handler is much more complicated. We have two 386 * consecutive handler areas with 32 instructions space each. 387 * Since they aren't used at the same time, we can overflow in the 388 * other one.To keep things simple, we first assume linear space, 389 * then we relocate it to the final handler layout as needed. 390 */ 391 static u32 final_handler[64] __cpuinitdata; 392 393 /* 394 * Hazards 395 * 396 * From the IDT errata for the QED RM5230 (Nevada), processor revision 1.0: 397 * 2. A timing hazard exists for the TLBP instruction. 398 * 399 * stalling_instruction 400 * TLBP 401 * 402 * The JTLB is being read for the TLBP throughout the stall generated by the 403 * previous instruction. This is not really correct as the stalling instruction 404 * can modify the address used to access the JTLB. The failure symptom is that 405 * the TLBP instruction will use an address created for the stalling instruction 406 * and not the address held in C0_ENHI and thus report the wrong results. 407 * 408 * The software work-around is to not allow the instruction preceding the TLBP 409 * to stall - make it an NOP or some other instruction guaranteed not to stall. 410 * 411 * Errata 2 will not be fixed. This errata is also on the R5000. 412 * 413 * As if we MIPS hackers wouldn't know how to nop pipelines happy ... 414 */ 415 static void __cpuinit __maybe_unused build_tlb_probe_entry(u32 **p) 416 { 417 switch (current_cpu_type()) { 418 /* Found by experiment: R4600 v2.0/R4700 needs this, too. */ 419 case CPU_R4600: 420 case CPU_R4700: 421 case CPU_R5000: 422 case CPU_R5000A: 423 case CPU_NEVADA: 424 uasm_i_nop(p); 425 uasm_i_tlbp(p); 426 break; 427 428 default: 429 uasm_i_tlbp(p); 430 break; 431 } 432 } 433 434 /* 435 * Write random or indexed TLB entry, and care about the hazards from 436 * the preceding mtc0 and for the following eret. 437 */ 438 enum tlb_write_entry { tlb_random, tlb_indexed }; 439 440 static void __cpuinit build_tlb_write_entry(u32 **p, struct uasm_label **l, 441 struct uasm_reloc **r, 442 enum tlb_write_entry wmode) 443 { 444 void(*tlbw)(u32 **) = NULL; 445 446 switch (wmode) { 447 case tlb_random: tlbw = uasm_i_tlbwr; break; 448 case tlb_indexed: tlbw = uasm_i_tlbwi; break; 449 } 450 451 if (cpu_has_mips_r2) { 452 /* 453 * The architecture spec says an ehb is required here, 454 * but a number of cores do not have the hazard and 455 * using an ehb causes an expensive pipeline stall. 456 */ 457 switch (current_cpu_type()) { 458 case CPU_M14KC: 459 case CPU_74K: 460 break; 461 462 default: 463 uasm_i_ehb(p); 464 break; 465 } 466 tlbw(p); 467 return; 468 } 469 470 switch (current_cpu_type()) { 471 case CPU_R4000PC: 472 case CPU_R4000SC: 473 case CPU_R4000MC: 474 case CPU_R4400PC: 475 case CPU_R4400SC: 476 case CPU_R4400MC: 477 /* 478 * This branch uses up a mtc0 hazard nop slot and saves 479 * two nops after the tlbw instruction. 480 */ 481 uasm_il_bgezl(p, r, 0, label_tlbw_hazard); 482 tlbw(p); 483 uasm_l_tlbw_hazard(l, *p); 484 uasm_i_nop(p); 485 break; 486 487 case CPU_R4600: 488 case CPU_R4700: 489 case CPU_R5000: 490 case CPU_R5000A: 491 uasm_i_nop(p); 492 tlbw(p); 493 uasm_i_nop(p); 494 break; 495 496 case CPU_R4300: 497 case CPU_5KC: 498 case CPU_TX49XX: 499 case CPU_PR4450: 500 case CPU_XLR: 501 uasm_i_nop(p); 502 tlbw(p); 503 break; 504 505 case CPU_R10000: 506 case CPU_R12000: 507 case CPU_R14000: 508 case CPU_4KC: 509 case CPU_4KEC: 510 case CPU_M14KC: 511 case CPU_SB1: 512 case CPU_SB1A: 513 case CPU_4KSC: 514 case CPU_20KC: 515 case CPU_25KF: 516 case CPU_BMIPS32: 517 case CPU_BMIPS3300: 518 case CPU_BMIPS4350: 519 case CPU_BMIPS4380: 520 case CPU_BMIPS5000: 521 case CPU_LOONGSON2: 522 case CPU_R5500: 523 if (m4kc_tlbp_war()) 524 uasm_i_nop(p); 525 case CPU_ALCHEMY: 526 tlbw(p); 527 break; 528 529 case CPU_NEVADA: 530 uasm_i_nop(p); /* QED specifies 2 nops hazard */ 531 /* 532 * This branch uses up a mtc0 hazard nop slot and saves 533 * a nop after the tlbw instruction. 534 */ 535 uasm_il_bgezl(p, r, 0, label_tlbw_hazard); 536 tlbw(p); 537 uasm_l_tlbw_hazard(l, *p); 538 break; 539 540 case CPU_RM7000: 541 uasm_i_nop(p); 542 uasm_i_nop(p); 543 uasm_i_nop(p); 544 uasm_i_nop(p); 545 tlbw(p); 546 break; 547 548 case CPU_RM9000: 549 /* 550 * When the JTLB is updated by tlbwi or tlbwr, a subsequent 551 * use of the JTLB for instructions should not occur for 4 552 * cpu cycles and use for data translations should not occur 553 * for 3 cpu cycles. 554 */ 555 uasm_i_ssnop(p); 556 uasm_i_ssnop(p); 557 uasm_i_ssnop(p); 558 uasm_i_ssnop(p); 559 tlbw(p); 560 uasm_i_ssnop(p); 561 uasm_i_ssnop(p); 562 uasm_i_ssnop(p); 563 uasm_i_ssnop(p); 564 break; 565 566 case CPU_VR4111: 567 case CPU_VR4121: 568 case CPU_VR4122: 569 case CPU_VR4181: 570 case CPU_VR4181A: 571 uasm_i_nop(p); 572 uasm_i_nop(p); 573 tlbw(p); 574 uasm_i_nop(p); 575 uasm_i_nop(p); 576 break; 577 578 case CPU_VR4131: 579 case CPU_VR4133: 580 case CPU_R5432: 581 uasm_i_nop(p); 582 uasm_i_nop(p); 583 tlbw(p); 584 break; 585 586 case CPU_JZRISC: 587 tlbw(p); 588 uasm_i_nop(p); 589 break; 590 591 default: 592 panic("No TLB refill handler yet (CPU type: %d)", 593 current_cpu_data.cputype); 594 break; 595 } 596 } 597 598 static __cpuinit __maybe_unused void build_convert_pte_to_entrylo(u32 **p, 599 unsigned int reg) 600 { 601 if (cpu_has_rixi) { 602 UASM_i_ROTR(p, reg, reg, ilog2(_PAGE_GLOBAL)); 603 } else { 604 #ifdef CONFIG_64BIT_PHYS_ADDR 605 uasm_i_dsrl_safe(p, reg, reg, ilog2(_PAGE_GLOBAL)); 606 #else 607 UASM_i_SRL(p, reg, reg, ilog2(_PAGE_GLOBAL)); 608 #endif 609 } 610 } 611 612 #ifdef CONFIG_HUGETLB_PAGE 613 614 static __cpuinit void build_restore_pagemask(u32 **p, 615 struct uasm_reloc **r, 616 unsigned int tmp, 617 enum label_id lid, 618 int restore_scratch) 619 { 620 if (restore_scratch) { 621 /* Reset default page size */ 622 if (PM_DEFAULT_MASK >> 16) { 623 uasm_i_lui(p, tmp, PM_DEFAULT_MASK >> 16); 624 uasm_i_ori(p, tmp, tmp, PM_DEFAULT_MASK & 0xffff); 625 uasm_i_mtc0(p, tmp, C0_PAGEMASK); 626 uasm_il_b(p, r, lid); 627 } else if (PM_DEFAULT_MASK) { 628 uasm_i_ori(p, tmp, 0, PM_DEFAULT_MASK); 629 uasm_i_mtc0(p, tmp, C0_PAGEMASK); 630 uasm_il_b(p, r, lid); 631 } else { 632 uasm_i_mtc0(p, 0, C0_PAGEMASK); 633 uasm_il_b(p, r, lid); 634 } 635 if (scratch_reg > 0) 636 UASM_i_MFC0(p, 1, 31, scratch_reg); 637 else 638 UASM_i_LW(p, 1, scratchpad_offset(0), 0); 639 } else { 640 /* Reset default page size */ 641 if (PM_DEFAULT_MASK >> 16) { 642 uasm_i_lui(p, tmp, PM_DEFAULT_MASK >> 16); 643 uasm_i_ori(p, tmp, tmp, PM_DEFAULT_MASK & 0xffff); 644 uasm_il_b(p, r, lid); 645 uasm_i_mtc0(p, tmp, C0_PAGEMASK); 646 } else if (PM_DEFAULT_MASK) { 647 uasm_i_ori(p, tmp, 0, PM_DEFAULT_MASK); 648 uasm_il_b(p, r, lid); 649 uasm_i_mtc0(p, tmp, C0_PAGEMASK); 650 } else { 651 uasm_il_b(p, r, lid); 652 uasm_i_mtc0(p, 0, C0_PAGEMASK); 653 } 654 } 655 } 656 657 static __cpuinit void build_huge_tlb_write_entry(u32 **p, 658 struct uasm_label **l, 659 struct uasm_reloc **r, 660 unsigned int tmp, 661 enum tlb_write_entry wmode, 662 int restore_scratch) 663 { 664 /* Set huge page tlb entry size */ 665 uasm_i_lui(p, tmp, PM_HUGE_MASK >> 16); 666 uasm_i_ori(p, tmp, tmp, PM_HUGE_MASK & 0xffff); 667 uasm_i_mtc0(p, tmp, C0_PAGEMASK); 668 669 build_tlb_write_entry(p, l, r, wmode); 670 671 build_restore_pagemask(p, r, tmp, label_leave, restore_scratch); 672 } 673 674 /* 675 * Check if Huge PTE is present, if so then jump to LABEL. 676 */ 677 static void __cpuinit 678 build_is_huge_pte(u32 **p, struct uasm_reloc **r, unsigned int tmp, 679 unsigned int pmd, int lid) 680 { 681 UASM_i_LW(p, tmp, 0, pmd); 682 if (use_bbit_insns()) { 683 uasm_il_bbit1(p, r, tmp, ilog2(_PAGE_HUGE), lid); 684 } else { 685 uasm_i_andi(p, tmp, tmp, _PAGE_HUGE); 686 uasm_il_bnez(p, r, tmp, lid); 687 } 688 } 689 690 static __cpuinit void build_huge_update_entries(u32 **p, 691 unsigned int pte, 692 unsigned int tmp) 693 { 694 int small_sequence; 695 696 /* 697 * A huge PTE describes an area the size of the 698 * configured huge page size. This is twice the 699 * of the large TLB entry size we intend to use. 700 * A TLB entry half the size of the configured 701 * huge page size is configured into entrylo0 702 * and entrylo1 to cover the contiguous huge PTE 703 * address space. 704 */ 705 small_sequence = (HPAGE_SIZE >> 7) < 0x10000; 706 707 /* We can clobber tmp. It isn't used after this.*/ 708 if (!small_sequence) 709 uasm_i_lui(p, tmp, HPAGE_SIZE >> (7 + 16)); 710 711 build_convert_pte_to_entrylo(p, pte); 712 UASM_i_MTC0(p, pte, C0_ENTRYLO0); /* load it */ 713 /* convert to entrylo1 */ 714 if (small_sequence) 715 UASM_i_ADDIU(p, pte, pte, HPAGE_SIZE >> 7); 716 else 717 UASM_i_ADDU(p, pte, pte, tmp); 718 719 UASM_i_MTC0(p, pte, C0_ENTRYLO1); /* load it */ 720 } 721 722 static __cpuinit void build_huge_handler_tail(u32 **p, 723 struct uasm_reloc **r, 724 struct uasm_label **l, 725 unsigned int pte, 726 unsigned int ptr) 727 { 728 #ifdef CONFIG_SMP 729 UASM_i_SC(p, pte, 0, ptr); 730 uasm_il_beqz(p, r, pte, label_tlb_huge_update); 731 UASM_i_LW(p, pte, 0, ptr); /* Needed because SC killed our PTE */ 732 #else 733 UASM_i_SW(p, pte, 0, ptr); 734 #endif 735 build_huge_update_entries(p, pte, ptr); 736 build_huge_tlb_write_entry(p, l, r, pte, tlb_indexed, 0); 737 } 738 #endif /* CONFIG_HUGETLB_PAGE */ 739 740 #ifdef CONFIG_64BIT 741 /* 742 * TMP and PTR are scratch. 743 * TMP will be clobbered, PTR will hold the pmd entry. 744 */ 745 static void __cpuinit 746 build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r, 747 unsigned int tmp, unsigned int ptr) 748 { 749 #ifndef CONFIG_MIPS_PGD_C0_CONTEXT 750 long pgdc = (long)pgd_current; 751 #endif 752 /* 753 * The vmalloc handling is not in the hotpath. 754 */ 755 uasm_i_dmfc0(p, tmp, C0_BADVADDR); 756 757 if (check_for_high_segbits) { 758 /* 759 * The kernel currently implicitely assumes that the 760 * MIPS SEGBITS parameter for the processor is 761 * (PGDIR_SHIFT+PGDIR_BITS) or less, and will never 762 * allocate virtual addresses outside the maximum 763 * range for SEGBITS = (PGDIR_SHIFT+PGDIR_BITS). But 764 * that doesn't prevent user code from accessing the 765 * higher xuseg addresses. Here, we make sure that 766 * everything but the lower xuseg addresses goes down 767 * the module_alloc/vmalloc path. 768 */ 769 uasm_i_dsrl_safe(p, ptr, tmp, PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3); 770 uasm_il_bnez(p, r, ptr, label_vmalloc); 771 } else { 772 uasm_il_bltz(p, r, tmp, label_vmalloc); 773 } 774 /* No uasm_i_nop needed here, since the next insn doesn't touch TMP. */ 775 776 #ifdef CONFIG_MIPS_PGD_C0_CONTEXT 777 if (pgd_reg != -1) { 778 /* pgd is in pgd_reg */ 779 UASM_i_MFC0(p, ptr, 31, pgd_reg); 780 } else { 781 /* 782 * &pgd << 11 stored in CONTEXT [23..63]. 783 */ 784 UASM_i_MFC0(p, ptr, C0_CONTEXT); 785 786 /* Clear lower 23 bits of context. */ 787 uasm_i_dins(p, ptr, 0, 0, 23); 788 789 /* 1 0 1 0 1 << 6 xkphys cached */ 790 uasm_i_ori(p, ptr, ptr, 0x540); 791 uasm_i_drotr(p, ptr, ptr, 11); 792 } 793 #elif defined(CONFIG_SMP) 794 # ifdef CONFIG_MIPS_MT_SMTC 795 /* 796 * SMTC uses TCBind value as "CPU" index 797 */ 798 uasm_i_mfc0(p, ptr, C0_TCBIND); 799 uasm_i_dsrl_safe(p, ptr, ptr, 19); 800 # else 801 /* 802 * 64 bit SMP running in XKPHYS has smp_processor_id() << 3 803 * stored in CONTEXT. 804 */ 805 uasm_i_dmfc0(p, ptr, C0_CONTEXT); 806 uasm_i_dsrl_safe(p, ptr, ptr, 23); 807 # endif 808 UASM_i_LA_mostly(p, tmp, pgdc); 809 uasm_i_daddu(p, ptr, ptr, tmp); 810 uasm_i_dmfc0(p, tmp, C0_BADVADDR); 811 uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr); 812 #else 813 UASM_i_LA_mostly(p, ptr, pgdc); 814 uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr); 815 #endif 816 817 uasm_l_vmalloc_done(l, *p); 818 819 /* get pgd offset in bytes */ 820 uasm_i_dsrl_safe(p, tmp, tmp, PGDIR_SHIFT - 3); 821 822 uasm_i_andi(p, tmp, tmp, (PTRS_PER_PGD - 1)<<3); 823 uasm_i_daddu(p, ptr, ptr, tmp); /* add in pgd offset */ 824 #ifndef __PAGETABLE_PMD_FOLDED 825 uasm_i_dmfc0(p, tmp, C0_BADVADDR); /* get faulting address */ 826 uasm_i_ld(p, ptr, 0, ptr); /* get pmd pointer */ 827 uasm_i_dsrl_safe(p, tmp, tmp, PMD_SHIFT-3); /* get pmd offset in bytes */ 828 uasm_i_andi(p, tmp, tmp, (PTRS_PER_PMD - 1)<<3); 829 uasm_i_daddu(p, ptr, ptr, tmp); /* add in pmd offset */ 830 #endif 831 } 832 833 /* 834 * BVADDR is the faulting address, PTR is scratch. 835 * PTR will hold the pgd for vmalloc. 836 */ 837 static void __cpuinit 838 build_get_pgd_vmalloc64(u32 **p, struct uasm_label **l, struct uasm_reloc **r, 839 unsigned int bvaddr, unsigned int ptr, 840 enum vmalloc64_mode mode) 841 { 842 long swpd = (long)swapper_pg_dir; 843 int single_insn_swpd; 844 int did_vmalloc_branch = 0; 845 846 single_insn_swpd = uasm_in_compat_space_p(swpd) && !uasm_rel_lo(swpd); 847 848 uasm_l_vmalloc(l, *p); 849 850 if (mode != not_refill && check_for_high_segbits) { 851 if (single_insn_swpd) { 852 uasm_il_bltz(p, r, bvaddr, label_vmalloc_done); 853 uasm_i_lui(p, ptr, uasm_rel_hi(swpd)); 854 did_vmalloc_branch = 1; 855 /* fall through */ 856 } else { 857 uasm_il_bgez(p, r, bvaddr, label_large_segbits_fault); 858 } 859 } 860 if (!did_vmalloc_branch) { 861 if (uasm_in_compat_space_p(swpd) && !uasm_rel_lo(swpd)) { 862 uasm_il_b(p, r, label_vmalloc_done); 863 uasm_i_lui(p, ptr, uasm_rel_hi(swpd)); 864 } else { 865 UASM_i_LA_mostly(p, ptr, swpd); 866 uasm_il_b(p, r, label_vmalloc_done); 867 if (uasm_in_compat_space_p(swpd)) 868 uasm_i_addiu(p, ptr, ptr, uasm_rel_lo(swpd)); 869 else 870 uasm_i_daddiu(p, ptr, ptr, uasm_rel_lo(swpd)); 871 } 872 } 873 if (mode != not_refill && check_for_high_segbits) { 874 uasm_l_large_segbits_fault(l, *p); 875 /* 876 * We get here if we are an xsseg address, or if we are 877 * an xuseg address above (PGDIR_SHIFT+PGDIR_BITS) boundary. 878 * 879 * Ignoring xsseg (assume disabled so would generate 880 * (address errors?), the only remaining possibility 881 * is the upper xuseg addresses. On processors with 882 * TLB_SEGBITS <= PGDIR_SHIFT+PGDIR_BITS, these 883 * addresses would have taken an address error. We try 884 * to mimic that here by taking a load/istream page 885 * fault. 886 */ 887 UASM_i_LA(p, ptr, (unsigned long)tlb_do_page_fault_0); 888 uasm_i_jr(p, ptr); 889 890 if (mode == refill_scratch) { 891 if (scratch_reg > 0) 892 UASM_i_MFC0(p, 1, 31, scratch_reg); 893 else 894 UASM_i_LW(p, 1, scratchpad_offset(0), 0); 895 } else { 896 uasm_i_nop(p); 897 } 898 } 899 } 900 901 #else /* !CONFIG_64BIT */ 902 903 /* 904 * TMP and PTR are scratch. 905 * TMP will be clobbered, PTR will hold the pgd entry. 906 */ 907 static void __cpuinit __maybe_unused 908 build_get_pgde32(u32 **p, unsigned int tmp, unsigned int ptr) 909 { 910 long pgdc = (long)pgd_current; 911 912 /* 32 bit SMP has smp_processor_id() stored in CONTEXT. */ 913 #ifdef CONFIG_SMP 914 #ifdef CONFIG_MIPS_MT_SMTC 915 /* 916 * SMTC uses TCBind value as "CPU" index 917 */ 918 uasm_i_mfc0(p, ptr, C0_TCBIND); 919 UASM_i_LA_mostly(p, tmp, pgdc); 920 uasm_i_srl(p, ptr, ptr, 19); 921 #else 922 /* 923 * smp_processor_id() << 3 is stored in CONTEXT. 924 */ 925 uasm_i_mfc0(p, ptr, C0_CONTEXT); 926 UASM_i_LA_mostly(p, tmp, pgdc); 927 uasm_i_srl(p, ptr, ptr, 23); 928 #endif 929 uasm_i_addu(p, ptr, tmp, ptr); 930 #else 931 UASM_i_LA_mostly(p, ptr, pgdc); 932 #endif 933 uasm_i_mfc0(p, tmp, C0_BADVADDR); /* get faulting address */ 934 uasm_i_lw(p, ptr, uasm_rel_lo(pgdc), ptr); 935 936 if (cpu_has_mips_r2) { 937 uasm_i_ext(p, tmp, tmp, PGDIR_SHIFT, (32 - PGDIR_SHIFT)); 938 uasm_i_ins(p, ptr, tmp, PGD_T_LOG2, (32 - PGDIR_SHIFT)); 939 return; 940 } 941 942 uasm_i_srl(p, tmp, tmp, PGDIR_SHIFT); /* get pgd only bits */ 943 uasm_i_sll(p, tmp, tmp, PGD_T_LOG2); 944 uasm_i_addu(p, ptr, ptr, tmp); /* add in pgd offset */ 945 } 946 947 #endif /* !CONFIG_64BIT */ 948 949 static void __cpuinit build_adjust_context(u32 **p, unsigned int ctx) 950 { 951 unsigned int shift = 4 - (PTE_T_LOG2 + 1) + PAGE_SHIFT - 12; 952 unsigned int mask = (PTRS_PER_PTE / 2 - 1) << (PTE_T_LOG2 + 1); 953 954 switch (current_cpu_type()) { 955 case CPU_VR41XX: 956 case CPU_VR4111: 957 case CPU_VR4121: 958 case CPU_VR4122: 959 case CPU_VR4131: 960 case CPU_VR4181: 961 case CPU_VR4181A: 962 case CPU_VR4133: 963 shift += 2; 964 break; 965 966 default: 967 break; 968 } 969 970 if (shift) 971 UASM_i_SRL(p, ctx, ctx, shift); 972 uasm_i_andi(p, ctx, ctx, mask); 973 } 974 975 static void __cpuinit build_get_ptep(u32 **p, unsigned int tmp, unsigned int ptr) 976 { 977 if (cpu_has_mips_r2) { 978 /* PTE ptr offset is obtained from BadVAddr */ 979 UASM_i_MFC0(p, tmp, C0_BADVADDR); 980 UASM_i_LW(p, ptr, 0, ptr); 981 uasm_i_ext(p, tmp, tmp, PAGE_SHIFT+1, PGDIR_SHIFT-PAGE_SHIFT-1); 982 uasm_i_ins(p, ptr, tmp, PTE_T_LOG2+1, PGDIR_SHIFT-PAGE_SHIFT-1); 983 return; 984 } 985 986 /* 987 * Bug workaround for the Nevada. It seems as if under certain 988 * circumstances the move from cp0_context might produce a 989 * bogus result when the mfc0 instruction and its consumer are 990 * in a different cacheline or a load instruction, probably any 991 * memory reference, is between them. 992 */ 993 switch (current_cpu_type()) { 994 case CPU_NEVADA: 995 UASM_i_LW(p, ptr, 0, ptr); 996 GET_CONTEXT(p, tmp); /* get context reg */ 997 break; 998 999 default: 1000 GET_CONTEXT(p, tmp); /* get context reg */ 1001 UASM_i_LW(p, ptr, 0, ptr); 1002 break; 1003 } 1004 1005 build_adjust_context(p, tmp); 1006 UASM_i_ADDU(p, ptr, ptr, tmp); /* add in offset */ 1007 } 1008 1009 static void __cpuinit build_update_entries(u32 **p, unsigned int tmp, 1010 unsigned int ptep) 1011 { 1012 /* 1013 * 64bit address support (36bit on a 32bit CPU) in a 32bit 1014 * Kernel is a special case. Only a few CPUs use it. 1015 */ 1016 #ifdef CONFIG_64BIT_PHYS_ADDR 1017 if (cpu_has_64bits) { 1018 uasm_i_ld(p, tmp, 0, ptep); /* get even pte */ 1019 uasm_i_ld(p, ptep, sizeof(pte_t), ptep); /* get odd pte */ 1020 if (cpu_has_rixi) { 1021 UASM_i_ROTR(p, tmp, tmp, ilog2(_PAGE_GLOBAL)); 1022 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */ 1023 UASM_i_ROTR(p, ptep, ptep, ilog2(_PAGE_GLOBAL)); 1024 } else { 1025 uasm_i_dsrl_safe(p, tmp, tmp, ilog2(_PAGE_GLOBAL)); /* convert to entrylo0 */ 1026 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */ 1027 uasm_i_dsrl_safe(p, ptep, ptep, ilog2(_PAGE_GLOBAL)); /* convert to entrylo1 */ 1028 } 1029 UASM_i_MTC0(p, ptep, C0_ENTRYLO1); /* load it */ 1030 } else { 1031 int pte_off_even = sizeof(pte_t) / 2; 1032 int pte_off_odd = pte_off_even + sizeof(pte_t); 1033 1034 /* The pte entries are pre-shifted */ 1035 uasm_i_lw(p, tmp, pte_off_even, ptep); /* get even pte */ 1036 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */ 1037 uasm_i_lw(p, ptep, pte_off_odd, ptep); /* get odd pte */ 1038 UASM_i_MTC0(p, ptep, C0_ENTRYLO1); /* load it */ 1039 } 1040 #else 1041 UASM_i_LW(p, tmp, 0, ptep); /* get even pte */ 1042 UASM_i_LW(p, ptep, sizeof(pte_t), ptep); /* get odd pte */ 1043 if (r45k_bvahwbug()) 1044 build_tlb_probe_entry(p); 1045 if (cpu_has_rixi) { 1046 UASM_i_ROTR(p, tmp, tmp, ilog2(_PAGE_GLOBAL)); 1047 if (r4k_250MHZhwbug()) 1048 UASM_i_MTC0(p, 0, C0_ENTRYLO0); 1049 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */ 1050 UASM_i_ROTR(p, ptep, ptep, ilog2(_PAGE_GLOBAL)); 1051 } else { 1052 UASM_i_SRL(p, tmp, tmp, ilog2(_PAGE_GLOBAL)); /* convert to entrylo0 */ 1053 if (r4k_250MHZhwbug()) 1054 UASM_i_MTC0(p, 0, C0_ENTRYLO0); 1055 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */ 1056 UASM_i_SRL(p, ptep, ptep, ilog2(_PAGE_GLOBAL)); /* convert to entrylo1 */ 1057 if (r45k_bvahwbug()) 1058 uasm_i_mfc0(p, tmp, C0_INDEX); 1059 } 1060 if (r4k_250MHZhwbug()) 1061 UASM_i_MTC0(p, 0, C0_ENTRYLO1); 1062 UASM_i_MTC0(p, ptep, C0_ENTRYLO1); /* load it */ 1063 #endif 1064 } 1065 1066 struct mips_huge_tlb_info { 1067 int huge_pte; 1068 int restore_scratch; 1069 }; 1070 1071 static struct mips_huge_tlb_info __cpuinit 1072 build_fast_tlb_refill_handler (u32 **p, struct uasm_label **l, 1073 struct uasm_reloc **r, unsigned int tmp, 1074 unsigned int ptr, int c0_scratch) 1075 { 1076 struct mips_huge_tlb_info rv; 1077 unsigned int even, odd; 1078 int vmalloc_branch_delay_filled = 0; 1079 const int scratch = 1; /* Our extra working register */ 1080 1081 rv.huge_pte = scratch; 1082 rv.restore_scratch = 0; 1083 1084 if (check_for_high_segbits) { 1085 UASM_i_MFC0(p, tmp, C0_BADVADDR); 1086 1087 if (pgd_reg != -1) 1088 UASM_i_MFC0(p, ptr, 31, pgd_reg); 1089 else 1090 UASM_i_MFC0(p, ptr, C0_CONTEXT); 1091 1092 if (c0_scratch >= 0) 1093 UASM_i_MTC0(p, scratch, 31, c0_scratch); 1094 else 1095 UASM_i_SW(p, scratch, scratchpad_offset(0), 0); 1096 1097 uasm_i_dsrl_safe(p, scratch, tmp, 1098 PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3); 1099 uasm_il_bnez(p, r, scratch, label_vmalloc); 1100 1101 if (pgd_reg == -1) { 1102 vmalloc_branch_delay_filled = 1; 1103 /* Clear lower 23 bits of context. */ 1104 uasm_i_dins(p, ptr, 0, 0, 23); 1105 } 1106 } else { 1107 if (pgd_reg != -1) 1108 UASM_i_MFC0(p, ptr, 31, pgd_reg); 1109 else 1110 UASM_i_MFC0(p, ptr, C0_CONTEXT); 1111 1112 UASM_i_MFC0(p, tmp, C0_BADVADDR); 1113 1114 if (c0_scratch >= 0) 1115 UASM_i_MTC0(p, scratch, 31, c0_scratch); 1116 else 1117 UASM_i_SW(p, scratch, scratchpad_offset(0), 0); 1118 1119 if (pgd_reg == -1) 1120 /* Clear lower 23 bits of context. */ 1121 uasm_i_dins(p, ptr, 0, 0, 23); 1122 1123 uasm_il_bltz(p, r, tmp, label_vmalloc); 1124 } 1125 1126 if (pgd_reg == -1) { 1127 vmalloc_branch_delay_filled = 1; 1128 /* 1 0 1 0 1 << 6 xkphys cached */ 1129 uasm_i_ori(p, ptr, ptr, 0x540); 1130 uasm_i_drotr(p, ptr, ptr, 11); 1131 } 1132 1133 #ifdef __PAGETABLE_PMD_FOLDED 1134 #define LOC_PTEP scratch 1135 #else 1136 #define LOC_PTEP ptr 1137 #endif 1138 1139 if (!vmalloc_branch_delay_filled) 1140 /* get pgd offset in bytes */ 1141 uasm_i_dsrl_safe(p, scratch, tmp, PGDIR_SHIFT - 3); 1142 1143 uasm_l_vmalloc_done(l, *p); 1144 1145 /* 1146 * tmp ptr 1147 * fall-through case = badvaddr *pgd_current 1148 * vmalloc case = badvaddr swapper_pg_dir 1149 */ 1150 1151 if (vmalloc_branch_delay_filled) 1152 /* get pgd offset in bytes */ 1153 uasm_i_dsrl_safe(p, scratch, tmp, PGDIR_SHIFT - 3); 1154 1155 #ifdef __PAGETABLE_PMD_FOLDED 1156 GET_CONTEXT(p, tmp); /* get context reg */ 1157 #endif 1158 uasm_i_andi(p, scratch, scratch, (PTRS_PER_PGD - 1) << 3); 1159 1160 if (use_lwx_insns()) { 1161 UASM_i_LWX(p, LOC_PTEP, scratch, ptr); 1162 } else { 1163 uasm_i_daddu(p, ptr, ptr, scratch); /* add in pgd offset */ 1164 uasm_i_ld(p, LOC_PTEP, 0, ptr); /* get pmd pointer */ 1165 } 1166 1167 #ifndef __PAGETABLE_PMD_FOLDED 1168 /* get pmd offset in bytes */ 1169 uasm_i_dsrl_safe(p, scratch, tmp, PMD_SHIFT - 3); 1170 uasm_i_andi(p, scratch, scratch, (PTRS_PER_PMD - 1) << 3); 1171 GET_CONTEXT(p, tmp); /* get context reg */ 1172 1173 if (use_lwx_insns()) { 1174 UASM_i_LWX(p, scratch, scratch, ptr); 1175 } else { 1176 uasm_i_daddu(p, ptr, ptr, scratch); /* add in pmd offset */ 1177 UASM_i_LW(p, scratch, 0, ptr); 1178 } 1179 #endif 1180 /* Adjust the context during the load latency. */ 1181 build_adjust_context(p, tmp); 1182 1183 #ifdef CONFIG_HUGETLB_PAGE 1184 uasm_il_bbit1(p, r, scratch, ilog2(_PAGE_HUGE), label_tlb_huge_update); 1185 /* 1186 * The in the LWX case we don't want to do the load in the 1187 * delay slot. It cannot issue in the same cycle and may be 1188 * speculative and unneeded. 1189 */ 1190 if (use_lwx_insns()) 1191 uasm_i_nop(p); 1192 #endif /* CONFIG_HUGETLB_PAGE */ 1193 1194 1195 /* build_update_entries */ 1196 if (use_lwx_insns()) { 1197 even = ptr; 1198 odd = tmp; 1199 UASM_i_LWX(p, even, scratch, tmp); 1200 UASM_i_ADDIU(p, tmp, tmp, sizeof(pte_t)); 1201 UASM_i_LWX(p, odd, scratch, tmp); 1202 } else { 1203 UASM_i_ADDU(p, ptr, scratch, tmp); /* add in offset */ 1204 even = tmp; 1205 odd = ptr; 1206 UASM_i_LW(p, even, 0, ptr); /* get even pte */ 1207 UASM_i_LW(p, odd, sizeof(pte_t), ptr); /* get odd pte */ 1208 } 1209 if (cpu_has_rixi) { 1210 uasm_i_drotr(p, even, even, ilog2(_PAGE_GLOBAL)); 1211 UASM_i_MTC0(p, even, C0_ENTRYLO0); /* load it */ 1212 uasm_i_drotr(p, odd, odd, ilog2(_PAGE_GLOBAL)); 1213 } else { 1214 uasm_i_dsrl_safe(p, even, even, ilog2(_PAGE_GLOBAL)); 1215 UASM_i_MTC0(p, even, C0_ENTRYLO0); /* load it */ 1216 uasm_i_dsrl_safe(p, odd, odd, ilog2(_PAGE_GLOBAL)); 1217 } 1218 UASM_i_MTC0(p, odd, C0_ENTRYLO1); /* load it */ 1219 1220 if (c0_scratch >= 0) { 1221 UASM_i_MFC0(p, scratch, 31, c0_scratch); 1222 build_tlb_write_entry(p, l, r, tlb_random); 1223 uasm_l_leave(l, *p); 1224 rv.restore_scratch = 1; 1225 } else if (PAGE_SHIFT == 14 || PAGE_SHIFT == 13) { 1226 build_tlb_write_entry(p, l, r, tlb_random); 1227 uasm_l_leave(l, *p); 1228 UASM_i_LW(p, scratch, scratchpad_offset(0), 0); 1229 } else { 1230 UASM_i_LW(p, scratch, scratchpad_offset(0), 0); 1231 build_tlb_write_entry(p, l, r, tlb_random); 1232 uasm_l_leave(l, *p); 1233 rv.restore_scratch = 1; 1234 } 1235 1236 uasm_i_eret(p); /* return from trap */ 1237 1238 return rv; 1239 } 1240 1241 /* 1242 * For a 64-bit kernel, we are using the 64-bit XTLB refill exception 1243 * because EXL == 0. If we wrap, we can also use the 32 instruction 1244 * slots before the XTLB refill exception handler which belong to the 1245 * unused TLB refill exception. 1246 */ 1247 #define MIPS64_REFILL_INSNS 32 1248 1249 static void __cpuinit build_r4000_tlb_refill_handler(void) 1250 { 1251 u32 *p = tlb_handler; 1252 struct uasm_label *l = labels; 1253 struct uasm_reloc *r = relocs; 1254 u32 *f; 1255 unsigned int final_len; 1256 struct mips_huge_tlb_info htlb_info __maybe_unused; 1257 enum vmalloc64_mode vmalloc_mode __maybe_unused; 1258 1259 memset(tlb_handler, 0, sizeof(tlb_handler)); 1260 memset(labels, 0, sizeof(labels)); 1261 memset(relocs, 0, sizeof(relocs)); 1262 memset(final_handler, 0, sizeof(final_handler)); 1263 1264 if ((scratch_reg > 0 || scratchpad_available()) && use_bbit_insns()) { 1265 htlb_info = build_fast_tlb_refill_handler(&p, &l, &r, K0, K1, 1266 scratch_reg); 1267 vmalloc_mode = refill_scratch; 1268 } else { 1269 htlb_info.huge_pte = K0; 1270 htlb_info.restore_scratch = 0; 1271 vmalloc_mode = refill_noscratch; 1272 /* 1273 * create the plain linear handler 1274 */ 1275 if (bcm1250_m3_war()) { 1276 unsigned int segbits = 44; 1277 1278 uasm_i_dmfc0(&p, K0, C0_BADVADDR); 1279 uasm_i_dmfc0(&p, K1, C0_ENTRYHI); 1280 uasm_i_xor(&p, K0, K0, K1); 1281 uasm_i_dsrl_safe(&p, K1, K0, 62); 1282 uasm_i_dsrl_safe(&p, K0, K0, 12 + 1); 1283 uasm_i_dsll_safe(&p, K0, K0, 64 + 12 + 1 - segbits); 1284 uasm_i_or(&p, K0, K0, K1); 1285 uasm_il_bnez(&p, &r, K0, label_leave); 1286 /* No need for uasm_i_nop */ 1287 } 1288 1289 #ifdef CONFIG_64BIT 1290 build_get_pmde64(&p, &l, &r, K0, K1); /* get pmd in K1 */ 1291 #else 1292 build_get_pgde32(&p, K0, K1); /* get pgd in K1 */ 1293 #endif 1294 1295 #ifdef CONFIG_HUGETLB_PAGE 1296 build_is_huge_pte(&p, &r, K0, K1, label_tlb_huge_update); 1297 #endif 1298 1299 build_get_ptep(&p, K0, K1); 1300 build_update_entries(&p, K0, K1); 1301 build_tlb_write_entry(&p, &l, &r, tlb_random); 1302 uasm_l_leave(&l, p); 1303 uasm_i_eret(&p); /* return from trap */ 1304 } 1305 #ifdef CONFIG_HUGETLB_PAGE 1306 uasm_l_tlb_huge_update(&l, p); 1307 build_huge_update_entries(&p, htlb_info.huge_pte, K1); 1308 build_huge_tlb_write_entry(&p, &l, &r, K0, tlb_random, 1309 htlb_info.restore_scratch); 1310 #endif 1311 1312 #ifdef CONFIG_64BIT 1313 build_get_pgd_vmalloc64(&p, &l, &r, K0, K1, vmalloc_mode); 1314 #endif 1315 1316 /* 1317 * Overflow check: For the 64bit handler, we need at least one 1318 * free instruction slot for the wrap-around branch. In worst 1319 * case, if the intended insertion point is a delay slot, we 1320 * need three, with the second nop'ed and the third being 1321 * unused. 1322 */ 1323 /* Loongson2 ebase is different than r4k, we have more space */ 1324 #if defined(CONFIG_32BIT) || defined(CONFIG_CPU_LOONGSON2) 1325 if ((p - tlb_handler) > 64) 1326 panic("TLB refill handler space exceeded"); 1327 #else 1328 if (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 1) 1329 || (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 3) 1330 && uasm_insn_has_bdelay(relocs, 1331 tlb_handler + MIPS64_REFILL_INSNS - 3))) 1332 panic("TLB refill handler space exceeded"); 1333 #endif 1334 1335 /* 1336 * Now fold the handler in the TLB refill handler space. 1337 */ 1338 #if defined(CONFIG_32BIT) || defined(CONFIG_CPU_LOONGSON2) 1339 f = final_handler; 1340 /* Simplest case, just copy the handler. */ 1341 uasm_copy_handler(relocs, labels, tlb_handler, p, f); 1342 final_len = p - tlb_handler; 1343 #else /* CONFIG_64BIT */ 1344 f = final_handler + MIPS64_REFILL_INSNS; 1345 if ((p - tlb_handler) <= MIPS64_REFILL_INSNS) { 1346 /* Just copy the handler. */ 1347 uasm_copy_handler(relocs, labels, tlb_handler, p, f); 1348 final_len = p - tlb_handler; 1349 } else { 1350 #if defined(CONFIG_HUGETLB_PAGE) 1351 const enum label_id ls = label_tlb_huge_update; 1352 #else 1353 const enum label_id ls = label_vmalloc; 1354 #endif 1355 u32 *split; 1356 int ov = 0; 1357 int i; 1358 1359 for (i = 0; i < ARRAY_SIZE(labels) && labels[i].lab != ls; i++) 1360 ; 1361 BUG_ON(i == ARRAY_SIZE(labels)); 1362 split = labels[i].addr; 1363 1364 /* 1365 * See if we have overflown one way or the other. 1366 */ 1367 if (split > tlb_handler + MIPS64_REFILL_INSNS || 1368 split < p - MIPS64_REFILL_INSNS) 1369 ov = 1; 1370 1371 if (ov) { 1372 /* 1373 * Split two instructions before the end. One 1374 * for the branch and one for the instruction 1375 * in the delay slot. 1376 */ 1377 split = tlb_handler + MIPS64_REFILL_INSNS - 2; 1378 1379 /* 1380 * If the branch would fall in a delay slot, 1381 * we must back up an additional instruction 1382 * so that it is no longer in a delay slot. 1383 */ 1384 if (uasm_insn_has_bdelay(relocs, split - 1)) 1385 split--; 1386 } 1387 /* Copy first part of the handler. */ 1388 uasm_copy_handler(relocs, labels, tlb_handler, split, f); 1389 f += split - tlb_handler; 1390 1391 if (ov) { 1392 /* Insert branch. */ 1393 uasm_l_split(&l, final_handler); 1394 uasm_il_b(&f, &r, label_split); 1395 if (uasm_insn_has_bdelay(relocs, split)) 1396 uasm_i_nop(&f); 1397 else { 1398 uasm_copy_handler(relocs, labels, 1399 split, split + 1, f); 1400 uasm_move_labels(labels, f, f + 1, -1); 1401 f++; 1402 split++; 1403 } 1404 } 1405 1406 /* Copy the rest of the handler. */ 1407 uasm_copy_handler(relocs, labels, split, p, final_handler); 1408 final_len = (f - (final_handler + MIPS64_REFILL_INSNS)) + 1409 (p - split); 1410 } 1411 #endif /* CONFIG_64BIT */ 1412 1413 uasm_resolve_relocs(relocs, labels); 1414 pr_debug("Wrote TLB refill handler (%u instructions).\n", 1415 final_len); 1416 1417 memcpy((void *)ebase, final_handler, 0x100); 1418 1419 dump_handler((u32 *)ebase, 64); 1420 } 1421 1422 /* 1423 * 128 instructions for the fastpath handler is generous and should 1424 * never be exceeded. 1425 */ 1426 #define FASTPATH_SIZE 128 1427 1428 u32 handle_tlbl[FASTPATH_SIZE] __cacheline_aligned; 1429 u32 handle_tlbs[FASTPATH_SIZE] __cacheline_aligned; 1430 u32 handle_tlbm[FASTPATH_SIZE] __cacheline_aligned; 1431 #ifdef CONFIG_MIPS_PGD_C0_CONTEXT 1432 u32 tlbmiss_handler_setup_pgd[16] __cacheline_aligned; 1433 1434 static void __cpuinit build_r4000_setup_pgd(void) 1435 { 1436 const int a0 = 4; 1437 const int a1 = 5; 1438 u32 *p = tlbmiss_handler_setup_pgd; 1439 struct uasm_label *l = labels; 1440 struct uasm_reloc *r = relocs; 1441 1442 memset(tlbmiss_handler_setup_pgd, 0, sizeof(tlbmiss_handler_setup_pgd)); 1443 memset(labels, 0, sizeof(labels)); 1444 memset(relocs, 0, sizeof(relocs)); 1445 1446 pgd_reg = allocate_kscratch(); 1447 1448 if (pgd_reg == -1) { 1449 /* PGD << 11 in c0_Context */ 1450 /* 1451 * If it is a ckseg0 address, convert to a physical 1452 * address. Shifting right by 29 and adding 4 will 1453 * result in zero for these addresses. 1454 * 1455 */ 1456 UASM_i_SRA(&p, a1, a0, 29); 1457 UASM_i_ADDIU(&p, a1, a1, 4); 1458 uasm_il_bnez(&p, &r, a1, label_tlbl_goaround1); 1459 uasm_i_nop(&p); 1460 uasm_i_dinsm(&p, a0, 0, 29, 64 - 29); 1461 uasm_l_tlbl_goaround1(&l, p); 1462 UASM_i_SLL(&p, a0, a0, 11); 1463 uasm_i_jr(&p, 31); 1464 UASM_i_MTC0(&p, a0, C0_CONTEXT); 1465 } else { 1466 /* PGD in c0_KScratch */ 1467 uasm_i_jr(&p, 31); 1468 UASM_i_MTC0(&p, a0, 31, pgd_reg); 1469 } 1470 if (p - tlbmiss_handler_setup_pgd > ARRAY_SIZE(tlbmiss_handler_setup_pgd)) 1471 panic("tlbmiss_handler_setup_pgd space exceeded"); 1472 uasm_resolve_relocs(relocs, labels); 1473 pr_debug("Wrote tlbmiss_handler_setup_pgd (%u instructions).\n", 1474 (unsigned int)(p - tlbmiss_handler_setup_pgd)); 1475 1476 dump_handler(tlbmiss_handler_setup_pgd, 1477 ARRAY_SIZE(tlbmiss_handler_setup_pgd)); 1478 } 1479 #endif 1480 1481 static void __cpuinit 1482 iPTE_LW(u32 **p, unsigned int pte, unsigned int ptr) 1483 { 1484 #ifdef CONFIG_SMP 1485 # ifdef CONFIG_64BIT_PHYS_ADDR 1486 if (cpu_has_64bits) 1487 uasm_i_lld(p, pte, 0, ptr); 1488 else 1489 # endif 1490 UASM_i_LL(p, pte, 0, ptr); 1491 #else 1492 # ifdef CONFIG_64BIT_PHYS_ADDR 1493 if (cpu_has_64bits) 1494 uasm_i_ld(p, pte, 0, ptr); 1495 else 1496 # endif 1497 UASM_i_LW(p, pte, 0, ptr); 1498 #endif 1499 } 1500 1501 static void __cpuinit 1502 iPTE_SW(u32 **p, struct uasm_reloc **r, unsigned int pte, unsigned int ptr, 1503 unsigned int mode) 1504 { 1505 #ifdef CONFIG_64BIT_PHYS_ADDR 1506 unsigned int hwmode = mode & (_PAGE_VALID | _PAGE_DIRTY); 1507 #endif 1508 1509 uasm_i_ori(p, pte, pte, mode); 1510 #ifdef CONFIG_SMP 1511 # ifdef CONFIG_64BIT_PHYS_ADDR 1512 if (cpu_has_64bits) 1513 uasm_i_scd(p, pte, 0, ptr); 1514 else 1515 # endif 1516 UASM_i_SC(p, pte, 0, ptr); 1517 1518 if (r10000_llsc_war()) 1519 uasm_il_beqzl(p, r, pte, label_smp_pgtable_change); 1520 else 1521 uasm_il_beqz(p, r, pte, label_smp_pgtable_change); 1522 1523 # ifdef CONFIG_64BIT_PHYS_ADDR 1524 if (!cpu_has_64bits) { 1525 /* no uasm_i_nop needed */ 1526 uasm_i_ll(p, pte, sizeof(pte_t) / 2, ptr); 1527 uasm_i_ori(p, pte, pte, hwmode); 1528 uasm_i_sc(p, pte, sizeof(pte_t) / 2, ptr); 1529 uasm_il_beqz(p, r, pte, label_smp_pgtable_change); 1530 /* no uasm_i_nop needed */ 1531 uasm_i_lw(p, pte, 0, ptr); 1532 } else 1533 uasm_i_nop(p); 1534 # else 1535 uasm_i_nop(p); 1536 # endif 1537 #else 1538 # ifdef CONFIG_64BIT_PHYS_ADDR 1539 if (cpu_has_64bits) 1540 uasm_i_sd(p, pte, 0, ptr); 1541 else 1542 # endif 1543 UASM_i_SW(p, pte, 0, ptr); 1544 1545 # ifdef CONFIG_64BIT_PHYS_ADDR 1546 if (!cpu_has_64bits) { 1547 uasm_i_lw(p, pte, sizeof(pte_t) / 2, ptr); 1548 uasm_i_ori(p, pte, pte, hwmode); 1549 uasm_i_sw(p, pte, sizeof(pte_t) / 2, ptr); 1550 uasm_i_lw(p, pte, 0, ptr); 1551 } 1552 # endif 1553 #endif 1554 } 1555 1556 /* 1557 * Check if PTE is present, if not then jump to LABEL. PTR points to 1558 * the page table where this PTE is located, PTE will be re-loaded 1559 * with it's original value. 1560 */ 1561 static void __cpuinit 1562 build_pte_present(u32 **p, struct uasm_reloc **r, 1563 int pte, int ptr, int scratch, enum label_id lid) 1564 { 1565 int t = scratch >= 0 ? scratch : pte; 1566 1567 if (cpu_has_rixi) { 1568 if (use_bbit_insns()) { 1569 uasm_il_bbit0(p, r, pte, ilog2(_PAGE_PRESENT), lid); 1570 uasm_i_nop(p); 1571 } else { 1572 uasm_i_andi(p, t, pte, _PAGE_PRESENT); 1573 uasm_il_beqz(p, r, t, lid); 1574 if (pte == t) 1575 /* You lose the SMP race :-(*/ 1576 iPTE_LW(p, pte, ptr); 1577 } 1578 } else { 1579 uasm_i_andi(p, t, pte, _PAGE_PRESENT | _PAGE_READ); 1580 uasm_i_xori(p, t, t, _PAGE_PRESENT | _PAGE_READ); 1581 uasm_il_bnez(p, r, t, lid); 1582 if (pte == t) 1583 /* You lose the SMP race :-(*/ 1584 iPTE_LW(p, pte, ptr); 1585 } 1586 } 1587 1588 /* Make PTE valid, store result in PTR. */ 1589 static void __cpuinit 1590 build_make_valid(u32 **p, struct uasm_reloc **r, unsigned int pte, 1591 unsigned int ptr) 1592 { 1593 unsigned int mode = _PAGE_VALID | _PAGE_ACCESSED; 1594 1595 iPTE_SW(p, r, pte, ptr, mode); 1596 } 1597 1598 /* 1599 * Check if PTE can be written to, if not branch to LABEL. Regardless 1600 * restore PTE with value from PTR when done. 1601 */ 1602 static void __cpuinit 1603 build_pte_writable(u32 **p, struct uasm_reloc **r, 1604 unsigned int pte, unsigned int ptr, int scratch, 1605 enum label_id lid) 1606 { 1607 int t = scratch >= 0 ? scratch : pte; 1608 1609 uasm_i_andi(p, t, pte, _PAGE_PRESENT | _PAGE_WRITE); 1610 uasm_i_xori(p, t, t, _PAGE_PRESENT | _PAGE_WRITE); 1611 uasm_il_bnez(p, r, t, lid); 1612 if (pte == t) 1613 /* You lose the SMP race :-(*/ 1614 iPTE_LW(p, pte, ptr); 1615 else 1616 uasm_i_nop(p); 1617 } 1618 1619 /* Make PTE writable, update software status bits as well, then store 1620 * at PTR. 1621 */ 1622 static void __cpuinit 1623 build_make_write(u32 **p, struct uasm_reloc **r, unsigned int pte, 1624 unsigned int ptr) 1625 { 1626 unsigned int mode = (_PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID 1627 | _PAGE_DIRTY); 1628 1629 iPTE_SW(p, r, pte, ptr, mode); 1630 } 1631 1632 /* 1633 * Check if PTE can be modified, if not branch to LABEL. Regardless 1634 * restore PTE with value from PTR when done. 1635 */ 1636 static void __cpuinit 1637 build_pte_modifiable(u32 **p, struct uasm_reloc **r, 1638 unsigned int pte, unsigned int ptr, int scratch, 1639 enum label_id lid) 1640 { 1641 if (use_bbit_insns()) { 1642 uasm_il_bbit0(p, r, pte, ilog2(_PAGE_WRITE), lid); 1643 uasm_i_nop(p); 1644 } else { 1645 int t = scratch >= 0 ? scratch : pte; 1646 uasm_i_andi(p, t, pte, _PAGE_WRITE); 1647 uasm_il_beqz(p, r, t, lid); 1648 if (pte == t) 1649 /* You lose the SMP race :-(*/ 1650 iPTE_LW(p, pte, ptr); 1651 } 1652 } 1653 1654 #ifndef CONFIG_MIPS_PGD_C0_CONTEXT 1655 1656 1657 /* 1658 * R3000 style TLB load/store/modify handlers. 1659 */ 1660 1661 /* 1662 * This places the pte into ENTRYLO0 and writes it with tlbwi. 1663 * Then it returns. 1664 */ 1665 static void __cpuinit 1666 build_r3000_pte_reload_tlbwi(u32 **p, unsigned int pte, unsigned int tmp) 1667 { 1668 uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */ 1669 uasm_i_mfc0(p, tmp, C0_EPC); /* cp0 delay */ 1670 uasm_i_tlbwi(p); 1671 uasm_i_jr(p, tmp); 1672 uasm_i_rfe(p); /* branch delay */ 1673 } 1674 1675 /* 1676 * This places the pte into ENTRYLO0 and writes it with tlbwi 1677 * or tlbwr as appropriate. This is because the index register 1678 * may have the probe fail bit set as a result of a trap on a 1679 * kseg2 access, i.e. without refill. Then it returns. 1680 */ 1681 static void __cpuinit 1682 build_r3000_tlb_reload_write(u32 **p, struct uasm_label **l, 1683 struct uasm_reloc **r, unsigned int pte, 1684 unsigned int tmp) 1685 { 1686 uasm_i_mfc0(p, tmp, C0_INDEX); 1687 uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */ 1688 uasm_il_bltz(p, r, tmp, label_r3000_write_probe_fail); /* cp0 delay */ 1689 uasm_i_mfc0(p, tmp, C0_EPC); /* branch delay */ 1690 uasm_i_tlbwi(p); /* cp0 delay */ 1691 uasm_i_jr(p, tmp); 1692 uasm_i_rfe(p); /* branch delay */ 1693 uasm_l_r3000_write_probe_fail(l, *p); 1694 uasm_i_tlbwr(p); /* cp0 delay */ 1695 uasm_i_jr(p, tmp); 1696 uasm_i_rfe(p); /* branch delay */ 1697 } 1698 1699 static void __cpuinit 1700 build_r3000_tlbchange_handler_head(u32 **p, unsigned int pte, 1701 unsigned int ptr) 1702 { 1703 long pgdc = (long)pgd_current; 1704 1705 uasm_i_mfc0(p, pte, C0_BADVADDR); 1706 uasm_i_lui(p, ptr, uasm_rel_hi(pgdc)); /* cp0 delay */ 1707 uasm_i_lw(p, ptr, uasm_rel_lo(pgdc), ptr); 1708 uasm_i_srl(p, pte, pte, 22); /* load delay */ 1709 uasm_i_sll(p, pte, pte, 2); 1710 uasm_i_addu(p, ptr, ptr, pte); 1711 uasm_i_mfc0(p, pte, C0_CONTEXT); 1712 uasm_i_lw(p, ptr, 0, ptr); /* cp0 delay */ 1713 uasm_i_andi(p, pte, pte, 0xffc); /* load delay */ 1714 uasm_i_addu(p, ptr, ptr, pte); 1715 uasm_i_lw(p, pte, 0, ptr); 1716 uasm_i_tlbp(p); /* load delay */ 1717 } 1718 1719 static void __cpuinit build_r3000_tlb_load_handler(void) 1720 { 1721 u32 *p = handle_tlbl; 1722 struct uasm_label *l = labels; 1723 struct uasm_reloc *r = relocs; 1724 1725 memset(handle_tlbl, 0, sizeof(handle_tlbl)); 1726 memset(labels, 0, sizeof(labels)); 1727 memset(relocs, 0, sizeof(relocs)); 1728 1729 build_r3000_tlbchange_handler_head(&p, K0, K1); 1730 build_pte_present(&p, &r, K0, K1, -1, label_nopage_tlbl); 1731 uasm_i_nop(&p); /* load delay */ 1732 build_make_valid(&p, &r, K0, K1); 1733 build_r3000_tlb_reload_write(&p, &l, &r, K0, K1); 1734 1735 uasm_l_nopage_tlbl(&l, p); 1736 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff); 1737 uasm_i_nop(&p); 1738 1739 if ((p - handle_tlbl) > FASTPATH_SIZE) 1740 panic("TLB load handler fastpath space exceeded"); 1741 1742 uasm_resolve_relocs(relocs, labels); 1743 pr_debug("Wrote TLB load handler fastpath (%u instructions).\n", 1744 (unsigned int)(p - handle_tlbl)); 1745 1746 dump_handler(handle_tlbl, ARRAY_SIZE(handle_tlbl)); 1747 } 1748 1749 static void __cpuinit build_r3000_tlb_store_handler(void) 1750 { 1751 u32 *p = handle_tlbs; 1752 struct uasm_label *l = labels; 1753 struct uasm_reloc *r = relocs; 1754 1755 memset(handle_tlbs, 0, sizeof(handle_tlbs)); 1756 memset(labels, 0, sizeof(labels)); 1757 memset(relocs, 0, sizeof(relocs)); 1758 1759 build_r3000_tlbchange_handler_head(&p, K0, K1); 1760 build_pte_writable(&p, &r, K0, K1, -1, label_nopage_tlbs); 1761 uasm_i_nop(&p); /* load delay */ 1762 build_make_write(&p, &r, K0, K1); 1763 build_r3000_tlb_reload_write(&p, &l, &r, K0, K1); 1764 1765 uasm_l_nopage_tlbs(&l, p); 1766 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); 1767 uasm_i_nop(&p); 1768 1769 if ((p - handle_tlbs) > FASTPATH_SIZE) 1770 panic("TLB store handler fastpath space exceeded"); 1771 1772 uasm_resolve_relocs(relocs, labels); 1773 pr_debug("Wrote TLB store handler fastpath (%u instructions).\n", 1774 (unsigned int)(p - handle_tlbs)); 1775 1776 dump_handler(handle_tlbs, ARRAY_SIZE(handle_tlbs)); 1777 } 1778 1779 static void __cpuinit build_r3000_tlb_modify_handler(void) 1780 { 1781 u32 *p = handle_tlbm; 1782 struct uasm_label *l = labels; 1783 struct uasm_reloc *r = relocs; 1784 1785 memset(handle_tlbm, 0, sizeof(handle_tlbm)); 1786 memset(labels, 0, sizeof(labels)); 1787 memset(relocs, 0, sizeof(relocs)); 1788 1789 build_r3000_tlbchange_handler_head(&p, K0, K1); 1790 build_pte_modifiable(&p, &r, K0, K1, -1, label_nopage_tlbm); 1791 uasm_i_nop(&p); /* load delay */ 1792 build_make_write(&p, &r, K0, K1); 1793 build_r3000_pte_reload_tlbwi(&p, K0, K1); 1794 1795 uasm_l_nopage_tlbm(&l, p); 1796 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); 1797 uasm_i_nop(&p); 1798 1799 if ((p - handle_tlbm) > FASTPATH_SIZE) 1800 panic("TLB modify handler fastpath space exceeded"); 1801 1802 uasm_resolve_relocs(relocs, labels); 1803 pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n", 1804 (unsigned int)(p - handle_tlbm)); 1805 1806 dump_handler(handle_tlbm, ARRAY_SIZE(handle_tlbm)); 1807 } 1808 #endif /* CONFIG_MIPS_PGD_C0_CONTEXT */ 1809 1810 /* 1811 * R4000 style TLB load/store/modify handlers. 1812 */ 1813 static struct work_registers __cpuinit 1814 build_r4000_tlbchange_handler_head(u32 **p, struct uasm_label **l, 1815 struct uasm_reloc **r) 1816 { 1817 struct work_registers wr = build_get_work_registers(p); 1818 1819 #ifdef CONFIG_64BIT 1820 build_get_pmde64(p, l, r, wr.r1, wr.r2); /* get pmd in ptr */ 1821 #else 1822 build_get_pgde32(p, wr.r1, wr.r2); /* get pgd in ptr */ 1823 #endif 1824 1825 #ifdef CONFIG_HUGETLB_PAGE 1826 /* 1827 * For huge tlb entries, pmd doesn't contain an address but 1828 * instead contains the tlb pte. Check the PAGE_HUGE bit and 1829 * see if we need to jump to huge tlb processing. 1830 */ 1831 build_is_huge_pte(p, r, wr.r1, wr.r2, label_tlb_huge_update); 1832 #endif 1833 1834 UASM_i_MFC0(p, wr.r1, C0_BADVADDR); 1835 UASM_i_LW(p, wr.r2, 0, wr.r2); 1836 UASM_i_SRL(p, wr.r1, wr.r1, PAGE_SHIFT + PTE_ORDER - PTE_T_LOG2); 1837 uasm_i_andi(p, wr.r1, wr.r1, (PTRS_PER_PTE - 1) << PTE_T_LOG2); 1838 UASM_i_ADDU(p, wr.r2, wr.r2, wr.r1); 1839 1840 #ifdef CONFIG_SMP 1841 uasm_l_smp_pgtable_change(l, *p); 1842 #endif 1843 iPTE_LW(p, wr.r1, wr.r2); /* get even pte */ 1844 if (!m4kc_tlbp_war()) 1845 build_tlb_probe_entry(p); 1846 return wr; 1847 } 1848 1849 static void __cpuinit 1850 build_r4000_tlbchange_handler_tail(u32 **p, struct uasm_label **l, 1851 struct uasm_reloc **r, unsigned int tmp, 1852 unsigned int ptr) 1853 { 1854 uasm_i_ori(p, ptr, ptr, sizeof(pte_t)); 1855 uasm_i_xori(p, ptr, ptr, sizeof(pte_t)); 1856 build_update_entries(p, tmp, ptr); 1857 build_tlb_write_entry(p, l, r, tlb_indexed); 1858 uasm_l_leave(l, *p); 1859 build_restore_work_registers(p); 1860 uasm_i_eret(p); /* return from trap */ 1861 1862 #ifdef CONFIG_64BIT 1863 build_get_pgd_vmalloc64(p, l, r, tmp, ptr, not_refill); 1864 #endif 1865 } 1866 1867 static void __cpuinit build_r4000_tlb_load_handler(void) 1868 { 1869 u32 *p = handle_tlbl; 1870 struct uasm_label *l = labels; 1871 struct uasm_reloc *r = relocs; 1872 struct work_registers wr; 1873 1874 memset(handle_tlbl, 0, sizeof(handle_tlbl)); 1875 memset(labels, 0, sizeof(labels)); 1876 memset(relocs, 0, sizeof(relocs)); 1877 1878 if (bcm1250_m3_war()) { 1879 unsigned int segbits = 44; 1880 1881 uasm_i_dmfc0(&p, K0, C0_BADVADDR); 1882 uasm_i_dmfc0(&p, K1, C0_ENTRYHI); 1883 uasm_i_xor(&p, K0, K0, K1); 1884 uasm_i_dsrl_safe(&p, K1, K0, 62); 1885 uasm_i_dsrl_safe(&p, K0, K0, 12 + 1); 1886 uasm_i_dsll_safe(&p, K0, K0, 64 + 12 + 1 - segbits); 1887 uasm_i_or(&p, K0, K0, K1); 1888 uasm_il_bnez(&p, &r, K0, label_leave); 1889 /* No need for uasm_i_nop */ 1890 } 1891 1892 wr = build_r4000_tlbchange_handler_head(&p, &l, &r); 1893 build_pte_present(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbl); 1894 if (m4kc_tlbp_war()) 1895 build_tlb_probe_entry(&p); 1896 1897 if (cpu_has_rixi) { 1898 /* 1899 * If the page is not _PAGE_VALID, RI or XI could not 1900 * have triggered it. Skip the expensive test.. 1901 */ 1902 if (use_bbit_insns()) { 1903 uasm_il_bbit0(&p, &r, wr.r1, ilog2(_PAGE_VALID), 1904 label_tlbl_goaround1); 1905 } else { 1906 uasm_i_andi(&p, wr.r3, wr.r1, _PAGE_VALID); 1907 uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround1); 1908 } 1909 uasm_i_nop(&p); 1910 1911 uasm_i_tlbr(&p); 1912 /* Examine entrylo 0 or 1 based on ptr. */ 1913 if (use_bbit_insns()) { 1914 uasm_i_bbit0(&p, wr.r2, ilog2(sizeof(pte_t)), 8); 1915 } else { 1916 uasm_i_andi(&p, wr.r3, wr.r2, sizeof(pte_t)); 1917 uasm_i_beqz(&p, wr.r3, 8); 1918 } 1919 /* load it in the delay slot*/ 1920 UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO0); 1921 /* load it if ptr is odd */ 1922 UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO1); 1923 /* 1924 * If the entryLo (now in wr.r3) is valid (bit 1), RI or 1925 * XI must have triggered it. 1926 */ 1927 if (use_bbit_insns()) { 1928 uasm_il_bbit1(&p, &r, wr.r3, 1, label_nopage_tlbl); 1929 uasm_i_nop(&p); 1930 uasm_l_tlbl_goaround1(&l, p); 1931 } else { 1932 uasm_i_andi(&p, wr.r3, wr.r3, 2); 1933 uasm_il_bnez(&p, &r, wr.r3, label_nopage_tlbl); 1934 uasm_i_nop(&p); 1935 } 1936 uasm_l_tlbl_goaround1(&l, p); 1937 } 1938 build_make_valid(&p, &r, wr.r1, wr.r2); 1939 build_r4000_tlbchange_handler_tail(&p, &l, &r, wr.r1, wr.r2); 1940 1941 #ifdef CONFIG_HUGETLB_PAGE 1942 /* 1943 * This is the entry point when build_r4000_tlbchange_handler_head 1944 * spots a huge page. 1945 */ 1946 uasm_l_tlb_huge_update(&l, p); 1947 iPTE_LW(&p, wr.r1, wr.r2); 1948 build_pte_present(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbl); 1949 build_tlb_probe_entry(&p); 1950 1951 if (cpu_has_rixi) { 1952 /* 1953 * If the page is not _PAGE_VALID, RI or XI could not 1954 * have triggered it. Skip the expensive test.. 1955 */ 1956 if (use_bbit_insns()) { 1957 uasm_il_bbit0(&p, &r, wr.r1, ilog2(_PAGE_VALID), 1958 label_tlbl_goaround2); 1959 } else { 1960 uasm_i_andi(&p, wr.r3, wr.r1, _PAGE_VALID); 1961 uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround2); 1962 } 1963 uasm_i_nop(&p); 1964 1965 uasm_i_tlbr(&p); 1966 /* Examine entrylo 0 or 1 based on ptr. */ 1967 if (use_bbit_insns()) { 1968 uasm_i_bbit0(&p, wr.r2, ilog2(sizeof(pte_t)), 8); 1969 } else { 1970 uasm_i_andi(&p, wr.r3, wr.r2, sizeof(pte_t)); 1971 uasm_i_beqz(&p, wr.r3, 8); 1972 } 1973 /* load it in the delay slot*/ 1974 UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO0); 1975 /* load it if ptr is odd */ 1976 UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO1); 1977 /* 1978 * If the entryLo (now in wr.r3) is valid (bit 1), RI or 1979 * XI must have triggered it. 1980 */ 1981 if (use_bbit_insns()) { 1982 uasm_il_bbit0(&p, &r, wr.r3, 1, label_tlbl_goaround2); 1983 } else { 1984 uasm_i_andi(&p, wr.r3, wr.r3, 2); 1985 uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround2); 1986 } 1987 if (PM_DEFAULT_MASK == 0) 1988 uasm_i_nop(&p); 1989 /* 1990 * We clobbered C0_PAGEMASK, restore it. On the other branch 1991 * it is restored in build_huge_tlb_write_entry. 1992 */ 1993 build_restore_pagemask(&p, &r, wr.r3, label_nopage_tlbl, 0); 1994 1995 uasm_l_tlbl_goaround2(&l, p); 1996 } 1997 uasm_i_ori(&p, wr.r1, wr.r1, (_PAGE_ACCESSED | _PAGE_VALID)); 1998 build_huge_handler_tail(&p, &r, &l, wr.r1, wr.r2); 1999 #endif 2000 2001 uasm_l_nopage_tlbl(&l, p); 2002 build_restore_work_registers(&p); 2003 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff); 2004 uasm_i_nop(&p); 2005 2006 if ((p - handle_tlbl) > FASTPATH_SIZE) 2007 panic("TLB load handler fastpath space exceeded"); 2008 2009 uasm_resolve_relocs(relocs, labels); 2010 pr_debug("Wrote TLB load handler fastpath (%u instructions).\n", 2011 (unsigned int)(p - handle_tlbl)); 2012 2013 dump_handler(handle_tlbl, ARRAY_SIZE(handle_tlbl)); 2014 } 2015 2016 static void __cpuinit build_r4000_tlb_store_handler(void) 2017 { 2018 u32 *p = handle_tlbs; 2019 struct uasm_label *l = labels; 2020 struct uasm_reloc *r = relocs; 2021 struct work_registers wr; 2022 2023 memset(handle_tlbs, 0, sizeof(handle_tlbs)); 2024 memset(labels, 0, sizeof(labels)); 2025 memset(relocs, 0, sizeof(relocs)); 2026 2027 wr = build_r4000_tlbchange_handler_head(&p, &l, &r); 2028 build_pte_writable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbs); 2029 if (m4kc_tlbp_war()) 2030 build_tlb_probe_entry(&p); 2031 build_make_write(&p, &r, wr.r1, wr.r2); 2032 build_r4000_tlbchange_handler_tail(&p, &l, &r, wr.r1, wr.r2); 2033 2034 #ifdef CONFIG_HUGETLB_PAGE 2035 /* 2036 * This is the entry point when 2037 * build_r4000_tlbchange_handler_head spots a huge page. 2038 */ 2039 uasm_l_tlb_huge_update(&l, p); 2040 iPTE_LW(&p, wr.r1, wr.r2); 2041 build_pte_writable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbs); 2042 build_tlb_probe_entry(&p); 2043 uasm_i_ori(&p, wr.r1, wr.r1, 2044 _PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID | _PAGE_DIRTY); 2045 build_huge_handler_tail(&p, &r, &l, wr.r1, wr.r2); 2046 #endif 2047 2048 uasm_l_nopage_tlbs(&l, p); 2049 build_restore_work_registers(&p); 2050 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); 2051 uasm_i_nop(&p); 2052 2053 if ((p - handle_tlbs) > FASTPATH_SIZE) 2054 panic("TLB store handler fastpath space exceeded"); 2055 2056 uasm_resolve_relocs(relocs, labels); 2057 pr_debug("Wrote TLB store handler fastpath (%u instructions).\n", 2058 (unsigned int)(p - handle_tlbs)); 2059 2060 dump_handler(handle_tlbs, ARRAY_SIZE(handle_tlbs)); 2061 } 2062 2063 static void __cpuinit build_r4000_tlb_modify_handler(void) 2064 { 2065 u32 *p = handle_tlbm; 2066 struct uasm_label *l = labels; 2067 struct uasm_reloc *r = relocs; 2068 struct work_registers wr; 2069 2070 memset(handle_tlbm, 0, sizeof(handle_tlbm)); 2071 memset(labels, 0, sizeof(labels)); 2072 memset(relocs, 0, sizeof(relocs)); 2073 2074 wr = build_r4000_tlbchange_handler_head(&p, &l, &r); 2075 build_pte_modifiable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbm); 2076 if (m4kc_tlbp_war()) 2077 build_tlb_probe_entry(&p); 2078 /* Present and writable bits set, set accessed and dirty bits. */ 2079 build_make_write(&p, &r, wr.r1, wr.r2); 2080 build_r4000_tlbchange_handler_tail(&p, &l, &r, wr.r1, wr.r2); 2081 2082 #ifdef CONFIG_HUGETLB_PAGE 2083 /* 2084 * This is the entry point when 2085 * build_r4000_tlbchange_handler_head spots a huge page. 2086 */ 2087 uasm_l_tlb_huge_update(&l, p); 2088 iPTE_LW(&p, wr.r1, wr.r2); 2089 build_pte_modifiable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbm); 2090 build_tlb_probe_entry(&p); 2091 uasm_i_ori(&p, wr.r1, wr.r1, 2092 _PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID | _PAGE_DIRTY); 2093 build_huge_handler_tail(&p, &r, &l, wr.r1, wr.r2); 2094 #endif 2095 2096 uasm_l_nopage_tlbm(&l, p); 2097 build_restore_work_registers(&p); 2098 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); 2099 uasm_i_nop(&p); 2100 2101 if ((p - handle_tlbm) > FASTPATH_SIZE) 2102 panic("TLB modify handler fastpath space exceeded"); 2103 2104 uasm_resolve_relocs(relocs, labels); 2105 pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n", 2106 (unsigned int)(p - handle_tlbm)); 2107 2108 dump_handler(handle_tlbm, ARRAY_SIZE(handle_tlbm)); 2109 } 2110 2111 void __cpuinit build_tlb_refill_handler(void) 2112 { 2113 /* 2114 * The refill handler is generated per-CPU, multi-node systems 2115 * may have local storage for it. The other handlers are only 2116 * needed once. 2117 */ 2118 static int run_once = 0; 2119 2120 #ifdef CONFIG_64BIT 2121 check_for_high_segbits = current_cpu_data.vmbits > (PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3); 2122 #endif 2123 2124 switch (current_cpu_type()) { 2125 case CPU_R2000: 2126 case CPU_R3000: 2127 case CPU_R3000A: 2128 case CPU_R3081E: 2129 case CPU_TX3912: 2130 case CPU_TX3922: 2131 case CPU_TX3927: 2132 #ifndef CONFIG_MIPS_PGD_C0_CONTEXT 2133 build_r3000_tlb_refill_handler(); 2134 if (!run_once) { 2135 build_r3000_tlb_load_handler(); 2136 build_r3000_tlb_store_handler(); 2137 build_r3000_tlb_modify_handler(); 2138 run_once++; 2139 } 2140 #else 2141 panic("No R3000 TLB refill handler"); 2142 #endif 2143 break; 2144 2145 case CPU_R6000: 2146 case CPU_R6000A: 2147 panic("No R6000 TLB refill handler yet"); 2148 break; 2149 2150 case CPU_R8000: 2151 panic("No R8000 TLB refill handler yet"); 2152 break; 2153 2154 default: 2155 if (!run_once) { 2156 scratch_reg = allocate_kscratch(); 2157 #ifdef CONFIG_MIPS_PGD_C0_CONTEXT 2158 build_r4000_setup_pgd(); 2159 #endif 2160 build_r4000_tlb_load_handler(); 2161 build_r4000_tlb_store_handler(); 2162 build_r4000_tlb_modify_handler(); 2163 run_once++; 2164 } 2165 build_r4000_tlb_refill_handler(); 2166 } 2167 } 2168 2169 void __cpuinit flush_tlb_handlers(void) 2170 { 2171 local_flush_icache_range((unsigned long)handle_tlbl, 2172 (unsigned long)handle_tlbl + sizeof(handle_tlbl)); 2173 local_flush_icache_range((unsigned long)handle_tlbs, 2174 (unsigned long)handle_tlbs + sizeof(handle_tlbs)); 2175 local_flush_icache_range((unsigned long)handle_tlbm, 2176 (unsigned long)handle_tlbm + sizeof(handle_tlbm)); 2177 #ifdef CONFIG_MIPS_PGD_C0_CONTEXT 2178 local_flush_icache_range((unsigned long)tlbmiss_handler_setup_pgd, 2179 (unsigned long)tlbmiss_handler_setup_pgd + sizeof(handle_tlbm)); 2180 #endif 2181 } 2182