1 /* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * Synthesize TLB refill handlers at runtime. 7 * 8 * Copyright (C) 2004, 2005, 2006, 2008 Thiemo Seufer 9 * Copyright (C) 2005, 2007, 2008, 2009 Maciej W. Rozycki 10 * Copyright (C) 2006 Ralf Baechle (ralf@linux-mips.org) 11 * Copyright (C) 2008, 2009 Cavium Networks, Inc. 12 * Copyright (C) 2011 MIPS Technologies, Inc. 13 * 14 * ... and the days got worse and worse and now you see 15 * I've gone completly out of my mind. 16 * 17 * They're coming to take me a away haha 18 * they're coming to take me a away hoho hihi haha 19 * to the funny farm where code is beautiful all the time ... 20 * 21 * (Condolences to Napoleon XIV) 22 */ 23 24 #include <linux/bug.h> 25 #include <linux/kernel.h> 26 #include <linux/types.h> 27 #include <linux/smp.h> 28 #include <linux/string.h> 29 #include <linux/init.h> 30 #include <linux/cache.h> 31 32 #include <asm/cacheflush.h> 33 #include <asm/pgtable.h> 34 #include <asm/war.h> 35 #include <asm/uasm.h> 36 #include <asm/setup.h> 37 38 /* 39 * TLB load/store/modify handlers. 40 * 41 * Only the fastpath gets synthesized at runtime, the slowpath for 42 * do_page_fault remains normal asm. 43 */ 44 extern void tlb_do_page_fault_0(void); 45 extern void tlb_do_page_fault_1(void); 46 47 struct work_registers { 48 int r1; 49 int r2; 50 int r3; 51 }; 52 53 struct tlb_reg_save { 54 unsigned long a; 55 unsigned long b; 56 } ____cacheline_aligned_in_smp; 57 58 static struct tlb_reg_save handler_reg_save[NR_CPUS]; 59 60 static inline int r45k_bvahwbug(void) 61 { 62 /* XXX: We should probe for the presence of this bug, but we don't. */ 63 return 0; 64 } 65 66 static inline int r4k_250MHZhwbug(void) 67 { 68 /* XXX: We should probe for the presence of this bug, but we don't. */ 69 return 0; 70 } 71 72 static inline int __maybe_unused bcm1250_m3_war(void) 73 { 74 return BCM1250_M3_WAR; 75 } 76 77 static inline int __maybe_unused r10000_llsc_war(void) 78 { 79 return R10000_LLSC_WAR; 80 } 81 82 static int use_bbit_insns(void) 83 { 84 switch (current_cpu_type()) { 85 case CPU_CAVIUM_OCTEON: 86 case CPU_CAVIUM_OCTEON_PLUS: 87 case CPU_CAVIUM_OCTEON2: 88 return 1; 89 default: 90 return 0; 91 } 92 } 93 94 static int use_lwx_insns(void) 95 { 96 switch (current_cpu_type()) { 97 case CPU_CAVIUM_OCTEON2: 98 return 1; 99 default: 100 return 0; 101 } 102 } 103 #if defined(CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE) && \ 104 CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE > 0 105 static bool scratchpad_available(void) 106 { 107 return true; 108 } 109 static int scratchpad_offset(int i) 110 { 111 /* 112 * CVMSEG starts at address -32768 and extends for 113 * CAVIUM_OCTEON_CVMSEG_SIZE 128 byte cache lines. 114 */ 115 i += 1; /* Kernel use starts at the top and works down. */ 116 return CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE * 128 - (8 * i) - 32768; 117 } 118 #else 119 static bool scratchpad_available(void) 120 { 121 return false; 122 } 123 static int scratchpad_offset(int i) 124 { 125 BUG(); 126 /* Really unreachable, but evidently some GCC want this. */ 127 return 0; 128 } 129 #endif 130 /* 131 * Found by experiment: At least some revisions of the 4kc throw under 132 * some circumstances a machine check exception, triggered by invalid 133 * values in the index register. Delaying the tlbp instruction until 134 * after the next branch, plus adding an additional nop in front of 135 * tlbwi/tlbwr avoids the invalid index register values. Nobody knows 136 * why; it's not an issue caused by the core RTL. 137 * 138 */ 139 static int m4kc_tlbp_war(void) 140 { 141 return (current_cpu_data.processor_id & 0xffff00) == 142 (PRID_COMP_MIPS | PRID_IMP_4KC); 143 } 144 145 /* Handle labels (which must be positive integers). */ 146 enum label_id { 147 label_second_part = 1, 148 label_leave, 149 label_vmalloc, 150 label_vmalloc_done, 151 label_tlbw_hazard_0, 152 label_split = label_tlbw_hazard_0 + 8, 153 label_tlbl_goaround1, 154 label_tlbl_goaround2, 155 label_nopage_tlbl, 156 label_nopage_tlbs, 157 label_nopage_tlbm, 158 label_smp_pgtable_change, 159 label_r3000_write_probe_fail, 160 label_large_segbits_fault, 161 #ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT 162 label_tlb_huge_update, 163 #endif 164 }; 165 166 UASM_L_LA(_second_part) 167 UASM_L_LA(_leave) 168 UASM_L_LA(_vmalloc) 169 UASM_L_LA(_vmalloc_done) 170 /* _tlbw_hazard_x is handled differently. */ 171 UASM_L_LA(_split) 172 UASM_L_LA(_tlbl_goaround1) 173 UASM_L_LA(_tlbl_goaround2) 174 UASM_L_LA(_nopage_tlbl) 175 UASM_L_LA(_nopage_tlbs) 176 UASM_L_LA(_nopage_tlbm) 177 UASM_L_LA(_smp_pgtable_change) 178 UASM_L_LA(_r3000_write_probe_fail) 179 UASM_L_LA(_large_segbits_fault) 180 #ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT 181 UASM_L_LA(_tlb_huge_update) 182 #endif 183 184 static int hazard_instance; 185 186 static void uasm_bgezl_hazard(u32 **p, struct uasm_reloc **r, int instance) 187 { 188 switch (instance) { 189 case 0 ... 7: 190 uasm_il_bgezl(p, r, 0, label_tlbw_hazard_0 + instance); 191 return; 192 default: 193 BUG(); 194 } 195 } 196 197 static void uasm_bgezl_label(struct uasm_label **l, u32 **p, int instance) 198 { 199 switch (instance) { 200 case 0 ... 7: 201 uasm_build_label(l, *p, label_tlbw_hazard_0 + instance); 202 break; 203 default: 204 BUG(); 205 } 206 } 207 208 /* 209 * pgtable bits are assigned dynamically depending on processor feature 210 * and statically based on kernel configuration. This spits out the actual 211 * values the kernel is using. Required to make sense from disassembled 212 * TLB exception handlers. 213 */ 214 static void output_pgtable_bits_defines(void) 215 { 216 #define pr_define(fmt, ...) \ 217 pr_debug("#define " fmt, ##__VA_ARGS__) 218 219 pr_debug("#include <asm/asm.h>\n"); 220 pr_debug("#include <asm/regdef.h>\n"); 221 pr_debug("\n"); 222 223 pr_define("_PAGE_PRESENT_SHIFT %d\n", _PAGE_PRESENT_SHIFT); 224 pr_define("_PAGE_READ_SHIFT %d\n", _PAGE_READ_SHIFT); 225 pr_define("_PAGE_WRITE_SHIFT %d\n", _PAGE_WRITE_SHIFT); 226 pr_define("_PAGE_ACCESSED_SHIFT %d\n", _PAGE_ACCESSED_SHIFT); 227 pr_define("_PAGE_MODIFIED_SHIFT %d\n", _PAGE_MODIFIED_SHIFT); 228 #ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT 229 pr_define("_PAGE_HUGE_SHIFT %d\n", _PAGE_HUGE_SHIFT); 230 pr_define("_PAGE_SPLITTING_SHIFT %d\n", _PAGE_SPLITTING_SHIFT); 231 #endif 232 if (cpu_has_rixi) { 233 #ifdef _PAGE_NO_EXEC_SHIFT 234 pr_define("_PAGE_NO_EXEC_SHIFT %d\n", _PAGE_NO_EXEC_SHIFT); 235 #endif 236 #ifdef _PAGE_NO_READ_SHIFT 237 pr_define("_PAGE_NO_READ_SHIFT %d\n", _PAGE_NO_READ_SHIFT); 238 #endif 239 } 240 pr_define("_PAGE_GLOBAL_SHIFT %d\n", _PAGE_GLOBAL_SHIFT); 241 pr_define("_PAGE_VALID_SHIFT %d\n", _PAGE_VALID_SHIFT); 242 pr_define("_PAGE_DIRTY_SHIFT %d\n", _PAGE_DIRTY_SHIFT); 243 pr_define("_PFN_SHIFT %d\n", _PFN_SHIFT); 244 pr_debug("\n"); 245 } 246 247 static inline void dump_handler(const char *symbol, const u32 *handler, int count) 248 { 249 int i; 250 251 pr_debug("LEAF(%s)\n", symbol); 252 253 pr_debug("\t.set push\n"); 254 pr_debug("\t.set noreorder\n"); 255 256 for (i = 0; i < count; i++) 257 pr_debug("\t.word\t0x%08x\t\t# %p\n", handler[i], &handler[i]); 258 259 pr_debug("\t.set\tpop\n"); 260 261 pr_debug("\tEND(%s)\n", symbol); 262 } 263 264 /* The only general purpose registers allowed in TLB handlers. */ 265 #define K0 26 266 #define K1 27 267 268 /* Some CP0 registers */ 269 #define C0_INDEX 0, 0 270 #define C0_ENTRYLO0 2, 0 271 #define C0_TCBIND 2, 2 272 #define C0_ENTRYLO1 3, 0 273 #define C0_CONTEXT 4, 0 274 #define C0_PAGEMASK 5, 0 275 #define C0_BADVADDR 8, 0 276 #define C0_ENTRYHI 10, 0 277 #define C0_EPC 14, 0 278 #define C0_XCONTEXT 20, 0 279 280 #ifdef CONFIG_64BIT 281 # define GET_CONTEXT(buf, reg) UASM_i_MFC0(buf, reg, C0_XCONTEXT) 282 #else 283 # define GET_CONTEXT(buf, reg) UASM_i_MFC0(buf, reg, C0_CONTEXT) 284 #endif 285 286 /* The worst case length of the handler is around 18 instructions for 287 * R3000-style TLBs and up to 63 instructions for R4000-style TLBs. 288 * Maximum space available is 32 instructions for R3000 and 64 289 * instructions for R4000. 290 * 291 * We deliberately chose a buffer size of 128, so we won't scribble 292 * over anything important on overflow before we panic. 293 */ 294 static u32 tlb_handler[128]; 295 296 /* simply assume worst case size for labels and relocs */ 297 static struct uasm_label labels[128]; 298 static struct uasm_reloc relocs[128]; 299 300 static int check_for_high_segbits; 301 302 static unsigned int kscratch_used_mask; 303 304 static inline int __maybe_unused c0_kscratch(void) 305 { 306 switch (current_cpu_type()) { 307 case CPU_XLP: 308 case CPU_XLR: 309 return 22; 310 default: 311 return 31; 312 } 313 } 314 315 static int allocate_kscratch(void) 316 { 317 int r; 318 unsigned int a = cpu_data[0].kscratch_mask & ~kscratch_used_mask; 319 320 r = ffs(a); 321 322 if (r == 0) 323 return -1; 324 325 r--; /* make it zero based */ 326 327 kscratch_used_mask |= (1 << r); 328 329 return r; 330 } 331 332 static int scratch_reg; 333 static int pgd_reg; 334 enum vmalloc64_mode {not_refill, refill_scratch, refill_noscratch}; 335 336 static struct work_registers build_get_work_registers(u32 **p) 337 { 338 struct work_registers r; 339 340 int smp_processor_id_reg; 341 int smp_processor_id_sel; 342 int smp_processor_id_shift; 343 344 if (scratch_reg >= 0) { 345 /* Save in CPU local C0_KScratch? */ 346 UASM_i_MTC0(p, 1, c0_kscratch(), scratch_reg); 347 r.r1 = K0; 348 r.r2 = K1; 349 r.r3 = 1; 350 return r; 351 } 352 353 if (num_possible_cpus() > 1) { 354 #ifdef CONFIG_MIPS_PGD_C0_CONTEXT 355 smp_processor_id_shift = 51; 356 smp_processor_id_reg = 20; /* XContext */ 357 smp_processor_id_sel = 0; 358 #else 359 # ifdef CONFIG_32BIT 360 smp_processor_id_shift = 25; 361 smp_processor_id_reg = 4; /* Context */ 362 smp_processor_id_sel = 0; 363 # endif 364 # ifdef CONFIG_64BIT 365 smp_processor_id_shift = 26; 366 smp_processor_id_reg = 4; /* Context */ 367 smp_processor_id_sel = 0; 368 # endif 369 #endif 370 /* Get smp_processor_id */ 371 UASM_i_MFC0(p, K0, smp_processor_id_reg, smp_processor_id_sel); 372 UASM_i_SRL_SAFE(p, K0, K0, smp_processor_id_shift); 373 374 /* handler_reg_save index in K0 */ 375 UASM_i_SLL(p, K0, K0, ilog2(sizeof(struct tlb_reg_save))); 376 377 UASM_i_LA(p, K1, (long)&handler_reg_save); 378 UASM_i_ADDU(p, K0, K0, K1); 379 } else { 380 UASM_i_LA(p, K0, (long)&handler_reg_save); 381 } 382 /* K0 now points to save area, save $1 and $2 */ 383 UASM_i_SW(p, 1, offsetof(struct tlb_reg_save, a), K0); 384 UASM_i_SW(p, 2, offsetof(struct tlb_reg_save, b), K0); 385 386 r.r1 = K1; 387 r.r2 = 1; 388 r.r3 = 2; 389 return r; 390 } 391 392 static void build_restore_work_registers(u32 **p) 393 { 394 if (scratch_reg >= 0) { 395 UASM_i_MFC0(p, 1, c0_kscratch(), scratch_reg); 396 return; 397 } 398 /* K0 already points to save area, restore $1 and $2 */ 399 UASM_i_LW(p, 1, offsetof(struct tlb_reg_save, a), K0); 400 UASM_i_LW(p, 2, offsetof(struct tlb_reg_save, b), K0); 401 } 402 403 #ifndef CONFIG_MIPS_PGD_C0_CONTEXT 404 405 /* 406 * CONFIG_MIPS_PGD_C0_CONTEXT implies 64 bit and lack of pgd_current, 407 * we cannot do r3000 under these circumstances. 408 * 409 * Declare pgd_current here instead of including mmu_context.h to avoid type 410 * conflicts for tlbmiss_handler_setup_pgd 411 */ 412 extern unsigned long pgd_current[]; 413 414 /* 415 * The R3000 TLB handler is simple. 416 */ 417 static void build_r3000_tlb_refill_handler(void) 418 { 419 long pgdc = (long)pgd_current; 420 u32 *p; 421 422 memset(tlb_handler, 0, sizeof(tlb_handler)); 423 p = tlb_handler; 424 425 uasm_i_mfc0(&p, K0, C0_BADVADDR); 426 uasm_i_lui(&p, K1, uasm_rel_hi(pgdc)); /* cp0 delay */ 427 uasm_i_lw(&p, K1, uasm_rel_lo(pgdc), K1); 428 uasm_i_srl(&p, K0, K0, 22); /* load delay */ 429 uasm_i_sll(&p, K0, K0, 2); 430 uasm_i_addu(&p, K1, K1, K0); 431 uasm_i_mfc0(&p, K0, C0_CONTEXT); 432 uasm_i_lw(&p, K1, 0, K1); /* cp0 delay */ 433 uasm_i_andi(&p, K0, K0, 0xffc); /* load delay */ 434 uasm_i_addu(&p, K1, K1, K0); 435 uasm_i_lw(&p, K0, 0, K1); 436 uasm_i_nop(&p); /* load delay */ 437 uasm_i_mtc0(&p, K0, C0_ENTRYLO0); 438 uasm_i_mfc0(&p, K1, C0_EPC); /* cp0 delay */ 439 uasm_i_tlbwr(&p); /* cp0 delay */ 440 uasm_i_jr(&p, K1); 441 uasm_i_rfe(&p); /* branch delay */ 442 443 if (p > tlb_handler + 32) 444 panic("TLB refill handler space exceeded"); 445 446 pr_debug("Wrote TLB refill handler (%u instructions).\n", 447 (unsigned int)(p - tlb_handler)); 448 449 memcpy((void *)ebase, tlb_handler, 0x80); 450 451 dump_handler("r3000_tlb_refill", (u32 *)ebase, 32); 452 } 453 #endif /* CONFIG_MIPS_PGD_C0_CONTEXT */ 454 455 /* 456 * The R4000 TLB handler is much more complicated. We have two 457 * consecutive handler areas with 32 instructions space each. 458 * Since they aren't used at the same time, we can overflow in the 459 * other one.To keep things simple, we first assume linear space, 460 * then we relocate it to the final handler layout as needed. 461 */ 462 static u32 final_handler[64]; 463 464 /* 465 * Hazards 466 * 467 * From the IDT errata for the QED RM5230 (Nevada), processor revision 1.0: 468 * 2. A timing hazard exists for the TLBP instruction. 469 * 470 * stalling_instruction 471 * TLBP 472 * 473 * The JTLB is being read for the TLBP throughout the stall generated by the 474 * previous instruction. This is not really correct as the stalling instruction 475 * can modify the address used to access the JTLB. The failure symptom is that 476 * the TLBP instruction will use an address created for the stalling instruction 477 * and not the address held in C0_ENHI and thus report the wrong results. 478 * 479 * The software work-around is to not allow the instruction preceding the TLBP 480 * to stall - make it an NOP or some other instruction guaranteed not to stall. 481 * 482 * Errata 2 will not be fixed. This errata is also on the R5000. 483 * 484 * As if we MIPS hackers wouldn't know how to nop pipelines happy ... 485 */ 486 static void __maybe_unused build_tlb_probe_entry(u32 **p) 487 { 488 switch (current_cpu_type()) { 489 /* Found by experiment: R4600 v2.0/R4700 needs this, too. */ 490 case CPU_R4600: 491 case CPU_R4700: 492 case CPU_R5000: 493 case CPU_NEVADA: 494 uasm_i_nop(p); 495 uasm_i_tlbp(p); 496 break; 497 498 default: 499 uasm_i_tlbp(p); 500 break; 501 } 502 } 503 504 /* 505 * Write random or indexed TLB entry, and care about the hazards from 506 * the preceding mtc0 and for the following eret. 507 */ 508 enum tlb_write_entry { tlb_random, tlb_indexed }; 509 510 static void build_tlb_write_entry(u32 **p, struct uasm_label **l, 511 struct uasm_reloc **r, 512 enum tlb_write_entry wmode) 513 { 514 void(*tlbw)(u32 **) = NULL; 515 516 switch (wmode) { 517 case tlb_random: tlbw = uasm_i_tlbwr; break; 518 case tlb_indexed: tlbw = uasm_i_tlbwi; break; 519 } 520 521 if (cpu_has_mips_r2) { 522 /* 523 * The architecture spec says an ehb is required here, 524 * but a number of cores do not have the hazard and 525 * using an ehb causes an expensive pipeline stall. 526 */ 527 switch (current_cpu_type()) { 528 case CPU_M14KC: 529 case CPU_74K: 530 break; 531 532 default: 533 uasm_i_ehb(p); 534 break; 535 } 536 tlbw(p); 537 return; 538 } 539 540 switch (current_cpu_type()) { 541 case CPU_R4000PC: 542 case CPU_R4000SC: 543 case CPU_R4000MC: 544 case CPU_R4400PC: 545 case CPU_R4400SC: 546 case CPU_R4400MC: 547 /* 548 * This branch uses up a mtc0 hazard nop slot and saves 549 * two nops after the tlbw instruction. 550 */ 551 uasm_bgezl_hazard(p, r, hazard_instance); 552 tlbw(p); 553 uasm_bgezl_label(l, p, hazard_instance); 554 hazard_instance++; 555 uasm_i_nop(p); 556 break; 557 558 case CPU_R4600: 559 case CPU_R4700: 560 uasm_i_nop(p); 561 tlbw(p); 562 uasm_i_nop(p); 563 break; 564 565 case CPU_R5000: 566 case CPU_NEVADA: 567 uasm_i_nop(p); /* QED specifies 2 nops hazard */ 568 uasm_i_nop(p); /* QED specifies 2 nops hazard */ 569 tlbw(p); 570 break; 571 572 case CPU_R4300: 573 case CPU_5KC: 574 case CPU_TX49XX: 575 case CPU_PR4450: 576 case CPU_XLR: 577 uasm_i_nop(p); 578 tlbw(p); 579 break; 580 581 case CPU_R10000: 582 case CPU_R12000: 583 case CPU_R14000: 584 case CPU_4KC: 585 case CPU_4KEC: 586 case CPU_M14KC: 587 case CPU_M14KEC: 588 case CPU_SB1: 589 case CPU_SB1A: 590 case CPU_4KSC: 591 case CPU_20KC: 592 case CPU_25KF: 593 case CPU_BMIPS32: 594 case CPU_BMIPS3300: 595 case CPU_BMIPS4350: 596 case CPU_BMIPS4380: 597 case CPU_BMIPS5000: 598 case CPU_LOONGSON2: 599 case CPU_R5500: 600 if (m4kc_tlbp_war()) 601 uasm_i_nop(p); 602 case CPU_ALCHEMY: 603 tlbw(p); 604 break; 605 606 case CPU_RM7000: 607 uasm_i_nop(p); 608 uasm_i_nop(p); 609 uasm_i_nop(p); 610 uasm_i_nop(p); 611 tlbw(p); 612 break; 613 614 case CPU_VR4111: 615 case CPU_VR4121: 616 case CPU_VR4122: 617 case CPU_VR4181: 618 case CPU_VR4181A: 619 uasm_i_nop(p); 620 uasm_i_nop(p); 621 tlbw(p); 622 uasm_i_nop(p); 623 uasm_i_nop(p); 624 break; 625 626 case CPU_VR4131: 627 case CPU_VR4133: 628 case CPU_R5432: 629 uasm_i_nop(p); 630 uasm_i_nop(p); 631 tlbw(p); 632 break; 633 634 case CPU_JZRISC: 635 tlbw(p); 636 uasm_i_nop(p); 637 break; 638 639 default: 640 panic("No TLB refill handler yet (CPU type: %d)", 641 current_cpu_data.cputype); 642 break; 643 } 644 } 645 646 static __maybe_unused void build_convert_pte_to_entrylo(u32 **p, 647 unsigned int reg) 648 { 649 if (cpu_has_rixi) { 650 UASM_i_ROTR(p, reg, reg, ilog2(_PAGE_GLOBAL)); 651 } else { 652 #ifdef CONFIG_64BIT_PHYS_ADDR 653 uasm_i_dsrl_safe(p, reg, reg, ilog2(_PAGE_GLOBAL)); 654 #else 655 UASM_i_SRL(p, reg, reg, ilog2(_PAGE_GLOBAL)); 656 #endif 657 } 658 } 659 660 #ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT 661 662 static void build_restore_pagemask(u32 **p, struct uasm_reloc **r, 663 unsigned int tmp, enum label_id lid, 664 int restore_scratch) 665 { 666 if (restore_scratch) { 667 /* Reset default page size */ 668 if (PM_DEFAULT_MASK >> 16) { 669 uasm_i_lui(p, tmp, PM_DEFAULT_MASK >> 16); 670 uasm_i_ori(p, tmp, tmp, PM_DEFAULT_MASK & 0xffff); 671 uasm_i_mtc0(p, tmp, C0_PAGEMASK); 672 uasm_il_b(p, r, lid); 673 } else if (PM_DEFAULT_MASK) { 674 uasm_i_ori(p, tmp, 0, PM_DEFAULT_MASK); 675 uasm_i_mtc0(p, tmp, C0_PAGEMASK); 676 uasm_il_b(p, r, lid); 677 } else { 678 uasm_i_mtc0(p, 0, C0_PAGEMASK); 679 uasm_il_b(p, r, lid); 680 } 681 if (scratch_reg >= 0) 682 UASM_i_MFC0(p, 1, c0_kscratch(), scratch_reg); 683 else 684 UASM_i_LW(p, 1, scratchpad_offset(0), 0); 685 } else { 686 /* Reset default page size */ 687 if (PM_DEFAULT_MASK >> 16) { 688 uasm_i_lui(p, tmp, PM_DEFAULT_MASK >> 16); 689 uasm_i_ori(p, tmp, tmp, PM_DEFAULT_MASK & 0xffff); 690 uasm_il_b(p, r, lid); 691 uasm_i_mtc0(p, tmp, C0_PAGEMASK); 692 } else if (PM_DEFAULT_MASK) { 693 uasm_i_ori(p, tmp, 0, PM_DEFAULT_MASK); 694 uasm_il_b(p, r, lid); 695 uasm_i_mtc0(p, tmp, C0_PAGEMASK); 696 } else { 697 uasm_il_b(p, r, lid); 698 uasm_i_mtc0(p, 0, C0_PAGEMASK); 699 } 700 } 701 } 702 703 static void build_huge_tlb_write_entry(u32 **p, struct uasm_label **l, 704 struct uasm_reloc **r, 705 unsigned int tmp, 706 enum tlb_write_entry wmode, 707 int restore_scratch) 708 { 709 /* Set huge page tlb entry size */ 710 uasm_i_lui(p, tmp, PM_HUGE_MASK >> 16); 711 uasm_i_ori(p, tmp, tmp, PM_HUGE_MASK & 0xffff); 712 uasm_i_mtc0(p, tmp, C0_PAGEMASK); 713 714 build_tlb_write_entry(p, l, r, wmode); 715 716 build_restore_pagemask(p, r, tmp, label_leave, restore_scratch); 717 } 718 719 /* 720 * Check if Huge PTE is present, if so then jump to LABEL. 721 */ 722 static void 723 build_is_huge_pte(u32 **p, struct uasm_reloc **r, unsigned int tmp, 724 unsigned int pmd, int lid) 725 { 726 UASM_i_LW(p, tmp, 0, pmd); 727 if (use_bbit_insns()) { 728 uasm_il_bbit1(p, r, tmp, ilog2(_PAGE_HUGE), lid); 729 } else { 730 uasm_i_andi(p, tmp, tmp, _PAGE_HUGE); 731 uasm_il_bnez(p, r, tmp, lid); 732 } 733 } 734 735 static void build_huge_update_entries(u32 **p, unsigned int pte, 736 unsigned int tmp) 737 { 738 int small_sequence; 739 740 /* 741 * A huge PTE describes an area the size of the 742 * configured huge page size. This is twice the 743 * of the large TLB entry size we intend to use. 744 * A TLB entry half the size of the configured 745 * huge page size is configured into entrylo0 746 * and entrylo1 to cover the contiguous huge PTE 747 * address space. 748 */ 749 small_sequence = (HPAGE_SIZE >> 7) < 0x10000; 750 751 /* We can clobber tmp. It isn't used after this.*/ 752 if (!small_sequence) 753 uasm_i_lui(p, tmp, HPAGE_SIZE >> (7 + 16)); 754 755 build_convert_pte_to_entrylo(p, pte); 756 UASM_i_MTC0(p, pte, C0_ENTRYLO0); /* load it */ 757 /* convert to entrylo1 */ 758 if (small_sequence) 759 UASM_i_ADDIU(p, pte, pte, HPAGE_SIZE >> 7); 760 else 761 UASM_i_ADDU(p, pte, pte, tmp); 762 763 UASM_i_MTC0(p, pte, C0_ENTRYLO1); /* load it */ 764 } 765 766 static void build_huge_handler_tail(u32 **p, struct uasm_reloc **r, 767 struct uasm_label **l, 768 unsigned int pte, 769 unsigned int ptr) 770 { 771 #ifdef CONFIG_SMP 772 UASM_i_SC(p, pte, 0, ptr); 773 uasm_il_beqz(p, r, pte, label_tlb_huge_update); 774 UASM_i_LW(p, pte, 0, ptr); /* Needed because SC killed our PTE */ 775 #else 776 UASM_i_SW(p, pte, 0, ptr); 777 #endif 778 build_huge_update_entries(p, pte, ptr); 779 build_huge_tlb_write_entry(p, l, r, pte, tlb_indexed, 0); 780 } 781 #endif /* CONFIG_MIPS_HUGE_TLB_SUPPORT */ 782 783 #ifdef CONFIG_64BIT 784 /* 785 * TMP and PTR are scratch. 786 * TMP will be clobbered, PTR will hold the pmd entry. 787 */ 788 static void 789 build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r, 790 unsigned int tmp, unsigned int ptr) 791 { 792 #ifndef CONFIG_MIPS_PGD_C0_CONTEXT 793 long pgdc = (long)pgd_current; 794 #endif 795 /* 796 * The vmalloc handling is not in the hotpath. 797 */ 798 uasm_i_dmfc0(p, tmp, C0_BADVADDR); 799 800 if (check_for_high_segbits) { 801 /* 802 * The kernel currently implicitely assumes that the 803 * MIPS SEGBITS parameter for the processor is 804 * (PGDIR_SHIFT+PGDIR_BITS) or less, and will never 805 * allocate virtual addresses outside the maximum 806 * range for SEGBITS = (PGDIR_SHIFT+PGDIR_BITS). But 807 * that doesn't prevent user code from accessing the 808 * higher xuseg addresses. Here, we make sure that 809 * everything but the lower xuseg addresses goes down 810 * the module_alloc/vmalloc path. 811 */ 812 uasm_i_dsrl_safe(p, ptr, tmp, PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3); 813 uasm_il_bnez(p, r, ptr, label_vmalloc); 814 } else { 815 uasm_il_bltz(p, r, tmp, label_vmalloc); 816 } 817 /* No uasm_i_nop needed here, since the next insn doesn't touch TMP. */ 818 819 #ifdef CONFIG_MIPS_PGD_C0_CONTEXT 820 if (pgd_reg != -1) { 821 /* pgd is in pgd_reg */ 822 UASM_i_MFC0(p, ptr, c0_kscratch(), pgd_reg); 823 } else { 824 /* 825 * &pgd << 11 stored in CONTEXT [23..63]. 826 */ 827 UASM_i_MFC0(p, ptr, C0_CONTEXT); 828 829 /* Clear lower 23 bits of context. */ 830 uasm_i_dins(p, ptr, 0, 0, 23); 831 832 /* 1 0 1 0 1 << 6 xkphys cached */ 833 uasm_i_ori(p, ptr, ptr, 0x540); 834 uasm_i_drotr(p, ptr, ptr, 11); 835 } 836 #elif defined(CONFIG_SMP) 837 # ifdef CONFIG_MIPS_MT_SMTC 838 /* 839 * SMTC uses TCBind value as "CPU" index 840 */ 841 uasm_i_mfc0(p, ptr, C0_TCBIND); 842 uasm_i_dsrl_safe(p, ptr, ptr, 19); 843 # else 844 /* 845 * 64 bit SMP running in XKPHYS has smp_processor_id() << 3 846 * stored in CONTEXT. 847 */ 848 uasm_i_dmfc0(p, ptr, C0_CONTEXT); 849 uasm_i_dsrl_safe(p, ptr, ptr, 23); 850 # endif 851 UASM_i_LA_mostly(p, tmp, pgdc); 852 uasm_i_daddu(p, ptr, ptr, tmp); 853 uasm_i_dmfc0(p, tmp, C0_BADVADDR); 854 uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr); 855 #else 856 UASM_i_LA_mostly(p, ptr, pgdc); 857 uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr); 858 #endif 859 860 uasm_l_vmalloc_done(l, *p); 861 862 /* get pgd offset in bytes */ 863 uasm_i_dsrl_safe(p, tmp, tmp, PGDIR_SHIFT - 3); 864 865 uasm_i_andi(p, tmp, tmp, (PTRS_PER_PGD - 1)<<3); 866 uasm_i_daddu(p, ptr, ptr, tmp); /* add in pgd offset */ 867 #ifndef __PAGETABLE_PMD_FOLDED 868 uasm_i_dmfc0(p, tmp, C0_BADVADDR); /* get faulting address */ 869 uasm_i_ld(p, ptr, 0, ptr); /* get pmd pointer */ 870 uasm_i_dsrl_safe(p, tmp, tmp, PMD_SHIFT-3); /* get pmd offset in bytes */ 871 uasm_i_andi(p, tmp, tmp, (PTRS_PER_PMD - 1)<<3); 872 uasm_i_daddu(p, ptr, ptr, tmp); /* add in pmd offset */ 873 #endif 874 } 875 876 /* 877 * BVADDR is the faulting address, PTR is scratch. 878 * PTR will hold the pgd for vmalloc. 879 */ 880 static void 881 build_get_pgd_vmalloc64(u32 **p, struct uasm_label **l, struct uasm_reloc **r, 882 unsigned int bvaddr, unsigned int ptr, 883 enum vmalloc64_mode mode) 884 { 885 long swpd = (long)swapper_pg_dir; 886 int single_insn_swpd; 887 int did_vmalloc_branch = 0; 888 889 single_insn_swpd = uasm_in_compat_space_p(swpd) && !uasm_rel_lo(swpd); 890 891 uasm_l_vmalloc(l, *p); 892 893 if (mode != not_refill && check_for_high_segbits) { 894 if (single_insn_swpd) { 895 uasm_il_bltz(p, r, bvaddr, label_vmalloc_done); 896 uasm_i_lui(p, ptr, uasm_rel_hi(swpd)); 897 did_vmalloc_branch = 1; 898 /* fall through */ 899 } else { 900 uasm_il_bgez(p, r, bvaddr, label_large_segbits_fault); 901 } 902 } 903 if (!did_vmalloc_branch) { 904 if (uasm_in_compat_space_p(swpd) && !uasm_rel_lo(swpd)) { 905 uasm_il_b(p, r, label_vmalloc_done); 906 uasm_i_lui(p, ptr, uasm_rel_hi(swpd)); 907 } else { 908 UASM_i_LA_mostly(p, ptr, swpd); 909 uasm_il_b(p, r, label_vmalloc_done); 910 if (uasm_in_compat_space_p(swpd)) 911 uasm_i_addiu(p, ptr, ptr, uasm_rel_lo(swpd)); 912 else 913 uasm_i_daddiu(p, ptr, ptr, uasm_rel_lo(swpd)); 914 } 915 } 916 if (mode != not_refill && check_for_high_segbits) { 917 uasm_l_large_segbits_fault(l, *p); 918 /* 919 * We get here if we are an xsseg address, or if we are 920 * an xuseg address above (PGDIR_SHIFT+PGDIR_BITS) boundary. 921 * 922 * Ignoring xsseg (assume disabled so would generate 923 * (address errors?), the only remaining possibility 924 * is the upper xuseg addresses. On processors with 925 * TLB_SEGBITS <= PGDIR_SHIFT+PGDIR_BITS, these 926 * addresses would have taken an address error. We try 927 * to mimic that here by taking a load/istream page 928 * fault. 929 */ 930 UASM_i_LA(p, ptr, (unsigned long)tlb_do_page_fault_0); 931 uasm_i_jr(p, ptr); 932 933 if (mode == refill_scratch) { 934 if (scratch_reg >= 0) 935 UASM_i_MFC0(p, 1, c0_kscratch(), scratch_reg); 936 else 937 UASM_i_LW(p, 1, scratchpad_offset(0), 0); 938 } else { 939 uasm_i_nop(p); 940 } 941 } 942 } 943 944 #else /* !CONFIG_64BIT */ 945 946 /* 947 * TMP and PTR are scratch. 948 * TMP will be clobbered, PTR will hold the pgd entry. 949 */ 950 static void __maybe_unused 951 build_get_pgde32(u32 **p, unsigned int tmp, unsigned int ptr) 952 { 953 long pgdc = (long)pgd_current; 954 955 /* 32 bit SMP has smp_processor_id() stored in CONTEXT. */ 956 #ifdef CONFIG_SMP 957 #ifdef CONFIG_MIPS_MT_SMTC 958 /* 959 * SMTC uses TCBind value as "CPU" index 960 */ 961 uasm_i_mfc0(p, ptr, C0_TCBIND); 962 UASM_i_LA_mostly(p, tmp, pgdc); 963 uasm_i_srl(p, ptr, ptr, 19); 964 #else 965 /* 966 * smp_processor_id() << 2 is stored in CONTEXT. 967 */ 968 uasm_i_mfc0(p, ptr, C0_CONTEXT); 969 UASM_i_LA_mostly(p, tmp, pgdc); 970 uasm_i_srl(p, ptr, ptr, 23); 971 #endif 972 uasm_i_addu(p, ptr, tmp, ptr); 973 #else 974 UASM_i_LA_mostly(p, ptr, pgdc); 975 #endif 976 uasm_i_mfc0(p, tmp, C0_BADVADDR); /* get faulting address */ 977 uasm_i_lw(p, ptr, uasm_rel_lo(pgdc), ptr); 978 uasm_i_srl(p, tmp, tmp, PGDIR_SHIFT); /* get pgd only bits */ 979 uasm_i_sll(p, tmp, tmp, PGD_T_LOG2); 980 uasm_i_addu(p, ptr, ptr, tmp); /* add in pgd offset */ 981 } 982 983 #endif /* !CONFIG_64BIT */ 984 985 static void build_adjust_context(u32 **p, unsigned int ctx) 986 { 987 unsigned int shift = 4 - (PTE_T_LOG2 + 1) + PAGE_SHIFT - 12; 988 unsigned int mask = (PTRS_PER_PTE / 2 - 1) << (PTE_T_LOG2 + 1); 989 990 switch (current_cpu_type()) { 991 case CPU_VR41XX: 992 case CPU_VR4111: 993 case CPU_VR4121: 994 case CPU_VR4122: 995 case CPU_VR4131: 996 case CPU_VR4181: 997 case CPU_VR4181A: 998 case CPU_VR4133: 999 shift += 2; 1000 break; 1001 1002 default: 1003 break; 1004 } 1005 1006 if (shift) 1007 UASM_i_SRL(p, ctx, ctx, shift); 1008 uasm_i_andi(p, ctx, ctx, mask); 1009 } 1010 1011 static void build_get_ptep(u32 **p, unsigned int tmp, unsigned int ptr) 1012 { 1013 /* 1014 * Bug workaround for the Nevada. It seems as if under certain 1015 * circumstances the move from cp0_context might produce a 1016 * bogus result when the mfc0 instruction and its consumer are 1017 * in a different cacheline or a load instruction, probably any 1018 * memory reference, is between them. 1019 */ 1020 switch (current_cpu_type()) { 1021 case CPU_NEVADA: 1022 UASM_i_LW(p, ptr, 0, ptr); 1023 GET_CONTEXT(p, tmp); /* get context reg */ 1024 break; 1025 1026 default: 1027 GET_CONTEXT(p, tmp); /* get context reg */ 1028 UASM_i_LW(p, ptr, 0, ptr); 1029 break; 1030 } 1031 1032 build_adjust_context(p, tmp); 1033 UASM_i_ADDU(p, ptr, ptr, tmp); /* add in offset */ 1034 } 1035 1036 static void build_update_entries(u32 **p, unsigned int tmp, unsigned int ptep) 1037 { 1038 /* 1039 * 64bit address support (36bit on a 32bit CPU) in a 32bit 1040 * Kernel is a special case. Only a few CPUs use it. 1041 */ 1042 #ifdef CONFIG_64BIT_PHYS_ADDR 1043 if (cpu_has_64bits) { 1044 uasm_i_ld(p, tmp, 0, ptep); /* get even pte */ 1045 uasm_i_ld(p, ptep, sizeof(pte_t), ptep); /* get odd pte */ 1046 if (cpu_has_rixi) { 1047 UASM_i_ROTR(p, tmp, tmp, ilog2(_PAGE_GLOBAL)); 1048 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */ 1049 UASM_i_ROTR(p, ptep, ptep, ilog2(_PAGE_GLOBAL)); 1050 } else { 1051 uasm_i_dsrl_safe(p, tmp, tmp, ilog2(_PAGE_GLOBAL)); /* convert to entrylo0 */ 1052 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */ 1053 uasm_i_dsrl_safe(p, ptep, ptep, ilog2(_PAGE_GLOBAL)); /* convert to entrylo1 */ 1054 } 1055 UASM_i_MTC0(p, ptep, C0_ENTRYLO1); /* load it */ 1056 } else { 1057 int pte_off_even = sizeof(pte_t) / 2; 1058 int pte_off_odd = pte_off_even + sizeof(pte_t); 1059 1060 /* The pte entries are pre-shifted */ 1061 uasm_i_lw(p, tmp, pte_off_even, ptep); /* get even pte */ 1062 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */ 1063 uasm_i_lw(p, ptep, pte_off_odd, ptep); /* get odd pte */ 1064 UASM_i_MTC0(p, ptep, C0_ENTRYLO1); /* load it */ 1065 } 1066 #else 1067 UASM_i_LW(p, tmp, 0, ptep); /* get even pte */ 1068 UASM_i_LW(p, ptep, sizeof(pte_t), ptep); /* get odd pte */ 1069 if (r45k_bvahwbug()) 1070 build_tlb_probe_entry(p); 1071 if (cpu_has_rixi) { 1072 UASM_i_ROTR(p, tmp, tmp, ilog2(_PAGE_GLOBAL)); 1073 if (r4k_250MHZhwbug()) 1074 UASM_i_MTC0(p, 0, C0_ENTRYLO0); 1075 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */ 1076 UASM_i_ROTR(p, ptep, ptep, ilog2(_PAGE_GLOBAL)); 1077 } else { 1078 UASM_i_SRL(p, tmp, tmp, ilog2(_PAGE_GLOBAL)); /* convert to entrylo0 */ 1079 if (r4k_250MHZhwbug()) 1080 UASM_i_MTC0(p, 0, C0_ENTRYLO0); 1081 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */ 1082 UASM_i_SRL(p, ptep, ptep, ilog2(_PAGE_GLOBAL)); /* convert to entrylo1 */ 1083 if (r45k_bvahwbug()) 1084 uasm_i_mfc0(p, tmp, C0_INDEX); 1085 } 1086 if (r4k_250MHZhwbug()) 1087 UASM_i_MTC0(p, 0, C0_ENTRYLO1); 1088 UASM_i_MTC0(p, ptep, C0_ENTRYLO1); /* load it */ 1089 #endif 1090 } 1091 1092 struct mips_huge_tlb_info { 1093 int huge_pte; 1094 int restore_scratch; 1095 }; 1096 1097 static struct mips_huge_tlb_info 1098 build_fast_tlb_refill_handler (u32 **p, struct uasm_label **l, 1099 struct uasm_reloc **r, unsigned int tmp, 1100 unsigned int ptr, int c0_scratch_reg) 1101 { 1102 struct mips_huge_tlb_info rv; 1103 unsigned int even, odd; 1104 int vmalloc_branch_delay_filled = 0; 1105 const int scratch = 1; /* Our extra working register */ 1106 1107 rv.huge_pte = scratch; 1108 rv.restore_scratch = 0; 1109 1110 if (check_for_high_segbits) { 1111 UASM_i_MFC0(p, tmp, C0_BADVADDR); 1112 1113 if (pgd_reg != -1) 1114 UASM_i_MFC0(p, ptr, c0_kscratch(), pgd_reg); 1115 else 1116 UASM_i_MFC0(p, ptr, C0_CONTEXT); 1117 1118 if (c0_scratch_reg >= 0) 1119 UASM_i_MTC0(p, scratch, c0_kscratch(), c0_scratch_reg); 1120 else 1121 UASM_i_SW(p, scratch, scratchpad_offset(0), 0); 1122 1123 uasm_i_dsrl_safe(p, scratch, tmp, 1124 PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3); 1125 uasm_il_bnez(p, r, scratch, label_vmalloc); 1126 1127 if (pgd_reg == -1) { 1128 vmalloc_branch_delay_filled = 1; 1129 /* Clear lower 23 bits of context. */ 1130 uasm_i_dins(p, ptr, 0, 0, 23); 1131 } 1132 } else { 1133 if (pgd_reg != -1) 1134 UASM_i_MFC0(p, ptr, c0_kscratch(), pgd_reg); 1135 else 1136 UASM_i_MFC0(p, ptr, C0_CONTEXT); 1137 1138 UASM_i_MFC0(p, tmp, C0_BADVADDR); 1139 1140 if (c0_scratch_reg >= 0) 1141 UASM_i_MTC0(p, scratch, c0_kscratch(), c0_scratch_reg); 1142 else 1143 UASM_i_SW(p, scratch, scratchpad_offset(0), 0); 1144 1145 if (pgd_reg == -1) 1146 /* Clear lower 23 bits of context. */ 1147 uasm_i_dins(p, ptr, 0, 0, 23); 1148 1149 uasm_il_bltz(p, r, tmp, label_vmalloc); 1150 } 1151 1152 if (pgd_reg == -1) { 1153 vmalloc_branch_delay_filled = 1; 1154 /* 1 0 1 0 1 << 6 xkphys cached */ 1155 uasm_i_ori(p, ptr, ptr, 0x540); 1156 uasm_i_drotr(p, ptr, ptr, 11); 1157 } 1158 1159 #ifdef __PAGETABLE_PMD_FOLDED 1160 #define LOC_PTEP scratch 1161 #else 1162 #define LOC_PTEP ptr 1163 #endif 1164 1165 if (!vmalloc_branch_delay_filled) 1166 /* get pgd offset in bytes */ 1167 uasm_i_dsrl_safe(p, scratch, tmp, PGDIR_SHIFT - 3); 1168 1169 uasm_l_vmalloc_done(l, *p); 1170 1171 /* 1172 * tmp ptr 1173 * fall-through case = badvaddr *pgd_current 1174 * vmalloc case = badvaddr swapper_pg_dir 1175 */ 1176 1177 if (vmalloc_branch_delay_filled) 1178 /* get pgd offset in bytes */ 1179 uasm_i_dsrl_safe(p, scratch, tmp, PGDIR_SHIFT - 3); 1180 1181 #ifdef __PAGETABLE_PMD_FOLDED 1182 GET_CONTEXT(p, tmp); /* get context reg */ 1183 #endif 1184 uasm_i_andi(p, scratch, scratch, (PTRS_PER_PGD - 1) << 3); 1185 1186 if (use_lwx_insns()) { 1187 UASM_i_LWX(p, LOC_PTEP, scratch, ptr); 1188 } else { 1189 uasm_i_daddu(p, ptr, ptr, scratch); /* add in pgd offset */ 1190 uasm_i_ld(p, LOC_PTEP, 0, ptr); /* get pmd pointer */ 1191 } 1192 1193 #ifndef __PAGETABLE_PMD_FOLDED 1194 /* get pmd offset in bytes */ 1195 uasm_i_dsrl_safe(p, scratch, tmp, PMD_SHIFT - 3); 1196 uasm_i_andi(p, scratch, scratch, (PTRS_PER_PMD - 1) << 3); 1197 GET_CONTEXT(p, tmp); /* get context reg */ 1198 1199 if (use_lwx_insns()) { 1200 UASM_i_LWX(p, scratch, scratch, ptr); 1201 } else { 1202 uasm_i_daddu(p, ptr, ptr, scratch); /* add in pmd offset */ 1203 UASM_i_LW(p, scratch, 0, ptr); 1204 } 1205 #endif 1206 /* Adjust the context during the load latency. */ 1207 build_adjust_context(p, tmp); 1208 1209 #ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT 1210 uasm_il_bbit1(p, r, scratch, ilog2(_PAGE_HUGE), label_tlb_huge_update); 1211 /* 1212 * The in the LWX case we don't want to do the load in the 1213 * delay slot. It cannot issue in the same cycle and may be 1214 * speculative and unneeded. 1215 */ 1216 if (use_lwx_insns()) 1217 uasm_i_nop(p); 1218 #endif /* CONFIG_MIPS_HUGE_TLB_SUPPORT */ 1219 1220 1221 /* build_update_entries */ 1222 if (use_lwx_insns()) { 1223 even = ptr; 1224 odd = tmp; 1225 UASM_i_LWX(p, even, scratch, tmp); 1226 UASM_i_ADDIU(p, tmp, tmp, sizeof(pte_t)); 1227 UASM_i_LWX(p, odd, scratch, tmp); 1228 } else { 1229 UASM_i_ADDU(p, ptr, scratch, tmp); /* add in offset */ 1230 even = tmp; 1231 odd = ptr; 1232 UASM_i_LW(p, even, 0, ptr); /* get even pte */ 1233 UASM_i_LW(p, odd, sizeof(pte_t), ptr); /* get odd pte */ 1234 } 1235 if (cpu_has_rixi) { 1236 uasm_i_drotr(p, even, even, ilog2(_PAGE_GLOBAL)); 1237 UASM_i_MTC0(p, even, C0_ENTRYLO0); /* load it */ 1238 uasm_i_drotr(p, odd, odd, ilog2(_PAGE_GLOBAL)); 1239 } else { 1240 uasm_i_dsrl_safe(p, even, even, ilog2(_PAGE_GLOBAL)); 1241 UASM_i_MTC0(p, even, C0_ENTRYLO0); /* load it */ 1242 uasm_i_dsrl_safe(p, odd, odd, ilog2(_PAGE_GLOBAL)); 1243 } 1244 UASM_i_MTC0(p, odd, C0_ENTRYLO1); /* load it */ 1245 1246 if (c0_scratch_reg >= 0) { 1247 UASM_i_MFC0(p, scratch, c0_kscratch(), c0_scratch_reg); 1248 build_tlb_write_entry(p, l, r, tlb_random); 1249 uasm_l_leave(l, *p); 1250 rv.restore_scratch = 1; 1251 } else if (PAGE_SHIFT == 14 || PAGE_SHIFT == 13) { 1252 build_tlb_write_entry(p, l, r, tlb_random); 1253 uasm_l_leave(l, *p); 1254 UASM_i_LW(p, scratch, scratchpad_offset(0), 0); 1255 } else { 1256 UASM_i_LW(p, scratch, scratchpad_offset(0), 0); 1257 build_tlb_write_entry(p, l, r, tlb_random); 1258 uasm_l_leave(l, *p); 1259 rv.restore_scratch = 1; 1260 } 1261 1262 uasm_i_eret(p); /* return from trap */ 1263 1264 return rv; 1265 } 1266 1267 /* 1268 * For a 64-bit kernel, we are using the 64-bit XTLB refill exception 1269 * because EXL == 0. If we wrap, we can also use the 32 instruction 1270 * slots before the XTLB refill exception handler which belong to the 1271 * unused TLB refill exception. 1272 */ 1273 #define MIPS64_REFILL_INSNS 32 1274 1275 static void build_r4000_tlb_refill_handler(void) 1276 { 1277 u32 *p = tlb_handler; 1278 struct uasm_label *l = labels; 1279 struct uasm_reloc *r = relocs; 1280 u32 *f; 1281 unsigned int final_len; 1282 struct mips_huge_tlb_info htlb_info __maybe_unused; 1283 enum vmalloc64_mode vmalloc_mode __maybe_unused; 1284 1285 memset(tlb_handler, 0, sizeof(tlb_handler)); 1286 memset(labels, 0, sizeof(labels)); 1287 memset(relocs, 0, sizeof(relocs)); 1288 memset(final_handler, 0, sizeof(final_handler)); 1289 1290 if ((scratch_reg >= 0 || scratchpad_available()) && use_bbit_insns()) { 1291 htlb_info = build_fast_tlb_refill_handler(&p, &l, &r, K0, K1, 1292 scratch_reg); 1293 vmalloc_mode = refill_scratch; 1294 } else { 1295 htlb_info.huge_pte = K0; 1296 htlb_info.restore_scratch = 0; 1297 vmalloc_mode = refill_noscratch; 1298 /* 1299 * create the plain linear handler 1300 */ 1301 if (bcm1250_m3_war()) { 1302 unsigned int segbits = 44; 1303 1304 uasm_i_dmfc0(&p, K0, C0_BADVADDR); 1305 uasm_i_dmfc0(&p, K1, C0_ENTRYHI); 1306 uasm_i_xor(&p, K0, K0, K1); 1307 uasm_i_dsrl_safe(&p, K1, K0, 62); 1308 uasm_i_dsrl_safe(&p, K0, K0, 12 + 1); 1309 uasm_i_dsll_safe(&p, K0, K0, 64 + 12 + 1 - segbits); 1310 uasm_i_or(&p, K0, K0, K1); 1311 uasm_il_bnez(&p, &r, K0, label_leave); 1312 /* No need for uasm_i_nop */ 1313 } 1314 1315 #ifdef CONFIG_64BIT 1316 build_get_pmde64(&p, &l, &r, K0, K1); /* get pmd in K1 */ 1317 #else 1318 build_get_pgde32(&p, K0, K1); /* get pgd in K1 */ 1319 #endif 1320 1321 #ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT 1322 build_is_huge_pte(&p, &r, K0, K1, label_tlb_huge_update); 1323 #endif 1324 1325 build_get_ptep(&p, K0, K1); 1326 build_update_entries(&p, K0, K1); 1327 build_tlb_write_entry(&p, &l, &r, tlb_random); 1328 uasm_l_leave(&l, p); 1329 uasm_i_eret(&p); /* return from trap */ 1330 } 1331 #ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT 1332 uasm_l_tlb_huge_update(&l, p); 1333 build_huge_update_entries(&p, htlb_info.huge_pte, K1); 1334 build_huge_tlb_write_entry(&p, &l, &r, K0, tlb_random, 1335 htlb_info.restore_scratch); 1336 #endif 1337 1338 #ifdef CONFIG_64BIT 1339 build_get_pgd_vmalloc64(&p, &l, &r, K0, K1, vmalloc_mode); 1340 #endif 1341 1342 /* 1343 * Overflow check: For the 64bit handler, we need at least one 1344 * free instruction slot for the wrap-around branch. In worst 1345 * case, if the intended insertion point is a delay slot, we 1346 * need three, with the second nop'ed and the third being 1347 * unused. 1348 */ 1349 /* Loongson2 ebase is different than r4k, we have more space */ 1350 #if defined(CONFIG_32BIT) || defined(CONFIG_CPU_LOONGSON2) 1351 if ((p - tlb_handler) > 64) 1352 panic("TLB refill handler space exceeded"); 1353 #else 1354 if (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 1) 1355 || (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 3) 1356 && uasm_insn_has_bdelay(relocs, 1357 tlb_handler + MIPS64_REFILL_INSNS - 3))) 1358 panic("TLB refill handler space exceeded"); 1359 #endif 1360 1361 /* 1362 * Now fold the handler in the TLB refill handler space. 1363 */ 1364 #if defined(CONFIG_32BIT) || defined(CONFIG_CPU_LOONGSON2) 1365 f = final_handler; 1366 /* Simplest case, just copy the handler. */ 1367 uasm_copy_handler(relocs, labels, tlb_handler, p, f); 1368 final_len = p - tlb_handler; 1369 #else /* CONFIG_64BIT */ 1370 f = final_handler + MIPS64_REFILL_INSNS; 1371 if ((p - tlb_handler) <= MIPS64_REFILL_INSNS) { 1372 /* Just copy the handler. */ 1373 uasm_copy_handler(relocs, labels, tlb_handler, p, f); 1374 final_len = p - tlb_handler; 1375 } else { 1376 #ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT 1377 const enum label_id ls = label_tlb_huge_update; 1378 #else 1379 const enum label_id ls = label_vmalloc; 1380 #endif 1381 u32 *split; 1382 int ov = 0; 1383 int i; 1384 1385 for (i = 0; i < ARRAY_SIZE(labels) && labels[i].lab != ls; i++) 1386 ; 1387 BUG_ON(i == ARRAY_SIZE(labels)); 1388 split = labels[i].addr; 1389 1390 /* 1391 * See if we have overflown one way or the other. 1392 */ 1393 if (split > tlb_handler + MIPS64_REFILL_INSNS || 1394 split < p - MIPS64_REFILL_INSNS) 1395 ov = 1; 1396 1397 if (ov) { 1398 /* 1399 * Split two instructions before the end. One 1400 * for the branch and one for the instruction 1401 * in the delay slot. 1402 */ 1403 split = tlb_handler + MIPS64_REFILL_INSNS - 2; 1404 1405 /* 1406 * If the branch would fall in a delay slot, 1407 * we must back up an additional instruction 1408 * so that it is no longer in a delay slot. 1409 */ 1410 if (uasm_insn_has_bdelay(relocs, split - 1)) 1411 split--; 1412 } 1413 /* Copy first part of the handler. */ 1414 uasm_copy_handler(relocs, labels, tlb_handler, split, f); 1415 f += split - tlb_handler; 1416 1417 if (ov) { 1418 /* Insert branch. */ 1419 uasm_l_split(&l, final_handler); 1420 uasm_il_b(&f, &r, label_split); 1421 if (uasm_insn_has_bdelay(relocs, split)) 1422 uasm_i_nop(&f); 1423 else { 1424 uasm_copy_handler(relocs, labels, 1425 split, split + 1, f); 1426 uasm_move_labels(labels, f, f + 1, -1); 1427 f++; 1428 split++; 1429 } 1430 } 1431 1432 /* Copy the rest of the handler. */ 1433 uasm_copy_handler(relocs, labels, split, p, final_handler); 1434 final_len = (f - (final_handler + MIPS64_REFILL_INSNS)) + 1435 (p - split); 1436 } 1437 #endif /* CONFIG_64BIT */ 1438 1439 uasm_resolve_relocs(relocs, labels); 1440 pr_debug("Wrote TLB refill handler (%u instructions).\n", 1441 final_len); 1442 1443 memcpy((void *)ebase, final_handler, 0x100); 1444 1445 dump_handler("r4000_tlb_refill", (u32 *)ebase, 64); 1446 } 1447 1448 extern u32 handle_tlbl[], handle_tlbl_end[]; 1449 extern u32 handle_tlbs[], handle_tlbs_end[]; 1450 extern u32 handle_tlbm[], handle_tlbm_end[]; 1451 1452 #ifdef CONFIG_MIPS_PGD_C0_CONTEXT 1453 extern u32 tlbmiss_handler_setup_pgd[], tlbmiss_handler_setup_pgd_end[]; 1454 1455 static void build_r4000_setup_pgd(void) 1456 { 1457 const int a0 = 4; 1458 const int a1 = 5; 1459 u32 *p = tlbmiss_handler_setup_pgd_array; 1460 const int tlbmiss_handler_setup_pgd_size = 1461 tlbmiss_handler_setup_pgd_end - tlbmiss_handler_setup_pgd; 1462 struct uasm_label *l = labels; 1463 struct uasm_reloc *r = relocs; 1464 1465 memset(tlbmiss_handler_setup_pgd, 0, tlbmiss_handler_setup_pgd_size * 1466 sizeof(tlbmiss_handler_setup_pgd[0])); 1467 memset(labels, 0, sizeof(labels)); 1468 memset(relocs, 0, sizeof(relocs)); 1469 1470 pgd_reg = allocate_kscratch(); 1471 1472 if (pgd_reg == -1) { 1473 /* PGD << 11 in c0_Context */ 1474 /* 1475 * If it is a ckseg0 address, convert to a physical 1476 * address. Shifting right by 29 and adding 4 will 1477 * result in zero for these addresses. 1478 * 1479 */ 1480 UASM_i_SRA(&p, a1, a0, 29); 1481 UASM_i_ADDIU(&p, a1, a1, 4); 1482 uasm_il_bnez(&p, &r, a1, label_tlbl_goaround1); 1483 uasm_i_nop(&p); 1484 uasm_i_dinsm(&p, a0, 0, 29, 64 - 29); 1485 uasm_l_tlbl_goaround1(&l, p); 1486 UASM_i_SLL(&p, a0, a0, 11); 1487 uasm_i_jr(&p, 31); 1488 UASM_i_MTC0(&p, a0, C0_CONTEXT); 1489 } else { 1490 /* PGD in c0_KScratch */ 1491 uasm_i_jr(&p, 31); 1492 UASM_i_MTC0(&p, a0, c0_kscratch(), pgd_reg); 1493 } 1494 if (p >= tlbmiss_handler_setup_pgd_end) 1495 panic("tlbmiss_handler_setup_pgd space exceeded"); 1496 1497 uasm_resolve_relocs(relocs, labels); 1498 pr_debug("Wrote tlbmiss_handler_setup_pgd (%u instructions).\n", 1499 (unsigned int)(p - tlbmiss_handler_setup_pgd)); 1500 1501 dump_handler("tlbmiss_handler", tlbmiss_handler_setup_pgd, 1502 tlbmiss_handler_setup_pgd_size); 1503 } 1504 #endif 1505 1506 static void 1507 iPTE_LW(u32 **p, unsigned int pte, unsigned int ptr) 1508 { 1509 #ifdef CONFIG_SMP 1510 # ifdef CONFIG_64BIT_PHYS_ADDR 1511 if (cpu_has_64bits) 1512 uasm_i_lld(p, pte, 0, ptr); 1513 else 1514 # endif 1515 UASM_i_LL(p, pte, 0, ptr); 1516 #else 1517 # ifdef CONFIG_64BIT_PHYS_ADDR 1518 if (cpu_has_64bits) 1519 uasm_i_ld(p, pte, 0, ptr); 1520 else 1521 # endif 1522 UASM_i_LW(p, pte, 0, ptr); 1523 #endif 1524 } 1525 1526 static void 1527 iPTE_SW(u32 **p, struct uasm_reloc **r, unsigned int pte, unsigned int ptr, 1528 unsigned int mode) 1529 { 1530 #ifdef CONFIG_64BIT_PHYS_ADDR 1531 unsigned int hwmode = mode & (_PAGE_VALID | _PAGE_DIRTY); 1532 #endif 1533 1534 uasm_i_ori(p, pte, pte, mode); 1535 #ifdef CONFIG_SMP 1536 # ifdef CONFIG_64BIT_PHYS_ADDR 1537 if (cpu_has_64bits) 1538 uasm_i_scd(p, pte, 0, ptr); 1539 else 1540 # endif 1541 UASM_i_SC(p, pte, 0, ptr); 1542 1543 if (r10000_llsc_war()) 1544 uasm_il_beqzl(p, r, pte, label_smp_pgtable_change); 1545 else 1546 uasm_il_beqz(p, r, pte, label_smp_pgtable_change); 1547 1548 # ifdef CONFIG_64BIT_PHYS_ADDR 1549 if (!cpu_has_64bits) { 1550 /* no uasm_i_nop needed */ 1551 uasm_i_ll(p, pte, sizeof(pte_t) / 2, ptr); 1552 uasm_i_ori(p, pte, pte, hwmode); 1553 uasm_i_sc(p, pte, sizeof(pte_t) / 2, ptr); 1554 uasm_il_beqz(p, r, pte, label_smp_pgtable_change); 1555 /* no uasm_i_nop needed */ 1556 uasm_i_lw(p, pte, 0, ptr); 1557 } else 1558 uasm_i_nop(p); 1559 # else 1560 uasm_i_nop(p); 1561 # endif 1562 #else 1563 # ifdef CONFIG_64BIT_PHYS_ADDR 1564 if (cpu_has_64bits) 1565 uasm_i_sd(p, pte, 0, ptr); 1566 else 1567 # endif 1568 UASM_i_SW(p, pte, 0, ptr); 1569 1570 # ifdef CONFIG_64BIT_PHYS_ADDR 1571 if (!cpu_has_64bits) { 1572 uasm_i_lw(p, pte, sizeof(pte_t) / 2, ptr); 1573 uasm_i_ori(p, pte, pte, hwmode); 1574 uasm_i_sw(p, pte, sizeof(pte_t) / 2, ptr); 1575 uasm_i_lw(p, pte, 0, ptr); 1576 } 1577 # endif 1578 #endif 1579 } 1580 1581 /* 1582 * Check if PTE is present, if not then jump to LABEL. PTR points to 1583 * the page table where this PTE is located, PTE will be re-loaded 1584 * with it's original value. 1585 */ 1586 static void 1587 build_pte_present(u32 **p, struct uasm_reloc **r, 1588 int pte, int ptr, int scratch, enum label_id lid) 1589 { 1590 int t = scratch >= 0 ? scratch : pte; 1591 1592 if (cpu_has_rixi) { 1593 if (use_bbit_insns()) { 1594 uasm_il_bbit0(p, r, pte, ilog2(_PAGE_PRESENT), lid); 1595 uasm_i_nop(p); 1596 } else { 1597 uasm_i_andi(p, t, pte, _PAGE_PRESENT); 1598 uasm_il_beqz(p, r, t, lid); 1599 if (pte == t) 1600 /* You lose the SMP race :-(*/ 1601 iPTE_LW(p, pte, ptr); 1602 } 1603 } else { 1604 uasm_i_andi(p, t, pte, _PAGE_PRESENT | _PAGE_READ); 1605 uasm_i_xori(p, t, t, _PAGE_PRESENT | _PAGE_READ); 1606 uasm_il_bnez(p, r, t, lid); 1607 if (pte == t) 1608 /* You lose the SMP race :-(*/ 1609 iPTE_LW(p, pte, ptr); 1610 } 1611 } 1612 1613 /* Make PTE valid, store result in PTR. */ 1614 static void 1615 build_make_valid(u32 **p, struct uasm_reloc **r, unsigned int pte, 1616 unsigned int ptr) 1617 { 1618 unsigned int mode = _PAGE_VALID | _PAGE_ACCESSED; 1619 1620 iPTE_SW(p, r, pte, ptr, mode); 1621 } 1622 1623 /* 1624 * Check if PTE can be written to, if not branch to LABEL. Regardless 1625 * restore PTE with value from PTR when done. 1626 */ 1627 static void 1628 build_pte_writable(u32 **p, struct uasm_reloc **r, 1629 unsigned int pte, unsigned int ptr, int scratch, 1630 enum label_id lid) 1631 { 1632 int t = scratch >= 0 ? scratch : pte; 1633 1634 uasm_i_andi(p, t, pte, _PAGE_PRESENT | _PAGE_WRITE); 1635 uasm_i_xori(p, t, t, _PAGE_PRESENT | _PAGE_WRITE); 1636 uasm_il_bnez(p, r, t, lid); 1637 if (pte == t) 1638 /* You lose the SMP race :-(*/ 1639 iPTE_LW(p, pte, ptr); 1640 else 1641 uasm_i_nop(p); 1642 } 1643 1644 /* Make PTE writable, update software status bits as well, then store 1645 * at PTR. 1646 */ 1647 static void 1648 build_make_write(u32 **p, struct uasm_reloc **r, unsigned int pte, 1649 unsigned int ptr) 1650 { 1651 unsigned int mode = (_PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID 1652 | _PAGE_DIRTY); 1653 1654 iPTE_SW(p, r, pte, ptr, mode); 1655 } 1656 1657 /* 1658 * Check if PTE can be modified, if not branch to LABEL. Regardless 1659 * restore PTE with value from PTR when done. 1660 */ 1661 static void 1662 build_pte_modifiable(u32 **p, struct uasm_reloc **r, 1663 unsigned int pte, unsigned int ptr, int scratch, 1664 enum label_id lid) 1665 { 1666 if (use_bbit_insns()) { 1667 uasm_il_bbit0(p, r, pte, ilog2(_PAGE_WRITE), lid); 1668 uasm_i_nop(p); 1669 } else { 1670 int t = scratch >= 0 ? scratch : pte; 1671 uasm_i_andi(p, t, pte, _PAGE_WRITE); 1672 uasm_il_beqz(p, r, t, lid); 1673 if (pte == t) 1674 /* You lose the SMP race :-(*/ 1675 iPTE_LW(p, pte, ptr); 1676 } 1677 } 1678 1679 #ifndef CONFIG_MIPS_PGD_C0_CONTEXT 1680 1681 1682 /* 1683 * R3000 style TLB load/store/modify handlers. 1684 */ 1685 1686 /* 1687 * This places the pte into ENTRYLO0 and writes it with tlbwi. 1688 * Then it returns. 1689 */ 1690 static void 1691 build_r3000_pte_reload_tlbwi(u32 **p, unsigned int pte, unsigned int tmp) 1692 { 1693 uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */ 1694 uasm_i_mfc0(p, tmp, C0_EPC); /* cp0 delay */ 1695 uasm_i_tlbwi(p); 1696 uasm_i_jr(p, tmp); 1697 uasm_i_rfe(p); /* branch delay */ 1698 } 1699 1700 /* 1701 * This places the pte into ENTRYLO0 and writes it with tlbwi 1702 * or tlbwr as appropriate. This is because the index register 1703 * may have the probe fail bit set as a result of a trap on a 1704 * kseg2 access, i.e. without refill. Then it returns. 1705 */ 1706 static void 1707 build_r3000_tlb_reload_write(u32 **p, struct uasm_label **l, 1708 struct uasm_reloc **r, unsigned int pte, 1709 unsigned int tmp) 1710 { 1711 uasm_i_mfc0(p, tmp, C0_INDEX); 1712 uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */ 1713 uasm_il_bltz(p, r, tmp, label_r3000_write_probe_fail); /* cp0 delay */ 1714 uasm_i_mfc0(p, tmp, C0_EPC); /* branch delay */ 1715 uasm_i_tlbwi(p); /* cp0 delay */ 1716 uasm_i_jr(p, tmp); 1717 uasm_i_rfe(p); /* branch delay */ 1718 uasm_l_r3000_write_probe_fail(l, *p); 1719 uasm_i_tlbwr(p); /* cp0 delay */ 1720 uasm_i_jr(p, tmp); 1721 uasm_i_rfe(p); /* branch delay */ 1722 } 1723 1724 static void 1725 build_r3000_tlbchange_handler_head(u32 **p, unsigned int pte, 1726 unsigned int ptr) 1727 { 1728 long pgdc = (long)pgd_current; 1729 1730 uasm_i_mfc0(p, pte, C0_BADVADDR); 1731 uasm_i_lui(p, ptr, uasm_rel_hi(pgdc)); /* cp0 delay */ 1732 uasm_i_lw(p, ptr, uasm_rel_lo(pgdc), ptr); 1733 uasm_i_srl(p, pte, pte, 22); /* load delay */ 1734 uasm_i_sll(p, pte, pte, 2); 1735 uasm_i_addu(p, ptr, ptr, pte); 1736 uasm_i_mfc0(p, pte, C0_CONTEXT); 1737 uasm_i_lw(p, ptr, 0, ptr); /* cp0 delay */ 1738 uasm_i_andi(p, pte, pte, 0xffc); /* load delay */ 1739 uasm_i_addu(p, ptr, ptr, pte); 1740 uasm_i_lw(p, pte, 0, ptr); 1741 uasm_i_tlbp(p); /* load delay */ 1742 } 1743 1744 static void build_r3000_tlb_load_handler(void) 1745 { 1746 u32 *p = handle_tlbl; 1747 const int handle_tlbl_size = handle_tlbl_end - handle_tlbl; 1748 struct uasm_label *l = labels; 1749 struct uasm_reloc *r = relocs; 1750 1751 memset(handle_tlbl, 0, handle_tlbl_size * sizeof(handle_tlbl[0])); 1752 memset(labels, 0, sizeof(labels)); 1753 memset(relocs, 0, sizeof(relocs)); 1754 1755 build_r3000_tlbchange_handler_head(&p, K0, K1); 1756 build_pte_present(&p, &r, K0, K1, -1, label_nopage_tlbl); 1757 uasm_i_nop(&p); /* load delay */ 1758 build_make_valid(&p, &r, K0, K1); 1759 build_r3000_tlb_reload_write(&p, &l, &r, K0, K1); 1760 1761 uasm_l_nopage_tlbl(&l, p); 1762 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff); 1763 uasm_i_nop(&p); 1764 1765 if (p >= handle_tlbl_end) 1766 panic("TLB load handler fastpath space exceeded"); 1767 1768 uasm_resolve_relocs(relocs, labels); 1769 pr_debug("Wrote TLB load handler fastpath (%u instructions).\n", 1770 (unsigned int)(p - handle_tlbl)); 1771 1772 dump_handler("r3000_tlb_load", handle_tlbl, handle_tlbl_size); 1773 } 1774 1775 static void build_r3000_tlb_store_handler(void) 1776 { 1777 u32 *p = handle_tlbs; 1778 const int handle_tlbs_size = handle_tlbs_end - handle_tlbs; 1779 struct uasm_label *l = labels; 1780 struct uasm_reloc *r = relocs; 1781 1782 memset(handle_tlbs, 0, handle_tlbs_size * sizeof(handle_tlbs[0])); 1783 memset(labels, 0, sizeof(labels)); 1784 memset(relocs, 0, sizeof(relocs)); 1785 1786 build_r3000_tlbchange_handler_head(&p, K0, K1); 1787 build_pte_writable(&p, &r, K0, K1, -1, label_nopage_tlbs); 1788 uasm_i_nop(&p); /* load delay */ 1789 build_make_write(&p, &r, K0, K1); 1790 build_r3000_tlb_reload_write(&p, &l, &r, K0, K1); 1791 1792 uasm_l_nopage_tlbs(&l, p); 1793 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); 1794 uasm_i_nop(&p); 1795 1796 if (p >= handle_tlbs) 1797 panic("TLB store handler fastpath space exceeded"); 1798 1799 uasm_resolve_relocs(relocs, labels); 1800 pr_debug("Wrote TLB store handler fastpath (%u instructions).\n", 1801 (unsigned int)(p - handle_tlbs)); 1802 1803 dump_handler("r3000_tlb_store", handle_tlbs, handle_tlbs_size); 1804 } 1805 1806 static void build_r3000_tlb_modify_handler(void) 1807 { 1808 u32 *p = handle_tlbm; 1809 const int handle_tlbm_size = handle_tlbm_end - handle_tlbm; 1810 struct uasm_label *l = labels; 1811 struct uasm_reloc *r = relocs; 1812 1813 memset(handle_tlbm, 0, handle_tlbm_size * sizeof(handle_tlbm[0])); 1814 memset(labels, 0, sizeof(labels)); 1815 memset(relocs, 0, sizeof(relocs)); 1816 1817 build_r3000_tlbchange_handler_head(&p, K0, K1); 1818 build_pte_modifiable(&p, &r, K0, K1, -1, label_nopage_tlbm); 1819 uasm_i_nop(&p); /* load delay */ 1820 build_make_write(&p, &r, K0, K1); 1821 build_r3000_pte_reload_tlbwi(&p, K0, K1); 1822 1823 uasm_l_nopage_tlbm(&l, p); 1824 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); 1825 uasm_i_nop(&p); 1826 1827 if (p >= handle_tlbm_end) 1828 panic("TLB modify handler fastpath space exceeded"); 1829 1830 uasm_resolve_relocs(relocs, labels); 1831 pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n", 1832 (unsigned int)(p - handle_tlbm)); 1833 1834 dump_handler("r3000_tlb_modify", handle_tlbm, handle_tlbm_size); 1835 } 1836 #endif /* CONFIG_MIPS_PGD_C0_CONTEXT */ 1837 1838 /* 1839 * R4000 style TLB load/store/modify handlers. 1840 */ 1841 static struct work_registers 1842 build_r4000_tlbchange_handler_head(u32 **p, struct uasm_label **l, 1843 struct uasm_reloc **r) 1844 { 1845 struct work_registers wr = build_get_work_registers(p); 1846 1847 #ifdef CONFIG_64BIT 1848 build_get_pmde64(p, l, r, wr.r1, wr.r2); /* get pmd in ptr */ 1849 #else 1850 build_get_pgde32(p, wr.r1, wr.r2); /* get pgd in ptr */ 1851 #endif 1852 1853 #ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT 1854 /* 1855 * For huge tlb entries, pmd doesn't contain an address but 1856 * instead contains the tlb pte. Check the PAGE_HUGE bit and 1857 * see if we need to jump to huge tlb processing. 1858 */ 1859 build_is_huge_pte(p, r, wr.r1, wr.r2, label_tlb_huge_update); 1860 #endif 1861 1862 UASM_i_MFC0(p, wr.r1, C0_BADVADDR); 1863 UASM_i_LW(p, wr.r2, 0, wr.r2); 1864 UASM_i_SRL(p, wr.r1, wr.r1, PAGE_SHIFT + PTE_ORDER - PTE_T_LOG2); 1865 uasm_i_andi(p, wr.r1, wr.r1, (PTRS_PER_PTE - 1) << PTE_T_LOG2); 1866 UASM_i_ADDU(p, wr.r2, wr.r2, wr.r1); 1867 1868 #ifdef CONFIG_SMP 1869 uasm_l_smp_pgtable_change(l, *p); 1870 #endif 1871 iPTE_LW(p, wr.r1, wr.r2); /* get even pte */ 1872 if (!m4kc_tlbp_war()) 1873 build_tlb_probe_entry(p); 1874 return wr; 1875 } 1876 1877 static void 1878 build_r4000_tlbchange_handler_tail(u32 **p, struct uasm_label **l, 1879 struct uasm_reloc **r, unsigned int tmp, 1880 unsigned int ptr) 1881 { 1882 uasm_i_ori(p, ptr, ptr, sizeof(pte_t)); 1883 uasm_i_xori(p, ptr, ptr, sizeof(pte_t)); 1884 build_update_entries(p, tmp, ptr); 1885 build_tlb_write_entry(p, l, r, tlb_indexed); 1886 uasm_l_leave(l, *p); 1887 build_restore_work_registers(p); 1888 uasm_i_eret(p); /* return from trap */ 1889 1890 #ifdef CONFIG_64BIT 1891 build_get_pgd_vmalloc64(p, l, r, tmp, ptr, not_refill); 1892 #endif 1893 } 1894 1895 static void build_r4000_tlb_load_handler(void) 1896 { 1897 u32 *p = handle_tlbl; 1898 const int handle_tlbl_size = handle_tlbl_end - handle_tlbl; 1899 struct uasm_label *l = labels; 1900 struct uasm_reloc *r = relocs; 1901 struct work_registers wr; 1902 1903 memset(handle_tlbl, 0, handle_tlbl_size * sizeof(handle_tlbl[0])); 1904 memset(labels, 0, sizeof(labels)); 1905 memset(relocs, 0, sizeof(relocs)); 1906 1907 if (bcm1250_m3_war()) { 1908 unsigned int segbits = 44; 1909 1910 uasm_i_dmfc0(&p, K0, C0_BADVADDR); 1911 uasm_i_dmfc0(&p, K1, C0_ENTRYHI); 1912 uasm_i_xor(&p, K0, K0, K1); 1913 uasm_i_dsrl_safe(&p, K1, K0, 62); 1914 uasm_i_dsrl_safe(&p, K0, K0, 12 + 1); 1915 uasm_i_dsll_safe(&p, K0, K0, 64 + 12 + 1 - segbits); 1916 uasm_i_or(&p, K0, K0, K1); 1917 uasm_il_bnez(&p, &r, K0, label_leave); 1918 /* No need for uasm_i_nop */ 1919 } 1920 1921 wr = build_r4000_tlbchange_handler_head(&p, &l, &r); 1922 build_pte_present(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbl); 1923 if (m4kc_tlbp_war()) 1924 build_tlb_probe_entry(&p); 1925 1926 if (cpu_has_rixi) { 1927 /* 1928 * If the page is not _PAGE_VALID, RI or XI could not 1929 * have triggered it. Skip the expensive test.. 1930 */ 1931 if (use_bbit_insns()) { 1932 uasm_il_bbit0(&p, &r, wr.r1, ilog2(_PAGE_VALID), 1933 label_tlbl_goaround1); 1934 } else { 1935 uasm_i_andi(&p, wr.r3, wr.r1, _PAGE_VALID); 1936 uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround1); 1937 } 1938 uasm_i_nop(&p); 1939 1940 uasm_i_tlbr(&p); 1941 1942 switch (current_cpu_type()) { 1943 default: 1944 if (cpu_has_mips_r2) { 1945 uasm_i_ehb(&p); 1946 1947 case CPU_CAVIUM_OCTEON: 1948 case CPU_CAVIUM_OCTEON_PLUS: 1949 case CPU_CAVIUM_OCTEON2: 1950 break; 1951 } 1952 } 1953 1954 /* Examine entrylo 0 or 1 based on ptr. */ 1955 if (use_bbit_insns()) { 1956 uasm_i_bbit0(&p, wr.r2, ilog2(sizeof(pte_t)), 8); 1957 } else { 1958 uasm_i_andi(&p, wr.r3, wr.r2, sizeof(pte_t)); 1959 uasm_i_beqz(&p, wr.r3, 8); 1960 } 1961 /* load it in the delay slot*/ 1962 UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO0); 1963 /* load it if ptr is odd */ 1964 UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO1); 1965 /* 1966 * If the entryLo (now in wr.r3) is valid (bit 1), RI or 1967 * XI must have triggered it. 1968 */ 1969 if (use_bbit_insns()) { 1970 uasm_il_bbit1(&p, &r, wr.r3, 1, label_nopage_tlbl); 1971 uasm_i_nop(&p); 1972 uasm_l_tlbl_goaround1(&l, p); 1973 } else { 1974 uasm_i_andi(&p, wr.r3, wr.r3, 2); 1975 uasm_il_bnez(&p, &r, wr.r3, label_nopage_tlbl); 1976 uasm_i_nop(&p); 1977 } 1978 uasm_l_tlbl_goaround1(&l, p); 1979 } 1980 build_make_valid(&p, &r, wr.r1, wr.r2); 1981 build_r4000_tlbchange_handler_tail(&p, &l, &r, wr.r1, wr.r2); 1982 1983 #ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT 1984 /* 1985 * This is the entry point when build_r4000_tlbchange_handler_head 1986 * spots a huge page. 1987 */ 1988 uasm_l_tlb_huge_update(&l, p); 1989 iPTE_LW(&p, wr.r1, wr.r2); 1990 build_pte_present(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbl); 1991 build_tlb_probe_entry(&p); 1992 1993 if (cpu_has_rixi) { 1994 /* 1995 * If the page is not _PAGE_VALID, RI or XI could not 1996 * have triggered it. Skip the expensive test.. 1997 */ 1998 if (use_bbit_insns()) { 1999 uasm_il_bbit0(&p, &r, wr.r1, ilog2(_PAGE_VALID), 2000 label_tlbl_goaround2); 2001 } else { 2002 uasm_i_andi(&p, wr.r3, wr.r1, _PAGE_VALID); 2003 uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround2); 2004 } 2005 uasm_i_nop(&p); 2006 2007 uasm_i_tlbr(&p); 2008 2009 switch (current_cpu_type()) { 2010 default: 2011 if (cpu_has_mips_r2) { 2012 uasm_i_ehb(&p); 2013 2014 case CPU_CAVIUM_OCTEON: 2015 case CPU_CAVIUM_OCTEON_PLUS: 2016 case CPU_CAVIUM_OCTEON2: 2017 break; 2018 } 2019 } 2020 2021 /* Examine entrylo 0 or 1 based on ptr. */ 2022 if (use_bbit_insns()) { 2023 uasm_i_bbit0(&p, wr.r2, ilog2(sizeof(pte_t)), 8); 2024 } else { 2025 uasm_i_andi(&p, wr.r3, wr.r2, sizeof(pte_t)); 2026 uasm_i_beqz(&p, wr.r3, 8); 2027 } 2028 /* load it in the delay slot*/ 2029 UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO0); 2030 /* load it if ptr is odd */ 2031 UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO1); 2032 /* 2033 * If the entryLo (now in wr.r3) is valid (bit 1), RI or 2034 * XI must have triggered it. 2035 */ 2036 if (use_bbit_insns()) { 2037 uasm_il_bbit0(&p, &r, wr.r3, 1, label_tlbl_goaround2); 2038 } else { 2039 uasm_i_andi(&p, wr.r3, wr.r3, 2); 2040 uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround2); 2041 } 2042 if (PM_DEFAULT_MASK == 0) 2043 uasm_i_nop(&p); 2044 /* 2045 * We clobbered C0_PAGEMASK, restore it. On the other branch 2046 * it is restored in build_huge_tlb_write_entry. 2047 */ 2048 build_restore_pagemask(&p, &r, wr.r3, label_nopage_tlbl, 0); 2049 2050 uasm_l_tlbl_goaround2(&l, p); 2051 } 2052 uasm_i_ori(&p, wr.r1, wr.r1, (_PAGE_ACCESSED | _PAGE_VALID)); 2053 build_huge_handler_tail(&p, &r, &l, wr.r1, wr.r2); 2054 #endif 2055 2056 uasm_l_nopage_tlbl(&l, p); 2057 build_restore_work_registers(&p); 2058 #ifdef CONFIG_CPU_MICROMIPS 2059 if ((unsigned long)tlb_do_page_fault_0 & 1) { 2060 uasm_i_lui(&p, K0, uasm_rel_hi((long)tlb_do_page_fault_0)); 2061 uasm_i_addiu(&p, K0, K0, uasm_rel_lo((long)tlb_do_page_fault_0)); 2062 uasm_i_jr(&p, K0); 2063 } else 2064 #endif 2065 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff); 2066 uasm_i_nop(&p); 2067 2068 if (p >= handle_tlbl_end) 2069 panic("TLB load handler fastpath space exceeded"); 2070 2071 uasm_resolve_relocs(relocs, labels); 2072 pr_debug("Wrote TLB load handler fastpath (%u instructions).\n", 2073 (unsigned int)(p - handle_tlbl)); 2074 2075 dump_handler("r4000_tlb_load", handle_tlbl, handle_tlbl_size); 2076 } 2077 2078 static void build_r4000_tlb_store_handler(void) 2079 { 2080 u32 *p = handle_tlbs; 2081 const int handle_tlbs_size = handle_tlbs_end - handle_tlbs; 2082 struct uasm_label *l = labels; 2083 struct uasm_reloc *r = relocs; 2084 struct work_registers wr; 2085 2086 memset(handle_tlbs, 0, handle_tlbs_size * sizeof(handle_tlbs[0])); 2087 memset(labels, 0, sizeof(labels)); 2088 memset(relocs, 0, sizeof(relocs)); 2089 2090 wr = build_r4000_tlbchange_handler_head(&p, &l, &r); 2091 build_pte_writable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbs); 2092 if (m4kc_tlbp_war()) 2093 build_tlb_probe_entry(&p); 2094 build_make_write(&p, &r, wr.r1, wr.r2); 2095 build_r4000_tlbchange_handler_tail(&p, &l, &r, wr.r1, wr.r2); 2096 2097 #ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT 2098 /* 2099 * This is the entry point when 2100 * build_r4000_tlbchange_handler_head spots a huge page. 2101 */ 2102 uasm_l_tlb_huge_update(&l, p); 2103 iPTE_LW(&p, wr.r1, wr.r2); 2104 build_pte_writable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbs); 2105 build_tlb_probe_entry(&p); 2106 uasm_i_ori(&p, wr.r1, wr.r1, 2107 _PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID | _PAGE_DIRTY); 2108 build_huge_handler_tail(&p, &r, &l, wr.r1, wr.r2); 2109 #endif 2110 2111 uasm_l_nopage_tlbs(&l, p); 2112 build_restore_work_registers(&p); 2113 #ifdef CONFIG_CPU_MICROMIPS 2114 if ((unsigned long)tlb_do_page_fault_1 & 1) { 2115 uasm_i_lui(&p, K0, uasm_rel_hi((long)tlb_do_page_fault_1)); 2116 uasm_i_addiu(&p, K0, K0, uasm_rel_lo((long)tlb_do_page_fault_1)); 2117 uasm_i_jr(&p, K0); 2118 } else 2119 #endif 2120 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); 2121 uasm_i_nop(&p); 2122 2123 if (p >= handle_tlbs_end) 2124 panic("TLB store handler fastpath space exceeded"); 2125 2126 uasm_resolve_relocs(relocs, labels); 2127 pr_debug("Wrote TLB store handler fastpath (%u instructions).\n", 2128 (unsigned int)(p - handle_tlbs)); 2129 2130 dump_handler("r4000_tlb_store", handle_tlbs, handle_tlbs_size); 2131 } 2132 2133 static void build_r4000_tlb_modify_handler(void) 2134 { 2135 u32 *p = handle_tlbm; 2136 const int handle_tlbm_size = handle_tlbm_end - handle_tlbm; 2137 struct uasm_label *l = labels; 2138 struct uasm_reloc *r = relocs; 2139 struct work_registers wr; 2140 2141 memset(handle_tlbm, 0, handle_tlbm_size * sizeof(handle_tlbm[0])); 2142 memset(labels, 0, sizeof(labels)); 2143 memset(relocs, 0, sizeof(relocs)); 2144 2145 wr = build_r4000_tlbchange_handler_head(&p, &l, &r); 2146 build_pte_modifiable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbm); 2147 if (m4kc_tlbp_war()) 2148 build_tlb_probe_entry(&p); 2149 /* Present and writable bits set, set accessed and dirty bits. */ 2150 build_make_write(&p, &r, wr.r1, wr.r2); 2151 build_r4000_tlbchange_handler_tail(&p, &l, &r, wr.r1, wr.r2); 2152 2153 #ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT 2154 /* 2155 * This is the entry point when 2156 * build_r4000_tlbchange_handler_head spots a huge page. 2157 */ 2158 uasm_l_tlb_huge_update(&l, p); 2159 iPTE_LW(&p, wr.r1, wr.r2); 2160 build_pte_modifiable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbm); 2161 build_tlb_probe_entry(&p); 2162 uasm_i_ori(&p, wr.r1, wr.r1, 2163 _PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID | _PAGE_DIRTY); 2164 build_huge_handler_tail(&p, &r, &l, wr.r1, wr.r2); 2165 #endif 2166 2167 uasm_l_nopage_tlbm(&l, p); 2168 build_restore_work_registers(&p); 2169 #ifdef CONFIG_CPU_MICROMIPS 2170 if ((unsigned long)tlb_do_page_fault_1 & 1) { 2171 uasm_i_lui(&p, K0, uasm_rel_hi((long)tlb_do_page_fault_1)); 2172 uasm_i_addiu(&p, K0, K0, uasm_rel_lo((long)tlb_do_page_fault_1)); 2173 uasm_i_jr(&p, K0); 2174 } else 2175 #endif 2176 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); 2177 uasm_i_nop(&p); 2178 2179 if (p >= handle_tlbm_end) 2180 panic("TLB modify handler fastpath space exceeded"); 2181 2182 uasm_resolve_relocs(relocs, labels); 2183 pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n", 2184 (unsigned int)(p - handle_tlbm)); 2185 2186 dump_handler("r4000_tlb_modify", handle_tlbm, handle_tlbm_size); 2187 } 2188 2189 static void flush_tlb_handlers(void) 2190 { 2191 local_flush_icache_range((unsigned long)handle_tlbl, 2192 (unsigned long)handle_tlbl_end); 2193 local_flush_icache_range((unsigned long)handle_tlbs, 2194 (unsigned long)handle_tlbs_end); 2195 local_flush_icache_range((unsigned long)handle_tlbm, 2196 (unsigned long)handle_tlbm_end); 2197 #ifdef CONFIG_MIPS_PGD_C0_CONTEXT 2198 local_flush_icache_range((unsigned long)tlbmiss_handler_setup_pgd, 2199 (unsigned long)tlbmiss_handler_setup_pgd_end); 2200 #endif 2201 } 2202 2203 void build_tlb_refill_handler(void) 2204 { 2205 /* 2206 * The refill handler is generated per-CPU, multi-node systems 2207 * may have local storage for it. The other handlers are only 2208 * needed once. 2209 */ 2210 static int run_once = 0; 2211 2212 output_pgtable_bits_defines(); 2213 2214 #ifdef CONFIG_64BIT 2215 check_for_high_segbits = current_cpu_data.vmbits > (PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3); 2216 #endif 2217 2218 switch (current_cpu_type()) { 2219 case CPU_R2000: 2220 case CPU_R3000: 2221 case CPU_R3000A: 2222 case CPU_R3081E: 2223 case CPU_TX3912: 2224 case CPU_TX3922: 2225 case CPU_TX3927: 2226 #ifndef CONFIG_MIPS_PGD_C0_CONTEXT 2227 if (cpu_has_local_ebase) 2228 build_r3000_tlb_refill_handler(); 2229 if (!run_once) { 2230 if (!cpu_has_local_ebase) 2231 build_r3000_tlb_refill_handler(); 2232 build_r3000_tlb_load_handler(); 2233 build_r3000_tlb_store_handler(); 2234 build_r3000_tlb_modify_handler(); 2235 flush_tlb_handlers(); 2236 run_once++; 2237 } 2238 #else 2239 panic("No R3000 TLB refill handler"); 2240 #endif 2241 break; 2242 2243 case CPU_R6000: 2244 case CPU_R6000A: 2245 panic("No R6000 TLB refill handler yet"); 2246 break; 2247 2248 case CPU_R8000: 2249 panic("No R8000 TLB refill handler yet"); 2250 break; 2251 2252 default: 2253 if (!run_once) { 2254 scratch_reg = allocate_kscratch(); 2255 #ifdef CONFIG_MIPS_PGD_C0_CONTEXT 2256 build_r4000_setup_pgd(); 2257 #endif 2258 build_r4000_tlb_load_handler(); 2259 build_r4000_tlb_store_handler(); 2260 build_r4000_tlb_modify_handler(); 2261 if (!cpu_has_local_ebase) 2262 build_r4000_tlb_refill_handler(); 2263 flush_tlb_handlers(); 2264 run_once++; 2265 } 2266 if (cpu_has_local_ebase) 2267 build_r4000_tlb_refill_handler(); 2268 } 2269 } 2270