1 /* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * Synthesize TLB refill handlers at runtime. 7 * 8 * Copyright (C) 2004, 2005, 2006, 2008 Thiemo Seufer 9 * Copyright (C) 2005, 2007, 2008, 2009 Maciej W. Rozycki 10 * Copyright (C) 2006 Ralf Baechle (ralf@linux-mips.org) 11 * Copyright (C) 2008, 2009 Cavium Networks, Inc. 12 * Copyright (C) 2011 MIPS Technologies, Inc. 13 * 14 * ... and the days got worse and worse and now you see 15 * I've gone completly out of my mind. 16 * 17 * They're coming to take me a away haha 18 * they're coming to take me a away hoho hihi haha 19 * to the funny farm where code is beautiful all the time ... 20 * 21 * (Condolences to Napoleon XIV) 22 */ 23 24 #include <linux/bug.h> 25 #include <linux/kernel.h> 26 #include <linux/types.h> 27 #include <linux/smp.h> 28 #include <linux/string.h> 29 #include <linux/init.h> 30 #include <linux/cache.h> 31 32 #include <asm/cacheflush.h> 33 #include <asm/pgtable.h> 34 #include <asm/war.h> 35 #include <asm/uasm.h> 36 #include <asm/setup.h> 37 38 /* 39 * TLB load/store/modify handlers. 40 * 41 * Only the fastpath gets synthesized at runtime, the slowpath for 42 * do_page_fault remains normal asm. 43 */ 44 extern void tlb_do_page_fault_0(void); 45 extern void tlb_do_page_fault_1(void); 46 47 struct work_registers { 48 int r1; 49 int r2; 50 int r3; 51 }; 52 53 struct tlb_reg_save { 54 unsigned long a; 55 unsigned long b; 56 } ____cacheline_aligned_in_smp; 57 58 static struct tlb_reg_save handler_reg_save[NR_CPUS]; 59 60 static inline int r45k_bvahwbug(void) 61 { 62 /* XXX: We should probe for the presence of this bug, but we don't. */ 63 return 0; 64 } 65 66 static inline int r4k_250MHZhwbug(void) 67 { 68 /* XXX: We should probe for the presence of this bug, but we don't. */ 69 return 0; 70 } 71 72 static inline int __maybe_unused bcm1250_m3_war(void) 73 { 74 return BCM1250_M3_WAR; 75 } 76 77 static inline int __maybe_unused r10000_llsc_war(void) 78 { 79 return R10000_LLSC_WAR; 80 } 81 82 static int use_bbit_insns(void) 83 { 84 switch (current_cpu_type()) { 85 case CPU_CAVIUM_OCTEON: 86 case CPU_CAVIUM_OCTEON_PLUS: 87 case CPU_CAVIUM_OCTEON2: 88 return 1; 89 default: 90 return 0; 91 } 92 } 93 94 static int use_lwx_insns(void) 95 { 96 switch (current_cpu_type()) { 97 case CPU_CAVIUM_OCTEON2: 98 return 1; 99 default: 100 return 0; 101 } 102 } 103 #if defined(CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE) && \ 104 CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE > 0 105 static bool scratchpad_available(void) 106 { 107 return true; 108 } 109 static int scratchpad_offset(int i) 110 { 111 /* 112 * CVMSEG starts at address -32768 and extends for 113 * CAVIUM_OCTEON_CVMSEG_SIZE 128 byte cache lines. 114 */ 115 i += 1; /* Kernel use starts at the top and works down. */ 116 return CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE * 128 - (8 * i) - 32768; 117 } 118 #else 119 static bool scratchpad_available(void) 120 { 121 return false; 122 } 123 static int scratchpad_offset(int i) 124 { 125 BUG(); 126 /* Really unreachable, but evidently some GCC want this. */ 127 return 0; 128 } 129 #endif 130 /* 131 * Found by experiment: At least some revisions of the 4kc throw under 132 * some circumstances a machine check exception, triggered by invalid 133 * values in the index register. Delaying the tlbp instruction until 134 * after the next branch, plus adding an additional nop in front of 135 * tlbwi/tlbwr avoids the invalid index register values. Nobody knows 136 * why; it's not an issue caused by the core RTL. 137 * 138 */ 139 static int __cpuinit m4kc_tlbp_war(void) 140 { 141 return (current_cpu_data.processor_id & 0xffff00) == 142 (PRID_COMP_MIPS | PRID_IMP_4KC); 143 } 144 145 /* Handle labels (which must be positive integers). */ 146 enum label_id { 147 label_second_part = 1, 148 label_leave, 149 label_vmalloc, 150 label_vmalloc_done, 151 label_tlbw_hazard_0, 152 label_split = label_tlbw_hazard_0 + 8, 153 label_tlbl_goaround1, 154 label_tlbl_goaround2, 155 label_nopage_tlbl, 156 label_nopage_tlbs, 157 label_nopage_tlbm, 158 label_smp_pgtable_change, 159 label_r3000_write_probe_fail, 160 label_large_segbits_fault, 161 #ifdef CONFIG_HUGETLB_PAGE 162 label_tlb_huge_update, 163 #endif 164 }; 165 166 UASM_L_LA(_second_part) 167 UASM_L_LA(_leave) 168 UASM_L_LA(_vmalloc) 169 UASM_L_LA(_vmalloc_done) 170 /* _tlbw_hazard_x is handled differently. */ 171 UASM_L_LA(_split) 172 UASM_L_LA(_tlbl_goaround1) 173 UASM_L_LA(_tlbl_goaround2) 174 UASM_L_LA(_nopage_tlbl) 175 UASM_L_LA(_nopage_tlbs) 176 UASM_L_LA(_nopage_tlbm) 177 UASM_L_LA(_smp_pgtable_change) 178 UASM_L_LA(_r3000_write_probe_fail) 179 UASM_L_LA(_large_segbits_fault) 180 #ifdef CONFIG_HUGETLB_PAGE 181 UASM_L_LA(_tlb_huge_update) 182 #endif 183 184 static int __cpuinitdata hazard_instance; 185 186 static void uasm_bgezl_hazard(u32 **p, struct uasm_reloc **r, int instance) 187 { 188 switch (instance) { 189 case 0 ... 7: 190 uasm_il_bgezl(p, r, 0, label_tlbw_hazard_0 + instance); 191 return; 192 default: 193 BUG(); 194 } 195 } 196 197 static void uasm_bgezl_label(struct uasm_label **l, u32 **p, int instance) 198 { 199 switch (instance) { 200 case 0 ... 7: 201 uasm_build_label(l, *p, label_tlbw_hazard_0 + instance); 202 break; 203 default: 204 BUG(); 205 } 206 } 207 208 /* 209 * For debug purposes. 210 */ 211 static inline void dump_handler(const u32 *handler, int count) 212 { 213 int i; 214 215 pr_debug("\t.set push\n"); 216 pr_debug("\t.set noreorder\n"); 217 218 for (i = 0; i < count; i++) 219 pr_debug("\t%p\t.word 0x%08x\n", &handler[i], handler[i]); 220 221 pr_debug("\t.set pop\n"); 222 } 223 224 /* The only general purpose registers allowed in TLB handlers. */ 225 #define K0 26 226 #define K1 27 227 228 /* Some CP0 registers */ 229 #define C0_INDEX 0, 0 230 #define C0_ENTRYLO0 2, 0 231 #define C0_TCBIND 2, 2 232 #define C0_ENTRYLO1 3, 0 233 #define C0_CONTEXT 4, 0 234 #define C0_PAGEMASK 5, 0 235 #define C0_BADVADDR 8, 0 236 #define C0_ENTRYHI 10, 0 237 #define C0_EPC 14, 0 238 #define C0_XCONTEXT 20, 0 239 240 #ifdef CONFIG_64BIT 241 # define GET_CONTEXT(buf, reg) UASM_i_MFC0(buf, reg, C0_XCONTEXT) 242 #else 243 # define GET_CONTEXT(buf, reg) UASM_i_MFC0(buf, reg, C0_CONTEXT) 244 #endif 245 246 /* The worst case length of the handler is around 18 instructions for 247 * R3000-style TLBs and up to 63 instructions for R4000-style TLBs. 248 * Maximum space available is 32 instructions for R3000 and 64 249 * instructions for R4000. 250 * 251 * We deliberately chose a buffer size of 128, so we won't scribble 252 * over anything important on overflow before we panic. 253 */ 254 static u32 tlb_handler[128] __cpuinitdata; 255 256 /* simply assume worst case size for labels and relocs */ 257 static struct uasm_label labels[128] __cpuinitdata; 258 static struct uasm_reloc relocs[128] __cpuinitdata; 259 260 #ifdef CONFIG_64BIT 261 static int check_for_high_segbits __cpuinitdata; 262 #endif 263 264 static int check_for_high_segbits __cpuinitdata; 265 266 static unsigned int kscratch_used_mask __cpuinitdata; 267 268 static int __cpuinit allocate_kscratch(void) 269 { 270 int r; 271 unsigned int a = cpu_data[0].kscratch_mask & ~kscratch_used_mask; 272 273 r = ffs(a); 274 275 if (r == 0) 276 return -1; 277 278 r--; /* make it zero based */ 279 280 kscratch_used_mask |= (1 << r); 281 282 return r; 283 } 284 285 static int scratch_reg __cpuinitdata; 286 static int pgd_reg __cpuinitdata; 287 enum vmalloc64_mode {not_refill, refill_scratch, refill_noscratch}; 288 289 static struct work_registers __cpuinit build_get_work_registers(u32 **p) 290 { 291 struct work_registers r; 292 293 int smp_processor_id_reg; 294 int smp_processor_id_sel; 295 int smp_processor_id_shift; 296 297 if (scratch_reg > 0) { 298 /* Save in CPU local C0_KScratch? */ 299 UASM_i_MTC0(p, 1, 31, scratch_reg); 300 r.r1 = K0; 301 r.r2 = K1; 302 r.r3 = 1; 303 return r; 304 } 305 306 if (num_possible_cpus() > 1) { 307 #ifdef CONFIG_MIPS_PGD_C0_CONTEXT 308 smp_processor_id_shift = 51; 309 smp_processor_id_reg = 20; /* XContext */ 310 smp_processor_id_sel = 0; 311 #else 312 # ifdef CONFIG_32BIT 313 smp_processor_id_shift = 25; 314 smp_processor_id_reg = 4; /* Context */ 315 smp_processor_id_sel = 0; 316 # endif 317 # ifdef CONFIG_64BIT 318 smp_processor_id_shift = 26; 319 smp_processor_id_reg = 4; /* Context */ 320 smp_processor_id_sel = 0; 321 # endif 322 #endif 323 /* Get smp_processor_id */ 324 UASM_i_MFC0(p, K0, smp_processor_id_reg, smp_processor_id_sel); 325 UASM_i_SRL_SAFE(p, K0, K0, smp_processor_id_shift); 326 327 /* handler_reg_save index in K0 */ 328 UASM_i_SLL(p, K0, K0, ilog2(sizeof(struct tlb_reg_save))); 329 330 UASM_i_LA(p, K1, (long)&handler_reg_save); 331 UASM_i_ADDU(p, K0, K0, K1); 332 } else { 333 UASM_i_LA(p, K0, (long)&handler_reg_save); 334 } 335 /* K0 now points to save area, save $1 and $2 */ 336 UASM_i_SW(p, 1, offsetof(struct tlb_reg_save, a), K0); 337 UASM_i_SW(p, 2, offsetof(struct tlb_reg_save, b), K0); 338 339 r.r1 = K1; 340 r.r2 = 1; 341 r.r3 = 2; 342 return r; 343 } 344 345 static void __cpuinit build_restore_work_registers(u32 **p) 346 { 347 if (scratch_reg > 0) { 348 UASM_i_MFC0(p, 1, 31, scratch_reg); 349 return; 350 } 351 /* K0 already points to save area, restore $1 and $2 */ 352 UASM_i_LW(p, 1, offsetof(struct tlb_reg_save, a), K0); 353 UASM_i_LW(p, 2, offsetof(struct tlb_reg_save, b), K0); 354 } 355 356 #ifndef CONFIG_MIPS_PGD_C0_CONTEXT 357 358 /* 359 * CONFIG_MIPS_PGD_C0_CONTEXT implies 64 bit and lack of pgd_current, 360 * we cannot do r3000 under these circumstances. 361 * 362 * Declare pgd_current here instead of including mmu_context.h to avoid type 363 * conflicts for tlbmiss_handler_setup_pgd 364 */ 365 extern unsigned long pgd_current[]; 366 367 /* 368 * The R3000 TLB handler is simple. 369 */ 370 static void __cpuinit build_r3000_tlb_refill_handler(void) 371 { 372 long pgdc = (long)pgd_current; 373 u32 *p; 374 375 memset(tlb_handler, 0, sizeof(tlb_handler)); 376 p = tlb_handler; 377 378 uasm_i_mfc0(&p, K0, C0_BADVADDR); 379 uasm_i_lui(&p, K1, uasm_rel_hi(pgdc)); /* cp0 delay */ 380 uasm_i_lw(&p, K1, uasm_rel_lo(pgdc), K1); 381 uasm_i_srl(&p, K0, K0, 22); /* load delay */ 382 uasm_i_sll(&p, K0, K0, 2); 383 uasm_i_addu(&p, K1, K1, K0); 384 uasm_i_mfc0(&p, K0, C0_CONTEXT); 385 uasm_i_lw(&p, K1, 0, K1); /* cp0 delay */ 386 uasm_i_andi(&p, K0, K0, 0xffc); /* load delay */ 387 uasm_i_addu(&p, K1, K1, K0); 388 uasm_i_lw(&p, K0, 0, K1); 389 uasm_i_nop(&p); /* load delay */ 390 uasm_i_mtc0(&p, K0, C0_ENTRYLO0); 391 uasm_i_mfc0(&p, K1, C0_EPC); /* cp0 delay */ 392 uasm_i_tlbwr(&p); /* cp0 delay */ 393 uasm_i_jr(&p, K1); 394 uasm_i_rfe(&p); /* branch delay */ 395 396 if (p > tlb_handler + 32) 397 panic("TLB refill handler space exceeded"); 398 399 pr_debug("Wrote TLB refill handler (%u instructions).\n", 400 (unsigned int)(p - tlb_handler)); 401 402 memcpy((void *)ebase, tlb_handler, 0x80); 403 404 dump_handler((u32 *)ebase, 32); 405 } 406 #endif /* CONFIG_MIPS_PGD_C0_CONTEXT */ 407 408 /* 409 * The R4000 TLB handler is much more complicated. We have two 410 * consecutive handler areas with 32 instructions space each. 411 * Since they aren't used at the same time, we can overflow in the 412 * other one.To keep things simple, we first assume linear space, 413 * then we relocate it to the final handler layout as needed. 414 */ 415 static u32 final_handler[64] __cpuinitdata; 416 417 /* 418 * Hazards 419 * 420 * From the IDT errata for the QED RM5230 (Nevada), processor revision 1.0: 421 * 2. A timing hazard exists for the TLBP instruction. 422 * 423 * stalling_instruction 424 * TLBP 425 * 426 * The JTLB is being read for the TLBP throughout the stall generated by the 427 * previous instruction. This is not really correct as the stalling instruction 428 * can modify the address used to access the JTLB. The failure symptom is that 429 * the TLBP instruction will use an address created for the stalling instruction 430 * and not the address held in C0_ENHI and thus report the wrong results. 431 * 432 * The software work-around is to not allow the instruction preceding the TLBP 433 * to stall - make it an NOP or some other instruction guaranteed not to stall. 434 * 435 * Errata 2 will not be fixed. This errata is also on the R5000. 436 * 437 * As if we MIPS hackers wouldn't know how to nop pipelines happy ... 438 */ 439 static void __cpuinit __maybe_unused build_tlb_probe_entry(u32 **p) 440 { 441 switch (current_cpu_type()) { 442 /* Found by experiment: R4600 v2.0/R4700 needs this, too. */ 443 case CPU_R4600: 444 case CPU_R4700: 445 case CPU_R5000: 446 case CPU_R5000A: 447 case CPU_NEVADA: 448 uasm_i_nop(p); 449 uasm_i_tlbp(p); 450 break; 451 452 default: 453 uasm_i_tlbp(p); 454 break; 455 } 456 } 457 458 /* 459 * Write random or indexed TLB entry, and care about the hazards from 460 * the preceding mtc0 and for the following eret. 461 */ 462 enum tlb_write_entry { tlb_random, tlb_indexed }; 463 464 static void __cpuinit build_tlb_write_entry(u32 **p, struct uasm_label **l, 465 struct uasm_reloc **r, 466 enum tlb_write_entry wmode) 467 { 468 void(*tlbw)(u32 **) = NULL; 469 470 switch (wmode) { 471 case tlb_random: tlbw = uasm_i_tlbwr; break; 472 case tlb_indexed: tlbw = uasm_i_tlbwi; break; 473 } 474 475 if (cpu_has_mips_r2) { 476 /* 477 * The architecture spec says an ehb is required here, 478 * but a number of cores do not have the hazard and 479 * using an ehb causes an expensive pipeline stall. 480 */ 481 switch (current_cpu_type()) { 482 case CPU_M14KC: 483 case CPU_74K: 484 break; 485 486 default: 487 uasm_i_ehb(p); 488 break; 489 } 490 tlbw(p); 491 return; 492 } 493 494 switch (current_cpu_type()) { 495 case CPU_R4000PC: 496 case CPU_R4000SC: 497 case CPU_R4000MC: 498 case CPU_R4400PC: 499 case CPU_R4400SC: 500 case CPU_R4400MC: 501 /* 502 * This branch uses up a mtc0 hazard nop slot and saves 503 * two nops after the tlbw instruction. 504 */ 505 uasm_bgezl_hazard(p, r, hazard_instance); 506 tlbw(p); 507 uasm_bgezl_label(l, p, hazard_instance); 508 hazard_instance++; 509 uasm_i_nop(p); 510 break; 511 512 case CPU_R4600: 513 case CPU_R4700: 514 uasm_i_nop(p); 515 tlbw(p); 516 uasm_i_nop(p); 517 break; 518 519 case CPU_R5000: 520 case CPU_R5000A: 521 case CPU_NEVADA: 522 uasm_i_nop(p); /* QED specifies 2 nops hazard */ 523 uasm_i_nop(p); /* QED specifies 2 nops hazard */ 524 tlbw(p); 525 break; 526 527 case CPU_R4300: 528 case CPU_5KC: 529 case CPU_TX49XX: 530 case CPU_PR4450: 531 case CPU_XLR: 532 uasm_i_nop(p); 533 tlbw(p); 534 break; 535 536 case CPU_R10000: 537 case CPU_R12000: 538 case CPU_R14000: 539 case CPU_4KC: 540 case CPU_4KEC: 541 case CPU_M14KC: 542 case CPU_SB1: 543 case CPU_SB1A: 544 case CPU_4KSC: 545 case CPU_20KC: 546 case CPU_25KF: 547 case CPU_BMIPS32: 548 case CPU_BMIPS3300: 549 case CPU_BMIPS4350: 550 case CPU_BMIPS4380: 551 case CPU_BMIPS5000: 552 case CPU_LOONGSON2: 553 case CPU_R5500: 554 if (m4kc_tlbp_war()) 555 uasm_i_nop(p); 556 case CPU_ALCHEMY: 557 tlbw(p); 558 break; 559 560 case CPU_RM7000: 561 uasm_i_nop(p); 562 uasm_i_nop(p); 563 uasm_i_nop(p); 564 uasm_i_nop(p); 565 tlbw(p); 566 break; 567 568 case CPU_RM9000: 569 /* 570 * When the JTLB is updated by tlbwi or tlbwr, a subsequent 571 * use of the JTLB for instructions should not occur for 4 572 * cpu cycles and use for data translations should not occur 573 * for 3 cpu cycles. 574 */ 575 uasm_i_ssnop(p); 576 uasm_i_ssnop(p); 577 uasm_i_ssnop(p); 578 uasm_i_ssnop(p); 579 tlbw(p); 580 uasm_i_ssnop(p); 581 uasm_i_ssnop(p); 582 uasm_i_ssnop(p); 583 uasm_i_ssnop(p); 584 break; 585 586 case CPU_VR4111: 587 case CPU_VR4121: 588 case CPU_VR4122: 589 case CPU_VR4181: 590 case CPU_VR4181A: 591 uasm_i_nop(p); 592 uasm_i_nop(p); 593 tlbw(p); 594 uasm_i_nop(p); 595 uasm_i_nop(p); 596 break; 597 598 case CPU_VR4131: 599 case CPU_VR4133: 600 case CPU_R5432: 601 uasm_i_nop(p); 602 uasm_i_nop(p); 603 tlbw(p); 604 break; 605 606 case CPU_JZRISC: 607 tlbw(p); 608 uasm_i_nop(p); 609 break; 610 611 default: 612 panic("No TLB refill handler yet (CPU type: %d)", 613 current_cpu_data.cputype); 614 break; 615 } 616 } 617 618 static __cpuinit __maybe_unused void build_convert_pte_to_entrylo(u32 **p, 619 unsigned int reg) 620 { 621 if (cpu_has_rixi) { 622 UASM_i_ROTR(p, reg, reg, ilog2(_PAGE_GLOBAL)); 623 } else { 624 #ifdef CONFIG_64BIT_PHYS_ADDR 625 uasm_i_dsrl_safe(p, reg, reg, ilog2(_PAGE_GLOBAL)); 626 #else 627 UASM_i_SRL(p, reg, reg, ilog2(_PAGE_GLOBAL)); 628 #endif 629 } 630 } 631 632 #ifdef CONFIG_HUGETLB_PAGE 633 634 static __cpuinit void build_restore_pagemask(u32 **p, 635 struct uasm_reloc **r, 636 unsigned int tmp, 637 enum label_id lid, 638 int restore_scratch) 639 { 640 if (restore_scratch) { 641 /* Reset default page size */ 642 if (PM_DEFAULT_MASK >> 16) { 643 uasm_i_lui(p, tmp, PM_DEFAULT_MASK >> 16); 644 uasm_i_ori(p, tmp, tmp, PM_DEFAULT_MASK & 0xffff); 645 uasm_i_mtc0(p, tmp, C0_PAGEMASK); 646 uasm_il_b(p, r, lid); 647 } else if (PM_DEFAULT_MASK) { 648 uasm_i_ori(p, tmp, 0, PM_DEFAULT_MASK); 649 uasm_i_mtc0(p, tmp, C0_PAGEMASK); 650 uasm_il_b(p, r, lid); 651 } else { 652 uasm_i_mtc0(p, 0, C0_PAGEMASK); 653 uasm_il_b(p, r, lid); 654 } 655 if (scratch_reg > 0) 656 UASM_i_MFC0(p, 1, 31, scratch_reg); 657 else 658 UASM_i_LW(p, 1, scratchpad_offset(0), 0); 659 } else { 660 /* Reset default page size */ 661 if (PM_DEFAULT_MASK >> 16) { 662 uasm_i_lui(p, tmp, PM_DEFAULT_MASK >> 16); 663 uasm_i_ori(p, tmp, tmp, PM_DEFAULT_MASK & 0xffff); 664 uasm_il_b(p, r, lid); 665 uasm_i_mtc0(p, tmp, C0_PAGEMASK); 666 } else if (PM_DEFAULT_MASK) { 667 uasm_i_ori(p, tmp, 0, PM_DEFAULT_MASK); 668 uasm_il_b(p, r, lid); 669 uasm_i_mtc0(p, tmp, C0_PAGEMASK); 670 } else { 671 uasm_il_b(p, r, lid); 672 uasm_i_mtc0(p, 0, C0_PAGEMASK); 673 } 674 } 675 } 676 677 static __cpuinit void build_huge_tlb_write_entry(u32 **p, 678 struct uasm_label **l, 679 struct uasm_reloc **r, 680 unsigned int tmp, 681 enum tlb_write_entry wmode, 682 int restore_scratch) 683 { 684 /* Set huge page tlb entry size */ 685 uasm_i_lui(p, tmp, PM_HUGE_MASK >> 16); 686 uasm_i_ori(p, tmp, tmp, PM_HUGE_MASK & 0xffff); 687 uasm_i_mtc0(p, tmp, C0_PAGEMASK); 688 689 build_tlb_write_entry(p, l, r, wmode); 690 691 build_restore_pagemask(p, r, tmp, label_leave, restore_scratch); 692 } 693 694 /* 695 * Check if Huge PTE is present, if so then jump to LABEL. 696 */ 697 static void __cpuinit 698 build_is_huge_pte(u32 **p, struct uasm_reloc **r, unsigned int tmp, 699 unsigned int pmd, int lid) 700 { 701 UASM_i_LW(p, tmp, 0, pmd); 702 if (use_bbit_insns()) { 703 uasm_il_bbit1(p, r, tmp, ilog2(_PAGE_HUGE), lid); 704 } else { 705 uasm_i_andi(p, tmp, tmp, _PAGE_HUGE); 706 uasm_il_bnez(p, r, tmp, lid); 707 } 708 } 709 710 static __cpuinit void build_huge_update_entries(u32 **p, 711 unsigned int pte, 712 unsigned int tmp) 713 { 714 int small_sequence; 715 716 /* 717 * A huge PTE describes an area the size of the 718 * configured huge page size. This is twice the 719 * of the large TLB entry size we intend to use. 720 * A TLB entry half the size of the configured 721 * huge page size is configured into entrylo0 722 * and entrylo1 to cover the contiguous huge PTE 723 * address space. 724 */ 725 small_sequence = (HPAGE_SIZE >> 7) < 0x10000; 726 727 /* We can clobber tmp. It isn't used after this.*/ 728 if (!small_sequence) 729 uasm_i_lui(p, tmp, HPAGE_SIZE >> (7 + 16)); 730 731 build_convert_pte_to_entrylo(p, pte); 732 UASM_i_MTC0(p, pte, C0_ENTRYLO0); /* load it */ 733 /* convert to entrylo1 */ 734 if (small_sequence) 735 UASM_i_ADDIU(p, pte, pte, HPAGE_SIZE >> 7); 736 else 737 UASM_i_ADDU(p, pte, pte, tmp); 738 739 UASM_i_MTC0(p, pte, C0_ENTRYLO1); /* load it */ 740 } 741 742 static __cpuinit void build_huge_handler_tail(u32 **p, 743 struct uasm_reloc **r, 744 struct uasm_label **l, 745 unsigned int pte, 746 unsigned int ptr) 747 { 748 #ifdef CONFIG_SMP 749 UASM_i_SC(p, pte, 0, ptr); 750 uasm_il_beqz(p, r, pte, label_tlb_huge_update); 751 UASM_i_LW(p, pte, 0, ptr); /* Needed because SC killed our PTE */ 752 #else 753 UASM_i_SW(p, pte, 0, ptr); 754 #endif 755 build_huge_update_entries(p, pte, ptr); 756 build_huge_tlb_write_entry(p, l, r, pte, tlb_indexed, 0); 757 } 758 #endif /* CONFIG_HUGETLB_PAGE */ 759 760 #ifdef CONFIG_64BIT 761 /* 762 * TMP and PTR are scratch. 763 * TMP will be clobbered, PTR will hold the pmd entry. 764 */ 765 static void __cpuinit 766 build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r, 767 unsigned int tmp, unsigned int ptr) 768 { 769 #ifndef CONFIG_MIPS_PGD_C0_CONTEXT 770 long pgdc = (long)pgd_current; 771 #endif 772 /* 773 * The vmalloc handling is not in the hotpath. 774 */ 775 uasm_i_dmfc0(p, tmp, C0_BADVADDR); 776 777 if (check_for_high_segbits) { 778 /* 779 * The kernel currently implicitely assumes that the 780 * MIPS SEGBITS parameter for the processor is 781 * (PGDIR_SHIFT+PGDIR_BITS) or less, and will never 782 * allocate virtual addresses outside the maximum 783 * range for SEGBITS = (PGDIR_SHIFT+PGDIR_BITS). But 784 * that doesn't prevent user code from accessing the 785 * higher xuseg addresses. Here, we make sure that 786 * everything but the lower xuseg addresses goes down 787 * the module_alloc/vmalloc path. 788 */ 789 uasm_i_dsrl_safe(p, ptr, tmp, PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3); 790 uasm_il_bnez(p, r, ptr, label_vmalloc); 791 } else { 792 uasm_il_bltz(p, r, tmp, label_vmalloc); 793 } 794 /* No uasm_i_nop needed here, since the next insn doesn't touch TMP. */ 795 796 #ifdef CONFIG_MIPS_PGD_C0_CONTEXT 797 if (pgd_reg != -1) { 798 /* pgd is in pgd_reg */ 799 UASM_i_MFC0(p, ptr, 31, pgd_reg); 800 } else { 801 /* 802 * &pgd << 11 stored in CONTEXT [23..63]. 803 */ 804 UASM_i_MFC0(p, ptr, C0_CONTEXT); 805 806 /* Clear lower 23 bits of context. */ 807 uasm_i_dins(p, ptr, 0, 0, 23); 808 809 /* 1 0 1 0 1 << 6 xkphys cached */ 810 uasm_i_ori(p, ptr, ptr, 0x540); 811 uasm_i_drotr(p, ptr, ptr, 11); 812 } 813 #elif defined(CONFIG_SMP) 814 # ifdef CONFIG_MIPS_MT_SMTC 815 /* 816 * SMTC uses TCBind value as "CPU" index 817 */ 818 uasm_i_mfc0(p, ptr, C0_TCBIND); 819 uasm_i_dsrl_safe(p, ptr, ptr, 19); 820 # else 821 /* 822 * 64 bit SMP running in XKPHYS has smp_processor_id() << 3 823 * stored in CONTEXT. 824 */ 825 uasm_i_dmfc0(p, ptr, C0_CONTEXT); 826 uasm_i_dsrl_safe(p, ptr, ptr, 23); 827 # endif 828 UASM_i_LA_mostly(p, tmp, pgdc); 829 uasm_i_daddu(p, ptr, ptr, tmp); 830 uasm_i_dmfc0(p, tmp, C0_BADVADDR); 831 uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr); 832 #else 833 UASM_i_LA_mostly(p, ptr, pgdc); 834 uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr); 835 #endif 836 837 uasm_l_vmalloc_done(l, *p); 838 839 /* get pgd offset in bytes */ 840 uasm_i_dsrl_safe(p, tmp, tmp, PGDIR_SHIFT - 3); 841 842 uasm_i_andi(p, tmp, tmp, (PTRS_PER_PGD - 1)<<3); 843 uasm_i_daddu(p, ptr, ptr, tmp); /* add in pgd offset */ 844 #ifndef __PAGETABLE_PMD_FOLDED 845 uasm_i_dmfc0(p, tmp, C0_BADVADDR); /* get faulting address */ 846 uasm_i_ld(p, ptr, 0, ptr); /* get pmd pointer */ 847 uasm_i_dsrl_safe(p, tmp, tmp, PMD_SHIFT-3); /* get pmd offset in bytes */ 848 uasm_i_andi(p, tmp, tmp, (PTRS_PER_PMD - 1)<<3); 849 uasm_i_daddu(p, ptr, ptr, tmp); /* add in pmd offset */ 850 #endif 851 } 852 853 /* 854 * BVADDR is the faulting address, PTR is scratch. 855 * PTR will hold the pgd for vmalloc. 856 */ 857 static void __cpuinit 858 build_get_pgd_vmalloc64(u32 **p, struct uasm_label **l, struct uasm_reloc **r, 859 unsigned int bvaddr, unsigned int ptr, 860 enum vmalloc64_mode mode) 861 { 862 long swpd = (long)swapper_pg_dir; 863 int single_insn_swpd; 864 int did_vmalloc_branch = 0; 865 866 single_insn_swpd = uasm_in_compat_space_p(swpd) && !uasm_rel_lo(swpd); 867 868 uasm_l_vmalloc(l, *p); 869 870 if (mode != not_refill && check_for_high_segbits) { 871 if (single_insn_swpd) { 872 uasm_il_bltz(p, r, bvaddr, label_vmalloc_done); 873 uasm_i_lui(p, ptr, uasm_rel_hi(swpd)); 874 did_vmalloc_branch = 1; 875 /* fall through */ 876 } else { 877 uasm_il_bgez(p, r, bvaddr, label_large_segbits_fault); 878 } 879 } 880 if (!did_vmalloc_branch) { 881 if (uasm_in_compat_space_p(swpd) && !uasm_rel_lo(swpd)) { 882 uasm_il_b(p, r, label_vmalloc_done); 883 uasm_i_lui(p, ptr, uasm_rel_hi(swpd)); 884 } else { 885 UASM_i_LA_mostly(p, ptr, swpd); 886 uasm_il_b(p, r, label_vmalloc_done); 887 if (uasm_in_compat_space_p(swpd)) 888 uasm_i_addiu(p, ptr, ptr, uasm_rel_lo(swpd)); 889 else 890 uasm_i_daddiu(p, ptr, ptr, uasm_rel_lo(swpd)); 891 } 892 } 893 if (mode != not_refill && check_for_high_segbits) { 894 uasm_l_large_segbits_fault(l, *p); 895 /* 896 * We get here if we are an xsseg address, or if we are 897 * an xuseg address above (PGDIR_SHIFT+PGDIR_BITS) boundary. 898 * 899 * Ignoring xsseg (assume disabled so would generate 900 * (address errors?), the only remaining possibility 901 * is the upper xuseg addresses. On processors with 902 * TLB_SEGBITS <= PGDIR_SHIFT+PGDIR_BITS, these 903 * addresses would have taken an address error. We try 904 * to mimic that here by taking a load/istream page 905 * fault. 906 */ 907 UASM_i_LA(p, ptr, (unsigned long)tlb_do_page_fault_0); 908 uasm_i_jr(p, ptr); 909 910 if (mode == refill_scratch) { 911 if (scratch_reg > 0) 912 UASM_i_MFC0(p, 1, 31, scratch_reg); 913 else 914 UASM_i_LW(p, 1, scratchpad_offset(0), 0); 915 } else { 916 uasm_i_nop(p); 917 } 918 } 919 } 920 921 #else /* !CONFIG_64BIT */ 922 923 /* 924 * TMP and PTR are scratch. 925 * TMP will be clobbered, PTR will hold the pgd entry. 926 */ 927 static void __cpuinit __maybe_unused 928 build_get_pgde32(u32 **p, unsigned int tmp, unsigned int ptr) 929 { 930 long pgdc = (long)pgd_current; 931 932 /* 32 bit SMP has smp_processor_id() stored in CONTEXT. */ 933 #ifdef CONFIG_SMP 934 #ifdef CONFIG_MIPS_MT_SMTC 935 /* 936 * SMTC uses TCBind value as "CPU" index 937 */ 938 uasm_i_mfc0(p, ptr, C0_TCBIND); 939 UASM_i_LA_mostly(p, tmp, pgdc); 940 uasm_i_srl(p, ptr, ptr, 19); 941 #else 942 /* 943 * smp_processor_id() << 3 is stored in CONTEXT. 944 */ 945 uasm_i_mfc0(p, ptr, C0_CONTEXT); 946 UASM_i_LA_mostly(p, tmp, pgdc); 947 uasm_i_srl(p, ptr, ptr, 23); 948 #endif 949 uasm_i_addu(p, ptr, tmp, ptr); 950 #else 951 UASM_i_LA_mostly(p, ptr, pgdc); 952 #endif 953 uasm_i_mfc0(p, tmp, C0_BADVADDR); /* get faulting address */ 954 uasm_i_lw(p, ptr, uasm_rel_lo(pgdc), ptr); 955 956 if (cpu_has_mips_r2) { 957 uasm_i_ext(p, tmp, tmp, PGDIR_SHIFT, (32 - PGDIR_SHIFT)); 958 uasm_i_ins(p, ptr, tmp, PGD_T_LOG2, (32 - PGDIR_SHIFT)); 959 return; 960 } 961 962 uasm_i_srl(p, tmp, tmp, PGDIR_SHIFT); /* get pgd only bits */ 963 uasm_i_sll(p, tmp, tmp, PGD_T_LOG2); 964 uasm_i_addu(p, ptr, ptr, tmp); /* add in pgd offset */ 965 } 966 967 #endif /* !CONFIG_64BIT */ 968 969 static void __cpuinit build_adjust_context(u32 **p, unsigned int ctx) 970 { 971 unsigned int shift = 4 - (PTE_T_LOG2 + 1) + PAGE_SHIFT - 12; 972 unsigned int mask = (PTRS_PER_PTE / 2 - 1) << (PTE_T_LOG2 + 1); 973 974 switch (current_cpu_type()) { 975 case CPU_VR41XX: 976 case CPU_VR4111: 977 case CPU_VR4121: 978 case CPU_VR4122: 979 case CPU_VR4131: 980 case CPU_VR4181: 981 case CPU_VR4181A: 982 case CPU_VR4133: 983 shift += 2; 984 break; 985 986 default: 987 break; 988 } 989 990 if (shift) 991 UASM_i_SRL(p, ctx, ctx, shift); 992 uasm_i_andi(p, ctx, ctx, mask); 993 } 994 995 static void __cpuinit build_get_ptep(u32 **p, unsigned int tmp, unsigned int ptr) 996 { 997 if (cpu_has_mips_r2) { 998 /* PTE ptr offset is obtained from BadVAddr */ 999 UASM_i_MFC0(p, tmp, C0_BADVADDR); 1000 UASM_i_LW(p, ptr, 0, ptr); 1001 uasm_i_ext(p, tmp, tmp, PAGE_SHIFT+1, PGDIR_SHIFT-PAGE_SHIFT-1); 1002 uasm_i_ins(p, ptr, tmp, PTE_T_LOG2+1, PGDIR_SHIFT-PAGE_SHIFT-1); 1003 return; 1004 } 1005 1006 /* 1007 * Bug workaround for the Nevada. It seems as if under certain 1008 * circumstances the move from cp0_context might produce a 1009 * bogus result when the mfc0 instruction and its consumer are 1010 * in a different cacheline or a load instruction, probably any 1011 * memory reference, is between them. 1012 */ 1013 switch (current_cpu_type()) { 1014 case CPU_NEVADA: 1015 UASM_i_LW(p, ptr, 0, ptr); 1016 GET_CONTEXT(p, tmp); /* get context reg */ 1017 break; 1018 1019 default: 1020 GET_CONTEXT(p, tmp); /* get context reg */ 1021 UASM_i_LW(p, ptr, 0, ptr); 1022 break; 1023 } 1024 1025 build_adjust_context(p, tmp); 1026 UASM_i_ADDU(p, ptr, ptr, tmp); /* add in offset */ 1027 } 1028 1029 static void __cpuinit build_update_entries(u32 **p, unsigned int tmp, 1030 unsigned int ptep) 1031 { 1032 /* 1033 * 64bit address support (36bit on a 32bit CPU) in a 32bit 1034 * Kernel is a special case. Only a few CPUs use it. 1035 */ 1036 #ifdef CONFIG_64BIT_PHYS_ADDR 1037 if (cpu_has_64bits) { 1038 uasm_i_ld(p, tmp, 0, ptep); /* get even pte */ 1039 uasm_i_ld(p, ptep, sizeof(pte_t), ptep); /* get odd pte */ 1040 if (cpu_has_rixi) { 1041 UASM_i_ROTR(p, tmp, tmp, ilog2(_PAGE_GLOBAL)); 1042 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */ 1043 UASM_i_ROTR(p, ptep, ptep, ilog2(_PAGE_GLOBAL)); 1044 } else { 1045 uasm_i_dsrl_safe(p, tmp, tmp, ilog2(_PAGE_GLOBAL)); /* convert to entrylo0 */ 1046 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */ 1047 uasm_i_dsrl_safe(p, ptep, ptep, ilog2(_PAGE_GLOBAL)); /* convert to entrylo1 */ 1048 } 1049 UASM_i_MTC0(p, ptep, C0_ENTRYLO1); /* load it */ 1050 } else { 1051 int pte_off_even = sizeof(pte_t) / 2; 1052 int pte_off_odd = pte_off_even + sizeof(pte_t); 1053 1054 /* The pte entries are pre-shifted */ 1055 uasm_i_lw(p, tmp, pte_off_even, ptep); /* get even pte */ 1056 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */ 1057 uasm_i_lw(p, ptep, pte_off_odd, ptep); /* get odd pte */ 1058 UASM_i_MTC0(p, ptep, C0_ENTRYLO1); /* load it */ 1059 } 1060 #else 1061 UASM_i_LW(p, tmp, 0, ptep); /* get even pte */ 1062 UASM_i_LW(p, ptep, sizeof(pte_t), ptep); /* get odd pte */ 1063 if (r45k_bvahwbug()) 1064 build_tlb_probe_entry(p); 1065 if (cpu_has_rixi) { 1066 UASM_i_ROTR(p, tmp, tmp, ilog2(_PAGE_GLOBAL)); 1067 if (r4k_250MHZhwbug()) 1068 UASM_i_MTC0(p, 0, C0_ENTRYLO0); 1069 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */ 1070 UASM_i_ROTR(p, ptep, ptep, ilog2(_PAGE_GLOBAL)); 1071 } else { 1072 UASM_i_SRL(p, tmp, tmp, ilog2(_PAGE_GLOBAL)); /* convert to entrylo0 */ 1073 if (r4k_250MHZhwbug()) 1074 UASM_i_MTC0(p, 0, C0_ENTRYLO0); 1075 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */ 1076 UASM_i_SRL(p, ptep, ptep, ilog2(_PAGE_GLOBAL)); /* convert to entrylo1 */ 1077 if (r45k_bvahwbug()) 1078 uasm_i_mfc0(p, tmp, C0_INDEX); 1079 } 1080 if (r4k_250MHZhwbug()) 1081 UASM_i_MTC0(p, 0, C0_ENTRYLO1); 1082 UASM_i_MTC0(p, ptep, C0_ENTRYLO1); /* load it */ 1083 #endif 1084 } 1085 1086 struct mips_huge_tlb_info { 1087 int huge_pte; 1088 int restore_scratch; 1089 }; 1090 1091 static struct mips_huge_tlb_info __cpuinit 1092 build_fast_tlb_refill_handler (u32 **p, struct uasm_label **l, 1093 struct uasm_reloc **r, unsigned int tmp, 1094 unsigned int ptr, int c0_scratch) 1095 { 1096 struct mips_huge_tlb_info rv; 1097 unsigned int even, odd; 1098 int vmalloc_branch_delay_filled = 0; 1099 const int scratch = 1; /* Our extra working register */ 1100 1101 rv.huge_pte = scratch; 1102 rv.restore_scratch = 0; 1103 1104 if (check_for_high_segbits) { 1105 UASM_i_MFC0(p, tmp, C0_BADVADDR); 1106 1107 if (pgd_reg != -1) 1108 UASM_i_MFC0(p, ptr, 31, pgd_reg); 1109 else 1110 UASM_i_MFC0(p, ptr, C0_CONTEXT); 1111 1112 if (c0_scratch >= 0) 1113 UASM_i_MTC0(p, scratch, 31, c0_scratch); 1114 else 1115 UASM_i_SW(p, scratch, scratchpad_offset(0), 0); 1116 1117 uasm_i_dsrl_safe(p, scratch, tmp, 1118 PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3); 1119 uasm_il_bnez(p, r, scratch, label_vmalloc); 1120 1121 if (pgd_reg == -1) { 1122 vmalloc_branch_delay_filled = 1; 1123 /* Clear lower 23 bits of context. */ 1124 uasm_i_dins(p, ptr, 0, 0, 23); 1125 } 1126 } else { 1127 if (pgd_reg != -1) 1128 UASM_i_MFC0(p, ptr, 31, pgd_reg); 1129 else 1130 UASM_i_MFC0(p, ptr, C0_CONTEXT); 1131 1132 UASM_i_MFC0(p, tmp, C0_BADVADDR); 1133 1134 if (c0_scratch >= 0) 1135 UASM_i_MTC0(p, scratch, 31, c0_scratch); 1136 else 1137 UASM_i_SW(p, scratch, scratchpad_offset(0), 0); 1138 1139 if (pgd_reg == -1) 1140 /* Clear lower 23 bits of context. */ 1141 uasm_i_dins(p, ptr, 0, 0, 23); 1142 1143 uasm_il_bltz(p, r, tmp, label_vmalloc); 1144 } 1145 1146 if (pgd_reg == -1) { 1147 vmalloc_branch_delay_filled = 1; 1148 /* 1 0 1 0 1 << 6 xkphys cached */ 1149 uasm_i_ori(p, ptr, ptr, 0x540); 1150 uasm_i_drotr(p, ptr, ptr, 11); 1151 } 1152 1153 #ifdef __PAGETABLE_PMD_FOLDED 1154 #define LOC_PTEP scratch 1155 #else 1156 #define LOC_PTEP ptr 1157 #endif 1158 1159 if (!vmalloc_branch_delay_filled) 1160 /* get pgd offset in bytes */ 1161 uasm_i_dsrl_safe(p, scratch, tmp, PGDIR_SHIFT - 3); 1162 1163 uasm_l_vmalloc_done(l, *p); 1164 1165 /* 1166 * tmp ptr 1167 * fall-through case = badvaddr *pgd_current 1168 * vmalloc case = badvaddr swapper_pg_dir 1169 */ 1170 1171 if (vmalloc_branch_delay_filled) 1172 /* get pgd offset in bytes */ 1173 uasm_i_dsrl_safe(p, scratch, tmp, PGDIR_SHIFT - 3); 1174 1175 #ifdef __PAGETABLE_PMD_FOLDED 1176 GET_CONTEXT(p, tmp); /* get context reg */ 1177 #endif 1178 uasm_i_andi(p, scratch, scratch, (PTRS_PER_PGD - 1) << 3); 1179 1180 if (use_lwx_insns()) { 1181 UASM_i_LWX(p, LOC_PTEP, scratch, ptr); 1182 } else { 1183 uasm_i_daddu(p, ptr, ptr, scratch); /* add in pgd offset */ 1184 uasm_i_ld(p, LOC_PTEP, 0, ptr); /* get pmd pointer */ 1185 } 1186 1187 #ifndef __PAGETABLE_PMD_FOLDED 1188 /* get pmd offset in bytes */ 1189 uasm_i_dsrl_safe(p, scratch, tmp, PMD_SHIFT - 3); 1190 uasm_i_andi(p, scratch, scratch, (PTRS_PER_PMD - 1) << 3); 1191 GET_CONTEXT(p, tmp); /* get context reg */ 1192 1193 if (use_lwx_insns()) { 1194 UASM_i_LWX(p, scratch, scratch, ptr); 1195 } else { 1196 uasm_i_daddu(p, ptr, ptr, scratch); /* add in pmd offset */ 1197 UASM_i_LW(p, scratch, 0, ptr); 1198 } 1199 #endif 1200 /* Adjust the context during the load latency. */ 1201 build_adjust_context(p, tmp); 1202 1203 #ifdef CONFIG_HUGETLB_PAGE 1204 uasm_il_bbit1(p, r, scratch, ilog2(_PAGE_HUGE), label_tlb_huge_update); 1205 /* 1206 * The in the LWX case we don't want to do the load in the 1207 * delay slot. It cannot issue in the same cycle and may be 1208 * speculative and unneeded. 1209 */ 1210 if (use_lwx_insns()) 1211 uasm_i_nop(p); 1212 #endif /* CONFIG_HUGETLB_PAGE */ 1213 1214 1215 /* build_update_entries */ 1216 if (use_lwx_insns()) { 1217 even = ptr; 1218 odd = tmp; 1219 UASM_i_LWX(p, even, scratch, tmp); 1220 UASM_i_ADDIU(p, tmp, tmp, sizeof(pte_t)); 1221 UASM_i_LWX(p, odd, scratch, tmp); 1222 } else { 1223 UASM_i_ADDU(p, ptr, scratch, tmp); /* add in offset */ 1224 even = tmp; 1225 odd = ptr; 1226 UASM_i_LW(p, even, 0, ptr); /* get even pte */ 1227 UASM_i_LW(p, odd, sizeof(pte_t), ptr); /* get odd pte */ 1228 } 1229 if (cpu_has_rixi) { 1230 uasm_i_drotr(p, even, even, ilog2(_PAGE_GLOBAL)); 1231 UASM_i_MTC0(p, even, C0_ENTRYLO0); /* load it */ 1232 uasm_i_drotr(p, odd, odd, ilog2(_PAGE_GLOBAL)); 1233 } else { 1234 uasm_i_dsrl_safe(p, even, even, ilog2(_PAGE_GLOBAL)); 1235 UASM_i_MTC0(p, even, C0_ENTRYLO0); /* load it */ 1236 uasm_i_dsrl_safe(p, odd, odd, ilog2(_PAGE_GLOBAL)); 1237 } 1238 UASM_i_MTC0(p, odd, C0_ENTRYLO1); /* load it */ 1239 1240 if (c0_scratch >= 0) { 1241 UASM_i_MFC0(p, scratch, 31, c0_scratch); 1242 build_tlb_write_entry(p, l, r, tlb_random); 1243 uasm_l_leave(l, *p); 1244 rv.restore_scratch = 1; 1245 } else if (PAGE_SHIFT == 14 || PAGE_SHIFT == 13) { 1246 build_tlb_write_entry(p, l, r, tlb_random); 1247 uasm_l_leave(l, *p); 1248 UASM_i_LW(p, scratch, scratchpad_offset(0), 0); 1249 } else { 1250 UASM_i_LW(p, scratch, scratchpad_offset(0), 0); 1251 build_tlb_write_entry(p, l, r, tlb_random); 1252 uasm_l_leave(l, *p); 1253 rv.restore_scratch = 1; 1254 } 1255 1256 uasm_i_eret(p); /* return from trap */ 1257 1258 return rv; 1259 } 1260 1261 /* 1262 * For a 64-bit kernel, we are using the 64-bit XTLB refill exception 1263 * because EXL == 0. If we wrap, we can also use the 32 instruction 1264 * slots before the XTLB refill exception handler which belong to the 1265 * unused TLB refill exception. 1266 */ 1267 #define MIPS64_REFILL_INSNS 32 1268 1269 static void __cpuinit build_r4000_tlb_refill_handler(void) 1270 { 1271 u32 *p = tlb_handler; 1272 struct uasm_label *l = labels; 1273 struct uasm_reloc *r = relocs; 1274 u32 *f; 1275 unsigned int final_len; 1276 struct mips_huge_tlb_info htlb_info __maybe_unused; 1277 enum vmalloc64_mode vmalloc_mode __maybe_unused; 1278 1279 memset(tlb_handler, 0, sizeof(tlb_handler)); 1280 memset(labels, 0, sizeof(labels)); 1281 memset(relocs, 0, sizeof(relocs)); 1282 memset(final_handler, 0, sizeof(final_handler)); 1283 1284 if ((scratch_reg > 0 || scratchpad_available()) && use_bbit_insns()) { 1285 htlb_info = build_fast_tlb_refill_handler(&p, &l, &r, K0, K1, 1286 scratch_reg); 1287 vmalloc_mode = refill_scratch; 1288 } else { 1289 htlb_info.huge_pte = K0; 1290 htlb_info.restore_scratch = 0; 1291 vmalloc_mode = refill_noscratch; 1292 /* 1293 * create the plain linear handler 1294 */ 1295 if (bcm1250_m3_war()) { 1296 unsigned int segbits = 44; 1297 1298 uasm_i_dmfc0(&p, K0, C0_BADVADDR); 1299 uasm_i_dmfc0(&p, K1, C0_ENTRYHI); 1300 uasm_i_xor(&p, K0, K0, K1); 1301 uasm_i_dsrl_safe(&p, K1, K0, 62); 1302 uasm_i_dsrl_safe(&p, K0, K0, 12 + 1); 1303 uasm_i_dsll_safe(&p, K0, K0, 64 + 12 + 1 - segbits); 1304 uasm_i_or(&p, K0, K0, K1); 1305 uasm_il_bnez(&p, &r, K0, label_leave); 1306 /* No need for uasm_i_nop */ 1307 } 1308 1309 #ifdef CONFIG_64BIT 1310 build_get_pmde64(&p, &l, &r, K0, K1); /* get pmd in K1 */ 1311 #else 1312 build_get_pgde32(&p, K0, K1); /* get pgd in K1 */ 1313 #endif 1314 1315 #ifdef CONFIG_HUGETLB_PAGE 1316 build_is_huge_pte(&p, &r, K0, K1, label_tlb_huge_update); 1317 #endif 1318 1319 build_get_ptep(&p, K0, K1); 1320 build_update_entries(&p, K0, K1); 1321 build_tlb_write_entry(&p, &l, &r, tlb_random); 1322 uasm_l_leave(&l, p); 1323 uasm_i_eret(&p); /* return from trap */ 1324 } 1325 #ifdef CONFIG_HUGETLB_PAGE 1326 uasm_l_tlb_huge_update(&l, p); 1327 build_huge_update_entries(&p, htlb_info.huge_pte, K1); 1328 build_huge_tlb_write_entry(&p, &l, &r, K0, tlb_random, 1329 htlb_info.restore_scratch); 1330 #endif 1331 1332 #ifdef CONFIG_64BIT 1333 build_get_pgd_vmalloc64(&p, &l, &r, K0, K1, vmalloc_mode); 1334 #endif 1335 1336 /* 1337 * Overflow check: For the 64bit handler, we need at least one 1338 * free instruction slot for the wrap-around branch. In worst 1339 * case, if the intended insertion point is a delay slot, we 1340 * need three, with the second nop'ed and the third being 1341 * unused. 1342 */ 1343 /* Loongson2 ebase is different than r4k, we have more space */ 1344 #if defined(CONFIG_32BIT) || defined(CONFIG_CPU_LOONGSON2) 1345 if ((p - tlb_handler) > 64) 1346 panic("TLB refill handler space exceeded"); 1347 #else 1348 if (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 1) 1349 || (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 3) 1350 && uasm_insn_has_bdelay(relocs, 1351 tlb_handler + MIPS64_REFILL_INSNS - 3))) 1352 panic("TLB refill handler space exceeded"); 1353 #endif 1354 1355 /* 1356 * Now fold the handler in the TLB refill handler space. 1357 */ 1358 #if defined(CONFIG_32BIT) || defined(CONFIG_CPU_LOONGSON2) 1359 f = final_handler; 1360 /* Simplest case, just copy the handler. */ 1361 uasm_copy_handler(relocs, labels, tlb_handler, p, f); 1362 final_len = p - tlb_handler; 1363 #else /* CONFIG_64BIT */ 1364 f = final_handler + MIPS64_REFILL_INSNS; 1365 if ((p - tlb_handler) <= MIPS64_REFILL_INSNS) { 1366 /* Just copy the handler. */ 1367 uasm_copy_handler(relocs, labels, tlb_handler, p, f); 1368 final_len = p - tlb_handler; 1369 } else { 1370 #if defined(CONFIG_HUGETLB_PAGE) 1371 const enum label_id ls = label_tlb_huge_update; 1372 #else 1373 const enum label_id ls = label_vmalloc; 1374 #endif 1375 u32 *split; 1376 int ov = 0; 1377 int i; 1378 1379 for (i = 0; i < ARRAY_SIZE(labels) && labels[i].lab != ls; i++) 1380 ; 1381 BUG_ON(i == ARRAY_SIZE(labels)); 1382 split = labels[i].addr; 1383 1384 /* 1385 * See if we have overflown one way or the other. 1386 */ 1387 if (split > tlb_handler + MIPS64_REFILL_INSNS || 1388 split < p - MIPS64_REFILL_INSNS) 1389 ov = 1; 1390 1391 if (ov) { 1392 /* 1393 * Split two instructions before the end. One 1394 * for the branch and one for the instruction 1395 * in the delay slot. 1396 */ 1397 split = tlb_handler + MIPS64_REFILL_INSNS - 2; 1398 1399 /* 1400 * If the branch would fall in a delay slot, 1401 * we must back up an additional instruction 1402 * so that it is no longer in a delay slot. 1403 */ 1404 if (uasm_insn_has_bdelay(relocs, split - 1)) 1405 split--; 1406 } 1407 /* Copy first part of the handler. */ 1408 uasm_copy_handler(relocs, labels, tlb_handler, split, f); 1409 f += split - tlb_handler; 1410 1411 if (ov) { 1412 /* Insert branch. */ 1413 uasm_l_split(&l, final_handler); 1414 uasm_il_b(&f, &r, label_split); 1415 if (uasm_insn_has_bdelay(relocs, split)) 1416 uasm_i_nop(&f); 1417 else { 1418 uasm_copy_handler(relocs, labels, 1419 split, split + 1, f); 1420 uasm_move_labels(labels, f, f + 1, -1); 1421 f++; 1422 split++; 1423 } 1424 } 1425 1426 /* Copy the rest of the handler. */ 1427 uasm_copy_handler(relocs, labels, split, p, final_handler); 1428 final_len = (f - (final_handler + MIPS64_REFILL_INSNS)) + 1429 (p - split); 1430 } 1431 #endif /* CONFIG_64BIT */ 1432 1433 uasm_resolve_relocs(relocs, labels); 1434 pr_debug("Wrote TLB refill handler (%u instructions).\n", 1435 final_len); 1436 1437 memcpy((void *)ebase, final_handler, 0x100); 1438 1439 dump_handler((u32 *)ebase, 64); 1440 } 1441 1442 /* 1443 * 128 instructions for the fastpath handler is generous and should 1444 * never be exceeded. 1445 */ 1446 #define FASTPATH_SIZE 128 1447 1448 u32 handle_tlbl[FASTPATH_SIZE] __cacheline_aligned; 1449 u32 handle_tlbs[FASTPATH_SIZE] __cacheline_aligned; 1450 u32 handle_tlbm[FASTPATH_SIZE] __cacheline_aligned; 1451 #ifdef CONFIG_MIPS_PGD_C0_CONTEXT 1452 u32 tlbmiss_handler_setup_pgd[16] __cacheline_aligned; 1453 1454 static void __cpuinit build_r4000_setup_pgd(void) 1455 { 1456 const int a0 = 4; 1457 const int a1 = 5; 1458 u32 *p = tlbmiss_handler_setup_pgd; 1459 struct uasm_label *l = labels; 1460 struct uasm_reloc *r = relocs; 1461 1462 memset(tlbmiss_handler_setup_pgd, 0, sizeof(tlbmiss_handler_setup_pgd)); 1463 memset(labels, 0, sizeof(labels)); 1464 memset(relocs, 0, sizeof(relocs)); 1465 1466 pgd_reg = allocate_kscratch(); 1467 1468 if (pgd_reg == -1) { 1469 /* PGD << 11 in c0_Context */ 1470 /* 1471 * If it is a ckseg0 address, convert to a physical 1472 * address. Shifting right by 29 and adding 4 will 1473 * result in zero for these addresses. 1474 * 1475 */ 1476 UASM_i_SRA(&p, a1, a0, 29); 1477 UASM_i_ADDIU(&p, a1, a1, 4); 1478 uasm_il_bnez(&p, &r, a1, label_tlbl_goaround1); 1479 uasm_i_nop(&p); 1480 uasm_i_dinsm(&p, a0, 0, 29, 64 - 29); 1481 uasm_l_tlbl_goaround1(&l, p); 1482 UASM_i_SLL(&p, a0, a0, 11); 1483 uasm_i_jr(&p, 31); 1484 UASM_i_MTC0(&p, a0, C0_CONTEXT); 1485 } else { 1486 /* PGD in c0_KScratch */ 1487 uasm_i_jr(&p, 31); 1488 UASM_i_MTC0(&p, a0, 31, pgd_reg); 1489 } 1490 if (p - tlbmiss_handler_setup_pgd > ARRAY_SIZE(tlbmiss_handler_setup_pgd)) 1491 panic("tlbmiss_handler_setup_pgd space exceeded"); 1492 uasm_resolve_relocs(relocs, labels); 1493 pr_debug("Wrote tlbmiss_handler_setup_pgd (%u instructions).\n", 1494 (unsigned int)(p - tlbmiss_handler_setup_pgd)); 1495 1496 dump_handler(tlbmiss_handler_setup_pgd, 1497 ARRAY_SIZE(tlbmiss_handler_setup_pgd)); 1498 } 1499 #endif 1500 1501 static void __cpuinit 1502 iPTE_LW(u32 **p, unsigned int pte, unsigned int ptr) 1503 { 1504 #ifdef CONFIG_SMP 1505 # ifdef CONFIG_64BIT_PHYS_ADDR 1506 if (cpu_has_64bits) 1507 uasm_i_lld(p, pte, 0, ptr); 1508 else 1509 # endif 1510 UASM_i_LL(p, pte, 0, ptr); 1511 #else 1512 # ifdef CONFIG_64BIT_PHYS_ADDR 1513 if (cpu_has_64bits) 1514 uasm_i_ld(p, pte, 0, ptr); 1515 else 1516 # endif 1517 UASM_i_LW(p, pte, 0, ptr); 1518 #endif 1519 } 1520 1521 static void __cpuinit 1522 iPTE_SW(u32 **p, struct uasm_reloc **r, unsigned int pte, unsigned int ptr, 1523 unsigned int mode) 1524 { 1525 #ifdef CONFIG_64BIT_PHYS_ADDR 1526 unsigned int hwmode = mode & (_PAGE_VALID | _PAGE_DIRTY); 1527 #endif 1528 1529 uasm_i_ori(p, pte, pte, mode); 1530 #ifdef CONFIG_SMP 1531 # ifdef CONFIG_64BIT_PHYS_ADDR 1532 if (cpu_has_64bits) 1533 uasm_i_scd(p, pte, 0, ptr); 1534 else 1535 # endif 1536 UASM_i_SC(p, pte, 0, ptr); 1537 1538 if (r10000_llsc_war()) 1539 uasm_il_beqzl(p, r, pte, label_smp_pgtable_change); 1540 else 1541 uasm_il_beqz(p, r, pte, label_smp_pgtable_change); 1542 1543 # ifdef CONFIG_64BIT_PHYS_ADDR 1544 if (!cpu_has_64bits) { 1545 /* no uasm_i_nop needed */ 1546 uasm_i_ll(p, pte, sizeof(pte_t) / 2, ptr); 1547 uasm_i_ori(p, pte, pte, hwmode); 1548 uasm_i_sc(p, pte, sizeof(pte_t) / 2, ptr); 1549 uasm_il_beqz(p, r, pte, label_smp_pgtable_change); 1550 /* no uasm_i_nop needed */ 1551 uasm_i_lw(p, pte, 0, ptr); 1552 } else 1553 uasm_i_nop(p); 1554 # else 1555 uasm_i_nop(p); 1556 # endif 1557 #else 1558 # ifdef CONFIG_64BIT_PHYS_ADDR 1559 if (cpu_has_64bits) 1560 uasm_i_sd(p, pte, 0, ptr); 1561 else 1562 # endif 1563 UASM_i_SW(p, pte, 0, ptr); 1564 1565 # ifdef CONFIG_64BIT_PHYS_ADDR 1566 if (!cpu_has_64bits) { 1567 uasm_i_lw(p, pte, sizeof(pte_t) / 2, ptr); 1568 uasm_i_ori(p, pte, pte, hwmode); 1569 uasm_i_sw(p, pte, sizeof(pte_t) / 2, ptr); 1570 uasm_i_lw(p, pte, 0, ptr); 1571 } 1572 # endif 1573 #endif 1574 } 1575 1576 /* 1577 * Check if PTE is present, if not then jump to LABEL. PTR points to 1578 * the page table where this PTE is located, PTE will be re-loaded 1579 * with it's original value. 1580 */ 1581 static void __cpuinit 1582 build_pte_present(u32 **p, struct uasm_reloc **r, 1583 int pte, int ptr, int scratch, enum label_id lid) 1584 { 1585 int t = scratch >= 0 ? scratch : pte; 1586 1587 if (cpu_has_rixi) { 1588 if (use_bbit_insns()) { 1589 uasm_il_bbit0(p, r, pte, ilog2(_PAGE_PRESENT), lid); 1590 uasm_i_nop(p); 1591 } else { 1592 uasm_i_andi(p, t, pte, _PAGE_PRESENT); 1593 uasm_il_beqz(p, r, t, lid); 1594 if (pte == t) 1595 /* You lose the SMP race :-(*/ 1596 iPTE_LW(p, pte, ptr); 1597 } 1598 } else { 1599 uasm_i_andi(p, t, pte, _PAGE_PRESENT | _PAGE_READ); 1600 uasm_i_xori(p, t, t, _PAGE_PRESENT | _PAGE_READ); 1601 uasm_il_bnez(p, r, t, lid); 1602 if (pte == t) 1603 /* You lose the SMP race :-(*/ 1604 iPTE_LW(p, pte, ptr); 1605 } 1606 } 1607 1608 /* Make PTE valid, store result in PTR. */ 1609 static void __cpuinit 1610 build_make_valid(u32 **p, struct uasm_reloc **r, unsigned int pte, 1611 unsigned int ptr) 1612 { 1613 unsigned int mode = _PAGE_VALID | _PAGE_ACCESSED; 1614 1615 iPTE_SW(p, r, pte, ptr, mode); 1616 } 1617 1618 /* 1619 * Check if PTE can be written to, if not branch to LABEL. Regardless 1620 * restore PTE with value from PTR when done. 1621 */ 1622 static void __cpuinit 1623 build_pte_writable(u32 **p, struct uasm_reloc **r, 1624 unsigned int pte, unsigned int ptr, int scratch, 1625 enum label_id lid) 1626 { 1627 int t = scratch >= 0 ? scratch : pte; 1628 1629 uasm_i_andi(p, t, pte, _PAGE_PRESENT | _PAGE_WRITE); 1630 uasm_i_xori(p, t, t, _PAGE_PRESENT | _PAGE_WRITE); 1631 uasm_il_bnez(p, r, t, lid); 1632 if (pte == t) 1633 /* You lose the SMP race :-(*/ 1634 iPTE_LW(p, pte, ptr); 1635 else 1636 uasm_i_nop(p); 1637 } 1638 1639 /* Make PTE writable, update software status bits as well, then store 1640 * at PTR. 1641 */ 1642 static void __cpuinit 1643 build_make_write(u32 **p, struct uasm_reloc **r, unsigned int pte, 1644 unsigned int ptr) 1645 { 1646 unsigned int mode = (_PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID 1647 | _PAGE_DIRTY); 1648 1649 iPTE_SW(p, r, pte, ptr, mode); 1650 } 1651 1652 /* 1653 * Check if PTE can be modified, if not branch to LABEL. Regardless 1654 * restore PTE with value from PTR when done. 1655 */ 1656 static void __cpuinit 1657 build_pte_modifiable(u32 **p, struct uasm_reloc **r, 1658 unsigned int pte, unsigned int ptr, int scratch, 1659 enum label_id lid) 1660 { 1661 if (use_bbit_insns()) { 1662 uasm_il_bbit0(p, r, pte, ilog2(_PAGE_WRITE), lid); 1663 uasm_i_nop(p); 1664 } else { 1665 int t = scratch >= 0 ? scratch : pte; 1666 uasm_i_andi(p, t, pte, _PAGE_WRITE); 1667 uasm_il_beqz(p, r, t, lid); 1668 if (pte == t) 1669 /* You lose the SMP race :-(*/ 1670 iPTE_LW(p, pte, ptr); 1671 } 1672 } 1673 1674 #ifndef CONFIG_MIPS_PGD_C0_CONTEXT 1675 1676 1677 /* 1678 * R3000 style TLB load/store/modify handlers. 1679 */ 1680 1681 /* 1682 * This places the pte into ENTRYLO0 and writes it with tlbwi. 1683 * Then it returns. 1684 */ 1685 static void __cpuinit 1686 build_r3000_pte_reload_tlbwi(u32 **p, unsigned int pte, unsigned int tmp) 1687 { 1688 uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */ 1689 uasm_i_mfc0(p, tmp, C0_EPC); /* cp0 delay */ 1690 uasm_i_tlbwi(p); 1691 uasm_i_jr(p, tmp); 1692 uasm_i_rfe(p); /* branch delay */ 1693 } 1694 1695 /* 1696 * This places the pte into ENTRYLO0 and writes it with tlbwi 1697 * or tlbwr as appropriate. This is because the index register 1698 * may have the probe fail bit set as a result of a trap on a 1699 * kseg2 access, i.e. without refill. Then it returns. 1700 */ 1701 static void __cpuinit 1702 build_r3000_tlb_reload_write(u32 **p, struct uasm_label **l, 1703 struct uasm_reloc **r, unsigned int pte, 1704 unsigned int tmp) 1705 { 1706 uasm_i_mfc0(p, tmp, C0_INDEX); 1707 uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */ 1708 uasm_il_bltz(p, r, tmp, label_r3000_write_probe_fail); /* cp0 delay */ 1709 uasm_i_mfc0(p, tmp, C0_EPC); /* branch delay */ 1710 uasm_i_tlbwi(p); /* cp0 delay */ 1711 uasm_i_jr(p, tmp); 1712 uasm_i_rfe(p); /* branch delay */ 1713 uasm_l_r3000_write_probe_fail(l, *p); 1714 uasm_i_tlbwr(p); /* cp0 delay */ 1715 uasm_i_jr(p, tmp); 1716 uasm_i_rfe(p); /* branch delay */ 1717 } 1718 1719 static void __cpuinit 1720 build_r3000_tlbchange_handler_head(u32 **p, unsigned int pte, 1721 unsigned int ptr) 1722 { 1723 long pgdc = (long)pgd_current; 1724 1725 uasm_i_mfc0(p, pte, C0_BADVADDR); 1726 uasm_i_lui(p, ptr, uasm_rel_hi(pgdc)); /* cp0 delay */ 1727 uasm_i_lw(p, ptr, uasm_rel_lo(pgdc), ptr); 1728 uasm_i_srl(p, pte, pte, 22); /* load delay */ 1729 uasm_i_sll(p, pte, pte, 2); 1730 uasm_i_addu(p, ptr, ptr, pte); 1731 uasm_i_mfc0(p, pte, C0_CONTEXT); 1732 uasm_i_lw(p, ptr, 0, ptr); /* cp0 delay */ 1733 uasm_i_andi(p, pte, pte, 0xffc); /* load delay */ 1734 uasm_i_addu(p, ptr, ptr, pte); 1735 uasm_i_lw(p, pte, 0, ptr); 1736 uasm_i_tlbp(p); /* load delay */ 1737 } 1738 1739 static void __cpuinit build_r3000_tlb_load_handler(void) 1740 { 1741 u32 *p = handle_tlbl; 1742 struct uasm_label *l = labels; 1743 struct uasm_reloc *r = relocs; 1744 1745 memset(handle_tlbl, 0, sizeof(handle_tlbl)); 1746 memset(labels, 0, sizeof(labels)); 1747 memset(relocs, 0, sizeof(relocs)); 1748 1749 build_r3000_tlbchange_handler_head(&p, K0, K1); 1750 build_pte_present(&p, &r, K0, K1, -1, label_nopage_tlbl); 1751 uasm_i_nop(&p); /* load delay */ 1752 build_make_valid(&p, &r, K0, K1); 1753 build_r3000_tlb_reload_write(&p, &l, &r, K0, K1); 1754 1755 uasm_l_nopage_tlbl(&l, p); 1756 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff); 1757 uasm_i_nop(&p); 1758 1759 if ((p - handle_tlbl) > FASTPATH_SIZE) 1760 panic("TLB load handler fastpath space exceeded"); 1761 1762 uasm_resolve_relocs(relocs, labels); 1763 pr_debug("Wrote TLB load handler fastpath (%u instructions).\n", 1764 (unsigned int)(p - handle_tlbl)); 1765 1766 dump_handler(handle_tlbl, ARRAY_SIZE(handle_tlbl)); 1767 } 1768 1769 static void __cpuinit build_r3000_tlb_store_handler(void) 1770 { 1771 u32 *p = handle_tlbs; 1772 struct uasm_label *l = labels; 1773 struct uasm_reloc *r = relocs; 1774 1775 memset(handle_tlbs, 0, sizeof(handle_tlbs)); 1776 memset(labels, 0, sizeof(labels)); 1777 memset(relocs, 0, sizeof(relocs)); 1778 1779 build_r3000_tlbchange_handler_head(&p, K0, K1); 1780 build_pte_writable(&p, &r, K0, K1, -1, label_nopage_tlbs); 1781 uasm_i_nop(&p); /* load delay */ 1782 build_make_write(&p, &r, K0, K1); 1783 build_r3000_tlb_reload_write(&p, &l, &r, K0, K1); 1784 1785 uasm_l_nopage_tlbs(&l, p); 1786 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); 1787 uasm_i_nop(&p); 1788 1789 if ((p - handle_tlbs) > FASTPATH_SIZE) 1790 panic("TLB store handler fastpath space exceeded"); 1791 1792 uasm_resolve_relocs(relocs, labels); 1793 pr_debug("Wrote TLB store handler fastpath (%u instructions).\n", 1794 (unsigned int)(p - handle_tlbs)); 1795 1796 dump_handler(handle_tlbs, ARRAY_SIZE(handle_tlbs)); 1797 } 1798 1799 static void __cpuinit build_r3000_tlb_modify_handler(void) 1800 { 1801 u32 *p = handle_tlbm; 1802 struct uasm_label *l = labels; 1803 struct uasm_reloc *r = relocs; 1804 1805 memset(handle_tlbm, 0, sizeof(handle_tlbm)); 1806 memset(labels, 0, sizeof(labels)); 1807 memset(relocs, 0, sizeof(relocs)); 1808 1809 build_r3000_tlbchange_handler_head(&p, K0, K1); 1810 build_pte_modifiable(&p, &r, K0, K1, -1, label_nopage_tlbm); 1811 uasm_i_nop(&p); /* load delay */ 1812 build_make_write(&p, &r, K0, K1); 1813 build_r3000_pte_reload_tlbwi(&p, K0, K1); 1814 1815 uasm_l_nopage_tlbm(&l, p); 1816 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); 1817 uasm_i_nop(&p); 1818 1819 if ((p - handle_tlbm) > FASTPATH_SIZE) 1820 panic("TLB modify handler fastpath space exceeded"); 1821 1822 uasm_resolve_relocs(relocs, labels); 1823 pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n", 1824 (unsigned int)(p - handle_tlbm)); 1825 1826 dump_handler(handle_tlbm, ARRAY_SIZE(handle_tlbm)); 1827 } 1828 #endif /* CONFIG_MIPS_PGD_C0_CONTEXT */ 1829 1830 /* 1831 * R4000 style TLB load/store/modify handlers. 1832 */ 1833 static struct work_registers __cpuinit 1834 build_r4000_tlbchange_handler_head(u32 **p, struct uasm_label **l, 1835 struct uasm_reloc **r) 1836 { 1837 struct work_registers wr = build_get_work_registers(p); 1838 1839 #ifdef CONFIG_64BIT 1840 build_get_pmde64(p, l, r, wr.r1, wr.r2); /* get pmd in ptr */ 1841 #else 1842 build_get_pgde32(p, wr.r1, wr.r2); /* get pgd in ptr */ 1843 #endif 1844 1845 #ifdef CONFIG_HUGETLB_PAGE 1846 /* 1847 * For huge tlb entries, pmd doesn't contain an address but 1848 * instead contains the tlb pte. Check the PAGE_HUGE bit and 1849 * see if we need to jump to huge tlb processing. 1850 */ 1851 build_is_huge_pte(p, r, wr.r1, wr.r2, label_tlb_huge_update); 1852 #endif 1853 1854 UASM_i_MFC0(p, wr.r1, C0_BADVADDR); 1855 UASM_i_LW(p, wr.r2, 0, wr.r2); 1856 UASM_i_SRL(p, wr.r1, wr.r1, PAGE_SHIFT + PTE_ORDER - PTE_T_LOG2); 1857 uasm_i_andi(p, wr.r1, wr.r1, (PTRS_PER_PTE - 1) << PTE_T_LOG2); 1858 UASM_i_ADDU(p, wr.r2, wr.r2, wr.r1); 1859 1860 #ifdef CONFIG_SMP 1861 uasm_l_smp_pgtable_change(l, *p); 1862 #endif 1863 iPTE_LW(p, wr.r1, wr.r2); /* get even pte */ 1864 if (!m4kc_tlbp_war()) 1865 build_tlb_probe_entry(p); 1866 return wr; 1867 } 1868 1869 static void __cpuinit 1870 build_r4000_tlbchange_handler_tail(u32 **p, struct uasm_label **l, 1871 struct uasm_reloc **r, unsigned int tmp, 1872 unsigned int ptr) 1873 { 1874 uasm_i_ori(p, ptr, ptr, sizeof(pte_t)); 1875 uasm_i_xori(p, ptr, ptr, sizeof(pte_t)); 1876 build_update_entries(p, tmp, ptr); 1877 build_tlb_write_entry(p, l, r, tlb_indexed); 1878 uasm_l_leave(l, *p); 1879 build_restore_work_registers(p); 1880 uasm_i_eret(p); /* return from trap */ 1881 1882 #ifdef CONFIG_64BIT 1883 build_get_pgd_vmalloc64(p, l, r, tmp, ptr, not_refill); 1884 #endif 1885 } 1886 1887 static void __cpuinit build_r4000_tlb_load_handler(void) 1888 { 1889 u32 *p = handle_tlbl; 1890 struct uasm_label *l = labels; 1891 struct uasm_reloc *r = relocs; 1892 struct work_registers wr; 1893 1894 memset(handle_tlbl, 0, sizeof(handle_tlbl)); 1895 memset(labels, 0, sizeof(labels)); 1896 memset(relocs, 0, sizeof(relocs)); 1897 1898 if (bcm1250_m3_war()) { 1899 unsigned int segbits = 44; 1900 1901 uasm_i_dmfc0(&p, K0, C0_BADVADDR); 1902 uasm_i_dmfc0(&p, K1, C0_ENTRYHI); 1903 uasm_i_xor(&p, K0, K0, K1); 1904 uasm_i_dsrl_safe(&p, K1, K0, 62); 1905 uasm_i_dsrl_safe(&p, K0, K0, 12 + 1); 1906 uasm_i_dsll_safe(&p, K0, K0, 64 + 12 + 1 - segbits); 1907 uasm_i_or(&p, K0, K0, K1); 1908 uasm_il_bnez(&p, &r, K0, label_leave); 1909 /* No need for uasm_i_nop */ 1910 } 1911 1912 wr = build_r4000_tlbchange_handler_head(&p, &l, &r); 1913 build_pte_present(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbl); 1914 if (m4kc_tlbp_war()) 1915 build_tlb_probe_entry(&p); 1916 1917 if (cpu_has_rixi) { 1918 /* 1919 * If the page is not _PAGE_VALID, RI or XI could not 1920 * have triggered it. Skip the expensive test.. 1921 */ 1922 if (use_bbit_insns()) { 1923 uasm_il_bbit0(&p, &r, wr.r1, ilog2(_PAGE_VALID), 1924 label_tlbl_goaround1); 1925 } else { 1926 uasm_i_andi(&p, wr.r3, wr.r1, _PAGE_VALID); 1927 uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround1); 1928 } 1929 uasm_i_nop(&p); 1930 1931 uasm_i_tlbr(&p); 1932 /* Examine entrylo 0 or 1 based on ptr. */ 1933 if (use_bbit_insns()) { 1934 uasm_i_bbit0(&p, wr.r2, ilog2(sizeof(pte_t)), 8); 1935 } else { 1936 uasm_i_andi(&p, wr.r3, wr.r2, sizeof(pte_t)); 1937 uasm_i_beqz(&p, wr.r3, 8); 1938 } 1939 /* load it in the delay slot*/ 1940 UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO0); 1941 /* load it if ptr is odd */ 1942 UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO1); 1943 /* 1944 * If the entryLo (now in wr.r3) is valid (bit 1), RI or 1945 * XI must have triggered it. 1946 */ 1947 if (use_bbit_insns()) { 1948 uasm_il_bbit1(&p, &r, wr.r3, 1, label_nopage_tlbl); 1949 uasm_i_nop(&p); 1950 uasm_l_tlbl_goaround1(&l, p); 1951 } else { 1952 uasm_i_andi(&p, wr.r3, wr.r3, 2); 1953 uasm_il_bnez(&p, &r, wr.r3, label_nopage_tlbl); 1954 uasm_i_nop(&p); 1955 } 1956 uasm_l_tlbl_goaround1(&l, p); 1957 } 1958 build_make_valid(&p, &r, wr.r1, wr.r2); 1959 build_r4000_tlbchange_handler_tail(&p, &l, &r, wr.r1, wr.r2); 1960 1961 #ifdef CONFIG_HUGETLB_PAGE 1962 /* 1963 * This is the entry point when build_r4000_tlbchange_handler_head 1964 * spots a huge page. 1965 */ 1966 uasm_l_tlb_huge_update(&l, p); 1967 iPTE_LW(&p, wr.r1, wr.r2); 1968 build_pte_present(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbl); 1969 build_tlb_probe_entry(&p); 1970 1971 if (cpu_has_rixi) { 1972 /* 1973 * If the page is not _PAGE_VALID, RI or XI could not 1974 * have triggered it. Skip the expensive test.. 1975 */ 1976 if (use_bbit_insns()) { 1977 uasm_il_bbit0(&p, &r, wr.r1, ilog2(_PAGE_VALID), 1978 label_tlbl_goaround2); 1979 } else { 1980 uasm_i_andi(&p, wr.r3, wr.r1, _PAGE_VALID); 1981 uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround2); 1982 } 1983 uasm_i_nop(&p); 1984 1985 uasm_i_tlbr(&p); 1986 /* Examine entrylo 0 or 1 based on ptr. */ 1987 if (use_bbit_insns()) { 1988 uasm_i_bbit0(&p, wr.r2, ilog2(sizeof(pte_t)), 8); 1989 } else { 1990 uasm_i_andi(&p, wr.r3, wr.r2, sizeof(pte_t)); 1991 uasm_i_beqz(&p, wr.r3, 8); 1992 } 1993 /* load it in the delay slot*/ 1994 UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO0); 1995 /* load it if ptr is odd */ 1996 UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO1); 1997 /* 1998 * If the entryLo (now in wr.r3) is valid (bit 1), RI or 1999 * XI must have triggered it. 2000 */ 2001 if (use_bbit_insns()) { 2002 uasm_il_bbit0(&p, &r, wr.r3, 1, label_tlbl_goaround2); 2003 } else { 2004 uasm_i_andi(&p, wr.r3, wr.r3, 2); 2005 uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround2); 2006 } 2007 if (PM_DEFAULT_MASK == 0) 2008 uasm_i_nop(&p); 2009 /* 2010 * We clobbered C0_PAGEMASK, restore it. On the other branch 2011 * it is restored in build_huge_tlb_write_entry. 2012 */ 2013 build_restore_pagemask(&p, &r, wr.r3, label_nopage_tlbl, 0); 2014 2015 uasm_l_tlbl_goaround2(&l, p); 2016 } 2017 uasm_i_ori(&p, wr.r1, wr.r1, (_PAGE_ACCESSED | _PAGE_VALID)); 2018 build_huge_handler_tail(&p, &r, &l, wr.r1, wr.r2); 2019 #endif 2020 2021 uasm_l_nopage_tlbl(&l, p); 2022 build_restore_work_registers(&p); 2023 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff); 2024 uasm_i_nop(&p); 2025 2026 if ((p - handle_tlbl) > FASTPATH_SIZE) 2027 panic("TLB load handler fastpath space exceeded"); 2028 2029 uasm_resolve_relocs(relocs, labels); 2030 pr_debug("Wrote TLB load handler fastpath (%u instructions).\n", 2031 (unsigned int)(p - handle_tlbl)); 2032 2033 dump_handler(handle_tlbl, ARRAY_SIZE(handle_tlbl)); 2034 } 2035 2036 static void __cpuinit build_r4000_tlb_store_handler(void) 2037 { 2038 u32 *p = handle_tlbs; 2039 struct uasm_label *l = labels; 2040 struct uasm_reloc *r = relocs; 2041 struct work_registers wr; 2042 2043 memset(handle_tlbs, 0, sizeof(handle_tlbs)); 2044 memset(labels, 0, sizeof(labels)); 2045 memset(relocs, 0, sizeof(relocs)); 2046 2047 wr = build_r4000_tlbchange_handler_head(&p, &l, &r); 2048 build_pte_writable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbs); 2049 if (m4kc_tlbp_war()) 2050 build_tlb_probe_entry(&p); 2051 build_make_write(&p, &r, wr.r1, wr.r2); 2052 build_r4000_tlbchange_handler_tail(&p, &l, &r, wr.r1, wr.r2); 2053 2054 #ifdef CONFIG_HUGETLB_PAGE 2055 /* 2056 * This is the entry point when 2057 * build_r4000_tlbchange_handler_head spots a huge page. 2058 */ 2059 uasm_l_tlb_huge_update(&l, p); 2060 iPTE_LW(&p, wr.r1, wr.r2); 2061 build_pte_writable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbs); 2062 build_tlb_probe_entry(&p); 2063 uasm_i_ori(&p, wr.r1, wr.r1, 2064 _PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID | _PAGE_DIRTY); 2065 build_huge_handler_tail(&p, &r, &l, wr.r1, wr.r2); 2066 #endif 2067 2068 uasm_l_nopage_tlbs(&l, p); 2069 build_restore_work_registers(&p); 2070 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); 2071 uasm_i_nop(&p); 2072 2073 if ((p - handle_tlbs) > FASTPATH_SIZE) 2074 panic("TLB store handler fastpath space exceeded"); 2075 2076 uasm_resolve_relocs(relocs, labels); 2077 pr_debug("Wrote TLB store handler fastpath (%u instructions).\n", 2078 (unsigned int)(p - handle_tlbs)); 2079 2080 dump_handler(handle_tlbs, ARRAY_SIZE(handle_tlbs)); 2081 } 2082 2083 static void __cpuinit build_r4000_tlb_modify_handler(void) 2084 { 2085 u32 *p = handle_tlbm; 2086 struct uasm_label *l = labels; 2087 struct uasm_reloc *r = relocs; 2088 struct work_registers wr; 2089 2090 memset(handle_tlbm, 0, sizeof(handle_tlbm)); 2091 memset(labels, 0, sizeof(labels)); 2092 memset(relocs, 0, sizeof(relocs)); 2093 2094 wr = build_r4000_tlbchange_handler_head(&p, &l, &r); 2095 build_pte_modifiable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbm); 2096 if (m4kc_tlbp_war()) 2097 build_tlb_probe_entry(&p); 2098 /* Present and writable bits set, set accessed and dirty bits. */ 2099 build_make_write(&p, &r, wr.r1, wr.r2); 2100 build_r4000_tlbchange_handler_tail(&p, &l, &r, wr.r1, wr.r2); 2101 2102 #ifdef CONFIG_HUGETLB_PAGE 2103 /* 2104 * This is the entry point when 2105 * build_r4000_tlbchange_handler_head spots a huge page. 2106 */ 2107 uasm_l_tlb_huge_update(&l, p); 2108 iPTE_LW(&p, wr.r1, wr.r2); 2109 build_pte_modifiable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbm); 2110 build_tlb_probe_entry(&p); 2111 uasm_i_ori(&p, wr.r1, wr.r1, 2112 _PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID | _PAGE_DIRTY); 2113 build_huge_handler_tail(&p, &r, &l, wr.r1, wr.r2); 2114 #endif 2115 2116 uasm_l_nopage_tlbm(&l, p); 2117 build_restore_work_registers(&p); 2118 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); 2119 uasm_i_nop(&p); 2120 2121 if ((p - handle_tlbm) > FASTPATH_SIZE) 2122 panic("TLB modify handler fastpath space exceeded"); 2123 2124 uasm_resolve_relocs(relocs, labels); 2125 pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n", 2126 (unsigned int)(p - handle_tlbm)); 2127 2128 dump_handler(handle_tlbm, ARRAY_SIZE(handle_tlbm)); 2129 } 2130 2131 void __cpuinit build_tlb_refill_handler(void) 2132 { 2133 /* 2134 * The refill handler is generated per-CPU, multi-node systems 2135 * may have local storage for it. The other handlers are only 2136 * needed once. 2137 */ 2138 static int run_once = 0; 2139 2140 #ifdef CONFIG_64BIT 2141 check_for_high_segbits = current_cpu_data.vmbits > (PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3); 2142 #endif 2143 2144 switch (current_cpu_type()) { 2145 case CPU_R2000: 2146 case CPU_R3000: 2147 case CPU_R3000A: 2148 case CPU_R3081E: 2149 case CPU_TX3912: 2150 case CPU_TX3922: 2151 case CPU_TX3927: 2152 #ifndef CONFIG_MIPS_PGD_C0_CONTEXT 2153 build_r3000_tlb_refill_handler(); 2154 if (!run_once) { 2155 build_r3000_tlb_load_handler(); 2156 build_r3000_tlb_store_handler(); 2157 build_r3000_tlb_modify_handler(); 2158 run_once++; 2159 } 2160 #else 2161 panic("No R3000 TLB refill handler"); 2162 #endif 2163 break; 2164 2165 case CPU_R6000: 2166 case CPU_R6000A: 2167 panic("No R6000 TLB refill handler yet"); 2168 break; 2169 2170 case CPU_R8000: 2171 panic("No R8000 TLB refill handler yet"); 2172 break; 2173 2174 default: 2175 if (!run_once) { 2176 scratch_reg = allocate_kscratch(); 2177 #ifdef CONFIG_MIPS_PGD_C0_CONTEXT 2178 build_r4000_setup_pgd(); 2179 #endif 2180 build_r4000_tlb_load_handler(); 2181 build_r4000_tlb_store_handler(); 2182 build_r4000_tlb_modify_handler(); 2183 run_once++; 2184 } 2185 build_r4000_tlb_refill_handler(); 2186 } 2187 } 2188 2189 void __cpuinit flush_tlb_handlers(void) 2190 { 2191 local_flush_icache_range((unsigned long)handle_tlbl, 2192 (unsigned long)handle_tlbl + sizeof(handle_tlbl)); 2193 local_flush_icache_range((unsigned long)handle_tlbs, 2194 (unsigned long)handle_tlbs + sizeof(handle_tlbs)); 2195 local_flush_icache_range((unsigned long)handle_tlbm, 2196 (unsigned long)handle_tlbm + sizeof(handle_tlbm)); 2197 #ifdef CONFIG_MIPS_PGD_C0_CONTEXT 2198 local_flush_icache_range((unsigned long)tlbmiss_handler_setup_pgd, 2199 (unsigned long)tlbmiss_handler_setup_pgd + sizeof(handle_tlbm)); 2200 #endif 2201 } 2202