1/* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * Copyright (C) 1994 - 2000, 2001, 2003 Ralf Baechle 7 * Copyright (C) 1999, 2000 Silicon Graphics, Inc. 8 * Copyright (C) 2002, 2007 Maciej W. Rozycki 9 * Copyright (C) 2001, 2012 MIPS Technologies, Inc. All rights reserved. 10 */ 11#include <linux/init.h> 12 13#include <asm/asm.h> 14#include <asm/asmmacro.h> 15#include <asm/cacheops.h> 16#include <asm/irqflags.h> 17#include <asm/regdef.h> 18#include <asm/fpregdef.h> 19#include <asm/mipsregs.h> 20#include <asm/stackframe.h> 21#include <asm/sync.h> 22#include <asm/thread_info.h> 23 24 __INIT 25 26/* 27 * General exception vector for all other CPUs. 28 * 29 * Be careful when changing this, it has to be at most 128 bytes 30 * to fit into space reserved for the exception handler. 31 */ 32NESTED(except_vec3_generic, 0, sp) 33 .set push 34 .set noat 35 mfc0 k1, CP0_CAUSE 36 andi k1, k1, 0x7c 37#ifdef CONFIG_64BIT 38 dsll k1, k1, 1 39#endif 40 PTR_L k0, exception_handlers(k1) 41 jr k0 42 .set pop 43 END(except_vec3_generic) 44 45/* 46 * General exception handler for CPUs with virtual coherency exception. 47 * 48 * Be careful when changing this, it has to be at most 256 (as a special 49 * exception) bytes to fit into space reserved for the exception handler. 50 */ 51NESTED(except_vec3_r4000, 0, sp) 52 .set push 53 .set arch=r4000 54 .set noat 55 mfc0 k1, CP0_CAUSE 56 li k0, 31<<2 57 andi k1, k1, 0x7c 58 .set push 59 .set noreorder 60 .set nomacro 61 beq k1, k0, handle_vced 62 li k0, 14<<2 63 beq k1, k0, handle_vcei 64#ifdef CONFIG_64BIT 65 dsll k1, k1, 1 66#endif 67 .set pop 68 PTR_L k0, exception_handlers(k1) 69 jr k0 70 71 /* 72 * Big shit, we now may have two dirty primary cache lines for the same 73 * physical address. We can safely invalidate the line pointed to by 74 * c0_badvaddr because after return from this exception handler the 75 * load / store will be re-executed. 76 */ 77handle_vced: 78 MFC0 k0, CP0_BADVADDR 79 li k1, -4 # Is this ... 80 and k0, k1 # ... really needed? 81 mtc0 zero, CP0_TAGLO 82 cache Index_Store_Tag_D, (k0) 83 cache Hit_Writeback_Inv_SD, (k0) 84#ifdef CONFIG_PROC_FS 85 PTR_LA k0, vced_count 86 lw k1, (k0) 87 addiu k1, 1 88 sw k1, (k0) 89#endif 90 eret 91 92handle_vcei: 93 MFC0 k0, CP0_BADVADDR 94 cache Hit_Writeback_Inv_SD, (k0) # also cleans pi 95#ifdef CONFIG_PROC_FS 96 PTR_LA k0, vcei_count 97 lw k1, (k0) 98 addiu k1, 1 99 sw k1, (k0) 100#endif 101 eret 102 .set pop 103 END(except_vec3_r4000) 104 105 __FINIT 106 107 .section .cpuidle.text,"ax" 108 /* Align to 32 bytes for the maximum idle interrupt region size. */ 109 .align 5 110LEAF(r4k_wait) 111 /* Keep the ISA bit clear for calculations on local labels here. */ 1120: .fill 0 113 /* Start of idle interrupt region. */ 114 local_irq_enable 115 /* 116 * If an interrupt lands here, before going idle on the next 117 * instruction, we must *NOT* go idle since the interrupt could 118 * have set TIF_NEED_RESCHED or caused a timer to need resched. 119 * Fall through -- see skipover_handler below -- and have the 120 * idle loop take care of things. 121 */ 1221: .fill 0 123 /* The R2 EI/EHB sequence takes 8 bytes, otherwise pad up. */ 124 .if 1b - 0b > 32 125 .error "overlong idle interrupt region" 126 .elseif 1b - 0b > 8 127 .align 4 128 .endif 1292: .fill 0 130 .equ r4k_wait_idle_size, 2b - 0b 131 /* End of idle interrupt region; size has to be a power of 2. */ 132 .set MIPS_ISA_ARCH_LEVEL_RAW 133r4k_wait_insn: 134 wait 135r4k_wait_exit: 136 .set mips0 137 local_irq_disable 138 jr ra 139 END(r4k_wait) 140 .previous 141 142 .macro BUILD_SKIPOVER_PROLOGUE handler 143 FEXPORT(skipover_\handler) 144 .set push 145 .set noat 146 MFC0 k0, CP0_EPC 147 /* Subtract/add 2 to let the ISA bit propagate through the mask. */ 148 PTR_LA k1, r4k_wait_insn - 2 149 ori k0, r4k_wait_idle_size - 2 150 .set noreorder 151 bne k0, k1, \handler 152 PTR_ADDIU k0, r4k_wait_exit - r4k_wait_insn + 2 153 .set reorder 154 MTC0 k0, CP0_EPC 155 .set pop 156 .endm 157 158 .align 5 159BUILD_SKIPOVER_PROLOGUE handle_int 160NESTED(handle_int, PT_SIZE, sp) 161 .cfi_signal_frame 162#ifdef CONFIG_TRACE_IRQFLAGS 163 /* 164 * Check to see if the interrupted code has just disabled 165 * interrupts and ignore this interrupt for now if so. 166 * 167 * local_irq_disable() disables interrupts and then calls 168 * trace_hardirqs_off() to track the state. If an interrupt is taken 169 * after interrupts are disabled but before the state is updated 170 * it will appear to restore_all that it is incorrectly returning with 171 * interrupts disabled 172 */ 173 .set push 174 .set noat 175 mfc0 k0, CP0_STATUS 176#if defined(CONFIG_CPU_R3000) 177 and k0, ST0_IEP 178 bnez k0, 1f 179 180 mfc0 k0, CP0_EPC 181 .set noreorder 182 j k0 183 rfe 184#else 185 and k0, ST0_IE 186 bnez k0, 1f 187 188 eret 189#endif 1901: 191 .set pop 192#endif 193 SAVE_ALL docfi=1 194 CLI 195 TRACE_IRQS_OFF 196 197 LONG_L s0, TI_REGS($28) 198 LONG_S sp, TI_REGS($28) 199 200 /* 201 * SAVE_ALL ensures we are using a valid kernel stack for the thread. 202 * Check if we are already using the IRQ stack. 203 */ 204 move s1, sp # Preserve the sp 205 206 /* Get IRQ stack for this CPU */ 207 ASM_CPUID_MFC0 k0, ASM_SMP_CPUID_REG 208#if defined(CONFIG_32BIT) || defined(KBUILD_64BIT_SYM32) 209 lui k1, %hi(irq_stack) 210#else 211 lui k1, %highest(irq_stack) 212 daddiu k1, %higher(irq_stack) 213 dsll k1, 16 214 daddiu k1, %hi(irq_stack) 215 dsll k1, 16 216#endif 217 LONG_SRL k0, SMP_CPUID_PTRSHIFT 218 LONG_ADDU k1, k0 219 LONG_L t0, %lo(irq_stack)(k1) 220 221 # Check if already on IRQ stack 222 PTR_LI t1, ~(_THREAD_SIZE-1) 223 and t1, t1, sp 224 beq t0, t1, 2f 225 226 /* Switch to IRQ stack */ 227 li t1, _IRQ_STACK_START 228 PTR_ADD sp, t0, t1 229 230 /* Save task's sp on IRQ stack so that unwinding can follow it */ 231 LONG_S s1, 0(sp) 2322: 233 jal plat_irq_dispatch 234 235 /* Restore sp */ 236 move sp, s1 237 238 j ret_from_irq 239#ifdef CONFIG_CPU_MICROMIPS 240 nop 241#endif 242 END(handle_int) 243 244 __INIT 245 246/* 247 * Special interrupt vector for MIPS64 ISA & embedded MIPS processors. 248 * This is a dedicated interrupt exception vector which reduces the 249 * interrupt processing overhead. The jump instruction will be replaced 250 * at the initialization time. 251 * 252 * Be careful when changing this, it has to be at most 128 bytes 253 * to fit into space reserved for the exception handler. 254 */ 255NESTED(except_vec4, 0, sp) 2561: j 1b /* Dummy, will be replaced */ 257 END(except_vec4) 258 259/* 260 * EJTAG debug exception handler. 261 * The EJTAG debug exception entry point is 0xbfc00480, which 262 * normally is in the boot PROM, so the boot PROM must do an 263 * unconditional jump to this vector. 264 */ 265NESTED(except_vec_ejtag_debug, 0, sp) 266 j ejtag_debug_handler 267#ifdef CONFIG_CPU_MICROMIPS 268 nop 269#endif 270 END(except_vec_ejtag_debug) 271 272 __FINIT 273 274/* 275 * Vectored interrupt handler. 276 * This prototype is copied to ebase + n*IntCtl.VS and patched 277 * to invoke the handler 278 */ 279BUILD_SKIPOVER_PROLOGUE except_vec_vi 280NESTED(except_vec_vi, 0, sp) 281 SAVE_SOME docfi=1 282 SAVE_AT docfi=1 283 .set push 284 .set noreorder 285 PTR_LA v1, except_vec_vi_handler 286 jr v1 287FEXPORT(except_vec_vi_ori) 288 ori v0, zero, 0 /* Offset in vi_handlers[] */ 289 .set pop 290 END(except_vec_vi) 291EXPORT(except_vec_vi_end) 292 293/* 294 * Common Vectored Interrupt code 295 * Complete the register saves and invoke the handler, $v0 holds 296 * offset into vi_handlers[] 297 */ 298NESTED(except_vec_vi_handler, 0, sp) 299 SAVE_TEMP 300 SAVE_STATIC 301 CLI 302#ifdef CONFIG_TRACE_IRQFLAGS 303 move s0, v0 304 TRACE_IRQS_OFF 305 move v0, s0 306#endif 307 308 LONG_L s0, TI_REGS($28) 309 LONG_S sp, TI_REGS($28) 310 311 /* 312 * SAVE_ALL ensures we are using a valid kernel stack for the thread. 313 * Check if we are already using the IRQ stack. 314 */ 315 move s1, sp # Preserve the sp 316 317 /* Get IRQ stack for this CPU */ 318 ASM_CPUID_MFC0 k0, ASM_SMP_CPUID_REG 319#if defined(CONFIG_32BIT) || defined(KBUILD_64BIT_SYM32) 320 lui k1, %hi(irq_stack) 321#else 322 lui k1, %highest(irq_stack) 323 daddiu k1, %higher(irq_stack) 324 dsll k1, 16 325 daddiu k1, %hi(irq_stack) 326 dsll k1, 16 327#endif 328 LONG_SRL k0, SMP_CPUID_PTRSHIFT 329 LONG_ADDU k1, k0 330 LONG_L t0, %lo(irq_stack)(k1) 331 332 # Check if already on IRQ stack 333 PTR_LI t1, ~(_THREAD_SIZE-1) 334 and t1, t1, sp 335 beq t0, t1, 2f 336 337 /* Switch to IRQ stack */ 338 li t1, _IRQ_STACK_START 339 PTR_ADD sp, t0, t1 340 341 /* Save task's sp on IRQ stack so that unwinding can follow it */ 342 LONG_S s1, 0(sp) 3432: 344 PTR_L v0, vi_handlers(v0) 345 jalr v0 346 347 /* Restore sp */ 348 move sp, s1 349 350 j ret_from_irq 351 END(except_vec_vi_handler) 352 353/* 354 * EJTAG debug exception handler. 355 */ 356NESTED(ejtag_debug_handler, PT_SIZE, sp) 357 .set push 358 .set noat 359 MTC0 k0, CP0_DESAVE 360 mfc0 k0, CP0_DEBUG 361 362 andi k0, k0, MIPS_DEBUG_DBP # Check for SDBBP. 363 beqz k0, ejtag_return 364 365#ifdef CONFIG_SMP 3661: PTR_LA k0, ejtag_debug_buffer_spinlock 367 __SYNC(full, loongson3_war) 3682: ll k0, 0(k0) 369 bnez k0, 2b 370 PTR_LA k0, ejtag_debug_buffer_spinlock 371 sc k0, 0(k0) 372 beqz k0, 1b 373# ifdef CONFIG_WEAK_REORDERING_BEYOND_LLSC 374 sync 375# endif 376 377 PTR_LA k0, ejtag_debug_buffer 378 LONG_S k1, 0(k0) 379 380 ASM_CPUID_MFC0 k1, ASM_SMP_CPUID_REG 381 PTR_SRL k1, SMP_CPUID_PTRSHIFT 382 PTR_SLL k1, LONGLOG 383 PTR_LA k0, ejtag_debug_buffer_per_cpu 384 PTR_ADDU k0, k1 385 386 PTR_LA k1, ejtag_debug_buffer 387 LONG_L k1, 0(k1) 388 LONG_S k1, 0(k0) 389 390 PTR_LA k0, ejtag_debug_buffer_spinlock 391 sw zero, 0(k0) 392#else 393 PTR_LA k0, ejtag_debug_buffer 394 LONG_S k1, 0(k0) 395#endif 396 397 SAVE_ALL 398 move a0, sp 399 jal ejtag_exception_handler 400 RESTORE_ALL 401 402#ifdef CONFIG_SMP 403 ASM_CPUID_MFC0 k1, ASM_SMP_CPUID_REG 404 PTR_SRL k1, SMP_CPUID_PTRSHIFT 405 PTR_SLL k1, LONGLOG 406 PTR_LA k0, ejtag_debug_buffer_per_cpu 407 PTR_ADDU k0, k1 408 LONG_L k1, 0(k0) 409#else 410 PTR_LA k0, ejtag_debug_buffer 411 LONG_L k1, 0(k0) 412#endif 413 414ejtag_return: 415 back_to_back_c0_hazard 416 MFC0 k0, CP0_DESAVE 417 .set mips32 418 deret 419 .set pop 420 END(ejtag_debug_handler) 421 422/* 423 * This buffer is reserved for the use of the EJTAG debug 424 * handler. 425 */ 426 .data 427EXPORT(ejtag_debug_buffer) 428 .fill LONGSIZE 429#ifdef CONFIG_SMP 430EXPORT(ejtag_debug_buffer_spinlock) 431 .fill LONGSIZE 432EXPORT(ejtag_debug_buffer_per_cpu) 433 .fill LONGSIZE * NR_CPUS 434#endif 435 .previous 436 437 __INIT 438 439/* 440 * NMI debug exception handler for MIPS reference boards. 441 * The NMI debug exception entry point is 0xbfc00000, which 442 * normally is in the boot PROM, so the boot PROM must do a 443 * unconditional jump to this vector. 444 */ 445NESTED(except_vec_nmi, 0, sp) 446 j nmi_handler 447#ifdef CONFIG_CPU_MICROMIPS 448 nop 449#endif 450 END(except_vec_nmi) 451 452 __FINIT 453 454NESTED(nmi_handler, PT_SIZE, sp) 455 .cfi_signal_frame 456 .set push 457 .set noat 458 /* 459 * Clear ERL - restore segment mapping 460 * Clear BEV - required for page fault exception handler to work 461 */ 462 mfc0 k0, CP0_STATUS 463 ori k0, k0, ST0_EXL 464 li k1, ~(ST0_BEV | ST0_ERL) 465 and k0, k0, k1 466 mtc0 k0, CP0_STATUS 467 _ehb 468 SAVE_ALL 469 move a0, sp 470 jal nmi_exception_handler 471 /* nmi_exception_handler never returns */ 472 .set pop 473 END(nmi_handler) 474 475 .macro __build_clear_none 476 .endm 477 478 .macro __build_clear_sti 479 TRACE_IRQS_ON 480 STI 481 .endm 482 483 .macro __build_clear_cli 484 CLI 485 TRACE_IRQS_OFF 486 .endm 487 488 .macro __build_clear_fpe 489 CLI 490 TRACE_IRQS_OFF 491 .set push 492 /* gas fails to assemble cfc1 for some archs (octeon).*/ \ 493 .set mips1 494 .set hardfloat 495 cfc1 a1, fcr31 496 .set pop 497 .endm 498 499 .macro __build_clear_msa_fpe 500 CLI 501 TRACE_IRQS_OFF 502 _cfcmsa a1, MSA_CSR 503 .endm 504 505 .macro __build_clear_ade 506 MFC0 t0, CP0_BADVADDR 507 PTR_S t0, PT_BVADDR(sp) 508 KMODE 509 .endm 510 511 .macro __build_clear_gsexc 512 .set push 513 /* 514 * We need to specify a selector to access the CP0.Diag1 (GSCause) 515 * register. All GSExc-equipped processors have MIPS32. 516 */ 517 .set mips32 518 mfc0 a1, CP0_DIAGNOSTIC1 519 .set pop 520 TRACE_IRQS_ON 521 STI 522 .endm 523 524 .macro __BUILD_silent exception 525 .endm 526 527 /* Gas tries to parse the ASM_PRINT argument as a string containing 528 string escapes and emits bogus warnings if it believes to 529 recognize an unknown escape code. So make the arguments 530 start with an n and gas will believe \n is ok ... */ 531 .macro __BUILD_verbose nexception 532 LONG_L a1, PT_EPC(sp) 533#ifdef CONFIG_32BIT 534 ASM_PRINT("Got \nexception at %08lx\012") 535#endif 536#ifdef CONFIG_64BIT 537 ASM_PRINT("Got \nexception at %016lx\012") 538#endif 539 .endm 540 541 .macro __BUILD_count exception 542 LONG_L t0,exception_count_\exception 543 LONG_ADDIU t0, 1 544 LONG_S t0,exception_count_\exception 545 .comm exception_count\exception, 8, 8 546 .endm 547 548 .macro __BUILD_HANDLER exception handler clear verbose ext 549 .align 5 550 NESTED(handle_\exception, PT_SIZE, sp) 551 .cfi_signal_frame 552 .set noat 553 SAVE_ALL 554 FEXPORT(handle_\exception\ext) 555 __build_clear_\clear 556 .set at 557 __BUILD_\verbose \exception 558 move a0, sp 559 jal do_\handler 560 j ret_from_exception 561 END(handle_\exception) 562 .endm 563 564 .macro BUILD_HANDLER exception handler clear verbose 565 __BUILD_HANDLER \exception \handler \clear \verbose _int 566 .endm 567 568 BUILD_HANDLER adel ade ade silent /* #4 */ 569 BUILD_HANDLER ades ade ade silent /* #5 */ 570 BUILD_HANDLER ibe be cli silent /* #6 */ 571 BUILD_HANDLER dbe be cli silent /* #7 */ 572 BUILD_HANDLER bp bp sti silent /* #9 */ 573 BUILD_HANDLER ri ri sti silent /* #10 */ 574 BUILD_HANDLER cpu cpu sti silent /* #11 */ 575 BUILD_HANDLER ov ov sti silent /* #12 */ 576 BUILD_HANDLER tr tr sti silent /* #13 */ 577 BUILD_HANDLER msa_fpe msa_fpe msa_fpe silent /* #14 */ 578#ifdef CONFIG_MIPS_FP_SUPPORT 579 BUILD_HANDLER fpe fpe fpe silent /* #15 */ 580#endif 581 BUILD_HANDLER ftlb ftlb none silent /* #16 */ 582 BUILD_HANDLER gsexc gsexc gsexc silent /* #16 */ 583 BUILD_HANDLER msa msa sti silent /* #21 */ 584 BUILD_HANDLER mdmx mdmx sti silent /* #22 */ 585#ifdef CONFIG_HARDWARE_WATCHPOINTS 586 /* 587 * For watch, interrupts will be enabled after the watch 588 * registers are read. 589 */ 590 BUILD_HANDLER watch watch cli silent /* #23 */ 591#else 592 BUILD_HANDLER watch watch sti verbose /* #23 */ 593#endif 594 BUILD_HANDLER mcheck mcheck cli verbose /* #24 */ 595 BUILD_HANDLER mt mt sti silent /* #25 */ 596 BUILD_HANDLER dsp dsp sti silent /* #26 */ 597 BUILD_HANDLER reserved reserved sti verbose /* others */ 598 599 .align 5 600 LEAF(handle_ri_rdhwr_tlbp) 601 .set push 602 .set noat 603 .set noreorder 604 /* check if TLB contains a entry for EPC */ 605 MFC0 k1, CP0_ENTRYHI 606 andi k1, MIPS_ENTRYHI_ASID | MIPS_ENTRYHI_ASIDX 607 MFC0 k0, CP0_EPC 608 PTR_SRL k0, _PAGE_SHIFT + 1 609 PTR_SLL k0, _PAGE_SHIFT + 1 610 or k1, k0 611 MTC0 k1, CP0_ENTRYHI 612 mtc0_tlbw_hazard 613 tlbp 614 tlb_probe_hazard 615 mfc0 k1, CP0_INDEX 616 .set pop 617 bltz k1, handle_ri /* slow path */ 618 /* fall thru */ 619 END(handle_ri_rdhwr_tlbp) 620 621 LEAF(handle_ri_rdhwr) 622 .set push 623 .set noat 624 .set noreorder 625 /* MIPS32: 0x7c03e83b: rdhwr v1,$29 */ 626 /* microMIPS: 0x007d6b3c: rdhwr v1,$29 */ 627 MFC0 k1, CP0_EPC 628#if defined(CONFIG_CPU_MICROMIPS) || defined(CONFIG_CPU_MIPS32_R2) || defined(CONFIG_CPU_MIPS64_R2) 629 and k0, k1, 1 630 beqz k0, 1f 631 xor k1, k0 632 lhu k0, (k1) 633 lhu k1, 2(k1) 634 ins k1, k0, 16, 16 635 lui k0, 0x007d 636 b docheck 637 ori k0, 0x6b3c 6381: 639 lui k0, 0x7c03 640 lw k1, (k1) 641 ori k0, 0xe83b 642#else 643 andi k0, k1, 1 644 bnez k0, handle_ri 645 lui k0, 0x7c03 646 lw k1, (k1) 647 ori k0, 0xe83b 648#endif 649 .set reorder 650docheck: 651 bne k0, k1, handle_ri /* if not ours */ 652 653isrdhwr: 654 /* The insn is rdhwr. No need to check CAUSE.BD here. */ 655 get_saved_sp /* k1 := current_thread_info */ 656 .set noreorder 657 MFC0 k0, CP0_EPC 658#if defined(CONFIG_CPU_R3000) 659 ori k1, _THREAD_MASK 660 xori k1, _THREAD_MASK 661 LONG_L v1, TI_TP_VALUE(k1) 662 LONG_ADDIU k0, 4 663 jr k0 664 rfe 665#else 666#ifndef CONFIG_CPU_DADDI_WORKAROUNDS 667 LONG_ADDIU k0, 4 /* stall on $k0 */ 668#else 669 .set at=v1 670 LONG_ADDIU k0, 4 671 .set noat 672#endif 673 MTC0 k0, CP0_EPC 674 /* I hope three instructions between MTC0 and ERET are enough... */ 675 ori k1, _THREAD_MASK 676 xori k1, _THREAD_MASK 677 LONG_L v1, TI_TP_VALUE(k1) 678 .set push 679 .set arch=r4000 680 eret 681 .set pop 682#endif 683 .set pop 684 END(handle_ri_rdhwr) 685 686#ifdef CONFIG_CPU_R4X00_BUGS64 687/* A temporary overflow handler used by check_daddi(). */ 688 689 __INIT 690 691 BUILD_HANDLER daddi_ov daddi_ov none silent /* #12 */ 692#endif 693