1/*- 2 * Copyright (c) 1989, 1990 William F. Jolitz. 3 * Copyright (c) 1990 The Regents of the University of California. 4 * All rights reserved. 5 * Copyright (c) 2014-2018 The FreeBSD Foundation 6 * All rights reserved. 7 * 8 * Portions of this software were developed by 9 * Konstantin Belousov <kib@FreeBSD.org> under sponsorship from 10 * the FreeBSD Foundation. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 3. Neither the name of the University nor the names of its contributors 21 * may be used to endorse or promote products derived from this software 22 * without specific prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 27 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 34 * SUCH DAMAGE. 35 * 36 * from: vector.s, 386BSD 0.1 unknown origin 37 */ 38 39/* 40 * Interrupt entry points for external interrupts triggered by I/O APICs 41 * as well as IPI handlers. 42 */ 43 44#include "opt_smp.h" 45 46#include "assym.inc" 47 48#include <machine/asmacros.h> 49#include <machine/specialreg.h> 50#include <x86/apicreg.h> 51 52 .text 53 SUPERALIGN_TEXT 54 /* End Of Interrupt to APIC */ 55as_lapic_eoi: 56 cmpl $0,x2apic_mode 57 jne 1f 58 movq lapic_map,%rax 59 movl $0,LA_EOI(%rax) 60 ret 611: 62 movl $MSR_APIC_EOI,%ecx 63 xorl %eax,%eax 64 xorl %edx,%edx 65 wrmsr 66 ret 67 68/* 69 * I/O Interrupt Entry Point. Rather than having one entry point for 70 * each interrupt source, we use one entry point for each 32-bit word 71 * in the ISR. The handler determines the highest bit set in the ISR, 72 * translates that into a vector, and passes the vector to the 73 * lapic_handle_intr() function. 74 */ 75 .macro ISR_VEC index, vec_name 76 INTR_HANDLER \vec_name 77 KMSAN_ENTER 78 cmpl $0,x2apic_mode 79 je 1f 80 movl $(MSR_APIC_ISR0 + \index),%ecx 81 rdmsr 82 jmp 2f 831: 84 movq lapic_map, %rdx /* pointer to local APIC */ 85 movl LA_ISR + 16 * (\index)(%rdx), %eax /* load ISR */ 862: 87 bsrl %eax, %eax /* index of highest set bit in ISR */ 88 jz 3f 89 addl $(32 * \index),%eax 90 movq %rsp, %rsi 91 movl %eax, %edi /* pass the IRQ */ 92 call lapic_handle_intr 933: 94 KMSAN_LEAVE 95 jmp doreti 96 .endm 97 98/* 99 * Handle "spurious INTerrupts". 100 * Notes: 101 * This is different than the "spurious INTerrupt" generated by an 102 * 8259 PIC for missing INTs. See the APIC documentation for details. 103 * This routine should NOT do an 'EOI' cycle. 104 */ 105 .text 106 SUPERALIGN_TEXT 107IDTVEC(spuriousint) 108 /* No EOI cycle used here */ 109 jmp doreti_iret 110 111 ISR_VEC 1, apic_isr1 112 ISR_VEC 2, apic_isr2 113 ISR_VEC 3, apic_isr3 114 ISR_VEC 4, apic_isr4 115 ISR_VEC 5, apic_isr5 116 ISR_VEC 6, apic_isr6 117 ISR_VEC 7, apic_isr7 118 119/* 120 * Local APIC periodic timer handler. 121 */ 122 INTR_HANDLER timerint 123 KMSAN_ENTER 124 movq %rsp, %rdi 125 call lapic_handle_timer 126 KMSAN_LEAVE 127 jmp doreti 128 129/* 130 * Local APIC CMCI handler. 131 */ 132 INTR_HANDLER cmcint 133 KMSAN_ENTER 134 call lapic_handle_cmc 135 KMSAN_LEAVE 136 jmp doreti 137 138/* 139 * Local APIC error interrupt handler. 140 */ 141 INTR_HANDLER errorint 142 KMSAN_ENTER 143 call lapic_handle_error 144 KMSAN_LEAVE 145 jmp doreti 146 147#ifdef XENHVM 148/* 149 * Xen event channel upcall interrupt handler. 150 * Only used when the hypervisor supports direct vector callbacks. 151 */ 152 INTR_HANDLER xen_intr_upcall 153 KMSAN_ENTER 154 movq %rsp, %rdi 155 call xen_arch_intr_handle_upcall 156 KMSAN_LEAVE 157 jmp doreti 158#endif 159 160/* 161 * Global address space TLB shootdown. 162 */ 163 .text 164 165 SUPERALIGN_TEXT 166/* 167 * IPI handler for cache and TLB shootdown 168 */ 169 INTR_HANDLER invlop 170 KMSAN_ENTER 171 call invlop_handler 172 call as_lapic_eoi 173 KMSAN_LEAVE 174 jmp ld_regs 175 176/* 177 * Handler for IPIs sent via the per-cpu IPI bitmap. 178 */ 179 INTR_HANDLER ipi_intr_bitmap_handler 180 call as_lapic_eoi 181 KMSAN_ENTER 182 call ipi_bitmap_handler 183 KMSAN_LEAVE 184 jmp doreti 185 186/* 187 * Executed by a CPU when it receives an IPI_STOP from another CPU. 188 */ 189 INTR_HANDLER cpustop 190 call as_lapic_eoi 191 KMSAN_ENTER 192 call cpustop_handler 193 KMSAN_LEAVE 194 jmp doreti 195 196/* 197 * Executed by a CPU when it receives an IPI_SUSPEND from another CPU. 198 */ 199 INTR_HANDLER cpususpend 200 KMSAN_ENTER 201 call cpususpend_handler 202 call as_lapic_eoi 203 KMSAN_LEAVE 204 jmp doreti 205 206/* 207 * Executed by a CPU when it receives an IPI_OFF from another CPU. 208 * Should never return 209 */ 210 INTR_HANDLER cpuoff 211 KMSAN_ENTER 212 call cpuoff_handler 213 call as_lapic_eoi 214 KMSAN_LEAVE 215 jmp doreti 216 217/* 218 * Executed by a CPU when it receives an IPI_SWI. 219 */ 220 INTR_HANDLER ipi_swi 221 call as_lapic_eoi 222 KMSAN_ENTER 223 call ipi_swi_handler 224 KMSAN_LEAVE 225 jmp doreti 226 227/* 228 * Executed by a CPU when it receives a RENDEZVOUS IPI from another CPU. 229 * 230 * - Calls the generic rendezvous action function. 231 */ 232 INTR_HANDLER rendezvous 233#ifdef COUNT_IPIS 234 movl PCPU(CPUID), %eax 235 movq ipi_rendezvous_counts(,%rax,8), %rax 236 incq (%rax) 237#endif 238 KMSAN_ENTER 239 call smp_rendezvous_action 240 call as_lapic_eoi 241 KMSAN_LEAVE 242 jmp doreti 243 244/* 245 * IPI handler whose purpose is to interrupt the CPU with minimum overhead. 246 * This is used by bhyve to force a host cpu executing in guest context to 247 * trap into the hypervisor. 248 * 249 * This handler is different from other IPI handlers in the following aspects: 250 * 251 * 1. It doesn't push a trapframe on the stack. 252 * 253 * This implies that a DDB backtrace involving 'justreturn' will skip the 254 * function that was interrupted by this handler. 255 * 256 * 2. It doesn't 'swapgs' when userspace is interrupted. 257 * 258 * The 'justreturn' handler does not access any pcpu data so it is not an 259 * issue. Moreover the 'justreturn' handler can only be interrupted by an NMI 260 * whose handler already doesn't trust GS.base when kernel code is interrupted. 261 */ 262 .text 263 SUPERALIGN_TEXT 264IDTVEC(justreturn) 265 pushq %rax 266 pushq %rcx 267 pushq %rdx 268 call as_lapic_eoi 269 popq %rdx 270 popq %rcx 271 popq %rax 272 jmp doreti_iret 273 274 INTR_HANDLER justreturn1 275 call as_lapic_eoi 276 jmp doreti 277