1/* SPDX-License-Identifier: GPL-2.0-only */ 2/* 3 * linux/arch/arm/vfp/vfphw.S 4 * 5 * Copyright (C) 2004 ARM Limited. 6 * Written by Deep Blue Solutions Limited. 7 * 8 * This code is called from the kernel's undefined instruction trap. 9 * r9 holds the return address for successful handling. 10 * lr holds the return address for unrecognised instructions. 11 * r10 points at the start of the private FP workspace in the thread structure 12 * sp points to a struct pt_regs (as defined in include/asm/proc/ptrace.h) 13 */ 14#include <linux/init.h> 15#include <linux/linkage.h> 16#include <asm/thread_info.h> 17#include <asm/vfpmacros.h> 18#include <linux/kern_levels.h> 19#include <asm/assembler.h> 20#include <asm/asm-offsets.h> 21 22 .macro DBGSTR, str 23#ifdef DEBUG 24 stmfd sp!, {r0-r3, ip, lr} 25 ldr r0, =1f 26 bl _printk 27 ldmfd sp!, {r0-r3, ip, lr} 28 29 .pushsection .rodata, "a" 301: .ascii KERN_DEBUG "VFP: \str\n" 31 .byte 0 32 .previous 33#endif 34 .endm 35 36 .macro DBGSTR1, str, arg 37#ifdef DEBUG 38 stmfd sp!, {r0-r3, ip, lr} 39 mov r1, \arg 40 ldr r0, =1f 41 bl _printk 42 ldmfd sp!, {r0-r3, ip, lr} 43 44 .pushsection .rodata, "a" 451: .ascii KERN_DEBUG "VFP: \str\n" 46 .byte 0 47 .previous 48#endif 49 .endm 50 51 .macro DBGSTR3, str, arg1, arg2, arg3 52#ifdef DEBUG 53 stmfd sp!, {r0-r3, ip, lr} 54 mov r3, \arg3 55 mov r2, \arg2 56 mov r1, \arg1 57 ldr r0, =1f 58 bl _printk 59 ldmfd sp!, {r0-r3, ip, lr} 60 61 .pushsection .rodata, "a" 621: .ascii KERN_DEBUG "VFP: \str\n" 63 .byte 0 64 .previous 65#endif 66 .endm 67 68 69@ VFP hardware support entry point. 70@ 71@ r0 = instruction opcode (32-bit ARM or two 16-bit Thumb) 72@ r2 = PC value to resume execution after successful emulation 73@ r9 = normal "successful" return address 74@ r10 = vfp_state union 75@ r11 = CPU number 76@ lr = unrecognised instruction return address 77@ IRQs enabled. 78ENTRY(vfp_support_entry) 79 DBGSTR3 "instr %08x pc %08x state %p", r0, r2, r10 80 81 .fpu vfpv2 82 VFPFMRX r1, FPEXC @ Is the VFP enabled? 83 DBGSTR1 "fpexc %08x", r1 84 tst r1, #FPEXC_EN 85 bne look_for_VFP_exceptions @ VFP is already enabled 86 87 DBGSTR1 "enable %x", r10 88 ldr r3, vfp_current_hw_state_address 89 orr r1, r1, #FPEXC_EN @ user FPEXC has the enable bit set 90 ldr r4, [r3, r11, lsl #2] @ vfp_current_hw_state pointer 91 bic r5, r1, #FPEXC_EX @ make sure exceptions are disabled 92 cmp r4, r10 @ this thread owns the hw context? 93#ifndef CONFIG_SMP 94 @ For UP, checking that this thread owns the hw context is 95 @ sufficient to determine that the hardware state is valid. 96 beq vfp_hw_state_valid 97 98 @ On UP, we lazily save the VFP context. As a different 99 @ thread wants ownership of the VFP hardware, save the old 100 @ state if there was a previous (valid) owner. 101 102 VFPFMXR FPEXC, r5 @ enable VFP, disable any pending 103 @ exceptions, so we can get at the 104 @ rest of it 105 106 DBGSTR1 "save old state %p", r4 107 cmp r4, #0 @ if the vfp_current_hw_state is NULL 108 beq vfp_reload_hw @ then the hw state needs reloading 109 VFPFSTMIA r4, r5 @ save the working registers 110 VFPFMRX r5, FPSCR @ current status 111#ifndef CONFIG_CPU_FEROCEON 112 tst r1, #FPEXC_EX @ is there additional state to save? 113 beq 1f 114 VFPFMRX r6, FPINST @ FPINST (only if FPEXC.EX is set) 115 tst r1, #FPEXC_FP2V @ is there an FPINST2 to read? 116 beq 1f 117 VFPFMRX r8, FPINST2 @ FPINST2 if needed (and present) 1181: 119#endif 120 stmia r4, {r1, r5, r6, r8} @ save FPEXC, FPSCR, FPINST, FPINST2 121vfp_reload_hw: 122 123#else 124 @ For SMP, if this thread does not own the hw context, then we 125 @ need to reload it. No need to save the old state as on SMP, 126 @ we always save the state when we switch away from a thread. 127 bne vfp_reload_hw 128 129 @ This thread has ownership of the current hardware context. 130 @ However, it may have been migrated to another CPU, in which 131 @ case the saved state is newer than the hardware context. 132 @ Check this by looking at the CPU number which the state was 133 @ last loaded onto. 134 ldr ip, [r10, #VFP_CPU] 135 teq ip, r11 136 beq vfp_hw_state_valid 137 138vfp_reload_hw: 139 @ We're loading this threads state into the VFP hardware. Update 140 @ the CPU number which contains the most up to date VFP context. 141 str r11, [r10, #VFP_CPU] 142 143 VFPFMXR FPEXC, r5 @ enable VFP, disable any pending 144 @ exceptions, so we can get at the 145 @ rest of it 146#endif 147 148 DBGSTR1 "load state %p", r10 149 str r10, [r3, r11, lsl #2] @ update the vfp_current_hw_state pointer 150 @ Load the saved state back into the VFP 151 VFPFLDMIA r10, r5 @ reload the working registers while 152 @ FPEXC is in a safe state 153 ldmia r10, {r1, r5, r6, r8} @ load FPEXC, FPSCR, FPINST, FPINST2 154#ifndef CONFIG_CPU_FEROCEON 155 tst r1, #FPEXC_EX @ is there additional state to restore? 156 beq 1f 157 VFPFMXR FPINST, r6 @ restore FPINST (only if FPEXC.EX is set) 158 tst r1, #FPEXC_FP2V @ is there an FPINST2 to write? 159 beq 1f 160 VFPFMXR FPINST2, r8 @ FPINST2 if needed (and present) 1611: 162#endif 163 VFPFMXR FPSCR, r5 @ restore status 164 165@ The context stored in the VFP hardware is up to date with this thread 166vfp_hw_state_valid: 167 tst r1, #FPEXC_EX 168 bne process_exception @ might as well handle the pending 169 @ exception before retrying branch 170 @ out before setting an FPEXC that 171 @ stops us reading stuff 172 VFPFMXR FPEXC, r1 @ Restore FPEXC last 173 sub r2, r2, #4 @ Retry current instruction - if Thumb 174 str r2, [sp, #S_PC] @ mode it's two 16-bit instructions, 175 @ else it's one 32-bit instruction, so 176 @ always subtract 4 from the following 177 @ instruction address. 178 dec_preempt_count_ti r10, r4 179 ret r9 @ we think we have handled things 180 181 182look_for_VFP_exceptions: 183 @ Check for synchronous or asynchronous exception 184 tst r1, #FPEXC_EX | FPEXC_DEX 185 bne process_exception 186 @ On some implementations of the VFP subarch 1, setting FPSCR.IXE 187 @ causes all the CDP instructions to be bounced synchronously without 188 @ setting the FPEXC.EX bit 189 VFPFMRX r5, FPSCR 190 tst r5, #FPSCR_IXE 191 bne process_exception 192 193 tst r5, #FPSCR_LENGTH_MASK 194 beq skip 195 orr r1, r1, #FPEXC_DEX 196 b process_exception 197skip: 198 199 @ Fall into hand on to next handler - appropriate coproc instr 200 @ not recognised by VFP 201 202 DBGSTR "not VFP" 203 dec_preempt_count_ti r10, r4 204 ret lr 205 206process_exception: 207 DBGSTR "bounce" 208 mov r2, sp @ nothing stacked - regdump is at TOS 209 mov lr, r9 @ setup for a return to the user code. 210 211 @ Now call the C code to package up the bounce to the support code 212 @ r0 holds the trigger instruction 213 @ r1 holds the FPEXC value 214 @ r2 pointer to register dump 215 b VFP_bounce @ we have handled this - the support 216 @ code will raise an exception if 217 @ required. If not, the user code will 218 @ retry the faulted instruction 219ENDPROC(vfp_support_entry) 220 221ENTRY(vfp_save_state) 222 @ Save the current VFP state 223 @ r0 - save location 224 @ r1 - FPEXC 225 DBGSTR1 "save VFP state %p", r0 226 VFPFSTMIA r0, r2 @ save the working registers 227 VFPFMRX r2, FPSCR @ current status 228 tst r1, #FPEXC_EX @ is there additional state to save? 229 beq 1f 230 VFPFMRX r3, FPINST @ FPINST (only if FPEXC.EX is set) 231 tst r1, #FPEXC_FP2V @ is there an FPINST2 to read? 232 beq 1f 233 VFPFMRX r12, FPINST2 @ FPINST2 if needed (and present) 2341: 235 stmia r0, {r1, r2, r3, r12} @ save FPEXC, FPSCR, FPINST, FPINST2 236 ret lr 237ENDPROC(vfp_save_state) 238 239 .align 240vfp_current_hw_state_address: 241 .word vfp_current_hw_state 242 243 .macro tbl_branch, base, tmp, shift 244#ifdef CONFIG_THUMB2_KERNEL 245 adr \tmp, 1f 246 add \tmp, \tmp, \base, lsl \shift 247 ret \tmp 248#else 249 add pc, pc, \base, lsl \shift 250 mov r0, r0 251#endif 2521: 253 .endm 254 255ENTRY(vfp_get_float) 256 tbl_branch r0, r3, #3 257 .fpu vfpv2 258 .irp dr,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15 2591: vmov r0, s\dr 260 ret lr 261 .org 1b + 8 262 .endr 263 .irp dr,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31 2641: vmov r0, s\dr 265 ret lr 266 .org 1b + 8 267 .endr 268ENDPROC(vfp_get_float) 269 270ENTRY(vfp_put_float) 271 tbl_branch r1, r3, #3 272 .fpu vfpv2 273 .irp dr,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15 2741: vmov s\dr, r0 275 ret lr 276 .org 1b + 8 277 .endr 278 .irp dr,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31 2791: vmov s\dr, r0 280 ret lr 281 .org 1b + 8 282 .endr 283ENDPROC(vfp_put_float) 284 285ENTRY(vfp_get_double) 286 tbl_branch r0, r3, #3 287 .fpu vfpv2 288 .irp dr,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15 2891: vmov r0, r1, d\dr 290 ret lr 291 .org 1b + 8 292 .endr 293#ifdef CONFIG_VFPv3 294 @ d16 - d31 registers 295 .fpu vfpv3 296 .irp dr,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31 2971: vmov r0, r1, d\dr 298 ret lr 299 .org 1b + 8 300 .endr 301#endif 302 303 @ virtual register 16 (or 32 if VFPv3) for compare with zero 304 mov r0, #0 305 mov r1, #0 306 ret lr 307ENDPROC(vfp_get_double) 308 309ENTRY(vfp_put_double) 310 tbl_branch r2, r3, #3 311 .fpu vfpv2 312 .irp dr,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15 3131: vmov d\dr, r0, r1 314 ret lr 315 .org 1b + 8 316 .endr 317#ifdef CONFIG_VFPv3 318 .fpu vfpv3 319 @ d16 - d31 registers 320 .irp dr,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31 3211: vmov d\dr, r0, r1 322 ret lr 323 .org 1b + 8 324 .endr 325#endif 326ENDPROC(vfp_put_double) 327