1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * In-kernel vector facility support functions 4 * 5 * Copyright IBM Corp. 2015 6 * Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com> 7 */ 8 9 #include <linux/export.h> 10 #include <linux/kernel.h> 11 #include <linux/cpu.h> 12 #include <linux/sched.h> 13 #include <asm/fpu.h> 14 15 void __kernel_fpu_begin(struct kernel_fpu *state, int flags) 16 { 17 __vector128 *vxrs = state->vxrs; 18 int mask; 19 20 /* 21 * Limit the save to the FPU/vector registers already 22 * in use by the previous context. 23 */ 24 flags &= state->hdr.mask; 25 if (flags & KERNEL_FPC) 26 fpu_stfpc(&state->hdr.fpc); 27 if (!cpu_has_vx()) { 28 if (flags & KERNEL_VXR_LOW) 29 save_fp_regs_vx(vxrs); 30 return; 31 } 32 mask = flags & KERNEL_VXR; 33 if (mask == KERNEL_VXR) { 34 vxrs += fpu_vstm(0, 15, vxrs); 35 vxrs += fpu_vstm(16, 31, vxrs); 36 return; 37 } 38 if (mask == KERNEL_VXR_MID) { 39 vxrs += fpu_vstm(8, 23, vxrs); 40 return; 41 } 42 mask = flags & KERNEL_VXR_LOW; 43 if (mask) { 44 if (mask == KERNEL_VXR_LOW) 45 vxrs += fpu_vstm(0, 15, vxrs); 46 else if (mask == KERNEL_VXR_V0V7) 47 vxrs += fpu_vstm(0, 7, vxrs); 48 else 49 vxrs += fpu_vstm(8, 15, vxrs); 50 } 51 mask = flags & KERNEL_VXR_HIGH; 52 if (mask) { 53 if (mask == KERNEL_VXR_HIGH) 54 vxrs += fpu_vstm(16, 31, vxrs); 55 else if (mask == KERNEL_VXR_V16V23) 56 vxrs += fpu_vstm(16, 23, vxrs); 57 else 58 vxrs += fpu_vstm(24, 31, vxrs); 59 } 60 } 61 EXPORT_SYMBOL(__kernel_fpu_begin); 62 63 void __kernel_fpu_end(struct kernel_fpu *state, int flags) 64 { 65 __vector128 *vxrs = state->vxrs; 66 int mask; 67 68 /* 69 * Limit the restore to the FPU/vector registers of the 70 * previous context that have been overwritten by the 71 * current context. 72 */ 73 flags &= state->hdr.mask; 74 if (flags & KERNEL_FPC) 75 fpu_lfpc(&state->hdr.fpc); 76 if (!cpu_has_vx()) { 77 if (flags & KERNEL_VXR_LOW) 78 load_fp_regs_vx(vxrs); 79 return; 80 } 81 mask = flags & KERNEL_VXR; 82 if (mask == KERNEL_VXR) { 83 vxrs += fpu_vlm(0, 15, vxrs); 84 vxrs += fpu_vlm(16, 31, vxrs); 85 return; 86 } 87 if (mask == KERNEL_VXR_MID) { 88 vxrs += fpu_vlm(8, 23, vxrs); 89 return; 90 } 91 mask = flags & KERNEL_VXR_LOW; 92 if (mask) { 93 if (mask == KERNEL_VXR_LOW) 94 vxrs += fpu_vlm(0, 15, vxrs); 95 else if (mask == KERNEL_VXR_V0V7) 96 vxrs += fpu_vlm(0, 7, vxrs); 97 else 98 vxrs += fpu_vlm(8, 15, vxrs); 99 } 100 mask = flags & KERNEL_VXR_HIGH; 101 if (mask) { 102 if (mask == KERNEL_VXR_HIGH) 103 vxrs += fpu_vlm(16, 31, vxrs); 104 else if (mask == KERNEL_VXR_V16V23) 105 vxrs += fpu_vlm(16, 23, vxrs); 106 else 107 vxrs += fpu_vlm(24, 31, vxrs); 108 } 109 } 110 EXPORT_SYMBOL(__kernel_fpu_end); 111 112 void load_fpu_state(struct fpu *state, int flags) 113 { 114 __vector128 *vxrs = &state->vxrs[0]; 115 int mask; 116 117 if (flags & KERNEL_FPC) 118 fpu_lfpc_safe(&state->fpc); 119 if (!cpu_has_vx()) { 120 if (flags & KERNEL_VXR_V0V7) 121 load_fp_regs_vx(state->vxrs); 122 return; 123 } 124 mask = flags & KERNEL_VXR; 125 if (mask == KERNEL_VXR) { 126 fpu_vlm(0, 15, &vxrs[0]); 127 fpu_vlm(16, 31, &vxrs[16]); 128 return; 129 } 130 if (mask == KERNEL_VXR_MID) { 131 fpu_vlm(8, 23, &vxrs[8]); 132 return; 133 } 134 mask = flags & KERNEL_VXR_LOW; 135 if (mask) { 136 if (mask == KERNEL_VXR_LOW) 137 fpu_vlm(0, 15, &vxrs[0]); 138 else if (mask == KERNEL_VXR_V0V7) 139 fpu_vlm(0, 7, &vxrs[0]); 140 else 141 fpu_vlm(8, 15, &vxrs[8]); 142 } 143 mask = flags & KERNEL_VXR_HIGH; 144 if (mask) { 145 if (mask == KERNEL_VXR_HIGH) 146 fpu_vlm(16, 31, &vxrs[16]); 147 else if (mask == KERNEL_VXR_V16V23) 148 fpu_vlm(16, 23, &vxrs[16]); 149 else 150 fpu_vlm(24, 31, &vxrs[24]); 151 } 152 } 153 154 void save_fpu_state(struct fpu *state, int flags) 155 { 156 __vector128 *vxrs = &state->vxrs[0]; 157 int mask; 158 159 if (flags & KERNEL_FPC) 160 fpu_stfpc(&state->fpc); 161 if (!cpu_has_vx()) { 162 if (flags & KERNEL_VXR_LOW) 163 save_fp_regs_vx(state->vxrs); 164 return; 165 } 166 mask = flags & KERNEL_VXR; 167 if (mask == KERNEL_VXR) { 168 fpu_vstm(0, 15, &vxrs[0]); 169 fpu_vstm(16, 31, &vxrs[16]); 170 return; 171 } 172 if (mask == KERNEL_VXR_MID) { 173 fpu_vstm(8, 23, &vxrs[8]); 174 return; 175 } 176 mask = flags & KERNEL_VXR_LOW; 177 if (mask) { 178 if (mask == KERNEL_VXR_LOW) 179 fpu_vstm(0, 15, &vxrs[0]); 180 else if (mask == KERNEL_VXR_V0V7) 181 fpu_vstm(0, 7, &vxrs[0]); 182 else 183 fpu_vstm(8, 15, &vxrs[8]); 184 } 185 mask = flags & KERNEL_VXR_HIGH; 186 if (mask) { 187 if (mask == KERNEL_VXR_HIGH) 188 fpu_vstm(16, 31, &vxrs[16]); 189 else if (mask == KERNEL_VXR_V16V23) 190 fpu_vstm(16, 23, &vxrs[16]); 191 else 192 fpu_vstm(24, 31, &vxrs[24]); 193 } 194 } 195 EXPORT_SYMBOL(save_fpu_state); 196