1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * In-kernel vector facility support functions
4 *
5 * Copyright IBM Corp. 2015
6 * Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com>
7 */
8
9 #include <linux/export.h>
10 #include <linux/kernel.h>
11 #include <linux/cpu.h>
12 #include <linux/sched.h>
13 #include <asm/fpu.h>
14
__kernel_fpu_begin(struct kernel_fpu * state,int flags)15 void __kernel_fpu_begin(struct kernel_fpu *state, int flags)
16 {
17 __vector128 *vxrs = state->vxrs;
18 int mask;
19
20 /*
21 * Limit the save to the FPU/vector registers already
22 * in use by the previous context.
23 */
24 flags &= state->hdr.mask;
25 if (flags & KERNEL_FPC)
26 fpu_stfpc(&state->hdr.fpc);
27 if (!cpu_has_vx()) {
28 if (flags & KERNEL_VXR_LOW)
29 save_fp_regs_vx(vxrs);
30 return;
31 }
32 mask = flags & KERNEL_VXR;
33 if (mask == KERNEL_VXR) {
34 vxrs += fpu_vstm(0, 15, vxrs);
35 vxrs += fpu_vstm(16, 31, vxrs);
36 return;
37 }
38 if (mask == KERNEL_VXR_MID) {
39 vxrs += fpu_vstm(8, 23, vxrs);
40 return;
41 }
42 mask = flags & KERNEL_VXR_LOW;
43 if (mask) {
44 if (mask == KERNEL_VXR_LOW)
45 vxrs += fpu_vstm(0, 15, vxrs);
46 else if (mask == KERNEL_VXR_V0V7)
47 vxrs += fpu_vstm(0, 7, vxrs);
48 else
49 vxrs += fpu_vstm(8, 15, vxrs);
50 }
51 mask = flags & KERNEL_VXR_HIGH;
52 if (mask) {
53 if (mask == KERNEL_VXR_HIGH)
54 vxrs += fpu_vstm(16, 31, vxrs);
55 else if (mask == KERNEL_VXR_V16V23)
56 vxrs += fpu_vstm(16, 23, vxrs);
57 else
58 vxrs += fpu_vstm(24, 31, vxrs);
59 }
60 }
61 EXPORT_SYMBOL(__kernel_fpu_begin);
62
__kernel_fpu_end(struct kernel_fpu * state,int flags)63 void __kernel_fpu_end(struct kernel_fpu *state, int flags)
64 {
65 __vector128 *vxrs = state->vxrs;
66 int mask;
67
68 /*
69 * Limit the restore to the FPU/vector registers of the
70 * previous context that have been overwritten by the
71 * current context.
72 */
73 flags &= state->hdr.mask;
74 if (flags & KERNEL_FPC)
75 fpu_lfpc(&state->hdr.fpc);
76 if (!cpu_has_vx()) {
77 if (flags & KERNEL_VXR_LOW)
78 load_fp_regs_vx(vxrs);
79 return;
80 }
81 mask = flags & KERNEL_VXR;
82 if (mask == KERNEL_VXR) {
83 vxrs += fpu_vlm(0, 15, vxrs);
84 vxrs += fpu_vlm(16, 31, vxrs);
85 return;
86 }
87 if (mask == KERNEL_VXR_MID) {
88 vxrs += fpu_vlm(8, 23, vxrs);
89 return;
90 }
91 mask = flags & KERNEL_VXR_LOW;
92 if (mask) {
93 if (mask == KERNEL_VXR_LOW)
94 vxrs += fpu_vlm(0, 15, vxrs);
95 else if (mask == KERNEL_VXR_V0V7)
96 vxrs += fpu_vlm(0, 7, vxrs);
97 else
98 vxrs += fpu_vlm(8, 15, vxrs);
99 }
100 mask = flags & KERNEL_VXR_HIGH;
101 if (mask) {
102 if (mask == KERNEL_VXR_HIGH)
103 vxrs += fpu_vlm(16, 31, vxrs);
104 else if (mask == KERNEL_VXR_V16V23)
105 vxrs += fpu_vlm(16, 23, vxrs);
106 else
107 vxrs += fpu_vlm(24, 31, vxrs);
108 }
109 }
110 EXPORT_SYMBOL(__kernel_fpu_end);
111
load_fpu_state(struct fpu * state,int flags)112 void load_fpu_state(struct fpu *state, int flags)
113 {
114 __vector128 *vxrs = &state->vxrs[0];
115 int mask;
116
117 if (flags & KERNEL_FPC)
118 fpu_lfpc_safe(&state->fpc);
119 if (!cpu_has_vx()) {
120 if (flags & KERNEL_VXR_V0V7)
121 load_fp_regs_vx(state->vxrs);
122 return;
123 }
124 mask = flags & KERNEL_VXR;
125 if (mask == KERNEL_VXR) {
126 fpu_vlm(0, 15, &vxrs[0]);
127 fpu_vlm(16, 31, &vxrs[16]);
128 return;
129 }
130 if (mask == KERNEL_VXR_MID) {
131 fpu_vlm(8, 23, &vxrs[8]);
132 return;
133 }
134 mask = flags & KERNEL_VXR_LOW;
135 if (mask) {
136 if (mask == KERNEL_VXR_LOW)
137 fpu_vlm(0, 15, &vxrs[0]);
138 else if (mask == KERNEL_VXR_V0V7)
139 fpu_vlm(0, 7, &vxrs[0]);
140 else
141 fpu_vlm(8, 15, &vxrs[8]);
142 }
143 mask = flags & KERNEL_VXR_HIGH;
144 if (mask) {
145 if (mask == KERNEL_VXR_HIGH)
146 fpu_vlm(16, 31, &vxrs[16]);
147 else if (mask == KERNEL_VXR_V16V23)
148 fpu_vlm(16, 23, &vxrs[16]);
149 else
150 fpu_vlm(24, 31, &vxrs[24]);
151 }
152 }
153
save_fpu_state(struct fpu * state,int flags)154 void save_fpu_state(struct fpu *state, int flags)
155 {
156 __vector128 *vxrs = &state->vxrs[0];
157 int mask;
158
159 if (flags & KERNEL_FPC)
160 fpu_stfpc(&state->fpc);
161 if (!cpu_has_vx()) {
162 if (flags & KERNEL_VXR_LOW)
163 save_fp_regs_vx(state->vxrs);
164 return;
165 }
166 mask = flags & KERNEL_VXR;
167 if (mask == KERNEL_VXR) {
168 fpu_vstm(0, 15, &vxrs[0]);
169 fpu_vstm(16, 31, &vxrs[16]);
170 return;
171 }
172 if (mask == KERNEL_VXR_MID) {
173 fpu_vstm(8, 23, &vxrs[8]);
174 return;
175 }
176 mask = flags & KERNEL_VXR_LOW;
177 if (mask) {
178 if (mask == KERNEL_VXR_LOW)
179 fpu_vstm(0, 15, &vxrs[0]);
180 else if (mask == KERNEL_VXR_V0V7)
181 fpu_vstm(0, 7, &vxrs[0]);
182 else
183 fpu_vstm(8, 15, &vxrs[8]);
184 }
185 mask = flags & KERNEL_VXR_HIGH;
186 if (mask) {
187 if (mask == KERNEL_VXR_HIGH)
188 fpu_vstm(16, 31, &vxrs[16]);
189 else if (mask == KERNEL_VXR_V16V23)
190 fpu_vstm(16, 23, &vxrs[16]);
191 else
192 fpu_vstm(24, 31, &vxrs[24]);
193 }
194 }
195 EXPORT_SYMBOL(save_fpu_state);
196