1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * Support for Floating Point and Vector Instructions
4 *
5 */
6
7 #ifndef __ASM_S390_FPU_INSN_H
8 #define __ASM_S390_FPU_INSN_H
9
10 #include <asm/fpu-insn-asm.h>
11
12 #ifndef __ASSEMBLER__
13
14 #include <linux/instrumented.h>
15 #include <linux/kmsan.h>
16 #include <asm/asm-extable.h>
17
18 asm(".include \"asm/insn-common-asm.h\"\n");
19 asm(".include \"asm/fpu-insn-asm.h\"\n");
20
21 /*
22 * Various small helper functions, which can and should be used within
23 * kernel fpu code sections. Each function represents only one floating
24 * point or vector instruction (except for helper functions which require
25 * exception handling).
26 *
27 * This allows to use floating point and vector instructions like C
28 * functions, which has the advantage that all supporting code, like
29 * e.g. loops, can be written in easy to read C code.
30 *
31 * Each of the helper functions provides support for code instrumentation,
32 * like e.g. KASAN. Therefore instrumentation is also covered automatically
33 * when using these functions.
34 *
35 * In order to ensure that code generated with the helper functions stays
36 * within kernel fpu sections, which are guarded with kernel_fpu_begin()
37 * and kernel_fpu_end() calls, each function has a mandatory "memory"
38 * barrier.
39 */
40
fpu_cefbr(u8 f1,s32 val)41 static __always_inline void fpu_cefbr(u8 f1, s32 val)
42 {
43 asm volatile("cefbr %[f1],%[val]"
44 :
45 : [f1] "I" (f1), [val] "d" (val)
46 : "memory");
47 }
48
fpu_cgebr(u8 f2,u8 mode)49 static __always_inline unsigned long fpu_cgebr(u8 f2, u8 mode)
50 {
51 unsigned long val;
52
53 asm volatile("cgebr %[val],%[mode],%[f2]"
54 : [val] "=d" (val)
55 : [f2] "I" (f2), [mode] "I" (mode)
56 : "memory");
57 return val;
58 }
59
fpu_debr(u8 f1,u8 f2)60 static __always_inline void fpu_debr(u8 f1, u8 f2)
61 {
62 asm volatile("debr %[f1],%[f2]"
63 :
64 : [f1] "I" (f1), [f2] "I" (f2)
65 : "memory");
66 }
67
fpu_ld(unsigned short fpr,freg_t * reg)68 static __always_inline void fpu_ld(unsigned short fpr, freg_t *reg)
69 {
70 instrument_read(reg, sizeof(*reg));
71 asm volatile("ld %[fpr],%[reg]"
72 :
73 : [fpr] "I" (fpr), [reg] "Q" (reg->ui)
74 : "memory");
75 }
76
fpu_ldgr(u8 f1,u32 val)77 static __always_inline void fpu_ldgr(u8 f1, u32 val)
78 {
79 asm volatile("ldgr %[f1],%[val]"
80 :
81 : [f1] "I" (f1), [val] "d" (val)
82 : "memory");
83 }
84
fpu_lfpc(unsigned int * fpc)85 static __always_inline void fpu_lfpc(unsigned int *fpc)
86 {
87 instrument_read(fpc, sizeof(*fpc));
88 asm volatile("lfpc %[fpc]"
89 :
90 : [fpc] "Q" (*fpc)
91 : "memory");
92 }
93
94 /**
95 * fpu_lfpc_safe - Load floating point control register safely.
96 * @fpc: new value for floating point control register
97 *
98 * Load floating point control register. This may lead to an exception,
99 * since a saved value may have been modified by user space (ptrace,
100 * signal return, kvm registers) to an invalid value. In such a case
101 * set the floating point control register to zero.
102 */
fpu_lfpc_safe(unsigned int * fpc)103 static inline void fpu_lfpc_safe(unsigned int *fpc)
104 {
105 instrument_read(fpc, sizeof(*fpc));
106 asm_inline volatile(
107 "0: lfpc %[fpc]\n"
108 "1:\n"
109 EX_TABLE_INSN_FPC(0b, 1b)
110 :
111 : [fpc] "Q" (*fpc)
112 : "memory");
113 }
114
fpu_std(unsigned short fpr,freg_t * reg)115 static __always_inline void fpu_std(unsigned short fpr, freg_t *reg)
116 {
117 instrument_write(reg, sizeof(*reg));
118 asm volatile("std %[fpr],%[reg]"
119 : [reg] "=Q" (reg->ui)
120 : [fpr] "I" (fpr)
121 : "memory");
122 }
123
fpu_sfpc(unsigned int fpc)124 static __always_inline void fpu_sfpc(unsigned int fpc)
125 {
126 asm volatile("sfpc %[fpc]"
127 :
128 : [fpc] "d" (fpc)
129 : "memory");
130 }
131
fpu_stfpc(unsigned int * fpc)132 static __always_inline void fpu_stfpc(unsigned int *fpc)
133 {
134 instrument_write(fpc, sizeof(*fpc));
135 asm volatile("stfpc %[fpc]"
136 : [fpc] "=Q" (*fpc)
137 :
138 : "memory");
139 }
140
fpu_vab(u8 v1,u8 v2,u8 v3)141 static __always_inline void fpu_vab(u8 v1, u8 v2, u8 v3)
142 {
143 asm volatile("VAB %[v1],%[v2],%[v3]"
144 :
145 : [v1] "I" (v1), [v2] "I" (v2), [v3] "I" (v3)
146 : "memory");
147 }
148
fpu_vcksm(u8 v1,u8 v2,u8 v3)149 static __always_inline void fpu_vcksm(u8 v1, u8 v2, u8 v3)
150 {
151 asm volatile("VCKSM %[v1],%[v2],%[v3]"
152 :
153 : [v1] "I" (v1), [v2] "I" (v2), [v3] "I" (v3)
154 : "memory");
155 }
156
fpu_vesravb(u8 v1,u8 v2,u8 v3)157 static __always_inline void fpu_vesravb(u8 v1, u8 v2, u8 v3)
158 {
159 asm volatile("VESRAVB %[v1],%[v2],%[v3]"
160 :
161 : [v1] "I" (v1), [v2] "I" (v2), [v3] "I" (v3)
162 : "memory");
163 }
164
fpu_vgfmag(u8 v1,u8 v2,u8 v3,u8 v4)165 static __always_inline void fpu_vgfmag(u8 v1, u8 v2, u8 v3, u8 v4)
166 {
167 asm volatile("VGFMAG %[v1],%[v2],%[v3],%[v4]"
168 :
169 : [v1] "I" (v1), [v2] "I" (v2), [v3] "I" (v3), [v4] "I" (v4)
170 : "memory");
171 }
172
fpu_vgfmg(u8 v1,u8 v2,u8 v3)173 static __always_inline void fpu_vgfmg(u8 v1, u8 v2, u8 v3)
174 {
175 asm volatile("VGFMG %[v1],%[v2],%[v3]"
176 :
177 : [v1] "I" (v1), [v2] "I" (v2), [v3] "I" (v3)
178 : "memory");
179 }
180
181 #ifdef CONFIG_CC_HAS_ASM_AOR_FORMAT_FLAGS
182
fpu_vl(u8 v1,const void * vxr)183 static __always_inline void fpu_vl(u8 v1, const void *vxr)
184 {
185 instrument_read(vxr, sizeof(__vector128));
186 asm volatile("VL %[v1],%O[vxr],,%R[vxr]"
187 :
188 : [vxr] "Q" (*(__vector128 *)vxr),
189 [v1] "I" (v1)
190 : "memory");
191 }
192
193 #else /* CONFIG_CC_HAS_ASM_AOR_FORMAT_FLAGS */
194
fpu_vl(u8 v1,const void * vxr)195 static __always_inline void fpu_vl(u8 v1, const void *vxr)
196 {
197 instrument_read(vxr, sizeof(__vector128));
198 asm volatile(
199 " la 1,%[vxr]\n"
200 " VL %[v1],0,,1"
201 :
202 : [vxr] "R" (*(__vector128 *)vxr),
203 [v1] "I" (v1)
204 : "memory", "1");
205 }
206
207 #endif /* CONFIG_CC_HAS_ASM_AOR_FORMAT_FLAGS */
208
fpu_vleib(u8 v,s16 val,u8 index)209 static __always_inline void fpu_vleib(u8 v, s16 val, u8 index)
210 {
211 asm volatile("VLEIB %[v],%[val],%[index]"
212 :
213 : [v] "I" (v), [val] "K" (val), [index] "I" (index)
214 : "memory");
215 }
216
fpu_vleig(u8 v,s16 val,u8 index)217 static __always_inline void fpu_vleig(u8 v, s16 val, u8 index)
218 {
219 asm volatile("VLEIG %[v],%[val],%[index]"
220 :
221 : [v] "I" (v), [val] "K" (val), [index] "I" (index)
222 : "memory");
223 }
224
fpu_vlgvf(u8 v,u16 index)225 static __always_inline u64 fpu_vlgvf(u8 v, u16 index)
226 {
227 u64 val;
228
229 asm volatile("VLGVF %[val],%[v],%[index]"
230 : [val] "=d" (val)
231 : [v] "I" (v), [index] "L" (index)
232 : "memory");
233 return val;
234 }
235
236 #ifdef CONFIG_CC_HAS_ASM_AOR_FORMAT_FLAGS
237
fpu_vll(u8 v1,u32 index,const void * vxr)238 static __always_inline void fpu_vll(u8 v1, u32 index, const void *vxr)
239 {
240 unsigned int size;
241
242 size = min(index + 1, sizeof(__vector128));
243 instrument_read(vxr, size);
244 asm volatile("VLL %[v1],%[index],%O[vxr],%R[vxr]"
245 :
246 : [vxr] "Q" (*(u8 *)vxr),
247 [index] "d" (index),
248 [v1] "I" (v1)
249 : "memory");
250 }
251
252 #else /* CONFIG_CC_HAS_ASM_AOR_FORMAT_FLAGS */
253
fpu_vll(u8 v1,u32 index,const void * vxr)254 static __always_inline void fpu_vll(u8 v1, u32 index, const void *vxr)
255 {
256 unsigned int size;
257
258 size = min(index + 1, sizeof(__vector128));
259 instrument_read(vxr, size);
260 asm volatile(
261 " la 1,%[vxr]\n"
262 " VLL %[v1],%[index],0,1"
263 :
264 : [vxr] "R" (*(u8 *)vxr),
265 [index] "d" (index),
266 [v1] "I" (v1)
267 : "memory", "1");
268 }
269
270 #endif /* CONFIG_CC_HAS_ASM_AOR_FORMAT_FLAGS */
271
272 #ifdef CONFIG_CC_HAS_ASM_AOR_FORMAT_FLAGS
273
274 #define fpu_vlm(_v1, _v3, _vxrs) \
275 ({ \
276 unsigned int size = ((_v3) - (_v1) + 1) * sizeof(__vector128); \
277 struct { \
278 __vector128 _v[(_v3) - (_v1) + 1]; \
279 } *_v = (void *)(_vxrs); \
280 \
281 instrument_read(_v, size); \
282 asm volatile("VLM %[v1],%[v3],%O[vxrs],%R[vxrs]" \
283 : \
284 : [vxrs] "Q" (*_v), \
285 [v1] "I" (_v1), [v3] "I" (_v3) \
286 : "memory"); \
287 (_v3) - (_v1) + 1; \
288 })
289
290 #else /* CONFIG_CC_HAS_ASM_AOR_FORMAT_FLAGS */
291
292 #define fpu_vlm(_v1, _v3, _vxrs) \
293 ({ \
294 unsigned int size = ((_v3) - (_v1) + 1) * sizeof(__vector128); \
295 struct { \
296 __vector128 _v[(_v3) - (_v1) + 1]; \
297 } *_v = (void *)(_vxrs); \
298 \
299 instrument_read(_v, size); \
300 asm volatile( \
301 " la 1,%[vxrs]\n" \
302 " VLM %[v1],%[v3],0,1" \
303 : \
304 : [vxrs] "R" (*_v), \
305 [v1] "I" (_v1), [v3] "I" (_v3) \
306 : "memory", "1"); \
307 (_v3) - (_v1) + 1; \
308 })
309
310 #endif /* CONFIG_CC_HAS_ASM_AOR_FORMAT_FLAGS */
311
fpu_vlr(u8 v1,u8 v2)312 static __always_inline void fpu_vlr(u8 v1, u8 v2)
313 {
314 asm volatile("VLR %[v1],%[v2]"
315 :
316 : [v1] "I" (v1), [v2] "I" (v2)
317 : "memory");
318 }
319
fpu_vlvgf(u8 v,u32 val,u16 index)320 static __always_inline void fpu_vlvgf(u8 v, u32 val, u16 index)
321 {
322 asm volatile("VLVGF %[v],%[val],%[index]"
323 :
324 : [v] "I" (v), [val] "d" (val), [index] "L" (index)
325 : "memory");
326 }
327
fpu_vn(u8 v1,u8 v2,u8 v3)328 static __always_inline void fpu_vn(u8 v1, u8 v2, u8 v3)
329 {
330 asm volatile("VN %[v1],%[v2],%[v3]"
331 :
332 : [v1] "I" (v1), [v2] "I" (v2), [v3] "I" (v3)
333 : "memory");
334 }
335
fpu_vperm(u8 v1,u8 v2,u8 v3,u8 v4)336 static __always_inline void fpu_vperm(u8 v1, u8 v2, u8 v3, u8 v4)
337 {
338 asm volatile("VPERM %[v1],%[v2],%[v3],%[v4]"
339 :
340 : [v1] "I" (v1), [v2] "I" (v2), [v3] "I" (v3), [v4] "I" (v4)
341 : "memory");
342 }
343
fpu_vrepib(u8 v1,s16 i2)344 static __always_inline void fpu_vrepib(u8 v1, s16 i2)
345 {
346 asm volatile("VREPIB %[v1],%[i2]"
347 :
348 : [v1] "I" (v1), [i2] "K" (i2)
349 : "memory");
350 }
351
fpu_vsrlb(u8 v1,u8 v2,u8 v3)352 static __always_inline void fpu_vsrlb(u8 v1, u8 v2, u8 v3)
353 {
354 asm volatile("VSRLB %[v1],%[v2],%[v3]"
355 :
356 : [v1] "I" (v1), [v2] "I" (v2), [v3] "I" (v3)
357 : "memory");
358 }
359
360 #ifdef CONFIG_CC_HAS_ASM_AOR_FORMAT_FLAGS
361
fpu_vst(u8 v1,const void * vxr)362 static __always_inline void fpu_vst(u8 v1, const void *vxr)
363 {
364 instrument_write(vxr, sizeof(__vector128));
365 asm volatile("VST %[v1],%O[vxr],,%R[vxr]"
366 : [vxr] "=Q" (*(__vector128 *)vxr)
367 : [v1] "I" (v1)
368 : "memory");
369 }
370
371 #else /* CONFIG_CC_HAS_ASM_AOR_FORMAT_FLAGS */
372
fpu_vst(u8 v1,const void * vxr)373 static __always_inline void fpu_vst(u8 v1, const void *vxr)
374 {
375 instrument_write(vxr, sizeof(__vector128));
376 asm volatile(
377 " la 1,%[vxr]\n"
378 " VST %[v1],0,,1"
379 : [vxr] "=R" (*(__vector128 *)vxr)
380 : [v1] "I" (v1)
381 : "memory", "1");
382 }
383
384 #endif /* CONFIG_CC_HAS_ASM_AOR_FORMAT_FLAGS */
385
386 #ifdef CONFIG_CC_HAS_ASM_AOR_FORMAT_FLAGS
387
fpu_vstl(u8 v1,u32 index,const void * vxr)388 static __always_inline void fpu_vstl(u8 v1, u32 index, const void *vxr)
389 {
390 unsigned int size;
391
392 size = min(index + 1, sizeof(__vector128));
393 instrument_write(vxr, size);
394 asm volatile("VSTL %[v1],%[index],%O[vxr],%R[vxr]"
395 : [vxr] "=Q" (*(u8 *)vxr)
396 : [index] "d" (index), [v1] "I" (v1)
397 : "memory");
398 kmsan_unpoison_memory(vxr, size);
399 }
400
401 #else /* CONFIG_CC_HAS_ASM_AOR_FORMAT_FLAGS */
402
fpu_vstl(u8 v1,u32 index,const void * vxr)403 static __always_inline void fpu_vstl(u8 v1, u32 index, const void *vxr)
404 {
405 unsigned int size;
406
407 size = min(index + 1, sizeof(__vector128));
408 instrument_write(vxr, size);
409 asm volatile(
410 " la 1,%[vxr]\n"
411 " VSTL %[v1],%[index],0,1"
412 : [vxr] "=R" (*(u8 *)vxr)
413 : [index] "d" (index), [v1] "I" (v1)
414 : "memory", "1");
415 kmsan_unpoison_memory(vxr, size);
416 }
417
418 #endif /* CONFIG_CC_HAS_ASM_AOR_FORMAT_FLAGS */
419
420 #ifdef CONFIG_CC_HAS_ASM_AOR_FORMAT_FLAGS
421
422 #define fpu_vstm(_v1, _v3, _vxrs) \
423 ({ \
424 unsigned int size = ((_v3) - (_v1) + 1) * sizeof(__vector128); \
425 struct { \
426 __vector128 _v[(_v3) - (_v1) + 1]; \
427 } *_v = (void *)(_vxrs); \
428 \
429 instrument_write(_v, size); \
430 asm volatile("VSTM %[v1],%[v3],%O[vxrs],%R[vxrs]" \
431 : [vxrs] "=Q" (*_v) \
432 : [v1] "I" (_v1), [v3] "I" (_v3) \
433 : "memory"); \
434 (_v3) - (_v1) + 1; \
435 })
436
437 #else /* CONFIG_CC_HAS_ASM_AOR_FORMAT_FLAGS */
438
439 #define fpu_vstm(_v1, _v3, _vxrs) \
440 ({ \
441 unsigned int size = ((_v3) - (_v1) + 1) * sizeof(__vector128); \
442 struct { \
443 __vector128 _v[(_v3) - (_v1) + 1]; \
444 } *_v = (void *)(_vxrs); \
445 \
446 instrument_write(_v, size); \
447 asm volatile( \
448 " la 1,%[vxrs]\n" \
449 " VSTM %[v1],%[v3],0,1" \
450 : [vxrs] "=R" (*_v) \
451 : [v1] "I" (_v1), [v3] "I" (_v3) \
452 : "memory", "1"); \
453 (_v3) - (_v1) + 1; \
454 })
455
456 #endif /* CONFIG_CC_HAS_ASM_AOR_FORMAT_FLAGS */
457
fpu_vupllf(u8 v1,u8 v2)458 static __always_inline void fpu_vupllf(u8 v1, u8 v2)
459 {
460 asm volatile("VUPLLF %[v1],%[v2]"
461 :
462 : [v1] "I" (v1), [v2] "I" (v2)
463 : "memory");
464 }
465
fpu_vx(u8 v1,u8 v2,u8 v3)466 static __always_inline void fpu_vx(u8 v1, u8 v2, u8 v3)
467 {
468 asm volatile("VX %[v1],%[v2],%[v3]"
469 :
470 : [v1] "I" (v1), [v2] "I" (v2), [v3] "I" (v3)
471 : "memory");
472 }
473
fpu_vzero(u8 v)474 static __always_inline void fpu_vzero(u8 v)
475 {
476 asm volatile("VZERO %[v]"
477 :
478 : [v] "I" (v)
479 : "memory");
480 }
481
482 #endif /* __ASSEMBLER__ */
483 #endif /* __ASM_S390_FPU_INSN_H */
484