xref: /linux/arch/s390/include/asm/fpu-insn.h (revision 3a2c4d55e32ad65efebdb6de44eef3bfa08bb49d) 
1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3  * Support for Floating Point and Vector Instructions
4  *
5  */
6 
7 #ifndef __ASM_S390_FPU_INSN_H
8 #define __ASM_S390_FPU_INSN_H
9 
10 #include <asm/fpu-insn-asm.h>
11 
12 #ifndef __ASSEMBLER__
13 
14 #include <linux/instrumented.h>
15 #include <linux/kmsan.h>
16 #include <asm/asm-extable.h>
17 
18 asm(".include \"asm/insn-common-asm.h\"\n");
19 asm(".include \"asm/fpu-insn-asm.h\"\n");
20 
21 /*
22  * Various small helper functions, which can and should be used within
23  * kernel fpu code sections. Each function represents only one floating
24  * point or vector instruction (except for helper functions which require
25  * exception handling).
26  *
27  * This allows to use floating point and vector instructions like C
28  * functions, which has the advantage that all supporting code, like
29  * e.g. loops, can be written in easy to read C code.
30  *
31  * Each of the helper functions provides support for code instrumentation,
32  * like e.g. KASAN. Therefore instrumentation is also covered automatically
33  * when using these functions.
34  *
35  * In order to ensure that code generated with the helper functions stays
36  * within kernel fpu sections, which are guarded with kernel_fpu_begin()
37  * and kernel_fpu_end() calls, each function has a mandatory "memory"
38  * barrier.
39  */
40 
41 static __always_inline void fpu_cefbr(u8 f1, s32 val)
42 {
43 	asm volatile("cefbr	%[f1],%[val]"
44 		     :
45 		     : [f1] "I" (f1), [val] "d" (val)
46 		     : "memory");
47 }
48 
49 static __always_inline unsigned long fpu_cgebr(u8 f2, u8 mode)
50 {
51 	unsigned long val;
52 
53 	asm volatile("cgebr	%[val],%[mode],%[f2]"
54 		     : [val] "=d" (val)
55 		     : [f2] "I" (f2), [mode] "I" (mode)
56 		     : "memory");
57 	return val;
58 }
59 
60 static __always_inline void fpu_debr(u8 f1, u8 f2)
61 {
62 	asm volatile("debr	%[f1],%[f2]"
63 		     :
64 		     : [f1] "I" (f1), [f2] "I" (f2)
65 		     : "memory");
66 }
67 
68 static __always_inline void fpu_ld(unsigned short fpr, freg_t *reg)
69 {
70 	instrument_read(reg, sizeof(*reg));
71 	asm volatile("ld	 %[fpr],%[reg]"
72 		     :
73 		     : [fpr] "I" (fpr), [reg] "Q" (reg->ui)
74 		     : "memory");
75 }
76 
77 static __always_inline void fpu_ldgr(u8 f1, u32 val)
78 {
79 	asm volatile("ldgr	%[f1],%[val]"
80 		     :
81 		     : [f1] "I" (f1), [val] "d" (val)
82 		     : "memory");
83 }
84 
85 static __always_inline void fpu_lfpc(unsigned int *fpc)
86 {
87 	instrument_read(fpc, sizeof(*fpc));
88 	asm volatile("lfpc	%[fpc]"
89 		     :
90 		     : [fpc] "Q" (*fpc)
91 		     : "memory");
92 }
93 
94 /**
95  * fpu_lfpc_safe - Load floating point control register safely.
96  * @fpc: new value for floating point control register
97  *
98  * Load floating point control register. This may lead to an exception,
99  * since a saved value may have been modified by user space (ptrace,
100  * signal return, kvm registers) to an invalid value. In such a case
101  * set the floating point control register to zero.
102  */
103 static inline void fpu_lfpc_safe(unsigned int *fpc)
104 {
105 	instrument_read(fpc, sizeof(*fpc));
106 	asm_inline volatile(
107 		"	lfpc	%[fpc]\n"
108 		"0:	nopr	%%r7\n"
109 		EX_TABLE_FPC(0b, 0b)
110 		:
111 		: [fpc] "Q" (*fpc)
112 		: "memory");
113 }
114 
115 static __always_inline void fpu_std(unsigned short fpr, freg_t *reg)
116 {
117 	instrument_write(reg, sizeof(*reg));
118 	asm volatile("std	 %[fpr],%[reg]"
119 		     : [reg] "=Q" (reg->ui)
120 		     : [fpr] "I" (fpr)
121 		     : "memory");
122 }
123 
124 static __always_inline void fpu_sfpc(unsigned int fpc)
125 {
126 	asm volatile("sfpc	%[fpc]"
127 		     :
128 		     : [fpc] "d" (fpc)
129 		     : "memory");
130 }
131 
132 static __always_inline void fpu_stfpc(unsigned int *fpc)
133 {
134 	instrument_write(fpc, sizeof(*fpc));
135 	asm volatile("stfpc	%[fpc]"
136 		     : [fpc] "=Q" (*fpc)
137 		     :
138 		     : "memory");
139 }
140 
141 static __always_inline void fpu_vab(u8 v1, u8 v2, u8 v3)
142 {
143 	asm volatile("VAB	%[v1],%[v2],%[v3]"
144 		     :
145 		     : [v1] "I" (v1), [v2] "I" (v2), [v3] "I" (v3)
146 		     : "memory");
147 }
148 
149 static __always_inline void fpu_vcksm(u8 v1, u8 v2, u8 v3)
150 {
151 	asm volatile("VCKSM	%[v1],%[v2],%[v3]"
152 		     :
153 		     : [v1] "I" (v1), [v2] "I" (v2), [v3] "I" (v3)
154 		     : "memory");
155 }
156 
157 static __always_inline void fpu_vesravb(u8 v1, u8 v2, u8 v3)
158 {
159 	asm volatile("VESRAVB	%[v1],%[v2],%[v3]"
160 		     :
161 		     : [v1] "I" (v1), [v2] "I" (v2), [v3] "I" (v3)
162 		     : "memory");
163 }
164 
165 static __always_inline void fpu_vgfmag(u8 v1, u8 v2, u8 v3, u8 v4)
166 {
167 	asm volatile("VGFMAG	%[v1],%[v2],%[v3],%[v4]"
168 		     :
169 		     : [v1] "I" (v1), [v2] "I" (v2), [v3] "I" (v3), [v4] "I" (v4)
170 		     : "memory");
171 }
172 
173 static __always_inline void fpu_vgfmg(u8 v1, u8 v2, u8 v3)
174 {
175 	asm volatile("VGFMG	%[v1],%[v2],%[v3]"
176 		     :
177 		     : [v1] "I" (v1), [v2] "I" (v2), [v3] "I" (v3)
178 		     : "memory");
179 }
180 
181 #ifdef CONFIG_CC_HAS_ASM_AOR_FORMAT_FLAGS
182 
183 static __always_inline void fpu_vl(u8 v1, const void *vxr)
184 {
185 	instrument_read(vxr, sizeof(__vector128));
186 	asm volatile("VL	%[v1],%O[vxr],,%R[vxr]"
187 		     :
188 		     : [vxr] "Q" (*(__vector128 *)vxr),
189 		       [v1] "I" (v1)
190 		     : "memory");
191 }
192 
193 #else /* CONFIG_CC_HAS_ASM_AOR_FORMAT_FLAGS */
194 
195 static __always_inline void fpu_vl(u8 v1, const void *vxr)
196 {
197 	instrument_read(vxr, sizeof(__vector128));
198 	asm volatile(
199 		"	la	1,%[vxr]\n"
200 		"	VL	%[v1],0,,1"
201 		:
202 		: [vxr] "R" (*(__vector128 *)vxr),
203 		  [v1] "I" (v1)
204 		: "memory", "1");
205 }
206 
207 #endif /* CONFIG_CC_HAS_ASM_AOR_FORMAT_FLAGS */
208 
209 static __always_inline void fpu_vleib(u8 v, s16 val, u8 index)
210 {
211 	asm volatile("VLEIB	%[v],%[val],%[index]"
212 		     :
213 		     : [v] "I" (v), [val] "K" (val), [index] "I" (index)
214 		     : "memory");
215 }
216 
217 static __always_inline void fpu_vleig(u8 v, s16 val, u8 index)
218 {
219 	asm volatile("VLEIG	%[v],%[val],%[index]"
220 		     :
221 		     : [v] "I" (v), [val] "K" (val), [index] "I" (index)
222 		     : "memory");
223 }
224 
225 static __always_inline u64 fpu_vlgvf(u8 v, u16 index)
226 {
227 	u64 val;
228 
229 	asm volatile("VLGVF	%[val],%[v],%[index]"
230 		     : [val] "=d" (val)
231 		     : [v] "I" (v), [index] "L" (index)
232 		     : "memory");
233 	return val;
234 }
235 
236 #ifdef CONFIG_CC_HAS_ASM_AOR_FORMAT_FLAGS
237 
238 static __always_inline void fpu_vll(u8 v1, u32 index, const void *vxr)
239 {
240 	unsigned int size;
241 
242 	size = min(index + 1, sizeof(__vector128));
243 	instrument_read(vxr, size);
244 	asm volatile("VLL	%[v1],%[index],%O[vxr],%R[vxr]"
245 		     :
246 		     : [vxr] "Q" (*(u8 *)vxr),
247 		       [index] "d" (index),
248 		       [v1] "I" (v1)
249 		     : "memory");
250 }
251 
252 #else /* CONFIG_CC_HAS_ASM_AOR_FORMAT_FLAGS */
253 
254 static __always_inline void fpu_vll(u8 v1, u32 index, const void *vxr)
255 {
256 	unsigned int size;
257 
258 	size = min(index + 1, sizeof(__vector128));
259 	instrument_read(vxr, size);
260 	asm volatile(
261 		"	la	1,%[vxr]\n"
262 		"	VLL	%[v1],%[index],0,1"
263 		:
264 		: [vxr] "R" (*(u8 *)vxr),
265 		  [index] "d" (index),
266 		  [v1] "I" (v1)
267 		: "memory", "1");
268 }
269 
270 #endif /* CONFIG_CC_HAS_ASM_AOR_FORMAT_FLAGS */
271 
272 #ifdef CONFIG_CC_HAS_ASM_AOR_FORMAT_FLAGS
273 
274 #define fpu_vlm(_v1, _v3, _vxrs)					\
275 ({									\
276 	unsigned int size = ((_v3) - (_v1) + 1) * sizeof(__vector128);	\
277 	struct {							\
278 		__vector128 _v[(_v3) - (_v1) + 1];			\
279 	} *_v = (void *)(_vxrs);					\
280 									\
281 	instrument_read(_v, size);					\
282 	asm volatile("VLM	%[v1],%[v3],%O[vxrs],%R[vxrs]"		\
283 		     :							\
284 		     : [vxrs] "Q" (*_v),				\
285 		       [v1] "I" (_v1), [v3] "I" (_v3)			\
286 		     : "memory");					\
287 	(_v3) - (_v1) + 1;						\
288 })
289 
290 #else /* CONFIG_CC_HAS_ASM_AOR_FORMAT_FLAGS */
291 
292 #define fpu_vlm(_v1, _v3, _vxrs)					\
293 ({									\
294 	unsigned int size = ((_v3) - (_v1) + 1) * sizeof(__vector128);	\
295 	struct {							\
296 		__vector128 _v[(_v3) - (_v1) + 1];			\
297 	} *_v = (void *)(_vxrs);					\
298 									\
299 	instrument_read(_v, size);					\
300 	asm volatile(							\
301 		"	la	1,%[vxrs]\n"				\
302 		"	VLM	%[v1],%[v3],0,1"			\
303 		:							\
304 		: [vxrs] "R" (*_v),					\
305 		  [v1] "I" (_v1), [v3] "I" (_v3)			\
306 		: "memory", "1");					\
307 	(_v3) - (_v1) + 1;						\
308 })
309 
310 #endif /* CONFIG_CC_HAS_ASM_AOR_FORMAT_FLAGS */
311 
312 static __always_inline void fpu_vlr(u8 v1, u8 v2)
313 {
314 	asm volatile("VLR	%[v1],%[v2]"
315 		     :
316 		     : [v1] "I" (v1), [v2] "I" (v2)
317 		     : "memory");
318 }
319 
320 static __always_inline void fpu_vlvgf(u8 v, u32 val, u16 index)
321 {
322 	asm volatile("VLVGF	%[v],%[val],%[index]"
323 		     :
324 		     : [v] "I" (v), [val] "d" (val), [index] "L" (index)
325 		     : "memory");
326 }
327 
328 static __always_inline void fpu_vn(u8 v1, u8 v2, u8 v3)
329 {
330 	asm volatile("VN	%[v1],%[v2],%[v3]"
331 		     :
332 		     : [v1] "I" (v1), [v2] "I" (v2), [v3] "I" (v3)
333 		     : "memory");
334 }
335 
336 static __always_inline void fpu_vperm(u8 v1, u8 v2, u8 v3, u8 v4)
337 {
338 	asm volatile("VPERM	%[v1],%[v2],%[v3],%[v4]"
339 		     :
340 		     : [v1] "I" (v1), [v2] "I" (v2), [v3] "I" (v3), [v4] "I" (v4)
341 		     : "memory");
342 }
343 
344 static __always_inline void fpu_vrepib(u8 v1, s16 i2)
345 {
346 	asm volatile("VREPIB	%[v1],%[i2]"
347 		     :
348 		     : [v1] "I" (v1), [i2] "K" (i2)
349 		     : "memory");
350 }
351 
352 static __always_inline void fpu_vsrlb(u8 v1, u8 v2, u8 v3)
353 {
354 	asm volatile("VSRLB	%[v1],%[v2],%[v3]"
355 		     :
356 		     : [v1] "I" (v1), [v2] "I" (v2), [v3] "I" (v3)
357 		     : "memory");
358 }
359 
360 #ifdef CONFIG_CC_HAS_ASM_AOR_FORMAT_FLAGS
361 
362 static __always_inline void fpu_vst(u8 v1, const void *vxr)
363 {
364 	instrument_write(vxr, sizeof(__vector128));
365 	asm volatile("VST	%[v1],%O[vxr],,%R[vxr]"
366 		     : [vxr] "=Q" (*(__vector128 *)vxr)
367 		     : [v1] "I" (v1)
368 		     : "memory");
369 }
370 
371 #else /* CONFIG_CC_HAS_ASM_AOR_FORMAT_FLAGS */
372 
373 static __always_inline void fpu_vst(u8 v1, const void *vxr)
374 {
375 	instrument_write(vxr, sizeof(__vector128));
376 	asm volatile(
377 		"	la	1,%[vxr]\n"
378 		"	VST	%[v1],0,,1"
379 		: [vxr] "=R" (*(__vector128 *)vxr)
380 		: [v1] "I" (v1)
381 		: "memory", "1");
382 }
383 
384 #endif /* CONFIG_CC_HAS_ASM_AOR_FORMAT_FLAGS */
385 
386 #ifdef CONFIG_CC_HAS_ASM_AOR_FORMAT_FLAGS
387 
388 static __always_inline void fpu_vstl(u8 v1, u32 index, const void *vxr)
389 {
390 	unsigned int size;
391 
392 	size = min(index + 1, sizeof(__vector128));
393 	instrument_write(vxr, size);
394 	asm volatile("VSTL	%[v1],%[index],%O[vxr],%R[vxr]"
395 		     : [vxr] "=Q" (*(u8 *)vxr)
396 		     : [index] "d" (index), [v1] "I" (v1)
397 		     : "memory");
398 	kmsan_unpoison_memory(vxr, size);
399 }
400 
401 #else /* CONFIG_CC_HAS_ASM_AOR_FORMAT_FLAGS */
402 
403 static __always_inline void fpu_vstl(u8 v1, u32 index, const void *vxr)
404 {
405 	unsigned int size;
406 
407 	size = min(index + 1, sizeof(__vector128));
408 	instrument_write(vxr, size);
409 	asm volatile(
410 		"	la	1,%[vxr]\n"
411 		"	VSTL	%[v1],%[index],0,1"
412 		: [vxr] "=R" (*(u8 *)vxr)
413 		: [index] "d" (index), [v1] "I" (v1)
414 		: "memory", "1");
415 	kmsan_unpoison_memory(vxr, size);
416 }
417 
418 #endif /* CONFIG_CC_HAS_ASM_AOR_FORMAT_FLAGS */
419 
420 #ifdef CONFIG_CC_HAS_ASM_AOR_FORMAT_FLAGS
421 
422 #define fpu_vstm(_v1, _v3, _vxrs)					\
423 ({									\
424 	unsigned int size = ((_v3) - (_v1) + 1) * sizeof(__vector128);	\
425 	struct {							\
426 		__vector128 _v[(_v3) - (_v1) + 1];			\
427 	} *_v = (void *)(_vxrs);					\
428 									\
429 	instrument_write(_v, size);					\
430 	asm volatile("VSTM	%[v1],%[v3],%O[vxrs],%R[vxrs]"		\
431 		     : [vxrs] "=Q" (*_v)				\
432 		     : [v1] "I" (_v1), [v3] "I" (_v3)			\
433 		     : "memory");					\
434 	(_v3) - (_v1) + 1;						\
435 })
436 
437 #else /* CONFIG_CC_HAS_ASM_AOR_FORMAT_FLAGS */
438 
439 #define fpu_vstm(_v1, _v3, _vxrs)					\
440 ({									\
441 	unsigned int size = ((_v3) - (_v1) + 1) * sizeof(__vector128);	\
442 	struct {							\
443 		__vector128 _v[(_v3) - (_v1) + 1];			\
444 	} *_v = (void *)(_vxrs);					\
445 									\
446 	instrument_write(_v, size);					\
447 	asm volatile(							\
448 		"	la	1,%[vxrs]\n"				\
449 		"	VSTM	%[v1],%[v3],0,1"			\
450 		: [vxrs] "=R" (*_v)					\
451 		: [v1] "I" (_v1), [v3] "I" (_v3)			\
452 		: "memory", "1");					\
453 	(_v3) - (_v1) + 1;						\
454 })
455 
456 #endif /* CONFIG_CC_HAS_ASM_AOR_FORMAT_FLAGS */
457 
458 static __always_inline void fpu_vupllf(u8 v1, u8 v2)
459 {
460 	asm volatile("VUPLLF	%[v1],%[v2]"
461 		     :
462 		     : [v1] "I" (v1), [v2] "I" (v2)
463 		     : "memory");
464 }
465 
466 static __always_inline void fpu_vx(u8 v1, u8 v2, u8 v3)
467 {
468 	asm volatile("VX	%[v1],%[v2],%[v3]"
469 		     :
470 		     : [v1] "I" (v1), [v2] "I" (v2), [v3] "I" (v3)
471 		     : "memory");
472 }
473 
474 static __always_inline void fpu_vzero(u8 v)
475 {
476 	asm volatile("VZERO	%[v]"
477 		     :
478 		     : [v] "I" (v)
479 		     : "memory");
480 }
481 
482 #endif /* __ASSEMBLER__ */
483 #endif	/* __ASM_S390_FPU_INSN_H */
484