1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2020 Western Digital Corporation or its affiliates.
4 */
5 #include <linux/kernel.h>
6 #include <linux/init.h>
7 #include <linux/mm.h>
8 #include <linux/module.h>
9 #include <linux/perf_event.h>
10 #include <linux/irq.h>
11 #include <linux/stringify.h>
12
13 #include <asm/processor.h>
14 #include <asm/ptrace.h>
15 #include <asm/csr.h>
16 #include <asm/entry-common.h>
17 #include <asm/hwprobe.h>
18 #include <asm/cpufeature.h>
19 #include <asm/vector.h>
20
21 #define INSN_MATCH_LB 0x3
22 #define INSN_MASK_LB 0x707f
23 #define INSN_MATCH_LH 0x1003
24 #define INSN_MASK_LH 0x707f
25 #define INSN_MATCH_LW 0x2003
26 #define INSN_MASK_LW 0x707f
27 #define INSN_MATCH_LD 0x3003
28 #define INSN_MASK_LD 0x707f
29 #define INSN_MATCH_LBU 0x4003
30 #define INSN_MASK_LBU 0x707f
31 #define INSN_MATCH_LHU 0x5003
32 #define INSN_MASK_LHU 0x707f
33 #define INSN_MATCH_LWU 0x6003
34 #define INSN_MASK_LWU 0x707f
35 #define INSN_MATCH_SB 0x23
36 #define INSN_MASK_SB 0x707f
37 #define INSN_MATCH_SH 0x1023
38 #define INSN_MASK_SH 0x707f
39 #define INSN_MATCH_SW 0x2023
40 #define INSN_MASK_SW 0x707f
41 #define INSN_MATCH_SD 0x3023
42 #define INSN_MASK_SD 0x707f
43
44 #define INSN_MATCH_FLW 0x2007
45 #define INSN_MASK_FLW 0x707f
46 #define INSN_MATCH_FLD 0x3007
47 #define INSN_MASK_FLD 0x707f
48 #define INSN_MATCH_FLQ 0x4007
49 #define INSN_MASK_FLQ 0x707f
50 #define INSN_MATCH_FSW 0x2027
51 #define INSN_MASK_FSW 0x707f
52 #define INSN_MATCH_FSD 0x3027
53 #define INSN_MASK_FSD 0x707f
54 #define INSN_MATCH_FSQ 0x4027
55 #define INSN_MASK_FSQ 0x707f
56
57 #define INSN_MATCH_C_LD 0x6000
58 #define INSN_MASK_C_LD 0xe003
59 #define INSN_MATCH_C_SD 0xe000
60 #define INSN_MASK_C_SD 0xe003
61 #define INSN_MATCH_C_LW 0x4000
62 #define INSN_MASK_C_LW 0xe003
63 #define INSN_MATCH_C_SW 0xc000
64 #define INSN_MASK_C_SW 0xe003
65 #define INSN_MATCH_C_LDSP 0x6002
66 #define INSN_MASK_C_LDSP 0xe003
67 #define INSN_MATCH_C_SDSP 0xe002
68 #define INSN_MASK_C_SDSP 0xe003
69 #define INSN_MATCH_C_LWSP 0x4002
70 #define INSN_MASK_C_LWSP 0xe003
71 #define INSN_MATCH_C_SWSP 0xc002
72 #define INSN_MASK_C_SWSP 0xe003
73
74 #define INSN_MATCH_C_FLD 0x2000
75 #define INSN_MASK_C_FLD 0xe003
76 #define INSN_MATCH_C_FLW 0x6000
77 #define INSN_MASK_C_FLW 0xe003
78 #define INSN_MATCH_C_FSD 0xa000
79 #define INSN_MASK_C_FSD 0xe003
80 #define INSN_MATCH_C_FSW 0xe000
81 #define INSN_MASK_C_FSW 0xe003
82 #define INSN_MATCH_C_FLDSP 0x2002
83 #define INSN_MASK_C_FLDSP 0xe003
84 #define INSN_MATCH_C_FSDSP 0xa002
85 #define INSN_MASK_C_FSDSP 0xe003
86 #define INSN_MATCH_C_FLWSP 0x6002
87 #define INSN_MASK_C_FLWSP 0xe003
88 #define INSN_MATCH_C_FSWSP 0xe002
89 #define INSN_MASK_C_FSWSP 0xe003
90
91 #define INSN_LEN(insn) ((((insn) & 0x3) < 0x3) ? 2 : 4)
92
93 #if defined(CONFIG_64BIT)
94 #define LOG_REGBYTES 3
95 #define XLEN 64
96 #else
97 #define LOG_REGBYTES 2
98 #define XLEN 32
99 #endif
100 #define REGBYTES (1 << LOG_REGBYTES)
101 #define XLEN_MINUS_16 ((XLEN) - 16)
102
103 #define SH_RD 7
104 #define SH_RS1 15
105 #define SH_RS2 20
106 #define SH_RS2C 2
107
108 #define RV_X(x, s, n) (((x) >> (s)) & ((1 << (n)) - 1))
109 #define RVC_LW_IMM(x) ((RV_X(x, 6, 1) << 2) | \
110 (RV_X(x, 10, 3) << 3) | \
111 (RV_X(x, 5, 1) << 6))
112 #define RVC_LD_IMM(x) ((RV_X(x, 10, 3) << 3) | \
113 (RV_X(x, 5, 2) << 6))
114 #define RVC_LWSP_IMM(x) ((RV_X(x, 4, 3) << 2) | \
115 (RV_X(x, 12, 1) << 5) | \
116 (RV_X(x, 2, 2) << 6))
117 #define RVC_LDSP_IMM(x) ((RV_X(x, 5, 2) << 3) | \
118 (RV_X(x, 12, 1) << 5) | \
119 (RV_X(x, 2, 3) << 6))
120 #define RVC_SWSP_IMM(x) ((RV_X(x, 9, 4) << 2) | \
121 (RV_X(x, 7, 2) << 6))
122 #define RVC_SDSP_IMM(x) ((RV_X(x, 10, 3) << 3) | \
123 (RV_X(x, 7, 3) << 6))
124 #define RVC_RS1S(insn) (8 + RV_X(insn, SH_RD, 3))
125 #define RVC_RS2S(insn) (8 + RV_X(insn, SH_RS2C, 3))
126 #define RVC_RS2(insn) RV_X(insn, SH_RS2C, 5)
127
128 #define SHIFT_RIGHT(x, y) \
129 ((y) < 0 ? ((x) << -(y)) : ((x) >> (y)))
130
131 #define REG_MASK \
132 ((1 << (5 + LOG_REGBYTES)) - (1 << LOG_REGBYTES))
133
134 #define REG_OFFSET(insn, pos) \
135 (SHIFT_RIGHT((insn), (pos) - LOG_REGBYTES) & REG_MASK)
136
137 #define REG_PTR(insn, pos, regs) \
138 (ulong *)((ulong)(regs) + REG_OFFSET(insn, pos))
139
140 #define GET_RS1(insn, regs) (*REG_PTR(insn, SH_RS1, regs))
141 #define GET_RS2(insn, regs) (*REG_PTR(insn, SH_RS2, regs))
142 #define GET_RS1S(insn, regs) (*REG_PTR(RVC_RS1S(insn), 0, regs))
143 #define GET_RS2S(insn, regs) (*REG_PTR(RVC_RS2S(insn), 0, regs))
144 #define GET_RS2C(insn, regs) (*REG_PTR(insn, SH_RS2C, regs))
145 #define GET_SP(regs) (*REG_PTR(2, 0, regs))
146 #define SET_RD(insn, regs, val) (*REG_PTR(insn, SH_RD, regs) = (val))
147 #define IMM_I(insn) ((s32)(insn) >> 20)
148 #define IMM_S(insn) (((s32)(insn) >> 25 << 5) | \
149 (s32)(((insn) >> 7) & 0x1f))
150 #define MASK_FUNCT3 0x7000
151
152 #define GET_PRECISION(insn) (((insn) >> 25) & 3)
153 #define GET_RM(insn) (((insn) >> 12) & 7)
154 #define PRECISION_S 0
155 #define PRECISION_D 1
156
157 #ifdef CONFIG_FPU
158
159 #define FP_GET_RD(insn) (insn >> 7 & 0x1F)
160
161 extern void put_f32_reg(unsigned long fp_reg, unsigned long value);
162
set_f32_rd(unsigned long insn,struct pt_regs * regs,unsigned long val)163 static int set_f32_rd(unsigned long insn, struct pt_regs *regs,
164 unsigned long val)
165 {
166 unsigned long fp_reg = FP_GET_RD(insn);
167
168 put_f32_reg(fp_reg, val);
169 regs->status |= SR_FS_DIRTY;
170
171 return 0;
172 }
173
174 extern void put_f64_reg(unsigned long fp_reg, unsigned long value);
175
set_f64_rd(unsigned long insn,struct pt_regs * regs,u64 val)176 static int set_f64_rd(unsigned long insn, struct pt_regs *regs, u64 val)
177 {
178 unsigned long fp_reg = FP_GET_RD(insn);
179 unsigned long value;
180
181 #if __riscv_xlen == 32
182 value = (unsigned long) &val;
183 #else
184 value = val;
185 #endif
186 put_f64_reg(fp_reg, value);
187 regs->status |= SR_FS_DIRTY;
188
189 return 0;
190 }
191
192 #if __riscv_xlen == 32
193 extern void get_f64_reg(unsigned long fp_reg, u64 *value);
194
get_f64_rs(unsigned long insn,u8 fp_reg_offset,struct pt_regs * regs)195 static u64 get_f64_rs(unsigned long insn, u8 fp_reg_offset,
196 struct pt_regs *regs)
197 {
198 unsigned long fp_reg = (insn >> fp_reg_offset) & 0x1F;
199 u64 val;
200
201 get_f64_reg(fp_reg, &val);
202 regs->status |= SR_FS_DIRTY;
203
204 return val;
205 }
206 #else
207
208 extern unsigned long get_f64_reg(unsigned long fp_reg);
209
get_f64_rs(unsigned long insn,u8 fp_reg_offset,struct pt_regs * regs)210 static unsigned long get_f64_rs(unsigned long insn, u8 fp_reg_offset,
211 struct pt_regs *regs)
212 {
213 unsigned long fp_reg = (insn >> fp_reg_offset) & 0x1F;
214 unsigned long val;
215
216 val = get_f64_reg(fp_reg);
217 regs->status |= SR_FS_DIRTY;
218
219 return val;
220 }
221
222 #endif
223
224 extern unsigned long get_f32_reg(unsigned long fp_reg);
225
get_f32_rs(unsigned long insn,u8 fp_reg_offset,struct pt_regs * regs)226 static unsigned long get_f32_rs(unsigned long insn, u8 fp_reg_offset,
227 struct pt_regs *regs)
228 {
229 unsigned long fp_reg = (insn >> fp_reg_offset) & 0x1F;
230 unsigned long val;
231
232 val = get_f32_reg(fp_reg);
233 regs->status |= SR_FS_DIRTY;
234
235 return val;
236 }
237
238 #else /* CONFIG_FPU */
set_f32_rd(unsigned long insn,struct pt_regs * regs,unsigned long val)239 static void set_f32_rd(unsigned long insn, struct pt_regs *regs,
240 unsigned long val) {}
241
set_f64_rd(unsigned long insn,struct pt_regs * regs,u64 val)242 static void set_f64_rd(unsigned long insn, struct pt_regs *regs, u64 val) {}
243
get_f64_rs(unsigned long insn,u8 fp_reg_offset,struct pt_regs * regs)244 static unsigned long get_f64_rs(unsigned long insn, u8 fp_reg_offset,
245 struct pt_regs *regs)
246 {
247 return 0;
248 }
249
get_f32_rs(unsigned long insn,u8 fp_reg_offset,struct pt_regs * regs)250 static unsigned long get_f32_rs(unsigned long insn, u8 fp_reg_offset,
251 struct pt_regs *regs)
252 {
253 return 0;
254 }
255
256 #endif
257
258 #define GET_F64_RS2(insn, regs) (get_f64_rs(insn, 20, regs))
259 #define GET_F64_RS2C(insn, regs) (get_f64_rs(insn, 2, regs))
260 #define GET_F64_RS2S(insn, regs) (get_f64_rs(RVC_RS2S(insn), 0, regs))
261
262 #define GET_F32_RS2(insn, regs) (get_f32_rs(insn, 20, regs))
263 #define GET_F32_RS2C(insn, regs) (get_f32_rs(insn, 2, regs))
264 #define GET_F32_RS2S(insn, regs) (get_f32_rs(RVC_RS2S(insn), 0, regs))
265
266 #define __read_insn(regs, insn, insn_addr, type) \
267 ({ \
268 int __ret; \
269 \
270 if (user_mode(regs)) { \
271 __ret = __get_user(insn, (type __user *) insn_addr); \
272 } else { \
273 insn = *(type *)insn_addr; \
274 __ret = 0; \
275 } \
276 \
277 __ret; \
278 })
279
get_insn(struct pt_regs * regs,ulong epc,ulong * r_insn)280 static inline int get_insn(struct pt_regs *regs, ulong epc, ulong *r_insn)
281 {
282 ulong insn = 0;
283
284 if (epc & 0x2) {
285 ulong tmp = 0;
286
287 if (__read_insn(regs, insn, epc, u16))
288 return -EFAULT;
289 /* __get_user() uses regular "lw" which sign extend the loaded
290 * value make sure to clear higher order bits in case we "or" it
291 * below with the upper 16 bits half.
292 */
293 insn &= GENMASK(15, 0);
294 if ((insn & __INSN_LENGTH_MASK) != __INSN_LENGTH_32) {
295 *r_insn = insn;
296 return 0;
297 }
298 epc += sizeof(u16);
299 if (__read_insn(regs, tmp, epc, u16))
300 return -EFAULT;
301 *r_insn = (tmp << 16) | insn;
302
303 return 0;
304 } else {
305 if (__read_insn(regs, insn, epc, u32))
306 return -EFAULT;
307 if ((insn & __INSN_LENGTH_MASK) == __INSN_LENGTH_32) {
308 *r_insn = insn;
309 return 0;
310 }
311 insn &= GENMASK(15, 0);
312 *r_insn = insn;
313
314 return 0;
315 }
316 }
317
318 union reg_data {
319 u8 data_bytes[8];
320 ulong data_ulong;
321 u64 data_u64;
322 };
323
324 /* sysctl hooks */
325 int unaligned_enabled __read_mostly = 1; /* Enabled by default */
326
327 #ifdef CONFIG_RISCV_VECTOR_MISALIGNED
handle_vector_misaligned_load(struct pt_regs * regs)328 static int handle_vector_misaligned_load(struct pt_regs *regs)
329 {
330 unsigned long epc = regs->epc;
331 unsigned long insn;
332
333 if (get_insn(regs, epc, &insn))
334 return -1;
335
336 /* Only return 0 when in check_vector_unaligned_access_emulated */
337 if (*this_cpu_ptr(&vector_misaligned_access) == RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN) {
338 *this_cpu_ptr(&vector_misaligned_access) = RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED;
339 regs->epc = epc + INSN_LEN(insn);
340 return 0;
341 }
342
343 /* If vector instruction we don't emulate it yet */
344 regs->epc = epc;
345 return -1;
346 }
347 #else
handle_vector_misaligned_load(struct pt_regs * regs)348 static int handle_vector_misaligned_load(struct pt_regs *regs)
349 {
350 return -1;
351 }
352 #endif
353
handle_scalar_misaligned_load(struct pt_regs * regs)354 static int handle_scalar_misaligned_load(struct pt_regs *regs)
355 {
356 union reg_data val;
357 unsigned long epc = regs->epc;
358 unsigned long insn;
359 unsigned long addr = regs->badaddr;
360 int fp = 0, shift = 0, len = 0;
361
362 perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, addr);
363
364 #ifdef CONFIG_RISCV_PROBE_UNALIGNED_ACCESS
365 *this_cpu_ptr(&misaligned_access_speed) = RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED;
366 #endif
367
368 if (!unaligned_enabled)
369 return -1;
370
371 if (user_mode(regs) && (current->thread.align_ctl & PR_UNALIGN_SIGBUS))
372 return -1;
373
374 if (get_insn(regs, epc, &insn))
375 return -1;
376
377 regs->epc = 0;
378
379 if ((insn & INSN_MASK_LW) == INSN_MATCH_LW) {
380 len = 4;
381 shift = 8 * (sizeof(unsigned long) - len);
382 #if defined(CONFIG_64BIT)
383 } else if ((insn & INSN_MASK_LD) == INSN_MATCH_LD) {
384 len = 8;
385 shift = 8 * (sizeof(unsigned long) - len);
386 } else if ((insn & INSN_MASK_LWU) == INSN_MATCH_LWU) {
387 len = 4;
388 #endif
389 } else if ((insn & INSN_MASK_FLD) == INSN_MATCH_FLD) {
390 fp = 1;
391 len = 8;
392 } else if ((insn & INSN_MASK_FLW) == INSN_MATCH_FLW) {
393 fp = 1;
394 len = 4;
395 } else if ((insn & INSN_MASK_LH) == INSN_MATCH_LH) {
396 len = 2;
397 shift = 8 * (sizeof(unsigned long) - len);
398 } else if ((insn & INSN_MASK_LHU) == INSN_MATCH_LHU) {
399 len = 2;
400 #if defined(CONFIG_64BIT)
401 } else if ((insn & INSN_MASK_C_LD) == INSN_MATCH_C_LD) {
402 len = 8;
403 shift = 8 * (sizeof(unsigned long) - len);
404 insn = RVC_RS2S(insn) << SH_RD;
405 } else if ((insn & INSN_MASK_C_LDSP) == INSN_MATCH_C_LDSP &&
406 ((insn >> SH_RD) & 0x1f)) {
407 len = 8;
408 shift = 8 * (sizeof(unsigned long) - len);
409 #endif
410 } else if ((insn & INSN_MASK_C_LW) == INSN_MATCH_C_LW) {
411 len = 4;
412 shift = 8 * (sizeof(unsigned long) - len);
413 insn = RVC_RS2S(insn) << SH_RD;
414 } else if ((insn & INSN_MASK_C_LWSP) == INSN_MATCH_C_LWSP &&
415 ((insn >> SH_RD) & 0x1f)) {
416 len = 4;
417 shift = 8 * (sizeof(unsigned long) - len);
418 } else if ((insn & INSN_MASK_C_FLD) == INSN_MATCH_C_FLD) {
419 fp = 1;
420 len = 8;
421 insn = RVC_RS2S(insn) << SH_RD;
422 } else if ((insn & INSN_MASK_C_FLDSP) == INSN_MATCH_C_FLDSP) {
423 fp = 1;
424 len = 8;
425 #if defined(CONFIG_32BIT)
426 } else if ((insn & INSN_MASK_C_FLW) == INSN_MATCH_C_FLW) {
427 fp = 1;
428 len = 4;
429 insn = RVC_RS2S(insn) << SH_RD;
430 } else if ((insn & INSN_MASK_C_FLWSP) == INSN_MATCH_C_FLWSP) {
431 fp = 1;
432 len = 4;
433 #endif
434 } else {
435 regs->epc = epc;
436 return -1;
437 }
438
439 if (!IS_ENABLED(CONFIG_FPU) && fp)
440 return -EOPNOTSUPP;
441
442 val.data_u64 = 0;
443 if (user_mode(regs)) {
444 if (copy_from_user(&val, (u8 __user *)addr, len))
445 return -1;
446 } else {
447 memcpy(&val, (u8 *)addr, len);
448 }
449
450 if (!fp)
451 SET_RD(insn, regs, val.data_ulong << shift >> shift);
452 else if (len == 8)
453 set_f64_rd(insn, regs, val.data_u64);
454 else
455 set_f32_rd(insn, regs, val.data_ulong);
456
457 regs->epc = epc + INSN_LEN(insn);
458
459 return 0;
460 }
461
handle_scalar_misaligned_store(struct pt_regs * regs)462 static int handle_scalar_misaligned_store(struct pt_regs *regs)
463 {
464 union reg_data val;
465 unsigned long epc = regs->epc;
466 unsigned long insn;
467 unsigned long addr = regs->badaddr;
468 int len = 0, fp = 0;
469
470 perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, addr);
471
472 if (!unaligned_enabled)
473 return -1;
474
475 if (user_mode(regs) && (current->thread.align_ctl & PR_UNALIGN_SIGBUS))
476 return -1;
477
478 if (get_insn(regs, epc, &insn))
479 return -1;
480
481 regs->epc = 0;
482
483 val.data_ulong = GET_RS2(insn, regs);
484
485 if ((insn & INSN_MASK_SW) == INSN_MATCH_SW) {
486 len = 4;
487 #if defined(CONFIG_64BIT)
488 } else if ((insn & INSN_MASK_SD) == INSN_MATCH_SD) {
489 len = 8;
490 #endif
491 } else if ((insn & INSN_MASK_FSD) == INSN_MATCH_FSD) {
492 fp = 1;
493 len = 8;
494 val.data_u64 = GET_F64_RS2(insn, regs);
495 } else if ((insn & INSN_MASK_FSW) == INSN_MATCH_FSW) {
496 fp = 1;
497 len = 4;
498 val.data_ulong = GET_F32_RS2(insn, regs);
499 } else if ((insn & INSN_MASK_SH) == INSN_MATCH_SH) {
500 len = 2;
501 #if defined(CONFIG_64BIT)
502 } else if ((insn & INSN_MASK_C_SD) == INSN_MATCH_C_SD) {
503 len = 8;
504 val.data_ulong = GET_RS2S(insn, regs);
505 } else if ((insn & INSN_MASK_C_SDSP) == INSN_MATCH_C_SDSP) {
506 len = 8;
507 val.data_ulong = GET_RS2C(insn, regs);
508 #endif
509 } else if ((insn & INSN_MASK_C_SW) == INSN_MATCH_C_SW) {
510 len = 4;
511 val.data_ulong = GET_RS2S(insn, regs);
512 } else if ((insn & INSN_MASK_C_SWSP) == INSN_MATCH_C_SWSP) {
513 len = 4;
514 val.data_ulong = GET_RS2C(insn, regs);
515 } else if ((insn & INSN_MASK_C_FSD) == INSN_MATCH_C_FSD) {
516 fp = 1;
517 len = 8;
518 val.data_u64 = GET_F64_RS2S(insn, regs);
519 } else if ((insn & INSN_MASK_C_FSDSP) == INSN_MATCH_C_FSDSP) {
520 fp = 1;
521 len = 8;
522 val.data_u64 = GET_F64_RS2C(insn, regs);
523 #if !defined(CONFIG_64BIT)
524 } else if ((insn & INSN_MASK_C_FSW) == INSN_MATCH_C_FSW) {
525 fp = 1;
526 len = 4;
527 val.data_ulong = GET_F32_RS2S(insn, regs);
528 } else if ((insn & INSN_MASK_C_FSWSP) == INSN_MATCH_C_FSWSP) {
529 fp = 1;
530 len = 4;
531 val.data_ulong = GET_F32_RS2C(insn, regs);
532 #endif
533 } else {
534 regs->epc = epc;
535 return -1;
536 }
537
538 if (!IS_ENABLED(CONFIG_FPU) && fp)
539 return -EOPNOTSUPP;
540
541 if (user_mode(regs)) {
542 if (copy_to_user((u8 __user *)addr, &val, len))
543 return -1;
544 } else {
545 memcpy((u8 *)addr, &val, len);
546 }
547
548 regs->epc = epc + INSN_LEN(insn);
549
550 return 0;
551 }
552
handle_misaligned_load(struct pt_regs * regs)553 int handle_misaligned_load(struct pt_regs *regs)
554 {
555 unsigned long epc = regs->epc;
556 unsigned long insn;
557
558 if (IS_ENABLED(CONFIG_RISCV_VECTOR_MISALIGNED)) {
559 if (get_insn(regs, epc, &insn))
560 return -1;
561
562 if (insn_is_vector(insn))
563 return handle_vector_misaligned_load(regs);
564 }
565
566 if (IS_ENABLED(CONFIG_RISCV_SCALAR_MISALIGNED))
567 return handle_scalar_misaligned_load(regs);
568
569 return -1;
570 }
571
handle_misaligned_store(struct pt_regs * regs)572 int handle_misaligned_store(struct pt_regs *regs)
573 {
574 if (IS_ENABLED(CONFIG_RISCV_SCALAR_MISALIGNED))
575 return handle_scalar_misaligned_store(regs);
576
577 return -1;
578 }
579
580 #ifdef CONFIG_RISCV_VECTOR_MISALIGNED
check_vector_unaligned_access_emulated(struct work_struct * work __always_unused)581 void check_vector_unaligned_access_emulated(struct work_struct *work __always_unused)
582 {
583 long *mas_ptr = this_cpu_ptr(&vector_misaligned_access);
584 unsigned long tmp_var;
585
586 *mas_ptr = RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN;
587
588 kernel_vector_begin();
589 /*
590 * In pre-13.0.0 versions of GCC, vector registers cannot appear in
591 * the clobber list. This inline asm clobbers v0, but since we do not
592 * currently build the kernel with V enabled, the v0 clobber arg is not
593 * needed (as the compiler will not emit vector code itself). If the kernel
594 * is changed to build with V enabled, the clobber arg will need to be
595 * added here.
596 */
597 __asm__ __volatile__ (
598 ".balign 4\n\t"
599 ".option push\n\t"
600 ".option arch, +zve32x\n\t"
601 " vsetivli zero, 1, e16, m1, ta, ma\n\t" // Vectors of 16b
602 " vle16.v v0, (%[ptr])\n\t" // Load bytes
603 ".option pop\n\t"
604 : : [ptr] "r" ((u8 *)&tmp_var + 1));
605 kernel_vector_end();
606 }
607
check_vector_unaligned_access_emulated_all_cpus(void)608 bool check_vector_unaligned_access_emulated_all_cpus(void)
609 {
610 int cpu;
611
612 if (!has_vector()) {
613 for_each_online_cpu(cpu)
614 per_cpu(vector_misaligned_access, cpu) = RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED;
615 return false;
616 }
617
618 schedule_on_each_cpu(check_vector_unaligned_access_emulated);
619
620 for_each_online_cpu(cpu)
621 if (per_cpu(vector_misaligned_access, cpu)
622 == RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN)
623 return false;
624
625 return true;
626 }
627 #else
check_vector_unaligned_access_emulated_all_cpus(void)628 bool check_vector_unaligned_access_emulated_all_cpus(void)
629 {
630 return false;
631 }
632 #endif
633
634 #ifdef CONFIG_RISCV_SCALAR_MISALIGNED
635
636 static bool unaligned_ctl __read_mostly;
637
check_unaligned_access_emulated(struct work_struct * work __always_unused)638 void check_unaligned_access_emulated(struct work_struct *work __always_unused)
639 {
640 int cpu = smp_processor_id();
641 long *mas_ptr = per_cpu_ptr(&misaligned_access_speed, cpu);
642 unsigned long tmp_var, tmp_val;
643
644 *mas_ptr = RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN;
645
646 __asm__ __volatile__ (
647 " "REG_L" %[tmp], 1(%[ptr])\n"
648 : [tmp] "=r" (tmp_val) : [ptr] "r" (&tmp_var) : "memory");
649
650 /*
651 * If unaligned_ctl is already set, this means that we detected that all
652 * CPUS uses emulated misaligned access at boot time. If that changed
653 * when hotplugging the new cpu, this is something we don't handle.
654 */
655 if (unlikely(unaligned_ctl && (*mas_ptr != RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED))) {
656 pr_crit("CPU misaligned accesses non homogeneous (expected all emulated)\n");
657 while (true)
658 cpu_relax();
659 }
660 }
661
check_unaligned_access_emulated_all_cpus(void)662 bool check_unaligned_access_emulated_all_cpus(void)
663 {
664 int cpu;
665
666 /*
667 * We can only support PR_UNALIGN controls if all CPUs have misaligned
668 * accesses emulated since tasks requesting such control can run on any
669 * CPU.
670 */
671 schedule_on_each_cpu(check_unaligned_access_emulated);
672
673 for_each_online_cpu(cpu)
674 if (per_cpu(misaligned_access_speed, cpu)
675 != RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED)
676 return false;
677
678 unaligned_ctl = true;
679 return true;
680 }
681
unaligned_ctl_available(void)682 bool unaligned_ctl_available(void)
683 {
684 return unaligned_ctl;
685 }
686 #else
check_unaligned_access_emulated_all_cpus(void)687 bool check_unaligned_access_emulated_all_cpus(void)
688 {
689 return false;
690 }
691 #endif
692