1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2020 Western Digital Corporation or its affiliates.
4 */
5 #include <linux/kernel.h>
6 #include <linux/init.h>
7 #include <linux/mm.h>
8 #include <linux/module.h>
9 #include <linux/perf_event.h>
10 #include <linux/irq.h>
11 #include <linux/stringify.h>
12
13 #include <asm/processor.h>
14 #include <asm/ptrace.h>
15 #include <asm/csr.h>
16 #include <asm/entry-common.h>
17 #include <asm/hwprobe.h>
18 #include <asm/cpufeature.h>
19 #include <asm/sbi.h>
20 #include <asm/vector.h>
21
22 #define INSN_MATCH_LB 0x3
23 #define INSN_MASK_LB 0x707f
24 #define INSN_MATCH_LH 0x1003
25 #define INSN_MASK_LH 0x707f
26 #define INSN_MATCH_LW 0x2003
27 #define INSN_MASK_LW 0x707f
28 #define INSN_MATCH_LD 0x3003
29 #define INSN_MASK_LD 0x707f
30 #define INSN_MATCH_LBU 0x4003
31 #define INSN_MASK_LBU 0x707f
32 #define INSN_MATCH_LHU 0x5003
33 #define INSN_MASK_LHU 0x707f
34 #define INSN_MATCH_LWU 0x6003
35 #define INSN_MASK_LWU 0x707f
36 #define INSN_MATCH_SB 0x23
37 #define INSN_MASK_SB 0x707f
38 #define INSN_MATCH_SH 0x1023
39 #define INSN_MASK_SH 0x707f
40 #define INSN_MATCH_SW 0x2023
41 #define INSN_MASK_SW 0x707f
42 #define INSN_MATCH_SD 0x3023
43 #define INSN_MASK_SD 0x707f
44
45 #define INSN_MATCH_FLW 0x2007
46 #define INSN_MASK_FLW 0x707f
47 #define INSN_MATCH_FLD 0x3007
48 #define INSN_MASK_FLD 0x707f
49 #define INSN_MATCH_FLQ 0x4007
50 #define INSN_MASK_FLQ 0x707f
51 #define INSN_MATCH_FSW 0x2027
52 #define INSN_MASK_FSW 0x707f
53 #define INSN_MATCH_FSD 0x3027
54 #define INSN_MASK_FSD 0x707f
55 #define INSN_MATCH_FSQ 0x4027
56 #define INSN_MASK_FSQ 0x707f
57
58 #define INSN_MATCH_C_LD 0x6000
59 #define INSN_MASK_C_LD 0xe003
60 #define INSN_MATCH_C_SD 0xe000
61 #define INSN_MASK_C_SD 0xe003
62 #define INSN_MATCH_C_LW 0x4000
63 #define INSN_MASK_C_LW 0xe003
64 #define INSN_MATCH_C_SW 0xc000
65 #define INSN_MASK_C_SW 0xe003
66 #define INSN_MATCH_C_LDSP 0x6002
67 #define INSN_MASK_C_LDSP 0xe003
68 #define INSN_MATCH_C_SDSP 0xe002
69 #define INSN_MASK_C_SDSP 0xe003
70 #define INSN_MATCH_C_LWSP 0x4002
71 #define INSN_MASK_C_LWSP 0xe003
72 #define INSN_MATCH_C_SWSP 0xc002
73 #define INSN_MASK_C_SWSP 0xe003
74
75 #define INSN_MATCH_C_FLD 0x2000
76 #define INSN_MASK_C_FLD 0xe003
77 #define INSN_MATCH_C_FLW 0x6000
78 #define INSN_MASK_C_FLW 0xe003
79 #define INSN_MATCH_C_FSD 0xa000
80 #define INSN_MASK_C_FSD 0xe003
81 #define INSN_MATCH_C_FSW 0xe000
82 #define INSN_MASK_C_FSW 0xe003
83 #define INSN_MATCH_C_FLDSP 0x2002
84 #define INSN_MASK_C_FLDSP 0xe003
85 #define INSN_MATCH_C_FSDSP 0xa002
86 #define INSN_MASK_C_FSDSP 0xe003
87 #define INSN_MATCH_C_FLWSP 0x6002
88 #define INSN_MASK_C_FLWSP 0xe003
89 #define INSN_MATCH_C_FSWSP 0xe002
90 #define INSN_MASK_C_FSWSP 0xe003
91
92 #define INSN_MATCH_C_LHU 0x8400
93 #define INSN_MASK_C_LHU 0xfc43
94 #define INSN_MATCH_C_LH 0x8440
95 #define INSN_MASK_C_LH 0xfc43
96 #define INSN_MATCH_C_SH 0x8c00
97 #define INSN_MASK_C_SH 0xfc43
98
99 #define INSN_LEN(insn) ((((insn) & 0x3) < 0x3) ? 2 : 4)
100
101 #if defined(CONFIG_64BIT)
102 #define LOG_REGBYTES 3
103 #define XLEN 64
104 #else
105 #define LOG_REGBYTES 2
106 #define XLEN 32
107 #endif
108 #define REGBYTES (1 << LOG_REGBYTES)
109 #define XLEN_MINUS_16 ((XLEN) - 16)
110
111 #define SH_RD 7
112 #define SH_RS1 15
113 #define SH_RS2 20
114 #define SH_RS2C 2
115
116 #define RV_X(x, s, n) (((x) >> (s)) & ((1 << (n)) - 1))
117 #define RVC_LW_IMM(x) ((RV_X(x, 6, 1) << 2) | \
118 (RV_X(x, 10, 3) << 3) | \
119 (RV_X(x, 5, 1) << 6))
120 #define RVC_LD_IMM(x) ((RV_X(x, 10, 3) << 3) | \
121 (RV_X(x, 5, 2) << 6))
122 #define RVC_LWSP_IMM(x) ((RV_X(x, 4, 3) << 2) | \
123 (RV_X(x, 12, 1) << 5) | \
124 (RV_X(x, 2, 2) << 6))
125 #define RVC_LDSP_IMM(x) ((RV_X(x, 5, 2) << 3) | \
126 (RV_X(x, 12, 1) << 5) | \
127 (RV_X(x, 2, 3) << 6))
128 #define RVC_SWSP_IMM(x) ((RV_X(x, 9, 4) << 2) | \
129 (RV_X(x, 7, 2) << 6))
130 #define RVC_SDSP_IMM(x) ((RV_X(x, 10, 3) << 3) | \
131 (RV_X(x, 7, 3) << 6))
132 #define RVC_RS1S(insn) (8 + RV_X(insn, SH_RD, 3))
133 #define RVC_RS2S(insn) (8 + RV_X(insn, SH_RS2C, 3))
134 #define RVC_RS2(insn) RV_X(insn, SH_RS2C, 5)
135
136 #define SHIFT_RIGHT(x, y) \
137 ((y) < 0 ? ((x) << -(y)) : ((x) >> (y)))
138
139 #define REG_MASK \
140 ((1 << (5 + LOG_REGBYTES)) - (1 << LOG_REGBYTES))
141
142 #define REG_OFFSET(insn, pos) \
143 (SHIFT_RIGHT((insn), (pos) - LOG_REGBYTES) & REG_MASK)
144
145 #define REG_PTR(insn, pos, regs) \
146 (ulong *)((ulong)(regs) + REG_OFFSET(insn, pos))
147
148 #define GET_RS1(insn, regs) (*REG_PTR(insn, SH_RS1, regs))
149 #define GET_RS2(insn, regs) (*REG_PTR(insn, SH_RS2, regs))
150 #define GET_RS1S(insn, regs) (*REG_PTR(RVC_RS1S(insn), 0, regs))
151 #define GET_RS2S(insn, regs) (*REG_PTR(RVC_RS2S(insn), 0, regs))
152 #define GET_RS2C(insn, regs) (*REG_PTR(insn, SH_RS2C, regs))
153 #define GET_SP(regs) (*REG_PTR(2, 0, regs))
154 #define SET_RD(insn, regs, val) (*REG_PTR(insn, SH_RD, regs) = (val))
155 #define IMM_I(insn) ((s32)(insn) >> 20)
156 #define IMM_S(insn) (((s32)(insn) >> 25 << 5) | \
157 (s32)(((insn) >> 7) & 0x1f))
158 #define MASK_FUNCT3 0x7000
159
160 #define GET_PRECISION(insn) (((insn) >> 25) & 3)
161 #define GET_RM(insn) (((insn) >> 12) & 7)
162 #define PRECISION_S 0
163 #define PRECISION_D 1
164
165 #ifdef CONFIG_FPU
166
167 #define FP_GET_RD(insn) (insn >> 7 & 0x1F)
168
169 extern void put_f32_reg(unsigned long fp_reg, unsigned long value);
170
set_f32_rd(unsigned long insn,struct pt_regs * regs,unsigned long val)171 static int set_f32_rd(unsigned long insn, struct pt_regs *regs,
172 unsigned long val)
173 {
174 unsigned long fp_reg = FP_GET_RD(insn);
175
176 put_f32_reg(fp_reg, val);
177 regs->status |= SR_FS_DIRTY;
178
179 return 0;
180 }
181
182 extern void put_f64_reg(unsigned long fp_reg, unsigned long value);
183
set_f64_rd(unsigned long insn,struct pt_regs * regs,u64 val)184 static int set_f64_rd(unsigned long insn, struct pt_regs *regs, u64 val)
185 {
186 unsigned long fp_reg = FP_GET_RD(insn);
187 unsigned long value;
188
189 #if __riscv_xlen == 32
190 value = (unsigned long) &val;
191 #else
192 value = val;
193 #endif
194 put_f64_reg(fp_reg, value);
195 regs->status |= SR_FS_DIRTY;
196
197 return 0;
198 }
199
200 #if __riscv_xlen == 32
201 extern void get_f64_reg(unsigned long fp_reg, u64 *value);
202
get_f64_rs(unsigned long insn,u8 fp_reg_offset,struct pt_regs * regs)203 static u64 get_f64_rs(unsigned long insn, u8 fp_reg_offset,
204 struct pt_regs *regs)
205 {
206 unsigned long fp_reg = (insn >> fp_reg_offset) & 0x1F;
207 u64 val;
208
209 get_f64_reg(fp_reg, &val);
210 regs->status |= SR_FS_DIRTY;
211
212 return val;
213 }
214 #else
215
216 extern unsigned long get_f64_reg(unsigned long fp_reg);
217
get_f64_rs(unsigned long insn,u8 fp_reg_offset,struct pt_regs * regs)218 static unsigned long get_f64_rs(unsigned long insn, u8 fp_reg_offset,
219 struct pt_regs *regs)
220 {
221 unsigned long fp_reg = (insn >> fp_reg_offset) & 0x1F;
222 unsigned long val;
223
224 val = get_f64_reg(fp_reg);
225 regs->status |= SR_FS_DIRTY;
226
227 return val;
228 }
229
230 #endif
231
232 extern unsigned long get_f32_reg(unsigned long fp_reg);
233
get_f32_rs(unsigned long insn,u8 fp_reg_offset,struct pt_regs * regs)234 static unsigned long get_f32_rs(unsigned long insn, u8 fp_reg_offset,
235 struct pt_regs *regs)
236 {
237 unsigned long fp_reg = (insn >> fp_reg_offset) & 0x1F;
238 unsigned long val;
239
240 val = get_f32_reg(fp_reg);
241 regs->status |= SR_FS_DIRTY;
242
243 return val;
244 }
245
246 #else /* CONFIG_FPU */
set_f32_rd(unsigned long insn,struct pt_regs * regs,unsigned long val)247 static void set_f32_rd(unsigned long insn, struct pt_regs *regs,
248 unsigned long val) {}
249
set_f64_rd(unsigned long insn,struct pt_regs * regs,u64 val)250 static void set_f64_rd(unsigned long insn, struct pt_regs *regs, u64 val) {}
251
get_f64_rs(unsigned long insn,u8 fp_reg_offset,struct pt_regs * regs)252 static unsigned long get_f64_rs(unsigned long insn, u8 fp_reg_offset,
253 struct pt_regs *regs)
254 {
255 return 0;
256 }
257
get_f32_rs(unsigned long insn,u8 fp_reg_offset,struct pt_regs * regs)258 static unsigned long get_f32_rs(unsigned long insn, u8 fp_reg_offset,
259 struct pt_regs *regs)
260 {
261 return 0;
262 }
263
264 #endif
265
266 #define GET_F64_RS2(insn, regs) (get_f64_rs(insn, 20, regs))
267 #define GET_F64_RS2C(insn, regs) (get_f64_rs(insn, 2, regs))
268 #define GET_F64_RS2S(insn, regs) (get_f64_rs(RVC_RS2S(insn), 0, regs))
269
270 #define GET_F32_RS2(insn, regs) (get_f32_rs(insn, 20, regs))
271 #define GET_F32_RS2C(insn, regs) (get_f32_rs(insn, 2, regs))
272 #define GET_F32_RS2S(insn, regs) (get_f32_rs(RVC_RS2S(insn), 0, regs))
273
274 #define __read_insn(regs, insn, insn_addr, type) \
275 ({ \
276 int __ret; \
277 \
278 if (user_mode(regs)) { \
279 __ret = get_user(insn, (type __user *) insn_addr); \
280 } else { \
281 insn = *(type *)insn_addr; \
282 __ret = 0; \
283 } \
284 \
285 __ret; \
286 })
287
get_insn(struct pt_regs * regs,ulong epc,ulong * r_insn)288 static inline int get_insn(struct pt_regs *regs, ulong epc, ulong *r_insn)
289 {
290 ulong insn = 0;
291
292 if (epc & 0x2) {
293 ulong tmp = 0;
294
295 if (__read_insn(regs, insn, epc, u16))
296 return -EFAULT;
297 /* __get_user() uses regular "lw" which sign extend the loaded
298 * value make sure to clear higher order bits in case we "or" it
299 * below with the upper 16 bits half.
300 */
301 insn &= GENMASK(15, 0);
302 if ((insn & __INSN_LENGTH_MASK) != __INSN_LENGTH_32) {
303 *r_insn = insn;
304 return 0;
305 }
306 epc += sizeof(u16);
307 if (__read_insn(regs, tmp, epc, u16))
308 return -EFAULT;
309 *r_insn = (tmp << 16) | insn;
310
311 return 0;
312 } else {
313 if (__read_insn(regs, insn, epc, u32))
314 return -EFAULT;
315 if ((insn & __INSN_LENGTH_MASK) == __INSN_LENGTH_32) {
316 *r_insn = insn;
317 return 0;
318 }
319 insn &= GENMASK(15, 0);
320 *r_insn = insn;
321
322 return 0;
323 }
324 }
325
326 union reg_data {
327 u8 data_bytes[8];
328 ulong data_ulong;
329 u64 data_u64;
330 };
331
332 /* sysctl hooks */
333 int unaligned_enabled __read_mostly = 1; /* Enabled by default */
334
335 #ifdef CONFIG_RISCV_VECTOR_MISALIGNED
handle_vector_misaligned_load(struct pt_regs * regs)336 static int handle_vector_misaligned_load(struct pt_regs *regs)
337 {
338 unsigned long epc = regs->epc;
339 unsigned long insn;
340
341 if (get_insn(regs, epc, &insn))
342 return -1;
343
344 /* Only return 0 when in check_vector_unaligned_access_emulated */
345 if (*this_cpu_ptr(&vector_misaligned_access) == RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN) {
346 *this_cpu_ptr(&vector_misaligned_access) = RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED;
347 regs->epc = epc + INSN_LEN(insn);
348 return 0;
349 }
350
351 /* If vector instruction we don't emulate it yet */
352 regs->epc = epc;
353 return -1;
354 }
355 #else
handle_vector_misaligned_load(struct pt_regs * regs)356 static int handle_vector_misaligned_load(struct pt_regs *regs)
357 {
358 return -1;
359 }
360 #endif
361
handle_scalar_misaligned_load(struct pt_regs * regs)362 static int handle_scalar_misaligned_load(struct pt_regs *regs)
363 {
364 union reg_data val;
365 unsigned long epc = regs->epc;
366 unsigned long insn;
367 unsigned long addr = regs->badaddr;
368 int fp = 0, shift = 0, len = 0;
369
370 perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, addr);
371
372 *this_cpu_ptr(&misaligned_access_speed) = RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED;
373
374 if (!unaligned_enabled)
375 return -1;
376
377 if (user_mode(regs) && (current->thread.align_ctl & PR_UNALIGN_SIGBUS))
378 return -1;
379
380 if (get_insn(regs, epc, &insn))
381 return -1;
382
383 regs->epc = 0;
384
385 if ((insn & INSN_MASK_LW) == INSN_MATCH_LW) {
386 len = 4;
387 shift = 8 * (sizeof(unsigned long) - len);
388 #if defined(CONFIG_64BIT)
389 } else if ((insn & INSN_MASK_LD) == INSN_MATCH_LD) {
390 len = 8;
391 shift = 8 * (sizeof(unsigned long) - len);
392 } else if ((insn & INSN_MASK_LWU) == INSN_MATCH_LWU) {
393 len = 4;
394 #endif
395 } else if ((insn & INSN_MASK_FLD) == INSN_MATCH_FLD) {
396 fp = 1;
397 len = 8;
398 } else if ((insn & INSN_MASK_FLW) == INSN_MATCH_FLW) {
399 fp = 1;
400 len = 4;
401 } else if ((insn & INSN_MASK_LH) == INSN_MATCH_LH) {
402 len = 2;
403 shift = 8 * (sizeof(unsigned long) - len);
404 } else if ((insn & INSN_MASK_LHU) == INSN_MATCH_LHU) {
405 len = 2;
406 #if defined(CONFIG_64BIT)
407 } else if ((insn & INSN_MASK_C_LD) == INSN_MATCH_C_LD) {
408 len = 8;
409 shift = 8 * (sizeof(unsigned long) - len);
410 insn = RVC_RS2S(insn) << SH_RD;
411 } else if ((insn & INSN_MASK_C_LDSP) == INSN_MATCH_C_LDSP &&
412 ((insn >> SH_RD) & 0x1f)) {
413 len = 8;
414 shift = 8 * (sizeof(unsigned long) - len);
415 #endif
416 } else if ((insn & INSN_MASK_C_LW) == INSN_MATCH_C_LW) {
417 len = 4;
418 shift = 8 * (sizeof(unsigned long) - len);
419 insn = RVC_RS2S(insn) << SH_RD;
420 } else if ((insn & INSN_MASK_C_LWSP) == INSN_MATCH_C_LWSP &&
421 ((insn >> SH_RD) & 0x1f)) {
422 len = 4;
423 shift = 8 * (sizeof(unsigned long) - len);
424 } else if ((insn & INSN_MASK_C_FLD) == INSN_MATCH_C_FLD) {
425 fp = 1;
426 len = 8;
427 insn = RVC_RS2S(insn) << SH_RD;
428 } else if ((insn & INSN_MASK_C_FLDSP) == INSN_MATCH_C_FLDSP) {
429 fp = 1;
430 len = 8;
431 #if defined(CONFIG_32BIT)
432 } else if ((insn & INSN_MASK_C_FLW) == INSN_MATCH_C_FLW) {
433 fp = 1;
434 len = 4;
435 insn = RVC_RS2S(insn) << SH_RD;
436 } else if ((insn & INSN_MASK_C_FLWSP) == INSN_MATCH_C_FLWSP) {
437 fp = 1;
438 len = 4;
439 #endif
440 } else if ((insn & INSN_MASK_C_LHU) == INSN_MATCH_C_LHU) {
441 len = 2;
442 insn = RVC_RS2S(insn) << SH_RD;
443 } else if ((insn & INSN_MASK_C_LH) == INSN_MATCH_C_LH) {
444 len = 2;
445 shift = 8 * (sizeof(ulong) - len);
446 insn = RVC_RS2S(insn) << SH_RD;
447 } else {
448 regs->epc = epc;
449 return -1;
450 }
451
452 if (!IS_ENABLED(CONFIG_FPU) && fp)
453 return -EOPNOTSUPP;
454
455 val.data_u64 = 0;
456 if (user_mode(regs)) {
457 if (copy_from_user(&val, (u8 __user *)addr, len))
458 return -1;
459 } else {
460 memcpy(&val, (u8 *)addr, len);
461 }
462
463 if (!fp)
464 SET_RD(insn, regs, val.data_ulong << shift >> shift);
465 else if (len == 8)
466 set_f64_rd(insn, regs, val.data_u64);
467 else
468 set_f32_rd(insn, regs, val.data_ulong);
469
470 regs->epc = epc + INSN_LEN(insn);
471
472 return 0;
473 }
474
handle_scalar_misaligned_store(struct pt_regs * regs)475 static int handle_scalar_misaligned_store(struct pt_regs *regs)
476 {
477 union reg_data val;
478 unsigned long epc = regs->epc;
479 unsigned long insn;
480 unsigned long addr = regs->badaddr;
481 int len = 0, fp = 0;
482
483 perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, addr);
484
485 if (!unaligned_enabled)
486 return -1;
487
488 if (user_mode(regs) && (current->thread.align_ctl & PR_UNALIGN_SIGBUS))
489 return -1;
490
491 if (get_insn(regs, epc, &insn))
492 return -1;
493
494 regs->epc = 0;
495
496 val.data_ulong = GET_RS2(insn, regs);
497
498 if ((insn & INSN_MASK_SW) == INSN_MATCH_SW) {
499 len = 4;
500 #if defined(CONFIG_64BIT)
501 } else if ((insn & INSN_MASK_SD) == INSN_MATCH_SD) {
502 len = 8;
503 #endif
504 } else if ((insn & INSN_MASK_FSD) == INSN_MATCH_FSD) {
505 fp = 1;
506 len = 8;
507 val.data_u64 = GET_F64_RS2(insn, regs);
508 } else if ((insn & INSN_MASK_FSW) == INSN_MATCH_FSW) {
509 fp = 1;
510 len = 4;
511 val.data_ulong = GET_F32_RS2(insn, regs);
512 } else if ((insn & INSN_MASK_SH) == INSN_MATCH_SH) {
513 len = 2;
514 #if defined(CONFIG_64BIT)
515 } else if ((insn & INSN_MASK_C_SD) == INSN_MATCH_C_SD) {
516 len = 8;
517 val.data_ulong = GET_RS2S(insn, regs);
518 } else if ((insn & INSN_MASK_C_SDSP) == INSN_MATCH_C_SDSP) {
519 len = 8;
520 val.data_ulong = GET_RS2C(insn, regs);
521 #endif
522 } else if ((insn & INSN_MASK_C_SW) == INSN_MATCH_C_SW) {
523 len = 4;
524 val.data_ulong = GET_RS2S(insn, regs);
525 } else if ((insn & INSN_MASK_C_SWSP) == INSN_MATCH_C_SWSP) {
526 len = 4;
527 val.data_ulong = GET_RS2C(insn, regs);
528 } else if ((insn & INSN_MASK_C_FSD) == INSN_MATCH_C_FSD) {
529 fp = 1;
530 len = 8;
531 val.data_u64 = GET_F64_RS2S(insn, regs);
532 } else if ((insn & INSN_MASK_C_FSDSP) == INSN_MATCH_C_FSDSP) {
533 fp = 1;
534 len = 8;
535 val.data_u64 = GET_F64_RS2C(insn, regs);
536 #if !defined(CONFIG_64BIT)
537 } else if ((insn & INSN_MASK_C_FSW) == INSN_MATCH_C_FSW) {
538 fp = 1;
539 len = 4;
540 val.data_ulong = GET_F32_RS2S(insn, regs);
541 } else if ((insn & INSN_MASK_C_FSWSP) == INSN_MATCH_C_FSWSP) {
542 fp = 1;
543 len = 4;
544 val.data_ulong = GET_F32_RS2C(insn, regs);
545 #endif
546 } else if ((insn & INSN_MASK_C_SH) == INSN_MATCH_C_SH) {
547 len = 2;
548 val.data_ulong = GET_RS2S(insn, regs);
549 } else {
550 regs->epc = epc;
551 return -1;
552 }
553
554 if (!IS_ENABLED(CONFIG_FPU) && fp)
555 return -EOPNOTSUPP;
556
557 if (user_mode(regs)) {
558 if (copy_to_user((u8 __user *)addr, &val, len))
559 return -1;
560 } else {
561 memcpy((u8 *)addr, &val, len);
562 }
563
564 regs->epc = epc + INSN_LEN(insn);
565
566 return 0;
567 }
568
handle_misaligned_load(struct pt_regs * regs)569 int handle_misaligned_load(struct pt_regs *regs)
570 {
571 unsigned long epc = regs->epc;
572 unsigned long insn;
573
574 if (IS_ENABLED(CONFIG_RISCV_VECTOR_MISALIGNED)) {
575 if (get_insn(regs, epc, &insn))
576 return -1;
577
578 if (insn_is_vector(insn))
579 return handle_vector_misaligned_load(regs);
580 }
581
582 if (IS_ENABLED(CONFIG_RISCV_SCALAR_MISALIGNED))
583 return handle_scalar_misaligned_load(regs);
584
585 return -1;
586 }
587
handle_misaligned_store(struct pt_regs * regs)588 int handle_misaligned_store(struct pt_regs *regs)
589 {
590 if (IS_ENABLED(CONFIG_RISCV_SCALAR_MISALIGNED))
591 return handle_scalar_misaligned_store(regs);
592
593 return -1;
594 }
595
596 #ifdef CONFIG_RISCV_VECTOR_MISALIGNED
check_vector_unaligned_access_emulated(struct work_struct * work __always_unused)597 void check_vector_unaligned_access_emulated(struct work_struct *work __always_unused)
598 {
599 long *mas_ptr = this_cpu_ptr(&vector_misaligned_access);
600 unsigned long tmp_var;
601
602 *mas_ptr = RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN;
603
604 kernel_vector_begin();
605 /*
606 * In pre-13.0.0 versions of GCC, vector registers cannot appear in
607 * the clobber list. This inline asm clobbers v0, but since we do not
608 * currently build the kernel with V enabled, the v0 clobber arg is not
609 * needed (as the compiler will not emit vector code itself). If the kernel
610 * is changed to build with V enabled, the clobber arg will need to be
611 * added here.
612 */
613 __asm__ __volatile__ (
614 ".balign 4\n\t"
615 ".option push\n\t"
616 ".option arch, +zve32x\n\t"
617 " vsetivli zero, 1, e16, m1, ta, ma\n\t" // Vectors of 16b
618 " vle16.v v0, (%[ptr])\n\t" // Load bytes
619 ".option pop\n\t"
620 : : [ptr] "r" ((u8 *)&tmp_var + 1));
621 kernel_vector_end();
622 }
623
check_vector_unaligned_access_emulated_all_cpus(void)624 bool __init check_vector_unaligned_access_emulated_all_cpus(void)
625 {
626 int cpu;
627
628 /*
629 * While being documented as very slow, schedule_on_each_cpu() is used since
630 * kernel_vector_begin() expects irqs to be enabled or it will panic()
631 */
632 schedule_on_each_cpu(check_vector_unaligned_access_emulated);
633
634 for_each_online_cpu(cpu)
635 if (per_cpu(vector_misaligned_access, cpu)
636 == RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN)
637 return false;
638
639 return true;
640 }
641 #else
check_vector_unaligned_access_emulated_all_cpus(void)642 bool __init check_vector_unaligned_access_emulated_all_cpus(void)
643 {
644 return false;
645 }
646 #endif
647
all_cpus_unaligned_scalar_access_emulated(void)648 static bool all_cpus_unaligned_scalar_access_emulated(void)
649 {
650 int cpu;
651
652 for_each_online_cpu(cpu)
653 if (per_cpu(misaligned_access_speed, cpu) !=
654 RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED)
655 return false;
656
657 return true;
658 }
659
660 #ifdef CONFIG_RISCV_SCALAR_MISALIGNED
661
662 static bool unaligned_ctl __read_mostly;
663
check_unaligned_access_emulated(void * arg __always_unused)664 static void check_unaligned_access_emulated(void *arg __always_unused)
665 {
666 int cpu = smp_processor_id();
667 long *mas_ptr = per_cpu_ptr(&misaligned_access_speed, cpu);
668 unsigned long tmp_var, tmp_val;
669
670 *mas_ptr = RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN;
671
672 __asm__ __volatile__ (
673 " "REG_L" %[tmp], 1(%[ptr])\n"
674 : [tmp] "=r" (tmp_val) : [ptr] "r" (&tmp_var) : "memory");
675 }
676
cpu_online_check_unaligned_access_emulated(unsigned int cpu)677 static int cpu_online_check_unaligned_access_emulated(unsigned int cpu)
678 {
679 long *mas_ptr = per_cpu_ptr(&misaligned_access_speed, cpu);
680
681 check_unaligned_access_emulated(NULL);
682
683 /*
684 * If unaligned_ctl is already set, this means that we detected that all
685 * CPUS uses emulated misaligned access at boot time. If that changed
686 * when hotplugging the new cpu, this is something we don't handle.
687 */
688 if (unlikely(unaligned_ctl && (*mas_ptr != RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED))) {
689 pr_crit("CPU misaligned accesses non homogeneous (expected all emulated)\n");
690 return -EINVAL;
691 }
692
693 return 0;
694 }
695
check_unaligned_access_emulated_all_cpus(void)696 bool __init check_unaligned_access_emulated_all_cpus(void)
697 {
698 /*
699 * We can only support PR_UNALIGN controls if all CPUs have misaligned
700 * accesses emulated since tasks requesting such control can run on any
701 * CPU.
702 */
703 on_each_cpu(check_unaligned_access_emulated, NULL, 1);
704
705 if (!all_cpus_unaligned_scalar_access_emulated())
706 return false;
707
708 unaligned_ctl = true;
709 return true;
710 }
711
unaligned_ctl_available(void)712 bool unaligned_ctl_available(void)
713 {
714 return unaligned_ctl;
715 }
716 #else
check_unaligned_access_emulated_all_cpus(void)717 bool __init check_unaligned_access_emulated_all_cpus(void)
718 {
719 return false;
720 }
cpu_online_check_unaligned_access_emulated(unsigned int cpu)721 static int cpu_online_check_unaligned_access_emulated(unsigned int cpu)
722 {
723 return 0;
724 }
725 #endif
726
727 static bool misaligned_traps_delegated;
728
729 #ifdef CONFIG_RISCV_SBI
730
cpu_online_sbi_unaligned_setup(unsigned int cpu)731 static int cpu_online_sbi_unaligned_setup(unsigned int cpu)
732 {
733 if (sbi_fwft_set(SBI_FWFT_MISALIGNED_EXC_DELEG, 1, 0) &&
734 misaligned_traps_delegated) {
735 pr_crit("Misaligned trap delegation non homogeneous (expected delegated)");
736 return -EINVAL;
737 }
738
739 return 0;
740 }
741
unaligned_access_init(void)742 void __init unaligned_access_init(void)
743 {
744 int ret;
745
746 ret = sbi_fwft_set_online_cpus(SBI_FWFT_MISALIGNED_EXC_DELEG, 1, 0);
747 if (ret)
748 return;
749
750 misaligned_traps_delegated = true;
751 pr_info("SBI misaligned access exception delegation ok\n");
752 /*
753 * Note that we don't have to take any specific action here, if
754 * the delegation is successful, then
755 * check_unaligned_access_emulated() will verify that indeed the
756 * platform traps on misaligned accesses.
757 */
758 }
759 #else
unaligned_access_init(void)760 void __init unaligned_access_init(void) {}
761
cpu_online_sbi_unaligned_setup(unsigned int cpu __always_unused)762 static int cpu_online_sbi_unaligned_setup(unsigned int cpu __always_unused)
763 {
764 return 0;
765 }
766
767 #endif
768
cpu_online_unaligned_access_init(unsigned int cpu)769 int cpu_online_unaligned_access_init(unsigned int cpu)
770 {
771 int ret;
772
773 ret = cpu_online_sbi_unaligned_setup(cpu);
774 if (ret)
775 return ret;
776
777 return cpu_online_check_unaligned_access_emulated(cpu);
778 }
779
misaligned_traps_can_delegate(void)780 bool misaligned_traps_can_delegate(void)
781 {
782 /*
783 * Either we successfully requested misaligned traps delegation for all
784 * CPUs, or the SBI does not implement the FWFT extension but delegated
785 * the exception by default.
786 */
787 return misaligned_traps_delegated ||
788 all_cpus_unaligned_scalar_access_emulated();
789 }
790 EXPORT_SYMBOL_GPL(misaligned_traps_can_delegate);
791