1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2020 Western Digital Corporation or its affiliates. 4 */ 5 #include <linux/kernel.h> 6 #include <linux/init.h> 7 #include <linux/mm.h> 8 #include <linux/module.h> 9 #include <linux/perf_event.h> 10 #include <linux/irq.h> 11 #include <linux/stringify.h> 12 13 #include <asm/processor.h> 14 #include <asm/ptrace.h> 15 #include <asm/csr.h> 16 #include <asm/entry-common.h> 17 #include <asm/hwprobe.h> 18 #include <asm/cpufeature.h> 19 #include <asm/vector.h> 20 21 #define INSN_MATCH_LB 0x3 22 #define INSN_MASK_LB 0x707f 23 #define INSN_MATCH_LH 0x1003 24 #define INSN_MASK_LH 0x707f 25 #define INSN_MATCH_LW 0x2003 26 #define INSN_MASK_LW 0x707f 27 #define INSN_MATCH_LD 0x3003 28 #define INSN_MASK_LD 0x707f 29 #define INSN_MATCH_LBU 0x4003 30 #define INSN_MASK_LBU 0x707f 31 #define INSN_MATCH_LHU 0x5003 32 #define INSN_MASK_LHU 0x707f 33 #define INSN_MATCH_LWU 0x6003 34 #define INSN_MASK_LWU 0x707f 35 #define INSN_MATCH_SB 0x23 36 #define INSN_MASK_SB 0x707f 37 #define INSN_MATCH_SH 0x1023 38 #define INSN_MASK_SH 0x707f 39 #define INSN_MATCH_SW 0x2023 40 #define INSN_MASK_SW 0x707f 41 #define INSN_MATCH_SD 0x3023 42 #define INSN_MASK_SD 0x707f 43 44 #define INSN_MATCH_FLW 0x2007 45 #define INSN_MASK_FLW 0x707f 46 #define INSN_MATCH_FLD 0x3007 47 #define INSN_MASK_FLD 0x707f 48 #define INSN_MATCH_FLQ 0x4007 49 #define INSN_MASK_FLQ 0x707f 50 #define INSN_MATCH_FSW 0x2027 51 #define INSN_MASK_FSW 0x707f 52 #define INSN_MATCH_FSD 0x3027 53 #define INSN_MASK_FSD 0x707f 54 #define INSN_MATCH_FSQ 0x4027 55 #define INSN_MASK_FSQ 0x707f 56 57 #define INSN_MATCH_C_LD 0x6000 58 #define INSN_MASK_C_LD 0xe003 59 #define INSN_MATCH_C_SD 0xe000 60 #define INSN_MASK_C_SD 0xe003 61 #define INSN_MATCH_C_LW 0x4000 62 #define INSN_MASK_C_LW 0xe003 63 #define INSN_MATCH_C_SW 0xc000 64 #define INSN_MASK_C_SW 0xe003 65 #define INSN_MATCH_C_LDSP 0x6002 66 #define INSN_MASK_C_LDSP 0xe003 67 #define INSN_MATCH_C_SDSP 0xe002 68 #define INSN_MASK_C_SDSP 0xe003 69 #define INSN_MATCH_C_LWSP 0x4002 70 #define INSN_MASK_C_LWSP 0xe003 71 #define INSN_MATCH_C_SWSP 0xc002 72 #define INSN_MASK_C_SWSP 0xe003 73 74 #define INSN_MATCH_C_FLD 0x2000 75 #define INSN_MASK_C_FLD 0xe003 76 #define INSN_MATCH_C_FLW 0x6000 77 #define INSN_MASK_C_FLW 0xe003 78 #define INSN_MATCH_C_FSD 0xa000 79 #define INSN_MASK_C_FSD 0xe003 80 #define INSN_MATCH_C_FSW 0xe000 81 #define INSN_MASK_C_FSW 0xe003 82 #define INSN_MATCH_C_FLDSP 0x2002 83 #define INSN_MASK_C_FLDSP 0xe003 84 #define INSN_MATCH_C_FSDSP 0xa002 85 #define INSN_MASK_C_FSDSP 0xe003 86 #define INSN_MATCH_C_FLWSP 0x6002 87 #define INSN_MASK_C_FLWSP 0xe003 88 #define INSN_MATCH_C_FSWSP 0xe002 89 #define INSN_MASK_C_FSWSP 0xe003 90 91 #define INSN_MATCH_C_LHU 0x8400 92 #define INSN_MASK_C_LHU 0xfc43 93 #define INSN_MATCH_C_LH 0x8440 94 #define INSN_MASK_C_LH 0xfc43 95 #define INSN_MATCH_C_SH 0x8c00 96 #define INSN_MASK_C_SH 0xfc43 97 98 #define INSN_LEN(insn) ((((insn) & 0x3) < 0x3) ? 2 : 4) 99 100 #if defined(CONFIG_64BIT) 101 #define LOG_REGBYTES 3 102 #define XLEN 64 103 #else 104 #define LOG_REGBYTES 2 105 #define XLEN 32 106 #endif 107 #define REGBYTES (1 << LOG_REGBYTES) 108 #define XLEN_MINUS_16 ((XLEN) - 16) 109 110 #define SH_RD 7 111 #define SH_RS1 15 112 #define SH_RS2 20 113 #define SH_RS2C 2 114 115 #define RV_X(x, s, n) (((x) >> (s)) & ((1 << (n)) - 1)) 116 #define RVC_LW_IMM(x) ((RV_X(x, 6, 1) << 2) | \ 117 (RV_X(x, 10, 3) << 3) | \ 118 (RV_X(x, 5, 1) << 6)) 119 #define RVC_LD_IMM(x) ((RV_X(x, 10, 3) << 3) | \ 120 (RV_X(x, 5, 2) << 6)) 121 #define RVC_LWSP_IMM(x) ((RV_X(x, 4, 3) << 2) | \ 122 (RV_X(x, 12, 1) << 5) | \ 123 (RV_X(x, 2, 2) << 6)) 124 #define RVC_LDSP_IMM(x) ((RV_X(x, 5, 2) << 3) | \ 125 (RV_X(x, 12, 1) << 5) | \ 126 (RV_X(x, 2, 3) << 6)) 127 #define RVC_SWSP_IMM(x) ((RV_X(x, 9, 4) << 2) | \ 128 (RV_X(x, 7, 2) << 6)) 129 #define RVC_SDSP_IMM(x) ((RV_X(x, 10, 3) << 3) | \ 130 (RV_X(x, 7, 3) << 6)) 131 #define RVC_RS1S(insn) (8 + RV_X(insn, SH_RD, 3)) 132 #define RVC_RS2S(insn) (8 + RV_X(insn, SH_RS2C, 3)) 133 #define RVC_RS2(insn) RV_X(insn, SH_RS2C, 5) 134 135 #define SHIFT_RIGHT(x, y) \ 136 ((y) < 0 ? ((x) << -(y)) : ((x) >> (y))) 137 138 #define REG_MASK \ 139 ((1 << (5 + LOG_REGBYTES)) - (1 << LOG_REGBYTES)) 140 141 #define REG_OFFSET(insn, pos) \ 142 (SHIFT_RIGHT((insn), (pos) - LOG_REGBYTES) & REG_MASK) 143 144 #define REG_PTR(insn, pos, regs) \ 145 (ulong *)((ulong)(regs) + REG_OFFSET(insn, pos)) 146 147 #define GET_RS1(insn, regs) (*REG_PTR(insn, SH_RS1, regs)) 148 #define GET_RS2(insn, regs) (*REG_PTR(insn, SH_RS2, regs)) 149 #define GET_RS1S(insn, regs) (*REG_PTR(RVC_RS1S(insn), 0, regs)) 150 #define GET_RS2S(insn, regs) (*REG_PTR(RVC_RS2S(insn), 0, regs)) 151 #define GET_RS2C(insn, regs) (*REG_PTR(insn, SH_RS2C, regs)) 152 #define GET_SP(regs) (*REG_PTR(2, 0, regs)) 153 #define SET_RD(insn, regs, val) (*REG_PTR(insn, SH_RD, regs) = (val)) 154 #define IMM_I(insn) ((s32)(insn) >> 20) 155 #define IMM_S(insn) (((s32)(insn) >> 25 << 5) | \ 156 (s32)(((insn) >> 7) & 0x1f)) 157 #define MASK_FUNCT3 0x7000 158 159 #define GET_PRECISION(insn) (((insn) >> 25) & 3) 160 #define GET_RM(insn) (((insn) >> 12) & 7) 161 #define PRECISION_S 0 162 #define PRECISION_D 1 163 164 #ifdef CONFIG_FPU 165 166 #define FP_GET_RD(insn) (insn >> 7 & 0x1F) 167 168 extern void put_f32_reg(unsigned long fp_reg, unsigned long value); 169 170 static int set_f32_rd(unsigned long insn, struct pt_regs *regs, 171 unsigned long val) 172 { 173 unsigned long fp_reg = FP_GET_RD(insn); 174 175 put_f32_reg(fp_reg, val); 176 regs->status |= SR_FS_DIRTY; 177 178 return 0; 179 } 180 181 extern void put_f64_reg(unsigned long fp_reg, unsigned long value); 182 183 static int set_f64_rd(unsigned long insn, struct pt_regs *regs, u64 val) 184 { 185 unsigned long fp_reg = FP_GET_RD(insn); 186 unsigned long value; 187 188 #if __riscv_xlen == 32 189 value = (unsigned long) &val; 190 #else 191 value = val; 192 #endif 193 put_f64_reg(fp_reg, value); 194 regs->status |= SR_FS_DIRTY; 195 196 return 0; 197 } 198 199 #if __riscv_xlen == 32 200 extern void get_f64_reg(unsigned long fp_reg, u64 *value); 201 202 static u64 get_f64_rs(unsigned long insn, u8 fp_reg_offset, 203 struct pt_regs *regs) 204 { 205 unsigned long fp_reg = (insn >> fp_reg_offset) & 0x1F; 206 u64 val; 207 208 get_f64_reg(fp_reg, &val); 209 regs->status |= SR_FS_DIRTY; 210 211 return val; 212 } 213 #else 214 215 extern unsigned long get_f64_reg(unsigned long fp_reg); 216 217 static unsigned long get_f64_rs(unsigned long insn, u8 fp_reg_offset, 218 struct pt_regs *regs) 219 { 220 unsigned long fp_reg = (insn >> fp_reg_offset) & 0x1F; 221 unsigned long val; 222 223 val = get_f64_reg(fp_reg); 224 regs->status |= SR_FS_DIRTY; 225 226 return val; 227 } 228 229 #endif 230 231 extern unsigned long get_f32_reg(unsigned long fp_reg); 232 233 static unsigned long get_f32_rs(unsigned long insn, u8 fp_reg_offset, 234 struct pt_regs *regs) 235 { 236 unsigned long fp_reg = (insn >> fp_reg_offset) & 0x1F; 237 unsigned long val; 238 239 val = get_f32_reg(fp_reg); 240 regs->status |= SR_FS_DIRTY; 241 242 return val; 243 } 244 245 #else /* CONFIG_FPU */ 246 static void set_f32_rd(unsigned long insn, struct pt_regs *regs, 247 unsigned long val) {} 248 249 static void set_f64_rd(unsigned long insn, struct pt_regs *regs, u64 val) {} 250 251 static unsigned long get_f64_rs(unsigned long insn, u8 fp_reg_offset, 252 struct pt_regs *regs) 253 { 254 return 0; 255 } 256 257 static unsigned long get_f32_rs(unsigned long insn, u8 fp_reg_offset, 258 struct pt_regs *regs) 259 { 260 return 0; 261 } 262 263 #endif 264 265 #define GET_F64_RS2(insn, regs) (get_f64_rs(insn, 20, regs)) 266 #define GET_F64_RS2C(insn, regs) (get_f64_rs(insn, 2, regs)) 267 #define GET_F64_RS2S(insn, regs) (get_f64_rs(RVC_RS2S(insn), 0, regs)) 268 269 #define GET_F32_RS2(insn, regs) (get_f32_rs(insn, 20, regs)) 270 #define GET_F32_RS2C(insn, regs) (get_f32_rs(insn, 2, regs)) 271 #define GET_F32_RS2S(insn, regs) (get_f32_rs(RVC_RS2S(insn), 0, regs)) 272 273 #define __read_insn(regs, insn, insn_addr, type) \ 274 ({ \ 275 int __ret; \ 276 \ 277 if (user_mode(regs)) { \ 278 __ret = get_user(insn, (type __user *) insn_addr); \ 279 } else { \ 280 insn = *(type *)insn_addr; \ 281 __ret = 0; \ 282 } \ 283 \ 284 __ret; \ 285 }) 286 287 static inline int get_insn(struct pt_regs *regs, ulong epc, ulong *r_insn) 288 { 289 ulong insn = 0; 290 291 if (epc & 0x2) { 292 ulong tmp = 0; 293 294 if (__read_insn(regs, insn, epc, u16)) 295 return -EFAULT; 296 /* __get_user() uses regular "lw" which sign extend the loaded 297 * value make sure to clear higher order bits in case we "or" it 298 * below with the upper 16 bits half. 299 */ 300 insn &= GENMASK(15, 0); 301 if ((insn & __INSN_LENGTH_MASK) != __INSN_LENGTH_32) { 302 *r_insn = insn; 303 return 0; 304 } 305 epc += sizeof(u16); 306 if (__read_insn(regs, tmp, epc, u16)) 307 return -EFAULT; 308 *r_insn = (tmp << 16) | insn; 309 310 return 0; 311 } else { 312 if (__read_insn(regs, insn, epc, u32)) 313 return -EFAULT; 314 if ((insn & __INSN_LENGTH_MASK) == __INSN_LENGTH_32) { 315 *r_insn = insn; 316 return 0; 317 } 318 insn &= GENMASK(15, 0); 319 *r_insn = insn; 320 321 return 0; 322 } 323 } 324 325 union reg_data { 326 u8 data_bytes[8]; 327 ulong data_ulong; 328 u64 data_u64; 329 }; 330 331 /* sysctl hooks */ 332 int unaligned_enabled __read_mostly = 1; /* Enabled by default */ 333 334 #ifdef CONFIG_RISCV_VECTOR_MISALIGNED 335 static int handle_vector_misaligned_load(struct pt_regs *regs) 336 { 337 unsigned long epc = regs->epc; 338 unsigned long insn; 339 340 if (get_insn(regs, epc, &insn)) 341 return -1; 342 343 /* Only return 0 when in check_vector_unaligned_access_emulated */ 344 if (*this_cpu_ptr(&vector_misaligned_access) == RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN) { 345 *this_cpu_ptr(&vector_misaligned_access) = RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED; 346 regs->epc = epc + INSN_LEN(insn); 347 return 0; 348 } 349 350 /* If vector instruction we don't emulate it yet */ 351 regs->epc = epc; 352 return -1; 353 } 354 #else 355 static int handle_vector_misaligned_load(struct pt_regs *regs) 356 { 357 return -1; 358 } 359 #endif 360 361 static int handle_scalar_misaligned_load(struct pt_regs *regs) 362 { 363 union reg_data val; 364 unsigned long epc = regs->epc; 365 unsigned long insn; 366 unsigned long addr = regs->badaddr; 367 int fp = 0, shift = 0, len = 0; 368 369 perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, addr); 370 371 #ifdef CONFIG_RISCV_PROBE_UNALIGNED_ACCESS 372 *this_cpu_ptr(&misaligned_access_speed) = RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED; 373 #endif 374 375 if (!unaligned_enabled) 376 return -1; 377 378 if (user_mode(regs) && (current->thread.align_ctl & PR_UNALIGN_SIGBUS)) 379 return -1; 380 381 if (get_insn(regs, epc, &insn)) 382 return -1; 383 384 regs->epc = 0; 385 386 if ((insn & INSN_MASK_LW) == INSN_MATCH_LW) { 387 len = 4; 388 shift = 8 * (sizeof(unsigned long) - len); 389 #if defined(CONFIG_64BIT) 390 } else if ((insn & INSN_MASK_LD) == INSN_MATCH_LD) { 391 len = 8; 392 shift = 8 * (sizeof(unsigned long) - len); 393 } else if ((insn & INSN_MASK_LWU) == INSN_MATCH_LWU) { 394 len = 4; 395 #endif 396 } else if ((insn & INSN_MASK_FLD) == INSN_MATCH_FLD) { 397 fp = 1; 398 len = 8; 399 } else if ((insn & INSN_MASK_FLW) == INSN_MATCH_FLW) { 400 fp = 1; 401 len = 4; 402 } else if ((insn & INSN_MASK_LH) == INSN_MATCH_LH) { 403 len = 2; 404 shift = 8 * (sizeof(unsigned long) - len); 405 } else if ((insn & INSN_MASK_LHU) == INSN_MATCH_LHU) { 406 len = 2; 407 #if defined(CONFIG_64BIT) 408 } else if ((insn & INSN_MASK_C_LD) == INSN_MATCH_C_LD) { 409 len = 8; 410 shift = 8 * (sizeof(unsigned long) - len); 411 insn = RVC_RS2S(insn) << SH_RD; 412 } else if ((insn & INSN_MASK_C_LDSP) == INSN_MATCH_C_LDSP && 413 ((insn >> SH_RD) & 0x1f)) { 414 len = 8; 415 shift = 8 * (sizeof(unsigned long) - len); 416 #endif 417 } else if ((insn & INSN_MASK_C_LW) == INSN_MATCH_C_LW) { 418 len = 4; 419 shift = 8 * (sizeof(unsigned long) - len); 420 insn = RVC_RS2S(insn) << SH_RD; 421 } else if ((insn & INSN_MASK_C_LWSP) == INSN_MATCH_C_LWSP && 422 ((insn >> SH_RD) & 0x1f)) { 423 len = 4; 424 shift = 8 * (sizeof(unsigned long) - len); 425 } else if ((insn & INSN_MASK_C_FLD) == INSN_MATCH_C_FLD) { 426 fp = 1; 427 len = 8; 428 insn = RVC_RS2S(insn) << SH_RD; 429 } else if ((insn & INSN_MASK_C_FLDSP) == INSN_MATCH_C_FLDSP) { 430 fp = 1; 431 len = 8; 432 #if defined(CONFIG_32BIT) 433 } else if ((insn & INSN_MASK_C_FLW) == INSN_MATCH_C_FLW) { 434 fp = 1; 435 len = 4; 436 insn = RVC_RS2S(insn) << SH_RD; 437 } else if ((insn & INSN_MASK_C_FLWSP) == INSN_MATCH_C_FLWSP) { 438 fp = 1; 439 len = 4; 440 #endif 441 } else if ((insn & INSN_MASK_C_LHU) == INSN_MATCH_C_LHU) { 442 len = 2; 443 insn = RVC_RS2S(insn) << SH_RD; 444 } else if ((insn & INSN_MASK_C_LH) == INSN_MATCH_C_LH) { 445 len = 2; 446 shift = 8 * (sizeof(ulong) - len); 447 insn = RVC_RS2S(insn) << SH_RD; 448 } else { 449 regs->epc = epc; 450 return -1; 451 } 452 453 if (!IS_ENABLED(CONFIG_FPU) && fp) 454 return -EOPNOTSUPP; 455 456 val.data_u64 = 0; 457 if (user_mode(regs)) { 458 if (copy_from_user(&val, (u8 __user *)addr, len)) 459 return -1; 460 } else { 461 memcpy(&val, (u8 *)addr, len); 462 } 463 464 if (!fp) 465 SET_RD(insn, regs, val.data_ulong << shift >> shift); 466 else if (len == 8) 467 set_f64_rd(insn, regs, val.data_u64); 468 else 469 set_f32_rd(insn, regs, val.data_ulong); 470 471 regs->epc = epc + INSN_LEN(insn); 472 473 return 0; 474 } 475 476 static int handle_scalar_misaligned_store(struct pt_regs *regs) 477 { 478 union reg_data val; 479 unsigned long epc = regs->epc; 480 unsigned long insn; 481 unsigned long addr = regs->badaddr; 482 int len = 0, fp = 0; 483 484 perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, addr); 485 486 if (!unaligned_enabled) 487 return -1; 488 489 if (user_mode(regs) && (current->thread.align_ctl & PR_UNALIGN_SIGBUS)) 490 return -1; 491 492 if (get_insn(regs, epc, &insn)) 493 return -1; 494 495 regs->epc = 0; 496 497 val.data_ulong = GET_RS2(insn, regs); 498 499 if ((insn & INSN_MASK_SW) == INSN_MATCH_SW) { 500 len = 4; 501 #if defined(CONFIG_64BIT) 502 } else if ((insn & INSN_MASK_SD) == INSN_MATCH_SD) { 503 len = 8; 504 #endif 505 } else if ((insn & INSN_MASK_FSD) == INSN_MATCH_FSD) { 506 fp = 1; 507 len = 8; 508 val.data_u64 = GET_F64_RS2(insn, regs); 509 } else if ((insn & INSN_MASK_FSW) == INSN_MATCH_FSW) { 510 fp = 1; 511 len = 4; 512 val.data_ulong = GET_F32_RS2(insn, regs); 513 } else if ((insn & INSN_MASK_SH) == INSN_MATCH_SH) { 514 len = 2; 515 #if defined(CONFIG_64BIT) 516 } else if ((insn & INSN_MASK_C_SD) == INSN_MATCH_C_SD) { 517 len = 8; 518 val.data_ulong = GET_RS2S(insn, regs); 519 } else if ((insn & INSN_MASK_C_SDSP) == INSN_MATCH_C_SDSP) { 520 len = 8; 521 val.data_ulong = GET_RS2C(insn, regs); 522 #endif 523 } else if ((insn & INSN_MASK_C_SW) == INSN_MATCH_C_SW) { 524 len = 4; 525 val.data_ulong = GET_RS2S(insn, regs); 526 } else if ((insn & INSN_MASK_C_SWSP) == INSN_MATCH_C_SWSP) { 527 len = 4; 528 val.data_ulong = GET_RS2C(insn, regs); 529 } else if ((insn & INSN_MASK_C_FSD) == INSN_MATCH_C_FSD) { 530 fp = 1; 531 len = 8; 532 val.data_u64 = GET_F64_RS2S(insn, regs); 533 } else if ((insn & INSN_MASK_C_FSDSP) == INSN_MATCH_C_FSDSP) { 534 fp = 1; 535 len = 8; 536 val.data_u64 = GET_F64_RS2C(insn, regs); 537 #if !defined(CONFIG_64BIT) 538 } else if ((insn & INSN_MASK_C_FSW) == INSN_MATCH_C_FSW) { 539 fp = 1; 540 len = 4; 541 val.data_ulong = GET_F32_RS2S(insn, regs); 542 } else if ((insn & INSN_MASK_C_FSWSP) == INSN_MATCH_C_FSWSP) { 543 fp = 1; 544 len = 4; 545 val.data_ulong = GET_F32_RS2C(insn, regs); 546 #endif 547 } else if ((insn & INSN_MASK_C_SH) == INSN_MATCH_C_SH) { 548 len = 2; 549 val.data_ulong = GET_RS2S(insn, regs); 550 } else { 551 regs->epc = epc; 552 return -1; 553 } 554 555 if (!IS_ENABLED(CONFIG_FPU) && fp) 556 return -EOPNOTSUPP; 557 558 if (user_mode(regs)) { 559 if (copy_to_user((u8 __user *)addr, &val, len)) 560 return -1; 561 } else { 562 memcpy((u8 *)addr, &val, len); 563 } 564 565 regs->epc = epc + INSN_LEN(insn); 566 567 return 0; 568 } 569 570 int handle_misaligned_load(struct pt_regs *regs) 571 { 572 unsigned long epc = regs->epc; 573 unsigned long insn; 574 575 if (IS_ENABLED(CONFIG_RISCV_VECTOR_MISALIGNED)) { 576 if (get_insn(regs, epc, &insn)) 577 return -1; 578 579 if (insn_is_vector(insn)) 580 return handle_vector_misaligned_load(regs); 581 } 582 583 if (IS_ENABLED(CONFIG_RISCV_SCALAR_MISALIGNED)) 584 return handle_scalar_misaligned_load(regs); 585 586 return -1; 587 } 588 589 int handle_misaligned_store(struct pt_regs *regs) 590 { 591 if (IS_ENABLED(CONFIG_RISCV_SCALAR_MISALIGNED)) 592 return handle_scalar_misaligned_store(regs); 593 594 return -1; 595 } 596 597 #ifdef CONFIG_RISCV_VECTOR_MISALIGNED 598 void check_vector_unaligned_access_emulated(struct work_struct *work __always_unused) 599 { 600 long *mas_ptr = this_cpu_ptr(&vector_misaligned_access); 601 unsigned long tmp_var; 602 603 *mas_ptr = RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN; 604 605 kernel_vector_begin(); 606 /* 607 * In pre-13.0.0 versions of GCC, vector registers cannot appear in 608 * the clobber list. This inline asm clobbers v0, but since we do not 609 * currently build the kernel with V enabled, the v0 clobber arg is not 610 * needed (as the compiler will not emit vector code itself). If the kernel 611 * is changed to build with V enabled, the clobber arg will need to be 612 * added here. 613 */ 614 __asm__ __volatile__ ( 615 ".balign 4\n\t" 616 ".option push\n\t" 617 ".option arch, +zve32x\n\t" 618 " vsetivli zero, 1, e16, m1, ta, ma\n\t" // Vectors of 16b 619 " vle16.v v0, (%[ptr])\n\t" // Load bytes 620 ".option pop\n\t" 621 : : [ptr] "r" ((u8 *)&tmp_var + 1)); 622 kernel_vector_end(); 623 } 624 625 bool __init check_vector_unaligned_access_emulated_all_cpus(void) 626 { 627 int cpu; 628 629 schedule_on_each_cpu(check_vector_unaligned_access_emulated); 630 631 for_each_online_cpu(cpu) 632 if (per_cpu(vector_misaligned_access, cpu) 633 == RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN) 634 return false; 635 636 return true; 637 } 638 #else 639 bool __init check_vector_unaligned_access_emulated_all_cpus(void) 640 { 641 return false; 642 } 643 #endif 644 645 #ifdef CONFIG_RISCV_SCALAR_MISALIGNED 646 647 static bool unaligned_ctl __read_mostly; 648 649 void check_unaligned_access_emulated(struct work_struct *work __always_unused) 650 { 651 int cpu = smp_processor_id(); 652 long *mas_ptr = per_cpu_ptr(&misaligned_access_speed, cpu); 653 unsigned long tmp_var, tmp_val; 654 655 *mas_ptr = RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN; 656 657 __asm__ __volatile__ ( 658 " "REG_L" %[tmp], 1(%[ptr])\n" 659 : [tmp] "=r" (tmp_val) : [ptr] "r" (&tmp_var) : "memory"); 660 661 /* 662 * If unaligned_ctl is already set, this means that we detected that all 663 * CPUS uses emulated misaligned access at boot time. If that changed 664 * when hotplugging the new cpu, this is something we don't handle. 665 */ 666 if (unlikely(unaligned_ctl && (*mas_ptr != RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED))) { 667 pr_crit("CPU misaligned accesses non homogeneous (expected all emulated)\n"); 668 while (true) 669 cpu_relax(); 670 } 671 } 672 673 bool __init check_unaligned_access_emulated_all_cpus(void) 674 { 675 int cpu; 676 677 /* 678 * We can only support PR_UNALIGN controls if all CPUs have misaligned 679 * accesses emulated since tasks requesting such control can run on any 680 * CPU. 681 */ 682 schedule_on_each_cpu(check_unaligned_access_emulated); 683 684 for_each_online_cpu(cpu) 685 if (per_cpu(misaligned_access_speed, cpu) 686 != RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED) 687 return false; 688 689 unaligned_ctl = true; 690 return true; 691 } 692 693 bool unaligned_ctl_available(void) 694 { 695 return unaligned_ctl; 696 } 697 #else 698 bool __init check_unaligned_access_emulated_all_cpus(void) 699 { 700 return false; 701 } 702 #endif 703