1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2020 Western Digital Corporation or its affiliates. 4 */ 5 #include <linux/kernel.h> 6 #include <linux/init.h> 7 #include <linux/mm.h> 8 #include <linux/module.h> 9 #include <linux/perf_event.h> 10 #include <linux/irq.h> 11 #include <linux/stringify.h> 12 13 #include <asm/processor.h> 14 #include <asm/ptrace.h> 15 #include <asm/csr.h> 16 #include <asm/entry-common.h> 17 #include <asm/hwprobe.h> 18 #include <asm/cpufeature.h> 19 #include <asm/vector.h> 20 21 #define INSN_MATCH_LB 0x3 22 #define INSN_MASK_LB 0x707f 23 #define INSN_MATCH_LH 0x1003 24 #define INSN_MASK_LH 0x707f 25 #define INSN_MATCH_LW 0x2003 26 #define INSN_MASK_LW 0x707f 27 #define INSN_MATCH_LD 0x3003 28 #define INSN_MASK_LD 0x707f 29 #define INSN_MATCH_LBU 0x4003 30 #define INSN_MASK_LBU 0x707f 31 #define INSN_MATCH_LHU 0x5003 32 #define INSN_MASK_LHU 0x707f 33 #define INSN_MATCH_LWU 0x6003 34 #define INSN_MASK_LWU 0x707f 35 #define INSN_MATCH_SB 0x23 36 #define INSN_MASK_SB 0x707f 37 #define INSN_MATCH_SH 0x1023 38 #define INSN_MASK_SH 0x707f 39 #define INSN_MATCH_SW 0x2023 40 #define INSN_MASK_SW 0x707f 41 #define INSN_MATCH_SD 0x3023 42 #define INSN_MASK_SD 0x707f 43 44 #define INSN_MATCH_FLW 0x2007 45 #define INSN_MASK_FLW 0x707f 46 #define INSN_MATCH_FLD 0x3007 47 #define INSN_MASK_FLD 0x707f 48 #define INSN_MATCH_FLQ 0x4007 49 #define INSN_MASK_FLQ 0x707f 50 #define INSN_MATCH_FSW 0x2027 51 #define INSN_MASK_FSW 0x707f 52 #define INSN_MATCH_FSD 0x3027 53 #define INSN_MASK_FSD 0x707f 54 #define INSN_MATCH_FSQ 0x4027 55 #define INSN_MASK_FSQ 0x707f 56 57 #define INSN_MATCH_C_LD 0x6000 58 #define INSN_MASK_C_LD 0xe003 59 #define INSN_MATCH_C_SD 0xe000 60 #define INSN_MASK_C_SD 0xe003 61 #define INSN_MATCH_C_LW 0x4000 62 #define INSN_MASK_C_LW 0xe003 63 #define INSN_MATCH_C_SW 0xc000 64 #define INSN_MASK_C_SW 0xe003 65 #define INSN_MATCH_C_LDSP 0x6002 66 #define INSN_MASK_C_LDSP 0xe003 67 #define INSN_MATCH_C_SDSP 0xe002 68 #define INSN_MASK_C_SDSP 0xe003 69 #define INSN_MATCH_C_LWSP 0x4002 70 #define INSN_MASK_C_LWSP 0xe003 71 #define INSN_MATCH_C_SWSP 0xc002 72 #define INSN_MASK_C_SWSP 0xe003 73 74 #define INSN_MATCH_C_FLD 0x2000 75 #define INSN_MASK_C_FLD 0xe003 76 #define INSN_MATCH_C_FLW 0x6000 77 #define INSN_MASK_C_FLW 0xe003 78 #define INSN_MATCH_C_FSD 0xa000 79 #define INSN_MASK_C_FSD 0xe003 80 #define INSN_MATCH_C_FSW 0xe000 81 #define INSN_MASK_C_FSW 0xe003 82 #define INSN_MATCH_C_FLDSP 0x2002 83 #define INSN_MASK_C_FLDSP 0xe003 84 #define INSN_MATCH_C_FSDSP 0xa002 85 #define INSN_MASK_C_FSDSP 0xe003 86 #define INSN_MATCH_C_FLWSP 0x6002 87 #define INSN_MASK_C_FLWSP 0xe003 88 #define INSN_MATCH_C_FSWSP 0xe002 89 #define INSN_MASK_C_FSWSP 0xe003 90 91 #define INSN_LEN(insn) ((((insn) & 0x3) < 0x3) ? 2 : 4) 92 93 #if defined(CONFIG_64BIT) 94 #define LOG_REGBYTES 3 95 #define XLEN 64 96 #else 97 #define LOG_REGBYTES 2 98 #define XLEN 32 99 #endif 100 #define REGBYTES (1 << LOG_REGBYTES) 101 #define XLEN_MINUS_16 ((XLEN) - 16) 102 103 #define SH_RD 7 104 #define SH_RS1 15 105 #define SH_RS2 20 106 #define SH_RS2C 2 107 108 #define RV_X(x, s, n) (((x) >> (s)) & ((1 << (n)) - 1)) 109 #define RVC_LW_IMM(x) ((RV_X(x, 6, 1) << 2) | \ 110 (RV_X(x, 10, 3) << 3) | \ 111 (RV_X(x, 5, 1) << 6)) 112 #define RVC_LD_IMM(x) ((RV_X(x, 10, 3) << 3) | \ 113 (RV_X(x, 5, 2) << 6)) 114 #define RVC_LWSP_IMM(x) ((RV_X(x, 4, 3) << 2) | \ 115 (RV_X(x, 12, 1) << 5) | \ 116 (RV_X(x, 2, 2) << 6)) 117 #define RVC_LDSP_IMM(x) ((RV_X(x, 5, 2) << 3) | \ 118 (RV_X(x, 12, 1) << 5) | \ 119 (RV_X(x, 2, 3) << 6)) 120 #define RVC_SWSP_IMM(x) ((RV_X(x, 9, 4) << 2) | \ 121 (RV_X(x, 7, 2) << 6)) 122 #define RVC_SDSP_IMM(x) ((RV_X(x, 10, 3) << 3) | \ 123 (RV_X(x, 7, 3) << 6)) 124 #define RVC_RS1S(insn) (8 + RV_X(insn, SH_RD, 3)) 125 #define RVC_RS2S(insn) (8 + RV_X(insn, SH_RS2C, 3)) 126 #define RVC_RS2(insn) RV_X(insn, SH_RS2C, 5) 127 128 #define SHIFT_RIGHT(x, y) \ 129 ((y) < 0 ? ((x) << -(y)) : ((x) >> (y))) 130 131 #define REG_MASK \ 132 ((1 << (5 + LOG_REGBYTES)) - (1 << LOG_REGBYTES)) 133 134 #define REG_OFFSET(insn, pos) \ 135 (SHIFT_RIGHT((insn), (pos) - LOG_REGBYTES) & REG_MASK) 136 137 #define REG_PTR(insn, pos, regs) \ 138 (ulong *)((ulong)(regs) + REG_OFFSET(insn, pos)) 139 140 #define GET_RM(insn) (((insn) >> 12) & 7) 141 142 #define GET_RS1(insn, regs) (*REG_PTR(insn, SH_RS1, regs)) 143 #define GET_RS2(insn, regs) (*REG_PTR(insn, SH_RS2, regs)) 144 #define GET_RS1S(insn, regs) (*REG_PTR(RVC_RS1S(insn), 0, regs)) 145 #define GET_RS2S(insn, regs) (*REG_PTR(RVC_RS2S(insn), 0, regs)) 146 #define GET_RS2C(insn, regs) (*REG_PTR(insn, SH_RS2C, regs)) 147 #define GET_SP(regs) (*REG_PTR(2, 0, regs)) 148 #define SET_RD(insn, regs, val) (*REG_PTR(insn, SH_RD, regs) = (val)) 149 #define IMM_I(insn) ((s32)(insn) >> 20) 150 #define IMM_S(insn) (((s32)(insn) >> 25 << 5) | \ 151 (s32)(((insn) >> 7) & 0x1f)) 152 #define MASK_FUNCT3 0x7000 153 154 #define GET_PRECISION(insn) (((insn) >> 25) & 3) 155 #define GET_RM(insn) (((insn) >> 12) & 7) 156 #define PRECISION_S 0 157 #define PRECISION_D 1 158 159 #ifdef CONFIG_FPU 160 161 #define FP_GET_RD(insn) (insn >> 7 & 0x1F) 162 163 extern void put_f32_reg(unsigned long fp_reg, unsigned long value); 164 165 static int set_f32_rd(unsigned long insn, struct pt_regs *regs, 166 unsigned long val) 167 { 168 unsigned long fp_reg = FP_GET_RD(insn); 169 170 put_f32_reg(fp_reg, val); 171 regs->status |= SR_FS_DIRTY; 172 173 return 0; 174 } 175 176 extern void put_f64_reg(unsigned long fp_reg, unsigned long value); 177 178 static int set_f64_rd(unsigned long insn, struct pt_regs *regs, u64 val) 179 { 180 unsigned long fp_reg = FP_GET_RD(insn); 181 unsigned long value; 182 183 #if __riscv_xlen == 32 184 value = (unsigned long) &val; 185 #else 186 value = val; 187 #endif 188 put_f64_reg(fp_reg, value); 189 regs->status |= SR_FS_DIRTY; 190 191 return 0; 192 } 193 194 #if __riscv_xlen == 32 195 extern void get_f64_reg(unsigned long fp_reg, u64 *value); 196 197 static u64 get_f64_rs(unsigned long insn, u8 fp_reg_offset, 198 struct pt_regs *regs) 199 { 200 unsigned long fp_reg = (insn >> fp_reg_offset) & 0x1F; 201 u64 val; 202 203 get_f64_reg(fp_reg, &val); 204 regs->status |= SR_FS_DIRTY; 205 206 return val; 207 } 208 #else 209 210 extern unsigned long get_f64_reg(unsigned long fp_reg); 211 212 static unsigned long get_f64_rs(unsigned long insn, u8 fp_reg_offset, 213 struct pt_regs *regs) 214 { 215 unsigned long fp_reg = (insn >> fp_reg_offset) & 0x1F; 216 unsigned long val; 217 218 val = get_f64_reg(fp_reg); 219 regs->status |= SR_FS_DIRTY; 220 221 return val; 222 } 223 224 #endif 225 226 extern unsigned long get_f32_reg(unsigned long fp_reg); 227 228 static unsigned long get_f32_rs(unsigned long insn, u8 fp_reg_offset, 229 struct pt_regs *regs) 230 { 231 unsigned long fp_reg = (insn >> fp_reg_offset) & 0x1F; 232 unsigned long val; 233 234 val = get_f32_reg(fp_reg); 235 regs->status |= SR_FS_DIRTY; 236 237 return val; 238 } 239 240 #else /* CONFIG_FPU */ 241 static void set_f32_rd(unsigned long insn, struct pt_regs *regs, 242 unsigned long val) {} 243 244 static void set_f64_rd(unsigned long insn, struct pt_regs *regs, u64 val) {} 245 246 static unsigned long get_f64_rs(unsigned long insn, u8 fp_reg_offset, 247 struct pt_regs *regs) 248 { 249 return 0; 250 } 251 252 static unsigned long get_f32_rs(unsigned long insn, u8 fp_reg_offset, 253 struct pt_regs *regs) 254 { 255 return 0; 256 } 257 258 #endif 259 260 #define GET_F64_RS2(insn, regs) (get_f64_rs(insn, 20, regs)) 261 #define GET_F64_RS2C(insn, regs) (get_f64_rs(insn, 2, regs)) 262 #define GET_F64_RS2S(insn, regs) (get_f64_rs(RVC_RS2S(insn), 0, regs)) 263 264 #define GET_F32_RS2(insn, regs) (get_f32_rs(insn, 20, regs)) 265 #define GET_F32_RS2C(insn, regs) (get_f32_rs(insn, 2, regs)) 266 #define GET_F32_RS2S(insn, regs) (get_f32_rs(RVC_RS2S(insn), 0, regs)) 267 268 #define __read_insn(regs, insn, insn_addr, type) \ 269 ({ \ 270 int __ret; \ 271 \ 272 if (user_mode(regs)) { \ 273 __ret = __get_user(insn, (type __user *) insn_addr); \ 274 } else { \ 275 insn = *(type *)insn_addr; \ 276 __ret = 0; \ 277 } \ 278 \ 279 __ret; \ 280 }) 281 282 static inline int get_insn(struct pt_regs *regs, ulong epc, ulong *r_insn) 283 { 284 ulong insn = 0; 285 286 if (epc & 0x2) { 287 ulong tmp = 0; 288 289 if (__read_insn(regs, insn, epc, u16)) 290 return -EFAULT; 291 /* __get_user() uses regular "lw" which sign extend the loaded 292 * value make sure to clear higher order bits in case we "or" it 293 * below with the upper 16 bits half. 294 */ 295 insn &= GENMASK(15, 0); 296 if ((insn & __INSN_LENGTH_MASK) != __INSN_LENGTH_32) { 297 *r_insn = insn; 298 return 0; 299 } 300 epc += sizeof(u16); 301 if (__read_insn(regs, tmp, epc, u16)) 302 return -EFAULT; 303 *r_insn = (tmp << 16) | insn; 304 305 return 0; 306 } else { 307 if (__read_insn(regs, insn, epc, u32)) 308 return -EFAULT; 309 if ((insn & __INSN_LENGTH_MASK) == __INSN_LENGTH_32) { 310 *r_insn = insn; 311 return 0; 312 } 313 insn &= GENMASK(15, 0); 314 *r_insn = insn; 315 316 return 0; 317 } 318 } 319 320 union reg_data { 321 u8 data_bytes[8]; 322 ulong data_ulong; 323 u64 data_u64; 324 }; 325 326 /* sysctl hooks */ 327 int unaligned_enabled __read_mostly = 1; /* Enabled by default */ 328 329 #ifdef CONFIG_RISCV_VECTOR_MISALIGNED 330 static int handle_vector_misaligned_load(struct pt_regs *regs) 331 { 332 unsigned long epc = regs->epc; 333 unsigned long insn; 334 335 if (get_insn(regs, epc, &insn)) 336 return -1; 337 338 /* Only return 0 when in check_vector_unaligned_access_emulated */ 339 if (*this_cpu_ptr(&vector_misaligned_access) == RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN) { 340 *this_cpu_ptr(&vector_misaligned_access) = RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED; 341 regs->epc = epc + INSN_LEN(insn); 342 return 0; 343 } 344 345 /* If vector instruction we don't emulate it yet */ 346 regs->epc = epc; 347 return -1; 348 } 349 #else 350 static int handle_vector_misaligned_load(struct pt_regs *regs) 351 { 352 return -1; 353 } 354 #endif 355 356 static int handle_scalar_misaligned_load(struct pt_regs *regs) 357 { 358 union reg_data val; 359 unsigned long epc = regs->epc; 360 unsigned long insn; 361 unsigned long addr = regs->badaddr; 362 int fp = 0, shift = 0, len = 0; 363 364 perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, addr); 365 366 #ifdef CONFIG_RISCV_PROBE_UNALIGNED_ACCESS 367 *this_cpu_ptr(&misaligned_access_speed) = RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED; 368 #endif 369 370 if (!unaligned_enabled) 371 return -1; 372 373 if (user_mode(regs) && (current->thread.align_ctl & PR_UNALIGN_SIGBUS)) 374 return -1; 375 376 if (get_insn(regs, epc, &insn)) 377 return -1; 378 379 regs->epc = 0; 380 381 if ((insn & INSN_MASK_LW) == INSN_MATCH_LW) { 382 len = 4; 383 shift = 8 * (sizeof(unsigned long) - len); 384 #if defined(CONFIG_64BIT) 385 } else if ((insn & INSN_MASK_LD) == INSN_MATCH_LD) { 386 len = 8; 387 shift = 8 * (sizeof(unsigned long) - len); 388 } else if ((insn & INSN_MASK_LWU) == INSN_MATCH_LWU) { 389 len = 4; 390 #endif 391 } else if ((insn & INSN_MASK_FLD) == INSN_MATCH_FLD) { 392 fp = 1; 393 len = 8; 394 } else if ((insn & INSN_MASK_FLW) == INSN_MATCH_FLW) { 395 fp = 1; 396 len = 4; 397 } else if ((insn & INSN_MASK_LH) == INSN_MATCH_LH) { 398 len = 2; 399 shift = 8 * (sizeof(unsigned long) - len); 400 } else if ((insn & INSN_MASK_LHU) == INSN_MATCH_LHU) { 401 len = 2; 402 #if defined(CONFIG_64BIT) 403 } else if ((insn & INSN_MASK_C_LD) == INSN_MATCH_C_LD) { 404 len = 8; 405 shift = 8 * (sizeof(unsigned long) - len); 406 insn = RVC_RS2S(insn) << SH_RD; 407 } else if ((insn & INSN_MASK_C_LDSP) == INSN_MATCH_C_LDSP && 408 ((insn >> SH_RD) & 0x1f)) { 409 len = 8; 410 shift = 8 * (sizeof(unsigned long) - len); 411 #endif 412 } else if ((insn & INSN_MASK_C_LW) == INSN_MATCH_C_LW) { 413 len = 4; 414 shift = 8 * (sizeof(unsigned long) - len); 415 insn = RVC_RS2S(insn) << SH_RD; 416 } else if ((insn & INSN_MASK_C_LWSP) == INSN_MATCH_C_LWSP && 417 ((insn >> SH_RD) & 0x1f)) { 418 len = 4; 419 shift = 8 * (sizeof(unsigned long) - len); 420 } else if ((insn & INSN_MASK_C_FLD) == INSN_MATCH_C_FLD) { 421 fp = 1; 422 len = 8; 423 insn = RVC_RS2S(insn) << SH_RD; 424 } else if ((insn & INSN_MASK_C_FLDSP) == INSN_MATCH_C_FLDSP) { 425 fp = 1; 426 len = 8; 427 #if defined(CONFIG_32BIT) 428 } else if ((insn & INSN_MASK_C_FLW) == INSN_MATCH_C_FLW) { 429 fp = 1; 430 len = 4; 431 insn = RVC_RS2S(insn) << SH_RD; 432 } else if ((insn & INSN_MASK_C_FLWSP) == INSN_MATCH_C_FLWSP) { 433 fp = 1; 434 len = 4; 435 #endif 436 } else { 437 regs->epc = epc; 438 return -1; 439 } 440 441 if (!IS_ENABLED(CONFIG_FPU) && fp) 442 return -EOPNOTSUPP; 443 444 val.data_u64 = 0; 445 if (user_mode(regs)) { 446 if (copy_from_user(&val, (u8 __user *)addr, len)) 447 return -1; 448 } else { 449 memcpy(&val, (u8 *)addr, len); 450 } 451 452 if (!fp) 453 SET_RD(insn, regs, val.data_ulong << shift >> shift); 454 else if (len == 8) 455 set_f64_rd(insn, regs, val.data_u64); 456 else 457 set_f32_rd(insn, regs, val.data_ulong); 458 459 regs->epc = epc + INSN_LEN(insn); 460 461 return 0; 462 } 463 464 static int handle_scalar_misaligned_store(struct pt_regs *regs) 465 { 466 union reg_data val; 467 unsigned long epc = regs->epc; 468 unsigned long insn; 469 unsigned long addr = regs->badaddr; 470 int len = 0, fp = 0; 471 472 perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, addr); 473 474 if (!unaligned_enabled) 475 return -1; 476 477 if (user_mode(regs) && (current->thread.align_ctl & PR_UNALIGN_SIGBUS)) 478 return -1; 479 480 if (get_insn(regs, epc, &insn)) 481 return -1; 482 483 regs->epc = 0; 484 485 val.data_ulong = GET_RS2(insn, regs); 486 487 if ((insn & INSN_MASK_SW) == INSN_MATCH_SW) { 488 len = 4; 489 #if defined(CONFIG_64BIT) 490 } else if ((insn & INSN_MASK_SD) == INSN_MATCH_SD) { 491 len = 8; 492 #endif 493 } else if ((insn & INSN_MASK_FSD) == INSN_MATCH_FSD) { 494 fp = 1; 495 len = 8; 496 val.data_u64 = GET_F64_RS2(insn, regs); 497 } else if ((insn & INSN_MASK_FSW) == INSN_MATCH_FSW) { 498 fp = 1; 499 len = 4; 500 val.data_ulong = GET_F32_RS2(insn, regs); 501 } else if ((insn & INSN_MASK_SH) == INSN_MATCH_SH) { 502 len = 2; 503 #if defined(CONFIG_64BIT) 504 } else if ((insn & INSN_MASK_C_SD) == INSN_MATCH_C_SD) { 505 len = 8; 506 val.data_ulong = GET_RS2S(insn, regs); 507 } else if ((insn & INSN_MASK_C_SDSP) == INSN_MATCH_C_SDSP) { 508 len = 8; 509 val.data_ulong = GET_RS2C(insn, regs); 510 #endif 511 } else if ((insn & INSN_MASK_C_SW) == INSN_MATCH_C_SW) { 512 len = 4; 513 val.data_ulong = GET_RS2S(insn, regs); 514 } else if ((insn & INSN_MASK_C_SWSP) == INSN_MATCH_C_SWSP) { 515 len = 4; 516 val.data_ulong = GET_RS2C(insn, regs); 517 } else if ((insn & INSN_MASK_C_FSD) == INSN_MATCH_C_FSD) { 518 fp = 1; 519 len = 8; 520 val.data_u64 = GET_F64_RS2S(insn, regs); 521 } else if ((insn & INSN_MASK_C_FSDSP) == INSN_MATCH_C_FSDSP) { 522 fp = 1; 523 len = 8; 524 val.data_u64 = GET_F64_RS2C(insn, regs); 525 #if !defined(CONFIG_64BIT) 526 } else if ((insn & INSN_MASK_C_FSW) == INSN_MATCH_C_FSW) { 527 fp = 1; 528 len = 4; 529 val.data_ulong = GET_F32_RS2S(insn, regs); 530 } else if ((insn & INSN_MASK_C_FSWSP) == INSN_MATCH_C_FSWSP) { 531 fp = 1; 532 len = 4; 533 val.data_ulong = GET_F32_RS2C(insn, regs); 534 #endif 535 } else { 536 regs->epc = epc; 537 return -1; 538 } 539 540 if (!IS_ENABLED(CONFIG_FPU) && fp) 541 return -EOPNOTSUPP; 542 543 if (user_mode(regs)) { 544 if (copy_to_user((u8 __user *)addr, &val, len)) 545 return -1; 546 } else { 547 memcpy((u8 *)addr, &val, len); 548 } 549 550 regs->epc = epc + INSN_LEN(insn); 551 552 return 0; 553 } 554 555 int handle_misaligned_load(struct pt_regs *regs) 556 { 557 unsigned long epc = regs->epc; 558 unsigned long insn; 559 560 if (IS_ENABLED(CONFIG_RISCV_VECTOR_MISALIGNED)) { 561 if (get_insn(regs, epc, &insn)) 562 return -1; 563 564 if (insn_is_vector(insn)) 565 return handle_vector_misaligned_load(regs); 566 } 567 568 if (IS_ENABLED(CONFIG_RISCV_SCALAR_MISALIGNED)) 569 return handle_scalar_misaligned_load(regs); 570 571 return -1; 572 } 573 574 int handle_misaligned_store(struct pt_regs *regs) 575 { 576 if (IS_ENABLED(CONFIG_RISCV_SCALAR_MISALIGNED)) 577 return handle_scalar_misaligned_store(regs); 578 579 return -1; 580 } 581 582 #ifdef CONFIG_RISCV_VECTOR_MISALIGNED 583 void check_vector_unaligned_access_emulated(struct work_struct *work __always_unused) 584 { 585 long *mas_ptr = this_cpu_ptr(&vector_misaligned_access); 586 unsigned long tmp_var; 587 588 *mas_ptr = RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN; 589 590 kernel_vector_begin(); 591 /* 592 * In pre-13.0.0 versions of GCC, vector registers cannot appear in 593 * the clobber list. This inline asm clobbers v0, but since we do not 594 * currently build the kernel with V enabled, the v0 clobber arg is not 595 * needed (as the compiler will not emit vector code itself). If the kernel 596 * is changed to build with V enabled, the clobber arg will need to be 597 * added here. 598 */ 599 __asm__ __volatile__ ( 600 ".balign 4\n\t" 601 ".option push\n\t" 602 ".option arch, +zve32x\n\t" 603 " vsetivli zero, 1, e16, m1, ta, ma\n\t" // Vectors of 16b 604 " vle16.v v0, (%[ptr])\n\t" // Load bytes 605 ".option pop\n\t" 606 : : [ptr] "r" ((u8 *)&tmp_var + 1)); 607 kernel_vector_end(); 608 } 609 610 bool check_vector_unaligned_access_emulated_all_cpus(void) 611 { 612 int cpu; 613 614 if (!has_vector()) { 615 for_each_online_cpu(cpu) 616 per_cpu(vector_misaligned_access, cpu) = RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED; 617 return false; 618 } 619 620 schedule_on_each_cpu(check_vector_unaligned_access_emulated); 621 622 for_each_online_cpu(cpu) 623 if (per_cpu(vector_misaligned_access, cpu) 624 == RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN) 625 return false; 626 627 return true; 628 } 629 #else 630 bool check_vector_unaligned_access_emulated_all_cpus(void) 631 { 632 return false; 633 } 634 #endif 635 636 #ifdef CONFIG_RISCV_SCALAR_MISALIGNED 637 638 static bool unaligned_ctl __read_mostly; 639 640 void check_unaligned_access_emulated(struct work_struct *work __always_unused) 641 { 642 int cpu = smp_processor_id(); 643 long *mas_ptr = per_cpu_ptr(&misaligned_access_speed, cpu); 644 unsigned long tmp_var, tmp_val; 645 646 *mas_ptr = RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN; 647 648 __asm__ __volatile__ ( 649 " "REG_L" %[tmp], 1(%[ptr])\n" 650 : [tmp] "=r" (tmp_val) : [ptr] "r" (&tmp_var) : "memory"); 651 652 /* 653 * If unaligned_ctl is already set, this means that we detected that all 654 * CPUS uses emulated misaligned access at boot time. If that changed 655 * when hotplugging the new cpu, this is something we don't handle. 656 */ 657 if (unlikely(unaligned_ctl && (*mas_ptr != RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED))) { 658 pr_crit("CPU misaligned accesses non homogeneous (expected all emulated)\n"); 659 while (true) 660 cpu_relax(); 661 } 662 } 663 664 bool check_unaligned_access_emulated_all_cpus(void) 665 { 666 int cpu; 667 668 /* 669 * We can only support PR_UNALIGN controls if all CPUs have misaligned 670 * accesses emulated since tasks requesting such control can run on any 671 * CPU. 672 */ 673 schedule_on_each_cpu(check_unaligned_access_emulated); 674 675 for_each_online_cpu(cpu) 676 if (per_cpu(misaligned_access_speed, cpu) 677 != RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED) 678 return false; 679 680 unaligned_ctl = true; 681 return true; 682 } 683 684 bool unaligned_ctl_available(void) 685 { 686 return unaligned_ctl; 687 } 688 #else 689 bool check_unaligned_access_emulated_all_cpus(void) 690 { 691 return false; 692 } 693 #endif 694