1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2020 Western Digital Corporation or its affiliates. 4 */ 5 #include <linux/kernel.h> 6 #include <linux/init.h> 7 #include <linux/mm.h> 8 #include <linux/module.h> 9 #include <linux/perf_event.h> 10 #include <linux/irq.h> 11 #include <linux/stringify.h> 12 13 #include <asm/processor.h> 14 #include <asm/ptrace.h> 15 #include <asm/csr.h> 16 #include <asm/entry-common.h> 17 #include <asm/hwprobe.h> 18 #include <asm/cpufeature.h> 19 #include <asm/vector.h> 20 21 #define INSN_MATCH_LB 0x3 22 #define INSN_MASK_LB 0x707f 23 #define INSN_MATCH_LH 0x1003 24 #define INSN_MASK_LH 0x707f 25 #define INSN_MATCH_LW 0x2003 26 #define INSN_MASK_LW 0x707f 27 #define INSN_MATCH_LD 0x3003 28 #define INSN_MASK_LD 0x707f 29 #define INSN_MATCH_LBU 0x4003 30 #define INSN_MASK_LBU 0x707f 31 #define INSN_MATCH_LHU 0x5003 32 #define INSN_MASK_LHU 0x707f 33 #define INSN_MATCH_LWU 0x6003 34 #define INSN_MASK_LWU 0x707f 35 #define INSN_MATCH_SB 0x23 36 #define INSN_MASK_SB 0x707f 37 #define INSN_MATCH_SH 0x1023 38 #define INSN_MASK_SH 0x707f 39 #define INSN_MATCH_SW 0x2023 40 #define INSN_MASK_SW 0x707f 41 #define INSN_MATCH_SD 0x3023 42 #define INSN_MASK_SD 0x707f 43 44 #define INSN_MATCH_FLW 0x2007 45 #define INSN_MASK_FLW 0x707f 46 #define INSN_MATCH_FLD 0x3007 47 #define INSN_MASK_FLD 0x707f 48 #define INSN_MATCH_FLQ 0x4007 49 #define INSN_MASK_FLQ 0x707f 50 #define INSN_MATCH_FSW 0x2027 51 #define INSN_MASK_FSW 0x707f 52 #define INSN_MATCH_FSD 0x3027 53 #define INSN_MASK_FSD 0x707f 54 #define INSN_MATCH_FSQ 0x4027 55 #define INSN_MASK_FSQ 0x707f 56 57 #define INSN_MATCH_C_LD 0x6000 58 #define INSN_MASK_C_LD 0xe003 59 #define INSN_MATCH_C_SD 0xe000 60 #define INSN_MASK_C_SD 0xe003 61 #define INSN_MATCH_C_LW 0x4000 62 #define INSN_MASK_C_LW 0xe003 63 #define INSN_MATCH_C_SW 0xc000 64 #define INSN_MASK_C_SW 0xe003 65 #define INSN_MATCH_C_LDSP 0x6002 66 #define INSN_MASK_C_LDSP 0xe003 67 #define INSN_MATCH_C_SDSP 0xe002 68 #define INSN_MASK_C_SDSP 0xe003 69 #define INSN_MATCH_C_LWSP 0x4002 70 #define INSN_MASK_C_LWSP 0xe003 71 #define INSN_MATCH_C_SWSP 0xc002 72 #define INSN_MASK_C_SWSP 0xe003 73 74 #define INSN_MATCH_C_FLD 0x2000 75 #define INSN_MASK_C_FLD 0xe003 76 #define INSN_MATCH_C_FLW 0x6000 77 #define INSN_MASK_C_FLW 0xe003 78 #define INSN_MATCH_C_FSD 0xa000 79 #define INSN_MASK_C_FSD 0xe003 80 #define INSN_MATCH_C_FSW 0xe000 81 #define INSN_MASK_C_FSW 0xe003 82 #define INSN_MATCH_C_FLDSP 0x2002 83 #define INSN_MASK_C_FLDSP 0xe003 84 #define INSN_MATCH_C_FSDSP 0xa002 85 #define INSN_MASK_C_FSDSP 0xe003 86 #define INSN_MATCH_C_FLWSP 0x6002 87 #define INSN_MASK_C_FLWSP 0xe003 88 #define INSN_MATCH_C_FSWSP 0xe002 89 #define INSN_MASK_C_FSWSP 0xe003 90 91 #define INSN_LEN(insn) ((((insn) & 0x3) < 0x3) ? 2 : 4) 92 93 #if defined(CONFIG_64BIT) 94 #define LOG_REGBYTES 3 95 #define XLEN 64 96 #else 97 #define LOG_REGBYTES 2 98 #define XLEN 32 99 #endif 100 #define REGBYTES (1 << LOG_REGBYTES) 101 #define XLEN_MINUS_16 ((XLEN) - 16) 102 103 #define SH_RD 7 104 #define SH_RS1 15 105 #define SH_RS2 20 106 #define SH_RS2C 2 107 108 #define RV_X(x, s, n) (((x) >> (s)) & ((1 << (n)) - 1)) 109 #define RVC_LW_IMM(x) ((RV_X(x, 6, 1) << 2) | \ 110 (RV_X(x, 10, 3) << 3) | \ 111 (RV_X(x, 5, 1) << 6)) 112 #define RVC_LD_IMM(x) ((RV_X(x, 10, 3) << 3) | \ 113 (RV_X(x, 5, 2) << 6)) 114 #define RVC_LWSP_IMM(x) ((RV_X(x, 4, 3) << 2) | \ 115 (RV_X(x, 12, 1) << 5) | \ 116 (RV_X(x, 2, 2) << 6)) 117 #define RVC_LDSP_IMM(x) ((RV_X(x, 5, 2) << 3) | \ 118 (RV_X(x, 12, 1) << 5) | \ 119 (RV_X(x, 2, 3) << 6)) 120 #define RVC_SWSP_IMM(x) ((RV_X(x, 9, 4) << 2) | \ 121 (RV_X(x, 7, 2) << 6)) 122 #define RVC_SDSP_IMM(x) ((RV_X(x, 10, 3) << 3) | \ 123 (RV_X(x, 7, 3) << 6)) 124 #define RVC_RS1S(insn) (8 + RV_X(insn, SH_RD, 3)) 125 #define RVC_RS2S(insn) (8 + RV_X(insn, SH_RS2C, 3)) 126 #define RVC_RS2(insn) RV_X(insn, SH_RS2C, 5) 127 128 #define SHIFT_RIGHT(x, y) \ 129 ((y) < 0 ? ((x) << -(y)) : ((x) >> (y))) 130 131 #define REG_MASK \ 132 ((1 << (5 + LOG_REGBYTES)) - (1 << LOG_REGBYTES)) 133 134 #define REG_OFFSET(insn, pos) \ 135 (SHIFT_RIGHT((insn), (pos) - LOG_REGBYTES) & REG_MASK) 136 137 #define REG_PTR(insn, pos, regs) \ 138 (ulong *)((ulong)(regs) + REG_OFFSET(insn, pos)) 139 140 #define GET_RS1(insn, regs) (*REG_PTR(insn, SH_RS1, regs)) 141 #define GET_RS2(insn, regs) (*REG_PTR(insn, SH_RS2, regs)) 142 #define GET_RS1S(insn, regs) (*REG_PTR(RVC_RS1S(insn), 0, regs)) 143 #define GET_RS2S(insn, regs) (*REG_PTR(RVC_RS2S(insn), 0, regs)) 144 #define GET_RS2C(insn, regs) (*REG_PTR(insn, SH_RS2C, regs)) 145 #define GET_SP(regs) (*REG_PTR(2, 0, regs)) 146 #define SET_RD(insn, regs, val) (*REG_PTR(insn, SH_RD, regs) = (val)) 147 #define IMM_I(insn) ((s32)(insn) >> 20) 148 #define IMM_S(insn) (((s32)(insn) >> 25 << 5) | \ 149 (s32)(((insn) >> 7) & 0x1f)) 150 #define MASK_FUNCT3 0x7000 151 152 #define GET_PRECISION(insn) (((insn) >> 25) & 3) 153 #define GET_RM(insn) (((insn) >> 12) & 7) 154 #define PRECISION_S 0 155 #define PRECISION_D 1 156 157 #ifdef CONFIG_FPU 158 159 #define FP_GET_RD(insn) (insn >> 7 & 0x1F) 160 161 extern void put_f32_reg(unsigned long fp_reg, unsigned long value); 162 163 static int set_f32_rd(unsigned long insn, struct pt_regs *regs, 164 unsigned long val) 165 { 166 unsigned long fp_reg = FP_GET_RD(insn); 167 168 put_f32_reg(fp_reg, val); 169 regs->status |= SR_FS_DIRTY; 170 171 return 0; 172 } 173 174 extern void put_f64_reg(unsigned long fp_reg, unsigned long value); 175 176 static int set_f64_rd(unsigned long insn, struct pt_regs *regs, u64 val) 177 { 178 unsigned long fp_reg = FP_GET_RD(insn); 179 unsigned long value; 180 181 #if __riscv_xlen == 32 182 value = (unsigned long) &val; 183 #else 184 value = val; 185 #endif 186 put_f64_reg(fp_reg, value); 187 regs->status |= SR_FS_DIRTY; 188 189 return 0; 190 } 191 192 #if __riscv_xlen == 32 193 extern void get_f64_reg(unsigned long fp_reg, u64 *value); 194 195 static u64 get_f64_rs(unsigned long insn, u8 fp_reg_offset, 196 struct pt_regs *regs) 197 { 198 unsigned long fp_reg = (insn >> fp_reg_offset) & 0x1F; 199 u64 val; 200 201 get_f64_reg(fp_reg, &val); 202 regs->status |= SR_FS_DIRTY; 203 204 return val; 205 } 206 #else 207 208 extern unsigned long get_f64_reg(unsigned long fp_reg); 209 210 static unsigned long get_f64_rs(unsigned long insn, u8 fp_reg_offset, 211 struct pt_regs *regs) 212 { 213 unsigned long fp_reg = (insn >> fp_reg_offset) & 0x1F; 214 unsigned long val; 215 216 val = get_f64_reg(fp_reg); 217 regs->status |= SR_FS_DIRTY; 218 219 return val; 220 } 221 222 #endif 223 224 extern unsigned long get_f32_reg(unsigned long fp_reg); 225 226 static unsigned long get_f32_rs(unsigned long insn, u8 fp_reg_offset, 227 struct pt_regs *regs) 228 { 229 unsigned long fp_reg = (insn >> fp_reg_offset) & 0x1F; 230 unsigned long val; 231 232 val = get_f32_reg(fp_reg); 233 regs->status |= SR_FS_DIRTY; 234 235 return val; 236 } 237 238 #else /* CONFIG_FPU */ 239 static void set_f32_rd(unsigned long insn, struct pt_regs *regs, 240 unsigned long val) {} 241 242 static void set_f64_rd(unsigned long insn, struct pt_regs *regs, u64 val) {} 243 244 static unsigned long get_f64_rs(unsigned long insn, u8 fp_reg_offset, 245 struct pt_regs *regs) 246 { 247 return 0; 248 } 249 250 static unsigned long get_f32_rs(unsigned long insn, u8 fp_reg_offset, 251 struct pt_regs *regs) 252 { 253 return 0; 254 } 255 256 #endif 257 258 #define GET_F64_RS2(insn, regs) (get_f64_rs(insn, 20, regs)) 259 #define GET_F64_RS2C(insn, regs) (get_f64_rs(insn, 2, regs)) 260 #define GET_F64_RS2S(insn, regs) (get_f64_rs(RVC_RS2S(insn), 0, regs)) 261 262 #define GET_F32_RS2(insn, regs) (get_f32_rs(insn, 20, regs)) 263 #define GET_F32_RS2C(insn, regs) (get_f32_rs(insn, 2, regs)) 264 #define GET_F32_RS2S(insn, regs) (get_f32_rs(RVC_RS2S(insn), 0, regs)) 265 266 #define __read_insn(regs, insn, insn_addr, type) \ 267 ({ \ 268 int __ret; \ 269 \ 270 if (user_mode(regs)) { \ 271 __ret = __get_user(insn, (type __user *) insn_addr); \ 272 } else { \ 273 insn = *(type *)insn_addr; \ 274 __ret = 0; \ 275 } \ 276 \ 277 __ret; \ 278 }) 279 280 static inline int get_insn(struct pt_regs *regs, ulong epc, ulong *r_insn) 281 { 282 ulong insn = 0; 283 284 if (epc & 0x2) { 285 ulong tmp = 0; 286 287 if (__read_insn(regs, insn, epc, u16)) 288 return -EFAULT; 289 /* __get_user() uses regular "lw" which sign extend the loaded 290 * value make sure to clear higher order bits in case we "or" it 291 * below with the upper 16 bits half. 292 */ 293 insn &= GENMASK(15, 0); 294 if ((insn & __INSN_LENGTH_MASK) != __INSN_LENGTH_32) { 295 *r_insn = insn; 296 return 0; 297 } 298 epc += sizeof(u16); 299 if (__read_insn(regs, tmp, epc, u16)) 300 return -EFAULT; 301 *r_insn = (tmp << 16) | insn; 302 303 return 0; 304 } else { 305 if (__read_insn(regs, insn, epc, u32)) 306 return -EFAULT; 307 if ((insn & __INSN_LENGTH_MASK) == __INSN_LENGTH_32) { 308 *r_insn = insn; 309 return 0; 310 } 311 insn &= GENMASK(15, 0); 312 *r_insn = insn; 313 314 return 0; 315 } 316 } 317 318 union reg_data { 319 u8 data_bytes[8]; 320 ulong data_ulong; 321 u64 data_u64; 322 }; 323 324 /* sysctl hooks */ 325 int unaligned_enabled __read_mostly = 1; /* Enabled by default */ 326 327 #ifdef CONFIG_RISCV_VECTOR_MISALIGNED 328 static int handle_vector_misaligned_load(struct pt_regs *regs) 329 { 330 unsigned long epc = regs->epc; 331 unsigned long insn; 332 333 if (get_insn(regs, epc, &insn)) 334 return -1; 335 336 /* Only return 0 when in check_vector_unaligned_access_emulated */ 337 if (*this_cpu_ptr(&vector_misaligned_access) == RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN) { 338 *this_cpu_ptr(&vector_misaligned_access) = RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED; 339 regs->epc = epc + INSN_LEN(insn); 340 return 0; 341 } 342 343 /* If vector instruction we don't emulate it yet */ 344 regs->epc = epc; 345 return -1; 346 } 347 #else 348 static int handle_vector_misaligned_load(struct pt_regs *regs) 349 { 350 return -1; 351 } 352 #endif 353 354 static int handle_scalar_misaligned_load(struct pt_regs *regs) 355 { 356 union reg_data val; 357 unsigned long epc = regs->epc; 358 unsigned long insn; 359 unsigned long addr = regs->badaddr; 360 int fp = 0, shift = 0, len = 0; 361 362 perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, addr); 363 364 #ifdef CONFIG_RISCV_PROBE_UNALIGNED_ACCESS 365 *this_cpu_ptr(&misaligned_access_speed) = RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED; 366 #endif 367 368 if (!unaligned_enabled) 369 return -1; 370 371 if (user_mode(regs) && (current->thread.align_ctl & PR_UNALIGN_SIGBUS)) 372 return -1; 373 374 if (get_insn(regs, epc, &insn)) 375 return -1; 376 377 regs->epc = 0; 378 379 if ((insn & INSN_MASK_LW) == INSN_MATCH_LW) { 380 len = 4; 381 shift = 8 * (sizeof(unsigned long) - len); 382 #if defined(CONFIG_64BIT) 383 } else if ((insn & INSN_MASK_LD) == INSN_MATCH_LD) { 384 len = 8; 385 shift = 8 * (sizeof(unsigned long) - len); 386 } else if ((insn & INSN_MASK_LWU) == INSN_MATCH_LWU) { 387 len = 4; 388 #endif 389 } else if ((insn & INSN_MASK_FLD) == INSN_MATCH_FLD) { 390 fp = 1; 391 len = 8; 392 } else if ((insn & INSN_MASK_FLW) == INSN_MATCH_FLW) { 393 fp = 1; 394 len = 4; 395 } else if ((insn & INSN_MASK_LH) == INSN_MATCH_LH) { 396 len = 2; 397 shift = 8 * (sizeof(unsigned long) - len); 398 } else if ((insn & INSN_MASK_LHU) == INSN_MATCH_LHU) { 399 len = 2; 400 #if defined(CONFIG_64BIT) 401 } else if ((insn & INSN_MASK_C_LD) == INSN_MATCH_C_LD) { 402 len = 8; 403 shift = 8 * (sizeof(unsigned long) - len); 404 insn = RVC_RS2S(insn) << SH_RD; 405 } else if ((insn & INSN_MASK_C_LDSP) == INSN_MATCH_C_LDSP && 406 ((insn >> SH_RD) & 0x1f)) { 407 len = 8; 408 shift = 8 * (sizeof(unsigned long) - len); 409 #endif 410 } else if ((insn & INSN_MASK_C_LW) == INSN_MATCH_C_LW) { 411 len = 4; 412 shift = 8 * (sizeof(unsigned long) - len); 413 insn = RVC_RS2S(insn) << SH_RD; 414 } else if ((insn & INSN_MASK_C_LWSP) == INSN_MATCH_C_LWSP && 415 ((insn >> SH_RD) & 0x1f)) { 416 len = 4; 417 shift = 8 * (sizeof(unsigned long) - len); 418 } else if ((insn & INSN_MASK_C_FLD) == INSN_MATCH_C_FLD) { 419 fp = 1; 420 len = 8; 421 insn = RVC_RS2S(insn) << SH_RD; 422 } else if ((insn & INSN_MASK_C_FLDSP) == INSN_MATCH_C_FLDSP) { 423 fp = 1; 424 len = 8; 425 #if defined(CONFIG_32BIT) 426 } else if ((insn & INSN_MASK_C_FLW) == INSN_MATCH_C_FLW) { 427 fp = 1; 428 len = 4; 429 insn = RVC_RS2S(insn) << SH_RD; 430 } else if ((insn & INSN_MASK_C_FLWSP) == INSN_MATCH_C_FLWSP) { 431 fp = 1; 432 len = 4; 433 #endif 434 } else { 435 regs->epc = epc; 436 return -1; 437 } 438 439 if (!IS_ENABLED(CONFIG_FPU) && fp) 440 return -EOPNOTSUPP; 441 442 val.data_u64 = 0; 443 if (user_mode(regs)) { 444 if (copy_from_user(&val, (u8 __user *)addr, len)) 445 return -1; 446 } else { 447 memcpy(&val, (u8 *)addr, len); 448 } 449 450 if (!fp) 451 SET_RD(insn, regs, val.data_ulong << shift >> shift); 452 else if (len == 8) 453 set_f64_rd(insn, regs, val.data_u64); 454 else 455 set_f32_rd(insn, regs, val.data_ulong); 456 457 regs->epc = epc + INSN_LEN(insn); 458 459 return 0; 460 } 461 462 static int handle_scalar_misaligned_store(struct pt_regs *regs) 463 { 464 union reg_data val; 465 unsigned long epc = regs->epc; 466 unsigned long insn; 467 unsigned long addr = regs->badaddr; 468 int len = 0, fp = 0; 469 470 perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, addr); 471 472 if (!unaligned_enabled) 473 return -1; 474 475 if (user_mode(regs) && (current->thread.align_ctl & PR_UNALIGN_SIGBUS)) 476 return -1; 477 478 if (get_insn(regs, epc, &insn)) 479 return -1; 480 481 regs->epc = 0; 482 483 val.data_ulong = GET_RS2(insn, regs); 484 485 if ((insn & INSN_MASK_SW) == INSN_MATCH_SW) { 486 len = 4; 487 #if defined(CONFIG_64BIT) 488 } else if ((insn & INSN_MASK_SD) == INSN_MATCH_SD) { 489 len = 8; 490 #endif 491 } else if ((insn & INSN_MASK_FSD) == INSN_MATCH_FSD) { 492 fp = 1; 493 len = 8; 494 val.data_u64 = GET_F64_RS2(insn, regs); 495 } else if ((insn & INSN_MASK_FSW) == INSN_MATCH_FSW) { 496 fp = 1; 497 len = 4; 498 val.data_ulong = GET_F32_RS2(insn, regs); 499 } else if ((insn & INSN_MASK_SH) == INSN_MATCH_SH) { 500 len = 2; 501 #if defined(CONFIG_64BIT) 502 } else if ((insn & INSN_MASK_C_SD) == INSN_MATCH_C_SD) { 503 len = 8; 504 val.data_ulong = GET_RS2S(insn, regs); 505 } else if ((insn & INSN_MASK_C_SDSP) == INSN_MATCH_C_SDSP) { 506 len = 8; 507 val.data_ulong = GET_RS2C(insn, regs); 508 #endif 509 } else if ((insn & INSN_MASK_C_SW) == INSN_MATCH_C_SW) { 510 len = 4; 511 val.data_ulong = GET_RS2S(insn, regs); 512 } else if ((insn & INSN_MASK_C_SWSP) == INSN_MATCH_C_SWSP) { 513 len = 4; 514 val.data_ulong = GET_RS2C(insn, regs); 515 } else if ((insn & INSN_MASK_C_FSD) == INSN_MATCH_C_FSD) { 516 fp = 1; 517 len = 8; 518 val.data_u64 = GET_F64_RS2S(insn, regs); 519 } else if ((insn & INSN_MASK_C_FSDSP) == INSN_MATCH_C_FSDSP) { 520 fp = 1; 521 len = 8; 522 val.data_u64 = GET_F64_RS2C(insn, regs); 523 #if !defined(CONFIG_64BIT) 524 } else if ((insn & INSN_MASK_C_FSW) == INSN_MATCH_C_FSW) { 525 fp = 1; 526 len = 4; 527 val.data_ulong = GET_F32_RS2S(insn, regs); 528 } else if ((insn & INSN_MASK_C_FSWSP) == INSN_MATCH_C_FSWSP) { 529 fp = 1; 530 len = 4; 531 val.data_ulong = GET_F32_RS2C(insn, regs); 532 #endif 533 } else { 534 regs->epc = epc; 535 return -1; 536 } 537 538 if (!IS_ENABLED(CONFIG_FPU) && fp) 539 return -EOPNOTSUPP; 540 541 if (user_mode(regs)) { 542 if (copy_to_user((u8 __user *)addr, &val, len)) 543 return -1; 544 } else { 545 memcpy((u8 *)addr, &val, len); 546 } 547 548 regs->epc = epc + INSN_LEN(insn); 549 550 return 0; 551 } 552 553 int handle_misaligned_load(struct pt_regs *regs) 554 { 555 unsigned long epc = regs->epc; 556 unsigned long insn; 557 558 if (IS_ENABLED(CONFIG_RISCV_VECTOR_MISALIGNED)) { 559 if (get_insn(regs, epc, &insn)) 560 return -1; 561 562 if (insn_is_vector(insn)) 563 return handle_vector_misaligned_load(regs); 564 } 565 566 if (IS_ENABLED(CONFIG_RISCV_SCALAR_MISALIGNED)) 567 return handle_scalar_misaligned_load(regs); 568 569 return -1; 570 } 571 572 int handle_misaligned_store(struct pt_regs *regs) 573 { 574 if (IS_ENABLED(CONFIG_RISCV_SCALAR_MISALIGNED)) 575 return handle_scalar_misaligned_store(regs); 576 577 return -1; 578 } 579 580 #ifdef CONFIG_RISCV_VECTOR_MISALIGNED 581 void check_vector_unaligned_access_emulated(struct work_struct *work __always_unused) 582 { 583 long *mas_ptr = this_cpu_ptr(&vector_misaligned_access); 584 unsigned long tmp_var; 585 586 *mas_ptr = RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN; 587 588 kernel_vector_begin(); 589 /* 590 * In pre-13.0.0 versions of GCC, vector registers cannot appear in 591 * the clobber list. This inline asm clobbers v0, but since we do not 592 * currently build the kernel with V enabled, the v0 clobber arg is not 593 * needed (as the compiler will not emit vector code itself). If the kernel 594 * is changed to build with V enabled, the clobber arg will need to be 595 * added here. 596 */ 597 __asm__ __volatile__ ( 598 ".balign 4\n\t" 599 ".option push\n\t" 600 ".option arch, +zve32x\n\t" 601 " vsetivli zero, 1, e16, m1, ta, ma\n\t" // Vectors of 16b 602 " vle16.v v0, (%[ptr])\n\t" // Load bytes 603 ".option pop\n\t" 604 : : [ptr] "r" ((u8 *)&tmp_var + 1)); 605 kernel_vector_end(); 606 } 607 608 bool check_vector_unaligned_access_emulated_all_cpus(void) 609 { 610 int cpu; 611 612 if (!has_vector()) { 613 for_each_online_cpu(cpu) 614 per_cpu(vector_misaligned_access, cpu) = RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED; 615 return false; 616 } 617 618 schedule_on_each_cpu(check_vector_unaligned_access_emulated); 619 620 for_each_online_cpu(cpu) 621 if (per_cpu(vector_misaligned_access, cpu) 622 == RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN) 623 return false; 624 625 return true; 626 } 627 #else 628 bool check_vector_unaligned_access_emulated_all_cpus(void) 629 { 630 return false; 631 } 632 #endif 633 634 #ifdef CONFIG_RISCV_SCALAR_MISALIGNED 635 636 static bool unaligned_ctl __read_mostly; 637 638 void check_unaligned_access_emulated(struct work_struct *work __always_unused) 639 { 640 int cpu = smp_processor_id(); 641 long *mas_ptr = per_cpu_ptr(&misaligned_access_speed, cpu); 642 unsigned long tmp_var, tmp_val; 643 644 *mas_ptr = RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN; 645 646 __asm__ __volatile__ ( 647 " "REG_L" %[tmp], 1(%[ptr])\n" 648 : [tmp] "=r" (tmp_val) : [ptr] "r" (&tmp_var) : "memory"); 649 650 /* 651 * If unaligned_ctl is already set, this means that we detected that all 652 * CPUS uses emulated misaligned access at boot time. If that changed 653 * when hotplugging the new cpu, this is something we don't handle. 654 */ 655 if (unlikely(unaligned_ctl && (*mas_ptr != RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED))) { 656 pr_crit("CPU misaligned accesses non homogeneous (expected all emulated)\n"); 657 while (true) 658 cpu_relax(); 659 } 660 } 661 662 bool check_unaligned_access_emulated_all_cpus(void) 663 { 664 int cpu; 665 666 /* 667 * We can only support PR_UNALIGN controls if all CPUs have misaligned 668 * accesses emulated since tasks requesting such control can run on any 669 * CPU. 670 */ 671 schedule_on_each_cpu(check_unaligned_access_emulated); 672 673 for_each_online_cpu(cpu) 674 if (per_cpu(misaligned_access_speed, cpu) 675 != RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED) 676 return false; 677 678 unaligned_ctl = true; 679 return true; 680 } 681 682 bool unaligned_ctl_available(void) 683 { 684 return unaligned_ctl; 685 } 686 #else 687 bool check_unaligned_access_emulated_all_cpus(void) 688 { 689 return false; 690 } 691 #endif 692